Merge pull request #9789 from yousong/automated-cherry-pick-of-#9785-upstream-release-3.4

Automated cherry pick of #9785: guest_deploy_task: log other deploy details with i18n

cmd/climc/shell/compute/hosts.go

@@ -407,12 +407,14 @@ func init() {
 	})
 
 	type HostAddNetIfOptions struct {
-		ID     string `help:"ID or Name of host"`
-		WIRE   string `help:"ID or Name of wire to attach"`
-		MAC    string `help:"Mac address of NIC"`
-		INDEX  int64  `help:"nic index"`
-		Type   string `help:"Nic type" choices:"admin|ipmi"`
-		IpAddr string `help:"IP address"`
+		ID        string `help:"ID or Name of host"`
+		WIRE      string `help:"ID or Name of wire to attach"`
+		MAC       string `help:"Mac address of NIC"`
+		INDEX     int64  `help:"nic index"`
+		Type      string `help:"Nic type" choices:"admin|ipmi"`
+		IpAddr    string `help:"IP address"`
+		Bridge    string `help:"Bridge of hostwire"`
+		Interface string `help:"Interface name, eg:eth0, en0"`
 	}
 	R(&HostAddNetIfOptions{}, "host-add-netif", "Host add a NIC", func(s *mcclient.ClientSession, args *HostAddNetIfOptions) error {
 		params := jsonutils.NewDict()
@@ -426,6 +428,12 @@ func init() {
 		if len(args.IpAddr) > 0 {
 			params.Add(jsonutils.NewString(args.IpAddr), "ip_addr")
 		}
+		if len(args.Bridge) > 0 {
+			params.Add(jsonutils.NewString(args.Bridge), "bridge")
+		}
+		if len(args.Interface) > 0 {
+			params.Add(jsonutils.NewString(args.Interface), "interface")
+		}
 		result, err := modules.Hosts.PerformAction(s, args.ID, "add-netif", params)
 		if err != nil {
 			return err

pkg/apis/input.go

@@ -201,6 +201,8 @@ type PerformStatusInput struct {
 	// 更改的目标状态值
 	// required:true
 	Status string `json:"status"`
+	// swagger:ignore
+	BlockJobsCount int `json:"block_jobs_count"`
 
 	// 更改状态的原因描述
 	// required:false

pkg/cloudcommon/db/opslog_const.go

@@ -47,8 +47,9 @@ const (
 	ACT_BACKUP_START        = "backup_start"
 	ACT_BACKUP_START_FAILED = "backup_start_fail"
 
-	ACT_FREEZE   = "freeze"
-	ACT_UNFREEZE = "unfreeze"
+	ACT_FREEZE      = "freeze"
+	ACT_FREEZE_FAIL = "freeze_fail"
+	ACT_UNFREEZE    = "unfreeze"
 
 	ACT_RESTARING    = "restarting"
 	ACT_RESTART_FAIL = "restart_fail"

pkg/cloudcommon/db/rbac.go

@@ -51,6 +51,8 @@ func isObjectRbacAllowed(model IModel, userCred mcclient.TokenCredential, action
 	case rbacutils.ScopeUser:
 		if ownerId != nil && objOwnerId != nil && (ownerId.GetUserId() == objOwnerId.GetUserId() || objOwnerId.GetUserId() == "" || (model.IsSharable(ownerId) && action == policy.PolicyActionGet)) {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && objOwnerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}
@@ -103,6 +105,8 @@ func isClassRbacAllowed(manager IModelManager, userCred mcclient.TokenCredential
 	case rbacutils.ScopeUser:
 		if ownerId != nil && ownerId.GetUserId() == objOwnerId.GetUserId() {
 			requireScope = rbacutils.ScopeUser
+		} else if ownerId != nil && ownerId.GetProjectDomainId() == objOwnerId.GetProjectDomainId() {
+			requireScope = rbacutils.ScopeDomain
 		} else {
 			requireScope = rbacutils.ScopeSystem
 		}

pkg/compute/models/guest_actions.go

@@ -788,6 +788,16 @@ func (self *SGuest) AllowPerformStart(ctx context.Context,
 func (self *SGuest) PerformStart(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject,
 	data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	if utils.IsInStringArray(self.Status, []string{api.VM_READY, api.VM_START_FAILED, api.VM_SAVE_DISK_FAILED, api.VM_SUSPEND}) {
+		if !self.guestDisksStorageTypeIsShared() {
+			host := self.GetHost()
+			guestsMem, err := host.GetNotReadyGuestsMemorySize()
+			if err != nil {
+				return nil, err
+			}
+			if float32(guestsMem+self.VmemSize) > host.GetVirtualMemorySize() {
+				return nil, httperrors.NewInsufficientResourceError("host virtual memory not enough")
+			}
+		}
 		if self.isAllDisksReady() {
 			var kwargs *jsonutils.JSONDict
 			if data != nil {
@@ -2745,7 +2755,7 @@ func (self *SGuest) PerformStatus(ctx context.Context, userCred mcclient.TokenCr
 	}
 
 	status := input.Status
-	if len(self.BackupHostId) > 0 && status == api.VM_RUNNING {
+	if len(self.BackupHostId) > 0 && status == api.VM_RUNNING && input.BlockJobsCount > 0 {
 		if len(self.GetMetadata("__mirror_job_status", userCred)) == 0 {
 			self.SetMetadata(ctx, "__mirror_job_status", "ready", userCred)
 		}
@@ -2786,6 +2796,27 @@ func (self *SGuest) PerformStop(ctx context.Context, userCred mcclient.TokenCred
 	}
 }
 
+func (self *SGuest) PerformFreeze(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformFreezeInput) (jsonutils.JSONObject, error) {
+	if self.Freezed {
+		return nil, httperrors.NewBadRequestError("virtual resource already freezed")
+	}
+	if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_STOP_FAILED}) {
+		return nil, self.StartGuestStopAndFreezeTask(ctx, userCred)
+	} else {
+		return self.SVirtualResourceBase.PerformFreeze(ctx, userCred, query, input)
+	}
+}
+
+func (self *SGuest) StartGuestStopAndFreezeTask(ctx context.Context, userCred mcclient.TokenCredential) error {
+	self.SetStatus(userCred, api.VM_START_STOP, "")
+	task, err := taskman.TaskManager.NewTask(ctx, "GuestStopAndFreezeTask", self, userCred, nil, "", "", nil)
+	if err != nil {
+		return err
+	}
+	task.ScheduleRun(nil)
+	return nil
+}
+
 func (self *SGuest) AllowPerformRestart(ctx context.Context,
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,

pkg/compute/models/guests.go

@@ -4654,7 +4654,7 @@ func (self *SGuestManager) switchBackupGuests(ctx context.Context, userCred mccl
 		log.Errorf("ReconcileBackupGuests failed fetch guests %s", err)
 		return
 	}
-	log.Infof("Guests count %d need reconcile with switch bakcup", len(guests))
+	log.Debugf("Guests count %d need reconcile with switch backup", len(guests))
 	for i := 0; i < len(guests); i++ {
 		val := guests[i].GetMetadataJson("switch_backup", userCred)
 		t, err := val.GetTime()

pkg/compute/models/hosts.go

@@ -1490,6 +1490,23 @@ func (self *SHost) GetRunningGuestCount() (int, error) {
 	return q.CountWithError()
 }
 
+func (self *SHost) GetNotReadyGuestsMemorySize() (int, error) {
+	guests := GuestManager.Query().SubQuery()
+	q := guests.Query(sqlchemy.COUNT("guest_count"),
+		sqlchemy.SUM("guest_vcpu_count", guests.Field("vcpu_count")),
+		sqlchemy.SUM("guest_vmem_size", guests.Field("vmem_size")))
+	cond := sqlchemy.OR(sqlchemy.Equals(q.Field("host_id"), self.Id),
+		sqlchemy.Equals(q.Field("backup_host_id"), self.Id))
+	q = q.Filter(cond)
+	q = q.NotEquals("status", api.VM_READY)
+	stat := SHostGuestResourceUsage{}
+	err := q.First(&stat)
+	if err != nil {
+		return -1, err
+	}
+	return stat.GuestVmemSize, nil
+}
+
 func (self *SHost) GetRunningGuestMemorySize() int {
 	res := self.getGuestsResource(api.VM_RUNNING)
 	if res != nil {
@@ -3085,7 +3102,7 @@ func (self *SHost) PostCreate(
 		}
 	}
 
-	keys := GetHostQuotaKeysFromCreateInput(input)
+	keys := GetHostQuotaKeysFromCreateInput(ownerId, input)
 	quota := SInfrasQuota{Host: 1}
 	quota.SetKeys(keys)
 	err = quotas.CancelPendingUsage(ctx, userCred, &quota, &quota, true)
@@ -3379,7 +3396,7 @@ func (manager *SHostManager) ValidateCreateData(
 		return input, errors.Wrap(err, "SEnabledStatusInfrasResourceBaseManager.ValidateCreateData")
 	}
 
-	keys := GetHostQuotaKeysFromCreateInput(input)
+	keys := GetHostQuotaKeysFromCreateInput(ownerId, input)
 	quota := SInfrasQuota{Host: 1}
 	quota.SetKeys(keys)
 	err = quotas.CheckSetPendingQuota(ctx, userCred, &quota)
@@ -5612,8 +5629,8 @@ func (host *SHost) GetChangeOwnerRequiredDomainIds() []string {
 	return requires
 }
 
-func GetHostQuotaKeysFromCreateInput(input api.HostCreateInput) quotas.SDomainRegionalCloudResourceKeys {
-	ownerId := &db.SOwnerId{DomainId: input.ProjectDomainId}
+func GetHostQuotaKeysFromCreateInput(owner mcclient.IIdentityProvider, input api.HostCreateInput) quotas.SDomainRegionalCloudResourceKeys {
+	ownerId := &db.SOwnerId{DomainId: owner.GetProjectDomainId()}
 	var zone *SZone
 	if len(input.ZoneId) > 0 {
 		zone = ZoneManager.FetchZoneById(input.ZoneId)

pkg/compute/models/vpcs.go

@@ -774,7 +774,7 @@ func (manager *SVpcManager) ValidateCreateData(
 		return input, httperrors.NewMissingParameterError("cidr")
 	}
 
-	keys := GetVpcQuotaKeysFromCreateInput(input)
+	keys := GetVpcQuotaKeysFromCreateInput(ownerId, input)
 	quota := &SInfrasQuota{Vpc: 1}
 	quota.SetKeys(keys)
 	err = quotas.CheckSetPendingQuota(ctx, userCred, quota)
@@ -792,7 +792,7 @@ func (self *SVpc) PostCreate(ctx context.Context, userCred mcclient.TokenCredent
 		log.Errorf("input unmarshal error %s", err)
 	} else {
 		pendingUsage := &SInfrasQuota{Vpc: 1}
-		keys := GetVpcQuotaKeysFromCreateInput(input)
+		keys := GetVpcQuotaKeysFromCreateInput(ownerId, input)
 		pendingUsage.SetKeys(keys)
 		quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, true)
 	}
@@ -1171,8 +1171,8 @@ func (self *SVpc) initWire(ctx context.Context, zone *SZone) (*SWire, error) {
 	return wire, nil
 }
 
-func GetVpcQuotaKeysFromCreateInput(input api.VpcCreateInput) quotas.SDomainRegionalCloudResourceKeys {
-	ownerId := &db.SOwnerId{DomainId: input.ProjectDomainId}
+func GetVpcQuotaKeysFromCreateInput(owner mcclient.IIdentityProvider, input api.VpcCreateInput) quotas.SDomainRegionalCloudResourceKeys {
+	ownerId := &db.SOwnerId{DomainId: owner.GetProjectDomainId()}
 	var region *SCloudregion
 	if len(input.CloudregionId) > 0 {
 		region = CloudregionManager.FetchRegionById(input.CloudregionId)

pkg/compute/tasks/guest_deploy_task.go

@@ -106,7 +106,7 @@ func (self *GuestDeployTask) OnDeployGuestComplete(ctx context.Context, obj db.I
 	}
 	if !_log {
 		// 如果 deploy 有其他事件，统一记在这里。
-		logclient.AddActionLogWithStartable(self, guest, "misc部署", "", self.UserCred, true)
+		logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_DEPLOY, action, self.UserCred, true)
 	}
 
 	if self.HasParentTask() {

pkg/compute/tasks/guest_stop_task.go

@@ -20,6 +20,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
+	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
@@ -33,6 +34,7 @@ type GuestStopTask struct {
 
 func init() {
 	taskman.RegisterTask(GuestStopTask{})
+	taskman.RegisterTask(GuestStopAndFreezeTask{})
 }
 
 func (self *GuestStopTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
@@ -80,3 +82,40 @@ func (self *GuestStopTask) OnGuestStopTaskCompleteFailed(ctx context.Context, gu
 	self.SetStageFailed(ctx, reason)
 	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_VM_STOP, reason.String(), self.UserCred, false)
 }
+
+type GuestStopAndFreezeTask struct {
+	SGuestBaseTask
+}
+
+func (self *GuestStopAndFreezeTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	self.SetStage("OnStopGuest", nil)
+	err := guest.StartGuestStopTask(ctx, self.UserCred, false, self.GetTaskId())
+	if err != nil {
+		self.OnStopGuestFailed(ctx, guest, jsonutils.NewString(err.Error()))
+	}
+}
+
+func (self *GuestStopAndFreezeTask) OnStopGuestFailed(ctx context.Context, guest *models.SGuest, reason jsonutils.JSONObject) {
+	db.OpsLog.LogEvent(guest, db.ACT_FREEZE_FAIL, reason.String(), self.UserCred)
+	self.SetStageFailed(ctx, reason)
+}
+
+func (self *GuestStopAndFreezeTask) OnStopGuest(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	self.SetStage("OnSyncStatus", nil)
+	guest.StartSyncstatus(ctx, self.UserCred, self.GetTaskId())
+}
+
+func (self *GuestStopAndFreezeTask) OnSyncStatus(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
+	_, err := guest.SVirtualResourceBase.PerformFreeze(ctx, self.UserCred, nil, apis.PerformFreezeInput{})
+	if err != nil {
+		self.OnStopGuestFailed(ctx, guest, jsonutils.NewString(err.Error()))
+		return
+	}
+	self.SetStageComplete(ctx, nil)
+}
+
+func (self *GuestStopAndFreezeTask) OnSyncStatusFailed(ctx context.Context, guest *models.SGuest, reason jsonutils.JSONObject) {
+	db.OpsLog.LogEvent(guest, db.ACT_FREEZE_FAIL, reason.String(), self.UserCred)
+	self.SetStageFailed(ctx, reason)
+}

pkg/compute/tasks/guest_syncstatus_task.go

@@ -68,8 +68,13 @@ func (self *GuestSyncstatusTask) OnGetStatusSucc(ctx context.Context, guest *mod
 	default:
 		statusStr = api.VM_UNKNOWN
 	}
+	blockJobsCount, err := body.Int("block_jobs_count")
+	if err != nil {
+		blockJobsCount = -1
+	}
 	input := apis.PerformStatusInput{
-		Status: statusStr,
+		Status:         statusStr,
+		BlockJobsCount: int(blockJobsCount),
 	}
 	guest.PerformStatus(ctx, self.UserCred, nil, input)
 	self.SetStageComplete(ctx, nil)

pkg/hostman/guestman/guesthandlers/guesthandler.go

@@ -106,8 +106,12 @@ func guestActions(f actionFunc) appsrv.FilterHandler {
 
 func getStatus(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	params, _, _ := appsrv.FetchEnv(ctx, w, r)
-	var status = guestman.GetGuestManager().Status(params["<sid>"])
-	appsrv.SendStruct(w, strDict{"status": status})
+	var status, blockJobsCount = guestman.GetGuestManager().StatusWithBlockJobsCount(params["<sid>"])
+	res := map[string]interface{}{
+		"status":           status,
+		"block_jobs_count": blockJobsCount,
+	}
+	appsrv.SendStruct(w, res)
 }
 
 func cpusetBalance(ctx context.Context, w http.ResponseWriter, r *http.Request) {

pkg/hostman/guestman/guestman.go

@@ -475,6 +475,19 @@ func (m *SGuestManager) Status(sid string) string {
 	return status
 }
 
+func (m *SGuestManager) StatusWithBlockJobsCount(sid string) (string, int) {
+	status := m.GetStatus(sid)
+	blockJobsCount := 0
+	if status == GUEST_RUNNING {
+		guest, _ := m.GetServer(sid)
+		if guest.Monitor == nil && !guest.IsStopping() {
+			guest.StartMonitor(context.Background())
+		}
+		blockJobsCount = guest.BlockJobsCount()
+	}
+	return status, blockJobsCount
+}
+
 func (m *SGuestManager) GetStatus(sid string) string {
 	if guest, ok := m.GetServer(sid); ok {
 		if guest.IsRunning() && guest.Monitor != nil && guest.IsMaster() {

pkg/multicloud/esxi/host.go

@@ -969,6 +969,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData
 					Device:    dev,
 				})
 			}
+			nicIndex += 1
 		}
 	}
 

pkg/notify/models/receiver.go

@@ -531,6 +531,8 @@ func (rm *SReceiverManager) FetchOwnerId(ctx context.Context, data jsonutils.JSO
 			return nil, errors.Wrap(err, "UserCacheManager.FetchUserByIdOrName")
 		}
 		ownerId := db.SOwnerId{
+			DomainId:     u.DomainId,
+			Domain:       u.Domain,
 			UserDomain:   u.Domain,
 			UserDomainId: u.DomainId,
 			UserId:       u.Id,

pkg/util/fileutils2/fileutils.go

@@ -178,6 +178,7 @@ func FileGetContents(file string) (string, error) {
 func GetFsFormat(diskPath string) string {
 	ret, err := procutils.NewCommand("blkid", "-o", "value", "-s", "TYPE", diskPath).Output()
 	if err != nil {
+		log.Errorf("failed exec blkid of dev %s: %s, %s", diskPath, err, ret)
 		return ""
 	}
 	var res string