fix(region): optimized guest list speed (#20725)

pkg/apis/compute/guests.go

@@ -191,18 +191,11 @@ type ServerDetails struct {
 	// 系统管理员可见的安全组规则
 	AdminSecurityRules string `json:"admin_security_rules"`
 
-	// list
-	AttachTime time.Time `json:"attach_time"`
-
 	// common
 	IsPrepaidRecycle bool `json:"is_prepaid_recycle"`
 
-	// 备份主机所在宿主机名称
-	BackupHostName string `json:"backup_host_name"`
-	// 备份主机所在宿主机状态
-	BackupHostStatus string `json:"backup_host_status"`
-	// 主备机同步状态
-	BackupGuestSyncStatus string `json:"backup_guest_sync_status"`
+	// 主备机信息
+	BackupInfo
 
 	// 是否可以回收
 	CanRecycle bool `json:"can_recycle"`
@@ -279,6 +272,15 @@ type ServerDetails struct {
 	Containers []*PodContainerDesc `json:"containers"`
 }
 
+type BackupInfo struct {
+	// 备份主机所在宿主机名称
+	BackupHostName string `json:"backup_host_name"`
+	// 备份主机所在宿主机状态
+	BackupHostStatus string `json:"backup_host_status"`
+	// 主备机同步状态
+	BackupGuestSyncStatus string `json:"backup_guest_sync_status"`
+}
+
 type PodContainerDesc struct {
 	Id    string `json:"id"`
 	Name  string `json:"name"`

pkg/apis/compute/host.go

@@ -291,6 +291,11 @@ type HostResourceInfo struct {
 	// 宿主机状态
 	HostStatus string `json:"host_status"`
 
+	HostResourceType string `json:"host_resource_type"`
+
+	// 宿主机计费类型
+	HostBillingType string `json:"host_billing_type"`
+
 	// 宿主机服务状态`
 	HostServiceStatus string `json:"host_service_status"`
 

pkg/compute/models/guest_queries.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"strings"
+	"time"
 
 	"yunion.io/x/cloudmux/pkg/cloudprovider"
 	"yunion.io/x/jsonutils"
@@ -28,9 +29,11 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	billing_api "yunion.io/x/onecloud/pkg/apis/billing"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
@@ -50,6 +53,7 @@ func (manager *SGuestManager) FetchCustomizeColumns(
 	encRows := manager.SEncryptedResourceManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	guestIds := make([]string, len(objs))
 	guests := make([]SGuest, len(objs))
+	backupHostIds := make([]string, len(objs))
 	for i := range objs {
 		rows[i] = api.ServerDetails{
 			VirtualResourceDetails: virtRows[i],
@@ -59,6 +63,7 @@ func (manager *SGuestManager) FetchCustomizeColumns(
 		}
 		guest := objs[i].(*SGuest)
 		guestIds[i] = guest.GetId()
+		backupHostIds[i] = guest.BackupHostId
 		guests[i] = *guest
 	}
 
@@ -282,12 +287,75 @@ func (manager *SGuestManager) FetchCustomizeColumns(
 		}
 	}
 
+	if len(fields) == 0 || fields.Contains("backup_host_name") || fields.Contains("backup_host_status") && len(backupHostIds) > 0 {
+		backups, _ := fetchGuestBackupInfo(backupHostIds)
+		meta := []db.SMetadata{}
+		db.Metadata.Query().In("obj_id", guestIds).Equals("obj_type", manager.Keyword()).Equals("key", api.MIRROR_JOB).All(&meta)
+		syncStatus := map[string]string{}
+		for _, v := range meta {
+			syncStatus[v.ObjId] = v.Value
+		}
+		if len(backups) > 0 || len(syncStatus) > 0 {
+			for i := range rows {
+				rows[i].BackupInfo, _ = backups[backupHostIds[i]]
+				rows[i].BackupGuestSyncStatus, _ = syncStatus[guestIds[i]]
+			}
+		}
+	}
+
+	if len(fields) == 0 || fields.Contains("container") {
+		containers, _ := fetchContainers(guestIds)
+		if len(containers) > 0 {
+			for i := range rows {
+				rows[i].Containers, _ = containers[guestIds[i]]
+			}
+		}
+	}
+
 	for i := range rows {
-		rows[i] = guests[i].moreExtraInfo(ctx, rows[i], userCred, query, fields, isList)
+		if len(fields) == 0 || fields.Contains("auto_delete_at") {
+			if guests[i].PendingDeleted {
+				pendingDeletedAt := guests[i].PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds))
+				rows[i].AutoDeleteAt = pendingDeletedAt
+			}
+		}
+		if len(fields) == 0 || fields.Contains("can_recycle") {
+			if guests[i].BillingType == billing_api.BILLING_TYPE_PREPAID && !guests[i].ExpiredAt.Before(time.Now()) && len(rows[i].ManagerId) > 0 {
+				rows[i].CanRecycle = true
+			}
+		}
+
+		rows[i].IsPrepaidRecycle = (rows[i].HostResourceType == api.HostResourceTypePrepaidRecycle && rows[i].HostBillingType == billing_api.BILLING_TYPE_PREPAID)
+
+		drv, _ := GetDriver(guests[i].Hypervisor, rows[i].Provider)
+		if drv != nil {
+			rows[i].CdromSupport, _ = drv.IsSupportCdrom(&guests[i])
+			rows[i].FloppySupport, _ = drv.IsSupportFloppy(&guests[i])
+			rows[i].MonitorUrl = drv.FetchMonitorUrl(ctx, &guests[i])
+		}
+
 		if len(guests[i].HostId) == 0 && guests[i].Status == api.VM_SCHEDULE_FAILED {
 			rows[i].Brand = "Unknown"
 			rows[i].Provider = "Unknown"
 		}
+
+		if !isList {
+			rows[i].Networks = guests[i].getNetworksDetails()
+			rows[i].VirtualIps = strings.Join(guests[i].getVirtualIPs(), ",")
+			rows[i].SecurityRules = guests[i].getSecurityGroupsRules()
+
+			osName := guests[i].GetOS()
+			if len(osName) > 0 {
+				rows[i].OsName = osName
+				if len(guests[i].OsType) == 0 {
+					rows[i].OsType = osName
+				}
+			}
+
+			if userCred.HasSystemAdminPrivilege() {
+				rows[i].AdminSecurityRules = guests[i].getAdminSecurityRules()
+			}
+		}
 	}
 
 	return rows
@@ -699,6 +767,40 @@ func fetchGuestGpuInstanceTypes(guestIds []string) (map[string]*GpuSpec, error)
 	return ret, nil
 }
 
+func fetchGuestBackupInfo(hostIds []string) (map[string]api.BackupInfo, error) {
+	ret := map[string]api.BackupInfo{}
+	hosts := []SHost{}
+	err := HostManager.Query().In("id", hostIds).All(&hosts)
+	if err != nil {
+		return nil, err
+	}
+	for _, host := range hosts {
+		ret[host.Id] = api.BackupInfo{BackupHostName: host.Name, BackupHostStatus: host.HostStatus}
+	}
+	return ret, nil
+}
+
+func fetchContainers(guestIds []string) (map[string][]*api.PodContainerDesc, error) {
+	ret := map[string][]*api.PodContainerDesc{}
+	containers := []SContainer{}
+	err := GetContainerManager().Query().In("guest_id", guestIds).All(&containers)
+	if err != nil {
+		return nil, err
+	}
+	for _, container := range containers {
+		_, ok := ret[container.GuestId]
+		if !ok {
+			ret[container.GuestId] = []*api.PodContainerDesc{}
+		}
+		desc := &api.PodContainerDesc{Id: container.GetId(), Name: container.GetName()}
+		if container.Spec != nil {
+			desc.Image = container.Spec.Image
+		}
+		ret[container.GuestId] = append(ret[container.GuestId], desc)
+	}
+	return ret, nil
+}
+
 func fetchGuestIsolatedDevices(guestIds []string) map[string][]api.SIsolatedDevice {
 	q := IsolatedDeviceManager.Query().In("guest_id", guestIds)
 	devs := make([]SIsolatedDevice, 0)

pkg/compute/models/guests.go

@@ -2537,100 +2537,6 @@ func (self *SGuest) getExtBandwidth() int {
 	return self.getBandwidth(true)
 }
 
-func (self *SGuest) moreExtraInfo(
-	ctx context.Context,
-	out api.ServerDetails,
-	userCred mcclient.TokenCredential,
-	query jsonutils.JSONObject,
-	fields stringutils2.SSortedStrings,
-	isList bool,
-) api.ServerDetails {
-	// extra.Add(jsonutils.NewInt(int64(self.getExtBandwidth())), "ext_bw")
-
-	if isList {
-		if query.Contains("group") {
-			groupId, _ := query.GetString("group")
-			q := GroupguestManager.Query().Equals("group_id", groupId).Equals("guest_id", self.Id)
-			var groupGuest SGroupguest
-			err := q.First(&groupGuest)
-			if err == nil {
-				out.AttachTime = groupGuest.CreatedAt
-			}
-		}
-	} else {
-		out.Networks = self.getNetworksDetails()
-		out.VirtualIps = strings.Join(self.getVirtualIPs(), ",")
-		out.SecurityRules = self.getSecurityGroupsRules()
-
-		osName := self.GetOS()
-		if len(osName) > 0 {
-			out.OsName = osName
-			if len(self.OsType) == 0 {
-				out.OsType = osName
-			}
-		}
-
-		if userCred.HasSystemAdminPrivilege() {
-			out.AdminSecurityRules = self.getAdminSecurityRules()
-		}
-
-	}
-
-	out.IsPrepaidRecycle = self.IsPrepaidRecycle()
-
-	if len(self.BackupHostId) > 0 && (len(fields) == 0 || fields.Contains("backup_host_name") || fields.Contains("backup_host_status")) {
-		backupHost := HostManager.FetchHostById(self.BackupHostId)
-		if backupHost != nil {
-			if len(fields) == 0 || fields.Contains("backup_host_name") {
-				out.BackupHostName = backupHost.Name
-			}
-			if len(fields) == 0 || fields.Contains("backup_host_status") {
-				out.BackupHostStatus = backupHost.HostStatus
-			}
-			out.BackupGuestSyncStatus = self.GetGuestBackupMirrorJobStatus(ctx, userCred)
-		}
-	}
-
-	if len(fields) == 0 || fields.Contains("can_recycle") {
-		err := self.CanPerformPrepaidRecycle()
-		if err == nil {
-			out.CanRecycle = true
-		}
-	}
-
-	if len(fields) == 0 || fields.Contains("auto_delete_at") {
-		if self.PendingDeleted {
-			pendingDeletedAt := self.PendingDeletedAt.Add(time.Second * time.Duration(options.Options.PendingDeleteExpireSeconds))
-			out.AutoDeleteAt = pendingDeletedAt
-		}
-	}
-
-	drv, _ := self.GetDriver()
-	if drv != nil {
-		out.CdromSupport, _ = drv.IsSupportCdrom(self)
-		out.FloppySupport, _ = drv.IsSupportFloppy(self)
-		out.MonitorUrl = drv.FetchMonitorUrl(ctx, self)
-	}
-
-	if drv != nil && drv.GetHypervisor() == api.HYPERVISOR_POD {
-		ctrs, _ := GetContainerManager().GetContainersByPod(self.GetId())
-		desc := make([]*api.PodContainerDesc, len(ctrs))
-		for i := range ctrs {
-			ctr := ctrs[i]
-			desc[i] = &api.PodContainerDesc{
-				Id:   ctr.GetId(),
-				Name: ctr.GetName(),
-			}
-			if ctr.Spec != nil {
-				desc[i].Image = ctr.Spec.Image
-			}
-		}
-		out.Containers = desc
-	}
-
-	return out
-}
-
 func (self *SGuestManager) GetMetadataHiddenKeys() []string {
 	return []string{
 		api.VM_METADATA_CREATE_PARAMS,

pkg/compute/models/hostresource.go

@@ -111,6 +111,8 @@ func (manager *SHostResourceBaseManager) FetchCustomizeColumns(
 			rows[i].HostAccessIp = host.AccessIp
 			rows[i].HostEIP = host.PublicIp
 			rows[i].ManagerId = host.ManagerId
+			rows[i].HostResourceType = host.ResourceType
+			rows[i].HostBillingType = host.BillingType
 			rows[i].ZoneId = host.ZoneId
 		}
 		zoneList[i] = &SZoneResourceBase{rows[i].ZoneId}

pkg/mcclient/options/compute/servers.go

@@ -53,7 +53,6 @@ type ServerListOptions struct {
 	UsableServerForEip string   `help:"Eip id or name"`
 	WithoutUserMeta    *bool    `help:"Show Servers without user metadata"`
 	EipAssociable      *bool    `help:"Show Servers can associate with eip"`
-	Group              string   `help:"Instance Group ID or Name"`
 	HostSn             string   `help:"Host SN"`
 	IpAddr             string   `help:"Fileter by ip"`
 	IpAddrs            []string `help:"Fileter by ips"`