Merge pull request #7908 from wanyaoqi/automated-cherry-pick-of-#7905-upstream-release-3.2

Automated cherry pick of #7905: webconsole: fix defer order to release zombie process

cmd/climc/shell/events.go

@@ -35,7 +35,10 @@ type BaseEventListOptions struct {
 	Action     []string `help:"Log action"`
 
 	User    string `help:"filter by operator user"`
-	Project string `help:"filter by owner project"`
+	Project string `help:"filter by operator user's project"`
+
+	OwnerProjectIds []string `help:"filter by owner project ids"`
+	OwnerDomainIds  []string `help:"filter by owner domain ids"`
 
 	PagingMarker string `help:"marker for pagination"`
 }
@@ -107,6 +110,12 @@ func doEventList(man modulebase.ResourceManager, s *mcclient.ClientSession, args
 	if len(args.Scope) > 0 {
 		params.Add(jsonutils.NewString(args.Scope), "scope")
 	}
+	if len(args.OwnerProjectIds) > 0 {
+		params.Add(jsonutils.NewStringArray(args.OwnerProjectIds), "owner_project_ids")
+	}
+	if len(args.OwnerDomainIds) > 0 {
+		params.Add(jsonutils.NewStringArray(args.OwnerDomainIds), "owner_domain_ids")
+	}
 	if len(args.PagingMarker) > 0 {
 		params.Add(jsonutils.NewString(args.PagingMarker), "paging_marker")
 	}

pkg/cloudcommon/db/opslog.go

@@ -16,6 +16,7 @@ package db
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"strconv"
 	"strings"
@@ -460,28 +461,7 @@ func (manager *SOpsLogManager) ListItemFilter(
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,
 ) (*sqlchemy.SQuery, error) {
-	/*userStrs := jsonutils.GetQueryStringArray(query, "user")
-	if len(userStrs) > 0 {
-		for i := range userStrs {
-			usrObj, err := DefaultUserFetcher(ctx, userStrs[i])
-			if err != nil {
-				if err == sql.ErrNoRows {
-					return nil, httperrors.NewResourceNotFoundError2("user", userStrs[i])
-				} else if err == sqlchemy.ErrDuplicateEntry {
-					return nil, httperrors.NewDuplicateNameError("user", userStrs[i])
-				} else {
-					return nil, httperrors.NewGeneralError(err)
-				}
-			}
-			userStrs[i] = usrObj.GetId()
-		}
-		if len(userStrs) == 1 {
-			q = q.Filter(sqlchemy.Equals(q.Field("user_id"), userStrs[0]))
-		} else {
-			q = q.Filter(sqlchemy.In(q.Field("user_id"), userStrs))
-		}
-	}
-	projStrs := jsonutils.GetQueryStringArray(query, "project")
+	projStrs := jsonutils.GetQueryStringArray(query, "owner_project_ids")
 	if len(projStrs) > 0 {
 		for i := range projStrs {
 			projObj, err := DefaultProjectFetcher(ctx, projStrs[i])
@@ -494,12 +474,23 @@ func (manager *SOpsLogManager) ListItemFilter(
 			}
 			projStrs[i] = projObj.GetId()
 		}
-		if len(projStrs) == 1 {
-			q = q.Filter(sqlchemy.Equals(q.Field("owner_tenant_id"), projStrs[0]))
-		} else {
-			q = q.Filter(sqlchemy.In(q.Field("owner_tenant_id"), projStrs))
+		q = q.Filter(sqlchemy.In(q.Field("owner_tenant_id"), projStrs))
+	}
+	domainStrs := jsonutils.GetQueryStringArray(query, "owner_domain_ids")
+	if len(domainStrs) > 0 {
+		for i := range domainStrs {
+			domainObj, err := DefaultDomainFetcher(ctx, domainStrs[i])
+			if err != nil {
+				if err == sql.ErrNoRows {
+					return nil, httperrors.NewResourceNotFoundError2("domain", domainStrs[i])
+				} else {
+					return nil, httperrors.NewGeneralError(err)
+				}
+			}
+			domainStrs[i] = domainObj.GetId()
 		}
-	}*/
+		q = q.Filter(sqlchemy.In(q.Field("owner_domain_id"), domainStrs))
+	}
 	objTypes := jsonutils.GetQueryStringArray(query, "obj_type")
 	if len(objTypes) > 0 {
 		if len(objTypes) == 1 {

pkg/cloudcommon/notifyclient/notify.go

@@ -126,7 +126,7 @@ func RawNotify(recipientId []string, isGroup bool, channel notify.TNotifyChannel
 	msg.Topic = topic
 	body, _ := getContent(event, "content", channel, data)
 	if len(body) == 0 {
-		body = data.String()
+		body, _ = data.GetString()
 	}
 	msg.Msg = body
 	// log.Debugf("send notification %s %s", topic, body)

pkg/compute/guestdrivers/esxi.go

@@ -368,3 +368,42 @@ func (self *SESXiGuestDriver) RequestAssociateEip(ctx context.Context, userCred
 func (self *SESXiGuestDriver) IsSupportCdrom(guest *models.SGuest) (bool, error) {
 	return false, nil
 }
+
+func (self *SESXiGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) {
+	ihost, err := host.GetIHost()
+	if err != nil {
+		return nil, err
+	}
+	ivm, err := ihost.GetIVMById(guest.GetExternalId())
+	if err != nil && errors.Cause(err) != errors.ErrNotFound {
+		return nil, err
+	}
+	// VM may be migrated by Vcenter, try to find VM from whole datacenter.
+	if err != nil {
+		ehost := ihost.(*esxi.SHost)
+		dc, err := ehost.GetDatacenter()
+		if err != nil {
+			return nil, errors.Wrapf(err, "ehost.GetDatacenter")
+		}
+		vm, err := dc.FetchVMById(guest.GetExternalId())
+		if err != nil {
+			log.Errorf("fail to find ivm by id %q in dc %q: %v", guest.GetExternalId(), dc.GetName(), err)
+			return nil, err
+		}
+		ihost = vm.GetIHost()
+		host = models.HostManager.FetchHostByExtId(ihost.GetGlobalId())
+		if host == nil {
+			return nil, errors.Wrapf(errors.ErrNotFound, "find ivm %q in ihost %q which is not existed here", guest.GetExternalId(), ihost.GetGlobalId())
+		}
+		ivm = vm
+	}
+	err = guest.SyncAllWithCloudVM(ctx, userCred, host, ivm)
+	if err != nil {
+		return nil, err
+	}
+
+	status := GetCloudVMStatus(ivm)
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString(status), "status")
+	return body, nil
+}

pkg/compute/models/dbinstances.go

@@ -1287,6 +1287,13 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred
 		return nil, nil, syncResult
 	}
 
+	for i := range dbInstances {
+		if taskman.TaskManager.IsInTask(&dbInstances[i]) {
+			syncResult.Error(fmt.Errorf("dbInstance %s(%s)in task", dbInstances[i].Name, dbInstances[i].Id))
+			return nil, nil, syncResult
+		}
+	}
+
 	removed := make([]SDBInstance, 0)
 	commondb := make([]SDBInstance, 0)
 	commonext := make([]cloudprovider.ICloudDBInstance, 0)

pkg/compute/models/elasticcache_instances.go

@@ -448,6 +448,13 @@ func (manager *SElasticcacheManager) SyncElasticcaches(ctx context.Context, user
 		return nil, nil, syncResult
 	}
 
+	for i := range dbInstances {
+		if taskman.TaskManager.IsInTask(&dbInstances[i]) {
+			syncResult.Error(fmt.Errorf("ElasticCacheInstance %s(%s)in task", dbInstances[i].Name, dbInstances[i].Id))
+			return nil, nil, syncResult
+		}
+	}
+
 	removed := make([]SElasticcache, 0)
 	commondb := make([]SElasticcache, 0)
 	commonext := make([]cloudprovider.ICloudElasticcache, 0)

pkg/compute/models/elasticips.go

@@ -421,8 +421,11 @@ func (self *SElasticip) SyncInstanceWithCloudEip(ctx context.Context, userCred m
 			case api.EIP_ASSOCIATE_TYPE_SERVER:
 				sq := HostManager.Query().SubQuery()
 				return q.Join(sq, sqlchemy.Equals(sq.Field("id"), q.Field("host_id"))).Filter(sqlchemy.Equals(sq.Field("manager_id"), self.ManagerId))
-			case api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY, api.EIP_ASSOCIATE_TYPE_LOADBALANCER:
+			case api.EIP_ASSOCIATE_TYPE_LOADBALANCER:
 				return q.Equals("manager_id", self.ManagerId)
+			case api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY:
+				sq := VpcManager.Query("id").Equals("manager_id", self.ManagerId)
+				return q.In("vpc_id", sq.SubQuery())
 			}
 			return q
 		})

pkg/compute/models/guest_actions.go

@@ -3619,9 +3619,11 @@ func (self *SGuest) SaveRenewInfo(
 	guestdisks := self.GetDisks()
 	for i := 0; i < len(guestdisks); i += 1 {
 		disk := guestdisks[i].GetDisk()
-		err = disk.SaveRenewInfo(ctx, userCred, bc, expireAt, billingType)
-		if err != nil {
-			return err
+		if disk.AutoDelete {
+			err = disk.SaveRenewInfo(ctx, userCred, bc, expireAt, billingType)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil

pkg/compute/models/guests.go

@@ -2730,7 +2730,7 @@ func getCloudNicNetwork(vnic cloudprovider.ICloudNic, host *SHost, ipList []stri
 	if vnet == nil {
 		if vnic.InClassicNetwork() {
 			region := host.GetRegion()
-			cloudprovider := region.GetCloudprovider()
+			cloudprovider := host.GetCloudprovider()
 			vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(cloudprovider, region)
 			if err != nil {
 				return nil, errors.Wrap(err, "NewVpcForClassicNetwork")
@@ -4220,6 +4220,14 @@ func (self *SGuest) GetShortDesc(ctx context.Context) *jsonutils.JSONDict {
 		billingInfo.SCloudProviderInfo = host.getCloudProviderInfo()
 	}
 
+	if len(self.BackupHostId) > 0 {
+		backupHost := HostManager.FetchHostById(self.BackupHostId)
+		if backupHost != nil {
+			desc.Set("backup_host", jsonutils.NewString(backupHost.Name))
+			desc.Set("backup_host_id", jsonutils.NewString(backupHost.Id))
+		}
+	}
+
 	if priceKey := self.GetMetadata("ext:price_key", nil); len(priceKey) > 0 {
 		billingInfo.PriceKey = priceKey
 	}

pkg/compute/models/hosts.go

@@ -1381,6 +1381,17 @@ func (self *SHost) GetGuests() []SGuest {
 	return guests
 }
 
+func (self *SHost) GetKvmGuests() []SGuest {
+	q := GuestManager.Query().Equals("host_id", self.Id).Equals("hypervisor", api.HYPERVISOR_KVM)
+	guests := make([]SGuest, 0)
+	err := db.FetchModelObjects(GuestManager, q, &guests)
+	if err != nil {
+		log.Errorf("GetGuests %s", err)
+		return nil
+	}
+	return guests
+}
+
 func (self *SHost) GetGuestCount() (int, error) {
 	q := self.GetGuestsQuery()
 	return q.CountWithError()
@@ -4809,7 +4820,7 @@ func (host *SHost) PerformHostMaintenance(ctx context.Context, userCred mcclient
 		preferHostId = host.Id
 	}
 
-	guests := host.GetGuests()
+	guests := host.GetKvmGuests()
 	for i := 0; i < len(guests); i++ {
 		lockman.LockObject(ctx, &guests[i])
 		defer lockman.ReleaseObject(ctx, &guests[i])
@@ -5229,3 +5240,15 @@ func (manager *SHostManager) ListItemExportKeys(ctx context.Context,
 	}
 	return q, nil
 }
+
+func (manager *SHostManager) FetchHostByExtId(extid string) *SHost {
+	host := SHost{}
+	host.SetModelManager(manager, &host)
+	err := manager.Query().Equals("external_id", extid).First(&host)
+	if err != nil {
+		log.Errorf("fetchHostByExtId fail %s", err)
+		return nil
+	} else {
+		return &host
+	}
+}

pkg/compute/models/loadbalancers.go

@@ -706,6 +706,13 @@ func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred
 		return nil, nil, syncResult
 	}
 
+	for i := range dbLbs {
+		if taskman.TaskManager.IsInTask(&dbLbs[i]) {
+			syncResult.Error(fmt.Errorf("loadbalancer %s(%s)in task", dbLbs[i].Name, dbLbs[i].Id))
+			return nil, nil, syncResult
+		}
+	}
+
 	removed := []SLoadbalancer{}
 	commondb := []SLoadbalancer{}
 	commonext := []cloudprovider.ICloudLoadbalancer{}

pkg/compute/models/vpcs.go

@@ -183,7 +183,7 @@ func (manager *SVpcManager) getVpcExternalIdForClassicNetwork(regionId, cloudpro
 func (manager *SVpcManager) GetOrCreateVpcForClassicNetwork(cloudprovider *SCloudprovider, region *SCloudregion) (*SVpc, error) {
 	externalId := manager.getVpcExternalIdForClassicNetwork(region.Id, cloudprovider.Id)
 	_vpc, err := db.FetchByExternalIdAndManagerId(manager, externalId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
-		return q.Equals("manager_id", region.ManagerId)
+		return q.Equals("manager_id", cloudprovider.Id)
 	})
 	if err == nil {
 		return _vpc.(*SVpc), nil
@@ -200,7 +200,7 @@ func (manager *SVpcManager) GetOrCreateVpcForClassicNetwork(cloudprovider *SClou
 	vpc.SetEnabled(false)
 	vpc.Status = api.VPC_STATUS_UNAVAILABLE
 	vpc.ExternalId = externalId
-	vpc.ManagerId = region.ManagerId
+	vpc.ManagerId = cloudprovider.Id
 	err = manager.TableSpec().Insert(vpc)
 	if err != nil {
 		return nil, errors.Wrap(err, "Insert vpc for classic network")

pkg/compute/tasks/disk_delete_task.go

@@ -101,10 +101,13 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD
 	)
 
 	storage = disk.GetStorage()
-	if storage != nil {
-		host = storage.GetMasterHost()
+	if storage == nil { // dirty data
+		self.OnGuestDiskDeleteComplete(ctx, disk, nil)
+		return
 	}
 
+	host = storage.GetMasterHost()
+
 	isPurge := false
 	if (host == nil || !host.GetEnabled()) && jsonutils.QueryBoolean(self.Params, "purge", false) {
 		isPurge = true
@@ -112,21 +115,25 @@ func (self *DiskDeleteTask) startDeleteDisk(ctx context.Context, disk *models.SD
 	disk.SetStatus(self.UserCred, api.DISK_DEALLOC, "")
 	if isPurge {
 		self.OnGuestDiskDeleteComplete(ctx, disk, nil)
+		return
+	}
+	if isNeed, _ := disk.IsNeedWaitSnapshotsDeleted(); isNeed { // for kvm rbd disk
+		self.OnGuestDiskDeleteComplete(ctx, disk, nil)
+		return
+	}
+	if len(disk.BackupStorageId) > 0 {
+		self.SetStage("OnMasterStorageDeleteDiskComplete", nil)
 	} else {
-		if isNeed, _ := disk.IsNeedWaitSnapshotsDeleted(); isNeed {
-			self.OnGuestDiskDeleteComplete(ctx, disk, nil)
-			return
-		}
-		if len(disk.BackupStorageId) > 0 {
-			self.SetStage("OnMasterStorageDeleteDiskComplete", nil)
-		} else {
-			self.SetStage("OnGuestDiskDeleteComplete", nil)
-		}
-		if host == nil {
-			self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString("fail to find master host"))
-		} else if err := host.GetHostDriver().RequestDeallocateDiskOnHost(ctx, host, storage, disk, self); err != nil {
-			self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString(err.Error()))
-		}
+		self.SetStage("OnGuestDiskDeleteComplete", nil)
+	}
+	if host == nil {
+		self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString("fail to find master host"))
+		return
+	}
+	err := host.GetHostDriver().RequestDeallocateDiskOnHost(ctx, host, storage, disk, self)
+	if err != nil {
+		self.OnGuestDiskDeleteCompleteFailed(ctx, disk, jsonutils.NewString(err.Error()))
+		return
 	}
 }
 

pkg/esxi/agent.go

@@ -133,7 +133,6 @@ func Start(app *appsrv.Application) error {
 }
 
 func (agent *SEsxiAgent) AddImageCacheHandler(prefix string, app *appsrv.Application) {
-	hostutils.InitWorkerManager()
 	app.AddHandler("POST",
 		fmt.Sprintf("%s/disks/image_cache", prefix),
 		auth.Authenticate(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {

pkg/esxi/options/options.go

@@ -19,18 +19,19 @@ import common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 type EsxiOptions struct {
 	common_options.CommonOptions
 
-	ListenInterface         string `help:"Master address of host server" default:"br0"`
-	ListenAddress           string `help:"Host serve IP address to select when multiple address bind to ListenInterface"`
-	EsxiAgentPath           string `default:"/opt/cloud/workspace/esxi_agent" help:"Path for esxi agent configuration files"`
-	ImageCachePath          string `help:"Path for storing image caches"`
-	ImageCacheLimit         int    `help:"Maximal storage space for image caching, in GB" default:"20"`
-	AgentTempPath           string `help:"Path for ESXI Agent"`
-	AgentTempLimit          int    `help:"Maximal storage space for ESXi agent, in GB" default:"20"`
-	LinuxDefaultRootUser    bool   `help:"Default account for Linux system is root" default:"false"`
-	WindowsDefaultAdminUser bool   `help:"Default account for Windows system is Administrator" default:"true"`
-	DefaultImageSaveFormat  string `help:"Default image save format, default is vmdk, canbe qcow2" default:"vmdk"`
-	Zone                    string `help:"Zone where the agent locates"`
-	DeployServerSocketPath  string `help:"Deploy server listen socket path" default:"/var/run/deploy.sock"`
+	ListenInterface          string `help:"Master address of host server" default:"br0"`
+	ListenAddress            string `help:"Host serve IP address to select when multiple address bind to ListenInterface"`
+	EsxiAgentPath            string `default:"/opt/cloud/workspace/esxi_agent" help:"Path for esxi agent configuration files"`
+	ImageCachePath           string `help:"Path for storing image caches"`
+	ImageCacheLimit          int    `help:"Maximal storage space for image caching, in GB" default:"20"`
+	AgentTempPath            string `help:"Path for ESXI Agent"`
+	AgentTempLimit           int    `help:"Maximal storage space for ESXi agent, in GB" default:"20"`
+	LinuxDefaultRootUser     bool   `help:"Default account for Linux system is root" default:"false"`
+	WindowsDefaultAdminUser  bool   `help:"Default account for Windows system is Administrator" default:"true"`
+	DefaultImageSaveFormat   string `help:"Default image save format, default is vmdk, canbe qcow2" default:"vmdk"`
+	Zone                     string `help:"Zone where the agent locates"`
+	DeployServerSocketPath   string `help:"Deploy server listen socket path" default:"/var/run/deploy.sock"`
+	HostDelayTaskWorkerCount int    `default:"8" help:"Host delay worker thread count, default is 8"`
 }
 
 var (

pkg/esxi/service/esxi_agent_service.go

@@ -30,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/esxi/options"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
+	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 )
 
 type SExsiAgentService struct {
@@ -70,6 +71,7 @@ func (s *SExsiAgentService) StartService() {
 	fsdriver.Init(nil)
 	deployclient.Init(options.Options.DeployServerSocketPath)
 
+	hostutils.InitWorkerManagerWithCount(options.Options.HostDelayTaskWorkerCount)
 	app := app_common.InitApp(&options.Options.BaseOptions, false)
 	handler.InitHandlers(app)
 

pkg/hostman/guestfs/fsdriver/linux.go

@@ -764,7 +764,7 @@ func (d *SUbuntuRootFs) GetReleaseInfo(rootFs IDiskPartition) *deployapi.Release
 	lines := strings.Split(string(rel), "\n")
 	for _, l := range lines {
 		if strings.HasPrefix(l, distroKey) {
-			version = strings.TrimSpace(l[len(distroKey) : len(l)-1])
+			version = strings.TrimSpace(l[len(distroKey):])
 		}
 	}
 	return deployapi.NewReleaseInfo(d.GetName(), version, d.GetArch(rootFs))

pkg/hostman/hostinfo/hostinfo.go

@@ -757,6 +757,7 @@ func (h *SHostInfo) tryCreateNetworkOnWire() {
 	params.Set("mask", jsonutils.NewInt(int64(mask)))
 	params.Set("is_on_premise", jsonutils.JSONTrue)
 	params.Set("server_type", jsonutils.NewString(api.NETWORK_TYPE_BAREMETAL))
+	params.Set("is_on_premise", jsonutils.JSONTrue)
 	ret, err := modules.Networks.PerformClassAction(
 		hostutils.GetComputeSession(context.Background()),
 		"try-create-network", params)
@@ -1295,6 +1296,9 @@ func (h *SHostInfo) onGetStorageInfoSucc(hoststorages []jsonutils.JSONObject) {
 
 func (h *SHostInfo) uploadStorageInfo() {
 	for _, s := range storageman.GetManager().Storages {
+		if err := s.SetStorageInfo(s.GetId(), s.GetStorageName(), s.GetStorageConf()); err != nil {
+			h.onFail(err)
+		}
 		res, err := s.SyncStorageInfo()
 		if err != nil {
 			h.onFail(err)

pkg/hostman/hostutils/hostutils.go

@@ -189,7 +189,11 @@ func DelayTaskWithWorker(
 }
 
 func InitWorkerManager() {
-	wm = workmanager.NewWorkManger(TaskFailed, TaskComplete, options.HostOptions.DefaultRequestWorkerCount)
+	InitWorkerManagerWithCount(options.HostOptions.DefaultRequestWorkerCount)
+}
+
+func InitWorkerManagerWithCount(count int) {
+	wm = workmanager.NewWorkManger(TaskFailed, TaskComplete, count)
 }
 
 func InitK8sWorkerManager() {

pkg/hostman/storageman/imagecache_rbd.go

@@ -24,7 +24,6 @@ import (
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
-	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/hostman/storageman/remotefile"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/procutils"
@@ -75,11 +74,11 @@ func (r *SRbdImageCache) Acquire(ctx context.Context, zone, srcUrl, format strin
 	}
 	r.imageName = localImageCache.GetName()
 	if !r.Load() {
-		log.Debugf("convert local image %s to rbd pool %s", r.imageId, r.Manager.GetPath())
+		log.Infof("convert local image %s to rbd pool %s", r.imageId, r.Manager.GetPath())
 		err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuImg(),
 			"convert", "-O", "raw", localImageCache.GetPath(), r.GetPath()).Run()
 		if err != nil {
-			log.Errorf("failed to convert image %s", options.HostOptions.ServersPath)
+			log.Errorf("failed to convert image %s", err)
 			return false
 		}
 	}

pkg/hostman/storageman/imagecachemanager_agent.go

@@ -225,12 +225,12 @@ func (c *SAgentImageCacheManager) perfetchTemplateVMImageCache(ctx context.Conte
 	if err != nil {
 		return nil, errors.Wrap(err, "host.GetDatacenter")
 	}
-	_, err = dc.GetTemplateVMById(data.ImageExternalId)
+	_, err = dc.FetchTemplateVMById(data.ImageExternalId)
 	if err != nil {
 		return nil, err
 	}
 	res := jsonutils.NewDict()
-	res.Add(jsonutils.NewString(data.ImageId), "image_id")
+	res.Add(jsonutils.NewString(data.ImageExternalId), "image_id")
 	return res, nil
 }
 

pkg/hostman/storageman/imagecachemanager_rbd.go

@@ -111,7 +111,7 @@ func (c *SRbdImageCacheManager) PrefetchImageCache(ctx context.Context, data int
 	if err != nil {
 		return nil, err
 	}
-	format := "qcow2"
+	format, _ := body.GetString("format")
 	srcUrl, _ := body.GetString("src_url")
 	zone, _ := body.GetString("zone")
 

pkg/hostman/storageman/storage_base.go

@@ -262,6 +262,7 @@ func (s *SBaseStorage) bindMountTo(sPath string) error {
 			return errors.Errorf("bind mount temp path to local image path failed %s", out)
 		}
 	}
+	log.Infof("bind mount %s -> %s", tempPath, sPath)
 	return nil
 }
 

pkg/hostman/storageman/storagehandler/storagehandler.go

@@ -66,7 +66,7 @@ func storageVerifyMountPoint(ctx context.Context, w http.ResponseWriter, r *http
 		hostutils.Response(ctx, w, httperrors.NewMissingParameterError("mount_point"))
 		return
 	}
-	output, err := procutils.NewCommand("mountpoint", mountPoint).Output()
+	output, err := procutils.NewRemoteCommandAsFarAsPossible("mountpoint", mountPoint).Output()
 	if err == nil {
 		appsrv.SendStruct(w, map[string]interface{}{"is_mount_point": true})
 	} else {

pkg/image/models/images.go

@@ -603,6 +603,16 @@ func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.To
 		if appParams != nil && appParams.Request.ContentLength > 0 {
 			return nil, httperrors.NewInvalidStatusError("cannot upload in status %s", self.Status)
 		}
+		if minDiskSize, err := data.Int("min_disk"); err == nil {
+			img, err := qemuimg.NewQemuImage(self.getLocalLocation())
+			if err != nil {
+				return nil, errors.Wrap(err, "open image")
+			}
+			virtualSizeMB := img.SizeBytes / 1024 / 1024
+			if virtualSizeMB > 0 && minDiskSize < virtualSizeMB {
+				return nil, httperrors.NewBadRequestError("min disk size must >= %v", virtualSizeMB)
+			}
+		}
 	} else {
 		appParams := appsrv.AppContextGetParams(ctx)
 		if appParams != nil {
@@ -654,7 +664,6 @@ func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.To
 		return nil, errors.Wrap(err, "SSharableVirtualResourceBase.ValidateUpdateData")
 	}
 	data.Update(jsonutils.Marshal(input))
-
 	return data, nil
 }
 

pkg/multicloud/esxi/datacenter.go

@@ -18,6 +18,8 @@ import (
 	"strings"
 
 	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/property"
+	"github.com/vmware/govmomi/view"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/types"
 
@@ -237,32 +239,85 @@ func (dc *SDatacenter) getDcObj() *object.Datacenter {
 	return object.NewDatacenter(dc.manager.client.Client, dc.object.Reference())
 }
 
-// fetchVms will identify if VM is a template and return two different arrays; the latter contains all template vms.
-func (dc *SDatacenter) fetchVms(vmRefs []types.ManagedObjectReference, all bool) ([]cloudprovider.ICloudVM, []*SVirtualMachine, error) {
-	var vms []mo.VirtualMachine
+func (dc *SDatacenter) fetchVms(vmRefs []types.ManagedObjectReference, all bool) ([]*SVirtualMachine, error) {
+	var movms []mo.VirtualMachine
 	if vmRefs != nil {
-		err := dc.manager.references2Objects(vmRefs, VIRTUAL_MACHINE_PROPS, &vms)
+		err := dc.manager.references2Objects(vmRefs, VIRTUAL_MACHINE_PROPS, &movms)
 		if err != nil {
-			return nil, nil, errors.Wrap(err, "dc.manager.references2Objects")
+			return nil, errors.Wrap(err, "dc.manager.references2Objects")
 		}
 	}
 
 	// avoid applying new memory and copying
-	retVms := make([]cloudprovider.ICloudVM, 0, len(vms)/2)
-	templateVMs := make([]*SVirtualMachine, 0, 2)
-	for i := 0; i < len(vms); i += 1 {
-		if all || !strings.HasPrefix(vms[i].Entity().Name, api.ESXI_IMAGE_CACHE_TMP_PREFIX) {
-			vmObj := NewVirtualMachine(dc.manager, &vms[i], dc)
-			if vms[i].Config != nil && vms[i].Config.Template {
-				templateVMs = append(templateVMs, vmObj)
-				continue
-			}
-			if vmObj != nil {
-				retVms = append(retVms, vmObj)
-			}
+	vms := make([]*SVirtualMachine, 0, len(movms))
+	for i := range movms {
+		if all || !strings.HasPrefix(movms[i].Entity().Name, api.ESXI_IMAGE_CACHE_TMP_PREFIX) {
+			vms = append(vms, NewVirtualMachine(dc.manager, &movms[i], dc))
 		}
 	}
-	return retVms, templateVMs, nil
+	return vms, nil
+}
+
+func (dc *SDatacenter) FetchVMs() ([]*SVirtualMachine, error) {
+	return dc.fetchVMs(property.Filter{})
+}
+
+func (dc *SDatacenter) FetchNoTemplateVMs() ([]*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["config.template"] = false
+	return dc.fetchVMs(filter)
+}
+
+func (dc *SDatacenter) fetchVMs(filter property.Filter) ([]*SVirtualMachine, error) {
+	odc := dc.getObjectDatacenter()
+	root := odc.Reference()
+	m := view.NewManager(dc.manager.client.Client)
+	v, err := m.CreateContainerView(dc.manager.context, root, []string{"VirtualMachine"}, true)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = v.Destroy(dc.manager.context)
+	}()
+	objs, err := v.Find(dc.manager.context, []string{"VirtualMachine"}, filter)
+	if err != nil {
+		return nil, err
+	}
+	vms, err := dc.fetchVms(objs, false)
+	return vms, err
+}
+
+func (dc *SDatacenter) FetchTemplateVMs() ([]*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["config.template"] = true
+	return dc.fetchVMs(filter)
+}
+
+func (dc *SDatacenter) FetchTemplateVMById(id string) (*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["config.template"] = true
+	filter["summary.config.uuid"] = id
+	vms, err := dc.fetchVMs(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(vms) == 0 {
+		return nil, errors.ErrNotFound
+	}
+	return vms[0], nil
+}
+
+func (dc *SDatacenter) FetchVMById(id string) (*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["summary.config.uuid"] = id
+	vms, err := dc.fetchVMs(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(vms) == 0 {
+		return nil, errors.ErrNotFound
+	}
+	return vms[0], nil
 }
 
 func (dc *SDatacenter) fetchDatastores(datastoreRefs []types.ManagedObjectReference) ([]cloudprovider.ICloudStorage, error) {
@@ -381,24 +436,3 @@ func (dc *SDatacenter) GetTemplateVMs() ([]*SVirtualMachine, error) {
 	}
 	return templateVms, nil
 }
-
-func (dc *SDatacenter) GetTemplateVMById(id string) (*SVirtualMachine, error) {
-	id = dc.manager.getPrivateId(id)
-	hosts, err := dc.GetIHosts()
-	if err != nil {
-		return nil, errors.Wrap(err, "SDatacenter.GetIHosts")
-	}
-	for _, ihost := range hosts {
-		host := ihost.(*SHost)
-		tvms, err := host.GetTemplateVMs()
-		if err != nil {
-			return nil, errors.Wrap(err, "host.GetTemplateVMs")
-		}
-		for i := range tvms {
-			if tvms[i].GetGlobalId() == id {
-				return tvms[i], nil
-			}
-		}
-	}
-	return nil, cloudprovider.ErrNotFound
-}

pkg/multicloud/esxi/host.go

@@ -179,6 +179,7 @@ func (self *SHost) fetchVMs(all bool) error {
 	}
 
 	MAX_TRIES := 3
+	var vms []*SVirtualMachine
 	for tried := 0; tried < MAX_TRIES; tried += 1 {
 		hostVms := self.getHostSystem().Vm
 		if len(hostVms) == 0 {
@@ -186,15 +187,20 @@ func (self *SHost) fetchVMs(all bool) error {
 			return nil
 		}
 
-		vms, templatevms, err := dc.fetchVms(hostVms, all)
+		vms, err = dc.fetchVms(hostVms, all)
 		if err != nil {
 			log.Errorf("dc.fetchVms fail %s", err)
 			time.Sleep(time.Second)
 			self.Refresh()
 			continue
 		}
-		self.vms = vms
-		self.tempalteVMs = templatevms
+	}
+	for _, vm := range vms {
+		if vm.IsTemplate() {
+			self.tempalteVMs = append(self.tempalteVMs, vm)
+		} else {
+			self.vms = append(self.vms, vm)
+		}
 	}
 	return nil
 }
@@ -684,9 +690,13 @@ func (self *SHost) CreateVM2(ctx context.Context, ds *SDatastore, params SCreate
 	if err != nil {
 		return nil, errors.Wrap(err, "SEsxiClient.FindHostByIp")
 	}
-	temvm, err := imgHost.GetTemplateVMById(imageInfo.ImageExternalId)
+	dc, err := imgHost.GetDatacenter()
 	if err != nil {
-		return nil, errors.Wrap(err, "SHost.GetTemplateVMById")
+		return nil, errors.Wrap(err, "host.GetDatacenter")
+	}
+	temvm, err := dc.FetchTemplateVMById(imageInfo.ImageExternalId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "datacenter.TemplateVMById for image %q and datacenter %q", imageInfo.ImageExternalId, dc.GetId())
 	}
 	return self.CloneVM(ctx, temvm, ds, params)
 }

pkg/multicloud/esxi/shell/virtualmachine.go

@@ -27,28 +27,53 @@ import (
 
 func init() {
 	type VirtualMachineListOptions struct {
-		HOSTIP   string `help:"Host IP"`
-		Template bool   `help:"Whether it is tempalte virtual machine"`
+		Datacenter string `help:"Datacenter"`
+		HostIP     string `help:"HostIP"`
+		Template   bool   `help:"Whether it is tempalte virtual machine, default:false"`
 	}
 	shellutils.R(&VirtualMachineListOptions{}, "vm-list", "List vms of a host", func(cli *esxi.SESXiClient, args *VirtualMachineListOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		if args.Template {
-			vms, err := host.GetTemplateVMs()
+		switch {
+		case len(args.HostIP) > 0:
+			host, err := cli.FindHostByIp(args.HostIP)
+			if err != nil {
+				return err
+			}
+			if args.Template {
+				vms, err := host.GetTemplateVMs()
+				if err != nil {
+					return err
+				}
+				printList(vms, []string{})
+				return nil
+			}
+			vms, err := host.GetIVMs2()
 			if err != nil {
 				return err
 			}
 			printList(vms, []string{})
 			return nil
+		case len(args.Datacenter) > 0:
+			dc, err := cli.FindDatacenterByMoId(args.Datacenter)
+			if err != nil {
+				return errors.Wrap(err, "FindDatacenterByMoId")
+			}
+			var vms []*esxi.SVirtualMachine
+			if args.Template {
+				vms, err = dc.FetchTemplateVMs()
+				if err != nil {
+					return errors.Wrap(err, "FetchTemplateVMs")
+				}
+			} else {
+				vms, err = dc.FetchNoTemplateVMs()
+				if err != nil {
+					return errors.Wrap(err, "FetchNoTemplateVMs")
+				}
+			}
+			printList(vms, []string{})
+			return nil
+		default:
+			return fmt.Errorf("Both Datacenter and HostIP cannot be empty")
 		}
-		vms, err := host.GetIVMs2()
-		if err != nil {
-			return err
-		}
-		printList(vms, []string{})
-		return nil
 	})
 
 	type VirtualMachineCloneOptions struct {
@@ -91,24 +116,45 @@ func init() {
 	})
 
 	type VirtualMachineShowOptions struct {
-		HOSTIP   string `help:"Host IP"`
-		VMID     string `help:"VM ID"`
-		Template bool
+		Datacenter string `help:"Datacenter"`
+		HostIP     string `help:"Host IP"`
+		VMID       string `help:"VM ID"`
+	}
+	getVM := func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) (*esxi.SVirtualMachine, error) {
+		var vm *esxi.SVirtualMachine
+		switch {
+		case len(args.HostIP) > 0:
+			host, err := cli.FindHostByIp(args.HostIP)
+			if err != nil {
+				return nil, errors.Wrap(err, "FindHostByIp")
+			}
+			ivm, err := host.GetIVMById(args.VMID)
+			if err != nil && errors.Cause(err) != errors.ErrNotFound {
+				return nil, err
+			}
+			if err != nil {
+				vm, err = host.GetTemplateVMById(args.VMID)
+				if err != nil {
+					return nil, errors.Wrap(err, "GetTemplateVMById")
+				}
+			}
+			vm = ivm.(*esxi.SVirtualMachine)
+		case len(args.Datacenter) > 0:
+			dc, err := cli.FindDatacenterByMoId(args.Datacenter)
+			if err != nil {
+				return nil, errors.Wrap(err, "FindDatacenterByMoId")
+			}
+			vm, err = dc.FetchVMById(args.VMID)
+			if err != nil {
+				return nil, errors.Wrap(err, "FetchVMById")
+			}
+		default:
+			return nil, fmt.Errorf("Both Datacenter and HostIP cannot be empty")
+		}
+		return vm, nil
 	}
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-show", "Show vm details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		if args.Template {
-			vm, err := host.GetTemplateVMById(args.VMID)
-			if err != nil {
-				return err
-			}
-			printObject(vm)
-			return nil
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -117,11 +163,7 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-nics", "Show vm nics details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -134,11 +176,7 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-disks", "Show vm disks details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -151,17 +189,12 @@ func init() {
 	})
 
 	type VirtualMachineDiskResizeOptions struct {
-		HOSTIP  string `help:"host ip"`
-		VMID    string `help:"virtual machine UUID"`
-		DISKIDX int    `help:"disk index"`
-		SIZEGB  int64  `help:"new size of disk"`
+		VirtualMachineShowOptions
+		DISKIDX int   `help:"disk index"`
+		SIZEGB  int64 `help:"new size of disk"`
 	}
 	shellutils.R(&VirtualMachineDiskResizeOptions{}, "vm-disk-resize", "Resize a vm disk", func(cli *esxi.SESXiClient, args *VirtualMachineDiskResizeOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, &args.VirtualMachineShowOptions)
 		if err != nil {
 			return err
 		}
@@ -178,11 +211,7 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-vnc", "Show vm VNC details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -195,15 +224,11 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-file-status", "Show vm files details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
-		vm, err := host.GetIVMById(args.VMID)
-		if err != nil {
-			return err
-		}
-		err = vm.(*esxi.SVirtualMachine).CheckFileInfo(context.Background())
+		err = vm.CheckFileInfo(context.Background())
 		if err != nil {
 			return err
 		}

pkg/multicloud/esxi/storage.go

@@ -259,7 +259,14 @@ func (self *SDatastore) getVMs() ([]cloudprovider.ICloudVM, error) {
 	if len(vms) == 0 {
 		return nil, nil
 	}
-	ret, _, err := dc.fetchVms(vms, false)
+	svms, err := dc.fetchVms(vms, false)
+	if err != nil {
+		return nil, err
+	}
+	ret := make([]cloudprovider.ICloudVM, len(svms))
+	for i := range svms {
+		ret[i] = svms[i]
+	}
 	return ret, err
 }
 

pkg/multicloud/esxi/virtualmachine.go

@@ -72,10 +72,7 @@ func (d byDiskType) Less(i, j int) bool {
 
 func NewVirtualMachine(manager *SESXiClient, vm *mo.VirtualMachine, dc *SDatacenter) *SVirtualMachine {
 	svm := &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc)}
-	err := svm.fetchHardwareInfo()
-	if err != nil {
-		return nil
-	}
+	svm.fetchHardwareInfo()
 	return svm
 }
 
@@ -708,7 +705,7 @@ func (self *SVirtualMachine) UpdateUserData(userData string) error {
 	return nil
 }
 
-func (self *SVirtualMachine) fetchHardwareInfo() error {
+func (self *SVirtualMachine) fetchHardwareInfo() {
 	self.vnics = make([]SVirtualNIC, 0)
 	self.vdisks = make([]SVirtualDisk, 0)
 	self.cdroms = make([]SVirtualCdrom, 0)
@@ -722,7 +719,8 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 	}
 
 	if moVM == nil || moVM.Config == nil || moVM.Config.Hardware.Device == nil {
-		return errors.Error("invalid vm config")
+		log.Errorf("invalid vm config, moVM: %v", moVM)
+		return
 	}
 
 	for i := 0; i < len(moVM.Config.Hardware.Device); i += 1 {
@@ -750,7 +748,6 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 	sort.Slice(self.vdisks, func(i, j int) bool {
 		return self.vdisks[i].GetIndex() < self.vdisks[j].GetIndex()
 	})
-	return nil
 }
 
 func (self *SVirtualMachine) getVdev(key int32) SVirtualDevice {
@@ -1218,3 +1215,8 @@ func (self *SVirtualMachine) FindMinDiffKey(limit int32) int32 {
 	}
 	return limit
 }
+
+func (self *SVirtualMachine) IsTemplate() bool {
+	movm := self.getVirtualMachine()
+	return movm.Config != nil && movm.Config.Template
+}

pkg/webconsole/service/service.go

@@ -26,6 +26,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/signalutils"
 
 	api "yunion.io/x/onecloud/pkg/apis/webconsole"
 	"yunion.io/x/onecloud/pkg/appsrv"
@@ -66,9 +67,15 @@ func StartService() {
 
 	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, o.OnOptionsChange)
 
+	registerSigTraps()
 	start()
 }
 
+func registerSigTraps() {
+	signalutils.SetDumpStackSignal()
+	signalutils.StartTrap()
+}
+
 func start() {
 	baseOpts := &o.Options.BaseOptions
 	// commonOpts := &o.Options.CommonOptions

pkg/webconsole/session/pty_session.go

@@ -102,6 +102,10 @@ func (p *Pty) Resize(size *pty.Winsize) {
 func (p *Pty) Stop() (err error) {
 	var errs []error
 
+	defer func() {
+		p.Cmd, p.Pty = nil, nil
+	}()
+
 	defer func() {
 		err = errors.NewAggregate(errs)
 	}()
@@ -139,8 +143,5 @@ func (p *Pty) Stop() (err error) {
 		}
 	}()
 
-	defer func() {
-		p.Cmd, p.Pty = nil, nil
-	}()
 	return
 }