Automated cherry pick of #23377: Automated cherry pick of #23374: fix(region,host): guest add vnc screen dump support (#23378)

* fix(host): update nic confs inside vm on qga delete nics

Signed-off-by: wanyaoqi <wanyaoqi@yunion.cn>

* fix(region,host): guest add vnc screen dump support

---------

Signed-off-by: wanyaoqi <wanyaoqi@yunion.cn>

cmd/climc/shell/compute/servers.go

@@ -144,6 +144,8 @@ func init() {
 	cmd.Get("sshport", new(options.ServerIdOptions))
 	cmd.Get("qemu-info", new(options.ServerIdOptions))
 	cmd.Get("hardware-info", new(options.ServerIdOptions))
+	cmd.Get("screen-dump-show", new(options.ServerScreenDumpOptions))
+	cmd.BatchPerform("screen-dump", new(options.ServerIdsOptions))
 
 	cmd.GetProperty(&options.ServerStatusStatisticsOptions{})
 	cmd.GetProperty(&options.ServerProjectStatisticsOptions{})

cmd/climc/shell/compute/serverscreendumps.go

@@ -0,0 +1,23 @@
+package compute
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type ServerScreenDumpListOptions struct {
+	options.BaseListOptions
+	Server string `help:"Id or name of server"`
+}
+
+func (o *ServerScreenDumpListOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.Marshal(o), nil
+}
+
+func init() {
+	cmd := shell.NewResourceCmd(&modules.ServerScreenDumps)
+	cmd.List(new(ServerScreenDumpListOptions))
+}

pkg/apis/compute/guest_image.go

@@ -27,3 +27,26 @@ type SImagesInGuest struct {
 	RootImage  SSubImage
 	DataImages []SSubImage
 }
+
+type SGuestScreenDump struct {
+	S3AccessKey  string
+	S3SecretKey  string
+	S3Endpoint   string
+	S3BucketName string
+	S3ObjectName string
+	S3UseSSL     bool
+}
+
+type GuestScreenDumpListInput struct {
+	Server string
+}
+
+type GetDetailsGuestScreenDumpInput struct {
+	ObjectName string
+}
+
+type GetDetailsGuestScreenDumpOutput struct {
+	GuestId    string
+	Name       string
+	ScreenDump string
+}

pkg/cloudcommon/db/db_dispatcher.go

@@ -1077,12 +1077,15 @@ func (dispatcher *DBModelDispatcher) GetSpecific(ctx context.Context, idStr stri
 
 	funcName := fmt.Sprintf("GetDetails%s", specCamel)
 	funcValue := modelValue.MethodByName(funcName)
+	log.Errorf("MethodByName %s", funcName)
 	if !funcValue.IsValid() || funcValue.IsNil() {
+		log.Errorf("MethodByName2 %s", funcName)
 		return nil, httperrors.NewSpecNotFoundError("%s %s %s not found", dispatcher.Keyword(), idStr, spec)
 	}
 
 	outs, err := callFunc(funcValue, funcName, params...)
 	if err != nil {
+		log.Errorf("MethodByName4 %s", funcName)
 		return nil, err
 	}
 	if len(outs) != 2 {
@@ -1092,6 +1095,7 @@ func (dispatcher *DBModelDispatcher) GetSpecific(ctx context.Context, idStr stri
 	resVal := outs[0]
 	errVal := outs[1].Interface()
 	if !gotypes.IsNil(errVal) {
+		log.Errorf("MethodByName3 %s", funcName)
 		return nil, errVal.(error)
 	} else {
 		if gotypes.IsNil(resVal.Interface()) {

pkg/cloudcommon/db/opslog_const.go

@@ -267,6 +267,7 @@ const (
 	ACT_GUEST_CREATE_FROM_IMPORT_SUCC    = "guest_create_from_import_succ"
 	ACT_GUEST_CREATE_FROM_IMPORT_FAIL    = "guest_create_from_import_fail"
 	ACT_GUEST_PANICKED                   = "guest_panicked"
+	ACT_GUEST_SCREEN_DUMP                = "guest_screen_dump"
 	ACT_HOST_MAINTENANCE                 = "host_maintenance"
 	ACT_HOST_DOWN                        = "host_down"
 

pkg/cloudcommon/options/options.go

@@ -163,6 +163,7 @@ type CommonOptions struct {
 
 type HostCommonOptions struct {
 	CommonOptions
+	S3ScreenDumpOptions
 
 	ExecutorSocketPath     string `help:"Executor socket path" default:"/var/run/onecloud/exec.sock"`
 	DeployServerSocketPath string `help:"Deploy server listen socket path" default:"/var/run/onecloud/deploy.sock"`
@@ -176,6 +177,15 @@ type HostCommonOptions struct {
 	Qcow2Preallocation            string `help:"Qcow2 image create preallocation" default:"metadata" choices:"disable|metadata|falloc|full"`
 }
 
+type S3ScreenDumpOptions struct {
+	S3AccessKey              string `help:"s3 access key"`
+	S3SecretKey              string `help:"s3 secret key"`
+	S3Endpoint               string `help:"s3 endpoint"`
+	S3UseSSL                 bool   `help:"s3 access use ssl"`
+	S3BucketName             string `help:"s3 bucket name" default:"onecloud-screendump"`
+	S3BucketLifecycleKeepDay int    `help:"s3 bucket lifecycle keep day" default:"180"`
+}
+
 type DBOptions struct {
 	SqlConnection string `help:"SQL connection string" alias:"connection"`
 

pkg/compute/guestdrivers/base.go

@@ -522,6 +522,10 @@ func (drv *SBaseGuestDriver) RequestQgaCommand(ctx context.Context, userCred mcc
 	return nil, httperrors.ErrNotImplemented
 }
 
+func (self *SBaseGuestDriver) RequestGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) (jsonutils.JSONObject, error) {
+	return nil, httperrors.ErrNotImplemented
+}
+
 func (drv *SBaseGuestDriver) FetchMonitorUrl(ctx context.Context, guest *models.SGuest) string {
 	s := auth.GetAdminSessionWithPublic(ctx, consts.GetRegion())
 	tsdbURL, err := tsdb.GetDefaultServiceSourceURL(s, options.Options.MonitorEndpointType)

pkg/compute/guestdrivers/kvm.go

@@ -1201,6 +1201,17 @@ func (self *SKVMGuestDriver) RequestQgaCommand(ctx context.Context, userCred mcc
 	return res, nil
 }
 
+func (self *SKVMGuestDriver) RequestGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *models.SHost, guest *models.SGuest) (jsonutils.JSONObject, error) {
+	url := fmt.Sprintf("%s/servers/%s/guest-screen-dump", host.ManagerUri, guest.Id)
+	httpClient := httputils.GetDefaultClient()
+	header := mcclient.GetTokenHeaders(userCred)
+	_, res, err := httputils.JSONRequest(httpClient, ctx, "POST", url, header, nil, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "host request")
+	}
+	return res, nil
+}
+
 func (self *SKVMGuestDriver) FetchMonitorUrl(ctx context.Context, guest *models.SGuest) string {
 	if options.Options.KvmMonitorAgentUseMetadataService && !guest.IsSriov() {
 		var metadataIp string

pkg/compute/models/guest_actions.go

@@ -148,15 +148,27 @@ func (self *SGuest) PerformEvent(ctx context.Context, userCred mcclient.TokenCre
 	}
 	if event == "GUEST_PANICKED" {
 		kwargs := jsonutils.NewDict()
-		kwargs.Set("reason", data)
+		kwargs.Set("reason", jsonutils.NewString(event))
+		if data.Contains("screen_dump_info") {
+			screenDumpInfo := api.SGuestScreenDump{}
+			if err := data.Unmarshal(&screenDumpInfo, "screen_dump_info"); err != nil {
+				log.Errorf("failed unmarshal screen_dump_info %s", err)
+			} else {
+				kwargs.Set("screen_dump_name", jsonutils.NewString(screenDumpInfo.S3ObjectName))
+				if _, err := self.SaveGuestScreenDump(ctx, userCred, &screenDumpInfo); err != nil {
+					log.Errorf("SaveGuestScreenDump failed %s", err)
+				}
+			}
+		}
 
-		db.OpsLog.LogEvent(self, db.ACT_GUEST_PANICKED, data.String(), userCred)
-		logclient.AddSimpleActionLog(self, logclient.ACT_GUEST_PANICKED, data.String(), userCred, true)
+		db.OpsLog.LogEvent(self, db.ACT_GUEST_PANICKED, kwargs.String(), userCred)
+		logclient.AddSimpleActionLog(self, logclient.ACT_GUEST_PANICKED, kwargs.String(), userCred, true)
 		notifyclient.EventNotify(ctx, userCred, notifyclient.SEventNotifyParam{
 			Obj:    self,
 			Action: notifyclient.ActionServerPanicked,
 			IsFail: true,
 		})
+
 	}
 	return nil, nil
 }

pkg/compute/models/guestdrivers.go

@@ -255,6 +255,8 @@ type IGuestDriver interface {
 
 	BeforeDetachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error
 	BeforeAttachIsolatedDevice(ctx context.Context, cred mcclient.TokenCredential, guest *SGuest, dev *SIsolatedDevice) error
+
+	RequestGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject, host *SHost, guest *SGuest) (jsonutils.JSONObject, error)
 }
 
 var guestDrivers map[string]IGuestDriver

pkg/compute/models/guestscreendump.go

@@ -0,0 +1,186 @@
+package models
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"yunion.io/x/cloudmux/pkg/multicloud/objectstore"
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type SGuestScreenDumpManager struct {
+	db.SResourceBaseManager
+}
+
+var GuestScreenDumpManager *SGuestScreenDumpManager
+
+func init() {
+	db.InitManager(func() {
+		GuestScreenDumpManager = &SGuestScreenDumpManager{
+			SResourceBaseManager: db.NewResourceBaseManager(
+				SGuestScreenDump{},
+				"guest_screen_dumps_tbl",
+				"guest_screen_dump",
+				"guest_screen_dumps",
+			),
+		}
+		GuestScreenDumpManager.SetVirtualObject(GuestScreenDumpManager)
+		GuestScreenDumpManager.TableSpec().AddIndex(true, "guest_id")
+	})
+}
+
+type SGuestScreenDump struct {
+	db.SResourceBase
+
+	RowId   int64  `primary:"true" auto_increment:"true" list:"user"`
+	GuestId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	Name    string `width:"64" charset:"ascii" nullable:"true" list:"user"`
+
+	// s3 config
+	S3AccessKey  string `width:"64" charset:"ascii" nullable:"true"`
+	S3SecretKey  string `width:"64" charset:"ascii" nullable:"true"`
+	S3Endpoint   string `width:"64" charset:"ascii" nullable:"true" list:"user"`
+	S3BucketName string `width:"64" charset:"ascii" nullable:"true" list:"user"`
+	S3UseSsl     bool   `default:"false" list:"user" create:"optional"`
+}
+
+func (manager *SGuestScreenDumpManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.GuestScreenDumpListInput,
+) (*sqlchemy.SQuery, error) {
+	if query.Server != "" {
+		iGuest, err := GuestManager.FetchByIdOrName(ctx, userCred, query.Server)
+		if err != nil {
+			return q, errors.Wrap(err, "fetch guest")
+		}
+		q = q.Equals("guest_id", iGuest.GetId())
+	}
+	return q, nil
+}
+
+func (self *SGuestScreenDump) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return db.DeleteModel(ctx, userCred, self)
+}
+
+func (self *SGuest) SaveGuestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, screenDumpInfo *api.SGuestScreenDump) (*SGuestScreenDump, error) {
+	sd := new(SGuestScreenDump)
+	sd.SetModelManager(GuestScreenDumpManager, sd)
+	sd.GuestId = self.GetId()
+	sd.S3SecretKey = base64.StdEncoding.EncodeToString([]byte(screenDumpInfo.S3SecretKey))
+	sd.S3Endpoint = screenDumpInfo.S3Endpoint
+	sd.S3BucketName = screenDumpInfo.S3BucketName
+	sd.S3AccessKey = base64.StdEncoding.EncodeToString([]byte(screenDumpInfo.S3AccessKey))
+	sd.Name = screenDumpInfo.S3ObjectName
+
+	lockman.LockClass(ctx, GuestScreenDumpManager, self.ProjectId)
+	defer lockman.ReleaseClass(ctx, GuestScreenDumpManager, self.ProjectId)
+
+	err := GuestScreenDumpManager.TableSpec().Insert(ctx, sd)
+	if err != nil {
+		return nil, errors.Wrap(err, "save guest screen_dump")
+	}
+	db.OpsLog.LogEvent(self, db.ACT_GUEST_SCREEN_DUMP, sd.Name, userCred)
+	logclient.AddSimpleActionLog(self, logclient.ACT_GUEST_SCREEN_DUMP, sd.Name, userCred, true)
+	return sd, nil
+}
+
+func (self *SGuest) GetDetailsScreenDumpShow(ctx context.Context, userCred mcclient.TokenCredential, input *api.GetDetailsGuestScreenDumpInput) (*api.GetDetailsGuestScreenDumpOutput, error) {
+	if input.ObjectName == "" {
+		return nil, httperrors.NewMissingParameterError("object_name")
+	}
+	q := GuestScreenDumpManager.Query()
+	q = q.Equals("guest_id", self.Id)
+	q = q.Equals("name", input.ObjectName)
+
+	screenDump := new(SGuestScreenDump)
+	err := q.First(screenDump)
+	if err != nil {
+		return nil, errors.Wrap(err, "query screenDump")
+	}
+
+	ak, _ := base64.StdEncoding.DecodeString(screenDump.S3AccessKey)
+	sk, _ := base64.StdEncoding.DecodeString(screenDump.S3SecretKey)
+
+	url := screenDump.S3Endpoint
+	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
+		prefix := "http://"
+		if screenDump.S3UseSsl {
+			prefix = "https://"
+		}
+		url = prefix + url
+	}
+	cfg := objectstore.NewObjectStoreClientConfig(url, string(ak), string(sk))
+	s3Client, err := objectstore.NewObjectStoreClient(cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "new minio client")
+	}
+	bucket, err := s3Client.GetIBucketByName(screenDump.S3BucketName)
+	if err != nil {
+		return nil, errors.Wrapf(err, "get bucket %s", screenDump.S3BucketName)
+	}
+	irc, err := bucket.GetObject(ctx, screenDump.Name, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "get object %s", screenDump.Name)
+	}
+	defer irc.Close()
+	obj, err := io.ReadAll(irc)
+	if err != nil {
+		return nil, errors.Wrapf(err, "read object %s", screenDump.Name)
+	}
+	ret := new(api.GetDetailsGuestScreenDumpOutput)
+
+	contentType := http.DetectContentType(obj)
+	base64Encoded := base64.StdEncoding.EncodeToString(obj)
+	ret.ScreenDump = fmt.Sprintf("data:%s;base64,%s", contentType, base64Encoded)
+	ret.ScreenDump = base64.StdEncoding.EncodeToString(obj)
+	ret.GuestId = self.Id
+	ret.Name = screenDump.Name
+	return ret, nil
+}
+
+func (self *SGuest) PerformScreenDump(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject,
+	data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	if self.PowerStates != api.VM_POWER_STATES_ON {
+		return nil, httperrors.NewBadRequestError("can't use qga in vm status: %s", self.Status)
+	}
+
+	host, _ := self.GetHost()
+	driver, err := self.GetDriver()
+	if err != nil {
+		return nil, err
+	}
+	res, err := driver.RequestGuestScreenDump(ctx, userCred, nil, host, self)
+	if err != nil {
+		return nil, err
+	}
+	screenDumpInfo := api.SGuestScreenDump{}
+	if err := res.Unmarshal(&screenDumpInfo); err != nil {
+		return nil, errors.Wrap(err, "unmarshal screen dump info")
+	}
+	if _, err := self.SaveGuestScreenDump(ctx, userCred, &screenDumpInfo); err != nil {
+		return nil, errors.Wrap(err, "failed save ")
+	}
+	input := &api.GetDetailsGuestScreenDumpInput{
+		ObjectName: screenDumpInfo.S3ObjectName,
+	}
+	ret, err := self.GetDetailsScreenDumpShow(ctx, userCred, input)
+	if err != nil {
+		return nil, err
+	}
+	return jsonutils.Marshal(ret), err
+}

pkg/compute/service/handlers.go

@@ -166,6 +166,7 @@ func InitHandlers(app *appsrv.Application) {
 		models.RouteTableAssociationManager,
 		models.RouteTableRouteSetManager,
 		models.InterVpcNetworkRouteSetManager,
+		models.GuestScreenDumpManager,
 
 		models.SchedpolicyManager,
 		models.DynamicschedtagManager,

pkg/hostman/guestman/guest-agent.go

@@ -147,7 +147,7 @@ func (m *SGuestManager) QgaGetOsInfo(sid string) (jsonutils.JSONObject, error) {
 	return jsonutils.Marshal(res), nil
 }
 
-func (guest *SKVMGuestInstance) QgaAddNicsConfigure(addNics []*desc.SGuestNetwork) error {
+func (guest *SKVMGuestInstance) QgaDeployNicsConfigure(nics []*desc.SGuestNetwork) error {
 	if guest.guestAgent == nil {
 		if err := guest.InitQga(); err != nil {
 			return errors.Wrap(err, "init qga")
@@ -156,5 +156,5 @@ func (guest *SKVMGuestInstance) QgaAddNicsConfigure(addNics []*desc.SGuestNetwor
 	if err := guest.guestAgent.GuestPing(1); err != nil {
 		return errors.Wrap(err, "Qga ping")
 	}
-	return guest.guestAgent.QgaDeployNics(deployapi.GuestNicsToServerNics(addNics))
+	return guest.guestAgent.QgaDeployNics(deployapi.GuestNicsToServerNics(nics))
 }

pkg/hostman/guestman/guesthandlers/guesthandler.go

@@ -112,6 +112,7 @@ func AddGuestTaskHandler(prefix string, app *appsrv.Application) {
 			"qga-set-network":          qgaSetNetwork,
 			"qga-get-os-info":          qgaGetOsInfo,
 			"start-rescue":             guestStartRescue,
+			"guest-screen-dump":        guestScreenDump,
 		} {
 			app.AddHandler("POST",
 				fmt.Sprintf("%s/%s/<sid>/%s", prefix, keyWord, action),
@@ -1019,6 +1020,11 @@ func qgaGetOsInfo(ctx context.Context, userCred mcclient.TokenCredential, sid st
 	return gm.QgaGetOsInfo(sid)
 }
 
+func guestScreenDump(ctx context.Context, userCred mcclient.TokenCredential, sid string, body jsonutils.JSONObject) (interface{}, error) {
+	gm := guestman.GetGuestManager()
+	return gm.RequestGuestScreenDump(sid)
+}
+
 // prepare rescue files
 func guestStartRescue(ctx context.Context, userCred mcclient.TokenCredential, sid string, body jsonutils.JSONObject) (interface{}, error) {
 	return guestman.GetGuestManager().GuestStartRescue(ctx, userCred, sid, body)

pkg/hostman/guestman/guestman.go

@@ -60,6 +60,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/storageman/lvmutils"
 	"yunion.io/x/onecloud/pkg/hostman/storageman/remotefile"
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/image/drivers/s3"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
 	"yunion.io/x/onecloud/pkg/util/cgrouputils"
@@ -1917,6 +1918,57 @@ func (m *SGuestManager) ProbeGuestInitStatus(sid string) *compute.HostUploadGues
 	return resp
 }
 
+func (m *SGuestManager) RequestGuestScreenDump(sid string) (jsonutils.JSONObject, error) {
+	guest, _ := m.GetServer(sid)
+	if guest == nil {
+		return nil, httperrors.NewNotFoundError("guest %s not found", sid)
+	}
+	kvmGuest, ok := guest.(*SKVMGuestInstance)
+	if !ok {
+		return nil, httperrors.NewBadRequestError("guest %s not kvm instance", sid)
+	}
+
+	screenDumpPath := kvmGuest.generateScreenDumpPath()
+	screenDumpName := filepath.Base(screenDumpPath)
+	c := make(chan interface{}, 0)
+	kvmGuest.Monitor.ScreenDump(screenDumpPath, func(res string) {
+		log.Infof("qmp screendump res %s", res)
+		if len(res) > 0 {
+			c <- errors.Errorf("qmp screen dump failed: %s", res)
+			return
+		}
+
+		if fileutils2.Exists(screenDumpPath) {
+			log.Infof("screendump success at %s", screenDumpPath)
+			_, err := s3.Put(context.Background(), screenDumpPath, screenDumpName, 50, 4, nil)
+			if err != nil {
+				log.Errorf("faild put screenDumpPath %s to s3 %s", screenDumpPath, err)
+				c <- err
+			} else {
+				screenDumpInfo := compute.SGuestScreenDump{
+					S3AccessKey:  options.HostOptions.S3AccessKey,
+					S3SecretKey:  options.HostOptions.S3SecretKey,
+					S3Endpoint:   options.HostOptions.S3Endpoint,
+					S3BucketName: options.HostOptions.S3BucketName,
+					S3ObjectName: screenDumpName,
+					S3UseSSL:     options.HostOptions.S3UseSSL,
+				}
+				c <- jsonutils.Marshal(screenDumpInfo)
+				log.Infof("put screendump %s success", screenDumpName)
+				os.Remove(screenDumpPath)
+			}
+		}
+	})
+	ret := <-c
+	switch ret.(type) {
+	case jsonutils.JSONObject:
+		return ret.(jsonutils.JSONObject), nil
+	case error:
+		return nil, ret.(error)
+	}
+	return nil, errors.Errorf("unknown ret of screendump")
+}
+
 func SyncGuestNicsTraffics(guestNicsTraffics map[string]map[string]compute.SNicTrafficRecord) {
 	session := hostutils.GetComputeSession(context.Background())
 	hostId := guestManager.host.GetHostId()

pkg/hostman/guestman/guesttasks.go

@@ -616,6 +616,7 @@ type SGuestNetworkSyncTask struct {
 	addNics []*desc.SGuestNetwork
 	errors  []error
 
+	delNicCnt   int
 	addNicMacs  []string
 	addNicConfs []*monitor.NetworkModify
 
@@ -625,30 +626,25 @@ type SGuestNetworkSyncTask struct {
 func (n *SGuestNetworkSyncTask) Start(callback func(...error)) {
 	n.callback = callback
 	if len(n.addNics) > 0 {
-		nics := make([]*desc.SGuestNetwork, 0)
-		nics = append(nics, n.guest.Desc.Nics...)
-		nics = append(nics, n.addNics...)
-		if err := n.guest.QgaAddNicsConfigure(nics); err != nil {
-			log.Errorf("QgaAddNicsConfigure failed %s", err)
-		} else {
-			addNicMacs := make([]string, 0)
-			addNicConfs := make([]*monitor.NetworkModify, 0)
-			for i := range n.addNics {
-				addNicMacs = append(addNicMacs, n.addNics[i].Mac)
-				netMod := &monitor.NetworkModify{}
-				if len(n.addNics[i].Ip) > 0 {
-					netMod.Ipmask = fmt.Sprintf("%s/%d", n.addNics[i].Ip, n.addNics[i].Masklen)
-					netMod.Gateway = n.addNics[i].Gateway
-				}
-				if len(n.addNics[i].Ip6) > 0 {
-					netMod.Ip6mask = fmt.Sprintf("%s/%d", n.addNics[i].Ip6, n.addNics[i].Masklen6)
-				}
-				addNicConfs = append(addNicConfs, netMod)
+		addNicMacs := make([]string, 0)
+		addNicConfs := make([]*monitor.NetworkModify, 0)
+		for i := range n.addNics {
+			addNicMacs = append(addNicMacs, n.addNics[i].Mac)
+			netMod := &monitor.NetworkModify{}
+			if len(n.addNics[i].Ip) > 0 {
+				netMod.Ipmask = fmt.Sprintf("%s/%d", n.addNics[i].Ip, n.addNics[i].Masklen)
+				netMod.Gateway = n.addNics[i].Gateway
 			}
-			n.addNicMacs = addNicMacs
-			n.addNicConfs = addNicConfs
+			if len(n.addNics[i].Ip6) > 0 {
+				netMod.Ip6mask = fmt.Sprintf("%s/%d", n.addNics[i].Ip6, n.addNics[i].Masklen6)
+			}
+			addNicConfs = append(addNicConfs, netMod)
 		}
+		n.addNicMacs = addNicMacs
+		n.addNicConfs = addNicConfs
 	}
+
+	n.delNicCnt = len(n.delNics)
 	n.syncNetworkConf()
 }
 
@@ -662,13 +658,22 @@ func (n *SGuestNetworkSyncTask) syncNetworkConf() {
 		n.addNics = n.addNics[:len(n.addNics)-1]
 		n.addNic(nic)
 	} else {
-		if len(n.addNicMacs) > 0 {
-			// try restart added nics, wait for added nic ready
-			time.Sleep(3 * time.Second)
-			if err := n.qgaRestartAddedNics(); err != nil {
-				log.Errorf("failed qgaRestartAddedNics")
+		func() {
+			if len(n.addNicMacs) > 0 || n.delNicCnt > 0 {
+				if err := n.guest.QgaDeployNicsConfigure(n.guest.Desc.Nics); err != nil {
+					log.Errorf("failed do QgaDeployNicsConfigure %s", err)
+					return
+				}
 			}
-		}
+			if len(n.addNicMacs) > 0 {
+				// try restart added nics, wait for added nic ready
+				time.Sleep(3 * time.Second)
+				if err := n.qgaRestartAddedNics(); err != nil {
+					log.Errorf("failed qgaRestartAddedNics %s", err)
+					return
+				}
+			}
+		}()
 
 		n.callback(n.errors...)
 	}
@@ -872,7 +877,7 @@ func (n *SGuestNetworkSyncTask) onDeviceAdd(nic *desc.SGuestNetwork) {
 func NewGuestNetworkSyncTask(
 	guest *SKVMGuestInstance, delNics, addNics []*desc.SGuestNetwork,
 ) *SGuestNetworkSyncTask {
-	return &SGuestNetworkSyncTask{guest, delNics, addNics, make([]error, 0), nil, nil, nil}
+	return &SGuestNetworkSyncTask{guest, delNics, addNics, make([]error, 0), 0, nil, nil, nil}
 }
 
 /**

pkg/hostman/guestman/qemu-kvm.go

@@ -58,6 +58,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
 	"yunion.io/x/onecloud/pkg/hostman/storageman/lvmutils"
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/image/drivers/s3"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
@@ -492,6 +493,10 @@ func (s *SKVMGuestInstance) getQemuLogPath() string {
 	return path.Join(s.HomeDir(), "qemu.log")
 }
 
+func (s *SKVMGuestInstance) generateScreenDumpPath() string {
+	return path.Join(s.HomeDir(), fmt.Sprintf("%s.%d", s.GetId(), time.Now().Unix()))
+}
+
 func (s *SKVMGuestInstance) RecycleDir() string {
 	return path.Join(s.manager.ServersPath, "recycle")
 }
@@ -1126,6 +1131,42 @@ func (s *SKVMGuestInstance) eventGuestPaniced(event *monitor.Event) {
 	if info, ok := event.Data["info"]; ok {
 		params.Set("info", jsonutils.Marshal(info))
 	}
+
+	screenDumpPath := s.generateScreenDumpPath()
+	screenDumpName := filepath.Base(screenDumpPath)
+
+	c := make(chan struct{})
+	s.Monitor.ScreenDump(screenDumpPath, func(res string) {
+		log.Infof("qmp screendump res %s", res)
+		if fileutils2.Exists(screenDumpPath) {
+			log.Infof("screendump success at %s", screenDumpPath)
+			_, err := s3.Put(context.Background(), screenDumpPath, screenDumpName, 50, 4, nil)
+			if err != nil {
+				log.Errorf("faild put screenDumpPath %s to s3 %s", screenDumpPath, err)
+			} else {
+				screenDumpInfo := api.SGuestScreenDump{
+					S3AccessKey:  options.HostOptions.S3AccessKey,
+					S3SecretKey:  options.HostOptions.S3SecretKey,
+					S3Endpoint:   options.HostOptions.S3Endpoint,
+					S3BucketName: options.HostOptions.S3BucketName,
+					S3ObjectName: screenDumpName,
+					S3UseSSL:     options.HostOptions.S3UseSSL,
+				}
+				params.Set("screen_dump_info", jsonutils.Marshal(screenDumpInfo))
+				log.Infof("put screendump %s success", screenDumpName)
+				os.Remove(screenDumpPath)
+			}
+		}
+		c <- struct{}{}
+	})
+	// wait screen dump
+	select {
+	case <-time.After(time.Second * 5):
+		log.Errorf("ScreenDump no response after 5 seconds")
+	case <-c:
+		break
+	}
+
 	params.Set("event", jsonutils.NewString(strings.Trim(event.Event, "\"")))
 	_, err := modules.Servers.PerformAction(
 		hostutils.GetComputeSession(context.Background()),

pkg/hostman/host_services.go

@@ -15,6 +15,8 @@
 package hostman
 
 import (
+	"fmt"
+	"strings"
 	"time"
 
 	execlient "yunion.io/x/executor/client"
@@ -40,6 +42,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/storageman"
 	"yunion.io/x/onecloud/pkg/hostman/storageman/diskhandlers"
 	"yunion.io/x/onecloud/pkg/hostman/storageman/storagehandler"
+	"yunion.io/x/onecloud/pkg/image/drivers/s3"
 	losetupman "yunion.io/x/onecloud/pkg/util/losetup/manager"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/qemuimg"
@@ -102,6 +105,8 @@ func (host *SHostService) RunService() {
 		log.Fatalf("Storage manager init error: %v", err)
 	}
 
+	initS3()
+
 	var guestChan chan struct{}
 
 	if err := guestman.Init(hostInstance, options.HostOptions.ServersPath, options.HostOptions.DeployConcurrent); err != nil {
@@ -167,6 +172,41 @@ func (host *SHostService) initHandlers(app *appsrv.Application) {
 	app_common.ExportOptionsHandler(app, &options.HostOptions)
 }
 
+func initS3() {
+	url := options.HostOptions.S3Endpoint
+	if len(url) == 0 {
+		return
+	}
+
+	if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
+		prefix := "http://"
+		if options.HostOptions.S3UseSSL {
+			prefix = "https://"
+		}
+		url = prefix + url
+	}
+	err := s3.Init(
+		url,
+		options.HostOptions.S3AccessKey,
+		options.HostOptions.S3SecretKey,
+		options.HostOptions.S3BucketName,
+		options.HostOptions.S3UseSSL,
+		"",
+	)
+	if err != nil {
+		log.Fatalf("failed init s3 client %s", err)
+	}
+
+	lifecycle := fmt.Sprintf(
+		`<LifecycleConfiguration><Rule><ID>%s</ID><Prefix></Prefix><Status>Enabled</Status><Expiration><Days>%d</Days></Expiration></Rule></LifecycleConfiguration>`,
+		options.HostOptions.S3BucketName, options.HostOptions.S3BucketLifecycleKeepDay,
+	)
+	err = s3.SetBucketLifecycle(lifecycle)
+	if err != nil {
+		log.Fatalf("failed set bucket lifecycle %s", lifecycle)
+	}
+}
+
 func StartService() {
 	var srv = &SHostService{}
 	srv.SServiceBase = &service.SServiceBase{

pkg/hostman/monitor/hmp.go

@@ -567,6 +567,10 @@ func (m *HmpMonitor) QueryMachines(callback QueryMachinesCallback) {
 	go callback(nil, "unsupported query machines for hmp")
 }
 
+func (m *HmpMonitor) ScreenDump(savePath string, callback StringCallback) {
+	m.HumanMonitorCommand(fmt.Sprintf("screendump %s", savePath), callback)
+}
+
 func (m *HmpMonitor) Quit(cb StringCallback) {
 	m.Query("quit", cb)
 }

pkg/hostman/monitor/monitor.go

@@ -252,6 +252,7 @@ type Monitor interface {
 	NetdevAdd(id, netType string, params map[string]string, callback StringCallback)
 	NetdevDel(id string, callback StringCallback)
 
+	ScreenDump(savePath string, callback StringCallback)
 	SaveState(statFilePath string, callback StringCallback)
 	QueryMachines(callback QueryMachinesCallback)
 	Quit(StringCallback)

pkg/hostman/monitor/qmp.go

@@ -987,6 +987,10 @@ func (m *QmpMonitor) CancelBlockJob(driveName string, force bool, callback Strin
 	m.HumanMonitorCommand(cmd, callback)
 }
 
+func (m *QmpMonitor) ScreenDump(savePath string, callback StringCallback) {
+	m.HumanMonitorCommand(fmt.Sprintf("screendump %s", savePath), callback)
+}
+
 func (m *QmpMonitor) BlockJobComplete(drive string, callback StringCallback) {
 	m.HumanMonitorCommand(fmt.Sprintf("block_job_complete %s", drive), callback)
 }

pkg/image/drivers/s3/minio.go

@@ -103,6 +103,13 @@ func ensureBucket() error {
 	return nil
 }
 
+func SetBucketLifecycle(lifecycle string) error {
+	if client == nil {
+		return ErrClientNotInit
+	}
+	return client.osc.SetBucketLifecycle(client.bucket, lifecycle)
+}
+
 func PutStream(ctx context.Context, file io.ReaderAt, fSize int64, objName string, partSizeMb int64, parallel int, progresser func(saved int64)) (string, error) {
 	if client == nil {
 		return "", ErrClientNotInit

pkg/mcclient/modules/compute/mod_screendump.go

@@ -0,0 +1,18 @@
+package compute
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+var (
+	ServerScreenDumps modulebase.ResourceManager
+)
+
+func init() {
+	ServerScreenDumps = modules.NewComputeManager("guest_screen_dump", "guest_screen_dumps",
+		[]string{"Guest_id", "Name", "Created_at", "S3_endpoint", "S3_bucket_name"},
+		[]string{})
+
+	modules.RegisterCompute(&ServerScreenDumps)
+}

pkg/mcclient/options/compute/servers.go

@@ -1520,6 +1520,15 @@ func (o *ServerVncOptions) Params() (jsonutils.JSONObject, error) {
 	return jsonutils.Marshal(o), nil
 }
 
+type ServerScreenDumpOptions struct {
+	ServerIdOptions
+	ObjectName string
+}
+
+func (o *ServerScreenDumpOptions) Params() (jsonutils.JSONObject, error) {
+	return jsonutils.Marshal(o), nil
+}
+
 type ServerIsoOptions struct {
 	ServerIdOptions
 	Ordinal int `help:"server iso ordinal, default 0"`

pkg/util/logclient/consts.go

@@ -144,6 +144,7 @@ const (
 	ACT_HOST_IMPORT_LIBVIRT_SERVERS = "host_import_libvirt_servers"
 	ACT_GUEST_CREATE_FROM_IMPORT    = "guest_create_from_import"
 	ACT_GUEST_PANICKED              = "guest_panicked"
+	ACT_GUEST_SCREEN_DUMP           = "guest_screen_dump"
 	ACT_HOST_MAINTAINING            = "host_maintaining"
 	ACT_HOST_UNMAINTENANCE          = "host_unmaintenance"