Merge pull request #7548 from wanyaoqi/automated-cherry-pick-of-#7547-upstream-release-3.3

Automated cherry pick of #7547: support diable usb kbd

cmd/climc/shell/compute/servers.go

@@ -1152,6 +1152,7 @@ func init() {
 		ID               string `help:"ID or name of VM"`
 		DisableIsaSerial string `help:"disable isa serial device" choices:"true|false"`
 		DisablePvpanic   string `help:"disable pvpanic device" choices:"true|false"`
+		DisableUsbKbd    string `help:"disable usb kbd" choices:"true|false"`
 	}
 
 	R(&ServerQemuParams{}, "server-set-qemu-params", "config qemu params", func(s *mcclient.ClientSession,
@@ -1163,6 +1164,9 @@ func init() {
 		if len(opts.DisablePvpanic) > 0 {
 			params.Set("disable_pvpanic", jsonutils.NewString(opts.DisablePvpanic))
 		}
+		if len(opts.DisableUsbKbd) > 0 {
+			params.Set("disable_usb_kbd", jsonutils.NewString(opts.DisableUsbKbd))
+		}
 		result, err := modules.Servers.PerformAction(s, opts.ID, "set-qemu-params", params)
 		if err != nil {
 			return err

cmd/climc/shell/image/images.go

@@ -54,6 +54,7 @@ type ImageOptionalOptions struct {
 	Hypervisor         []string `help:"Prefer hypervisor type" choices:"kvm|esxi|baremetal|container|openstack|ctyun"`
 	DiskDriver         string   `help:"Perfer disk driver" choices:"virtio|scsi|pvscsi|ide|sata"`
 	NetDriver          string   `help:"Preferred network driver" choices:"virtio|e1000|vmxnet3"`
+	DisableUsbKbd      bool     `help:"Disable usb keyboard on this image(for hypervisor kvm)"`
 }
 
 func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDict, args ImageOptionalOptions) error {
@@ -131,6 +132,9 @@ func addImageOptionalOptions(s *mcclient.ClientSession, params *jsonutils.JSONDi
 	if len(args.Hypervisor) > 0 {
 		params.Add(jsonutils.NewString(strings.Join(args.Hypervisor, ",")), "properties", "hypervisor")
 	}
+	if args.DisableUsbKbd {
+		params.Add(jsonutils.NewString("true"), "properties", "disable_usb_kbd")
+	}
 	return nil
 }
 

pkg/apigateway/handler/imageutils.go

@@ -0,0 +1,129 @@
+package handler
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"strconv"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func readImageForm(r *multipart.Reader) (map[string]string, *multipart.Part, error) {
+	params := make(map[string]string)
+
+	maxValueBytes := int64(10 << 20)
+	for {
+		p, err := r.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, nil, err
+		}
+
+		name := p.FormName()
+		if name == "" {
+			continue
+		}
+		filename := p.FileName()
+
+		var b bytes.Buffer
+
+		_, hasContentTypeHeader := p.Header["Content-Type"]
+		if !hasContentTypeHeader && filename == "" {
+			// value, store as string in memory
+			n, err := io.CopyN(&b, p, maxValueBytes+1)
+			if err != nil && err != io.EOF {
+				return nil, nil, err
+			}
+			maxValueBytes -= n
+			if maxValueBytes < 0 {
+				return nil, nil, multipart.ErrMessageTooLarge
+			}
+			params[name] = b.String()
+			continue
+		}
+
+		if name == "image" || name == "file" {
+			return params, p, nil
+		} else {
+			return nil, nil, fmt.Errorf("no file uploaded")
+		}
+	}
+	return nil, nil, fmt.Errorf("empty form")
+}
+
+func imageUploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	reader, e := r.MultipartReader()
+	if e != nil {
+		httperrors.InvalidInputError(w, "无效的表单")
+		return
+	}
+
+	p, f, e := readImageForm(reader)
+	if e != nil {
+		httperrors.InvalidInputError(w, "无效的表单")
+		return
+	}
+
+	params := jsonutils.NewDict()
+
+	name, ok := p["name"]
+	if !ok {
+		httperrors.InvalidInputError(w, "缺少镜像名称")
+		return
+	}
+	params.Add(jsonutils.NewString(name), "name")
+
+	_imageSize, ok := p["image_size"]
+	if !ok {
+		httperrors.InvalidInputError(w, "缺少文件信息")
+		return
+	}
+	imageSize, e := strconv.ParseInt(_imageSize, 10, 64)
+	if e != nil {
+		httperrors.InvalidInputError(w, "文件信息错误")
+		return
+	}
+
+	// add all other params
+	for k, v := range p {
+		if k == "name" || k == "image_size" {
+			continue
+		}
+		params.Add(jsonutils.NewString(v), k)
+	}
+
+	token := AppContextToken(ctx)
+	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+
+	res, e := modules.Images.Upload(s, params, f, imageSize)
+	if e != nil {
+		httperrors.GeneralServerError(w, e)
+		return
+	} else {
+		appsrv.SendJSON(w, res)
+	}
+}
+
+func uploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo {
+	log.Debugf("%s - %s", method, prefix)
+	hi := appsrv.SHandlerInfo{}
+	hi.SetMethod(method)
+	hi.SetPath(prefix)
+	hi.SetHandler(handler)
+	hi.SetProcessTimeout(6 * time.Hour)
+	hi.SetWorkerManager(GetUploaderWorker())
+	return &hi
+}

pkg/apigateway/handler/misc.go

@@ -20,6 +20,7 @@ import (
 	"encoding/csv"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -90,6 +91,10 @@ func (h *MiscHandler) Bind(app *appsrv.Application) {
 	app.AddHandler3(uploader)
 	app.AddHandler(GET, prefix+"downloads/<template_id>", FetchAuthToken(h.getDownloadsHandler))
 	app.AddHandler(POST, prefix+"piuploads", FetchAuthToken(h.postPIUploads)) // itsm process instances upload api
+	imageUploader := uploadHandlerInfo("POST", prefix+"/imageutils/upload", FetchAuthToken(imageUploadHandler))
+	app.AddHandler3(imageUploader)
+	s3upload := uploadHandlerInfo(POST, prefix+"s3uploads", FetchAuthToken(h.postS3UploadHandler))
+	app.AddHandler3(s3upload)
 }
 
 func UploadHandlerInfo(method, prefix string, handler func(context.Context, http.ResponseWriter, *http.Request)) *appsrv.SHandlerInfo {
@@ -509,6 +514,61 @@ func (mh *MiscHandler) postPIUploads(ctx context.Context, w http.ResponseWriter,
 	appsrv.SendJSON(w, resp)
 }
 
+func (mh *MiscHandler) postS3UploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	reader, e := r.MultipartReader()
+	if e != nil {
+		log.Debugf("postS3UploadHandler.MultipartReader %s", e)
+		httperrors.InvalidInputError(w, "invalid form")
+		return
+	}
+
+	p, f, e := readImageForm(reader)
+	if e != nil {
+		log.Debugf("postS3UploadHandler.readImageForm %s", e)
+		httperrors.InvalidInputError(w, "invalid form")
+		return
+	}
+
+	bucket_id, ok := p["bucket_id"]
+	if !ok {
+		httperrors.MissingParameterError(w, "bucket_id")
+		return
+	}
+
+	key, ok := p["key"]
+	if !ok {
+		httperrors.MissingParameterError(w, "key")
+		return
+	}
+
+	_content_length, ok := p["content_length"]
+	if !ok {
+		httperrors.MissingParameterError(w, "content_length")
+		return
+	}
+
+	content_length, e := strconv.ParseInt(_content_length, 10, 64)
+	if e != nil {
+		httperrors.InvalidInputError(w, "invalid content_length %s", _content_length)
+		return
+	}
+
+	storage_class, _ := p["storage_class"]
+	acl, _ := p["acl"]
+
+	token := AppContextToken(ctx)
+	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+
+	meta := http.Header{}
+	meta.Set("Content-Type", "application/octet-stream")
+	e = modules.Buckets.Upload(s, bucket_id, key, f, content_length, storage_class, acl, meta)
+	if e != nil {
+		httperrors.GeneralServerError(w, e)
+		return
+	}
+	appsrv.SendJSON(w, jsonutils.NewDict())
+}
+
 func writeCsv(records [][]string) (bytes.Buffer, error) {
 	var content bytes.Buffer
 	content.WriteString("\xEF\xBB\xBF") // 写入UTF-8 BOM, 防止office打开后中文乱码

pkg/apis/compute/api.go

@@ -471,7 +471,8 @@ type ServerCreateInput struct {
 
 	// swagger:ignore
 	OsType string `json:"os_type"`
-
+	// swagger:ignore
+	DisableUsbKbd bool `json:"disable_usb_kbd"`
 	// swagger:ignore
 	OsProfile jsonutils.JSONObject `json:"__os_profile__"`
 	// swagger:ignore

pkg/apis/compute/host.go

@@ -141,7 +141,7 @@ type HostDetails struct {
 	NonsystemGuests int `json:"nonsystem_guests"`
 	// 运行中云主机数量
 	// example: 2
-	RunningGuests int `json:"running_geusts"`
+	RunningGuests int `json:"running_guests"`
 	// CPU超分率
 	CpuCommitRate float64 `json:"cpu_commit_rate"`
 	// 内存超分率

pkg/apis/image/consts.go

@@ -50,6 +50,7 @@ const (
 	IMAGE_IS_READONLY         = "is_readonly"
 	IMAGE_PARTITION_TYPE      = "partition_type"
 	IMAGE_INSTALLED_CLOUDINIT = "installed_cloud_init"
+	IMAGE_DISABLE_USB_KBD     = "disable_usb_kbd"
 
 	IMAGE_STATUS_UPDATING = "updating"
 )

pkg/compute/models/capabilities.go

@@ -615,20 +615,6 @@ func getNetworkCountByFilter(region *SCloudregion, zone *SZone, domainId string,
 		region = zone.GetRegion()
 	}
 
-	/*vpcQuery := VpcManager.Query()
-	if len(domainId) > 0 {
-		ownerId := &db.SOwnerId{DomainId: domainId}
-		vpcQuery = VpcManager.FilterByOwner(vpcQuery, ownerId, rbacutils.ScopeDomain)
-	}
-	vpcs := vpcQuery.SubQuery()*/
-
-	wireQuery := WireManager.Query().Equals("vpc_id", api.DEFAULT_VPC_ID)
-	if len(domainId) > 0 {
-		ownerId := &db.SOwnerId{DomainId: domainId}
-		wireQuery = WireManager.FilterByOwner(wireQuery, ownerId, rbacutils.ScopeDomain)
-	}
-	wires := wireQuery.SubQuery()
-
 	networks := NetworkManager.Query().SubQuery()
 
 	q := networks.Query()
@@ -639,9 +625,11 @@ func getNetworkCountByFilter(region *SCloudregion, zone *SZone, domainId string,
 			q = q.IsFalse("is_public")
 		}
 	}
-	q = q.Join(wires, sqlchemy.Equals(networks.Field("wire_id"), wires.Field("id")))
 
-	if region != nil {
+	if zone != nil && !utils.IsInStringArray(region.Provider, api.REGIONAL_NETWORK_PROVIDERS) {
+		wires := WireManager.Query("id").Equals("zone_id", zone.Id)
+		q = q.In("wire_id", wires.SubQuery())
+	} else if region != nil {
 		if utils.IsInStringArray(region.Provider, api.REGIONAL_NETWORK_PROVIDERS) {
 			wires := WireManager.Query().SubQuery()
 			vpcs := VpcManager.Query().SubQuery()
@@ -651,28 +639,14 @@ func getNetworkCountByFilter(region *SCloudregion, zone *SZone, domainId string,
 			q = q.Filter(sqlchemy.In(q.Field("wire_id"), subq))
 		} else {
 			subq := getRegionZoneSubq(region)
-			q = q.Filter(sqlchemy.In(wires.Field("zone_id"), subq))
+			wires := WireManager.Query("id").In("zone_id", subq)
+			q = q.In("wire_id", wires.SubQuery())
 		}
 	}
-	if zone != nil && !utils.IsInStringArray(region.Provider, api.REGIONAL_NETWORK_PROVIDERS) {
-		q = q.Filter(sqlchemy.Equals(wires.Field("zone_id"), zone.Id))
-	}
+
 	if len(domainId) > 0 {
 		ownerId := &db.SOwnerId{DomainId: domainId}
 		q = NetworkManager.FilterByOwner(q, ownerId, rbacutils.ScopeDomain)
-		/*subq := getDomainManagerSubq(domainId)
-		q = q.Join(vpcs, sqlchemy.Equals(wires.Field("vpc_id"), vpcs.Field("id")))
-		q = q.Filter(sqlchemy.OR(
-			sqlchemy.In(vpcs.Field("manager_id"), subq),
-			sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id")),
-		))
-		if isPublic.Bool() {
-			q = q.Filter(sqlchemy.OR(
-				sqlchemy.Equals(q.Field("public_scope"), rbacutils.ScopeSystem),
-				sqlchemy.AND(
-					sqlchemy.Equals(q.Field("public_scope"), rbacutils.ScopeDomain),
-					sqlchemy.Equals(q.Field("domain_id"), domainId))))
-		}*/
 	}
 	if len(serverType) > 0 {
 		q = q.Filter(sqlchemy.Equals(networks.Field("server_type"), serverType))

pkg/compute/models/cloudaccounts.go

@@ -819,7 +819,7 @@ func (manager *SCloudaccountManager) ValidateCreateData(
 		input.AutoCreateProject = &forceAutoCreateProject
 	}
 
-	if len(input.ProjectId) > 0 {
+	if len(input.Project) > 0 {
 		var proj *db.STenant
 		proj, input.ProjectizedResourceInput, err = db.ValidateProjectizedResourceInput(ctx, input.ProjectizedResourceInput)
 		if err != nil {

pkg/compute/models/guest_actions.go

@@ -3047,6 +3047,13 @@ func (self *SGuest) PerformSetQemuParams(ctx context.Context, userCred mcclient.
 			return nil, err
 		}
 	}
+	usbKbd, err := data.GetString("disable_usb_kbd")
+	if err == nil {
+		err = self.SetMetadata(ctx, "disable_usb_kbd", usbKbd, userCred)
+		if err != nil {
+			return nil, err
+		}
+	}
 	return nil, nil
 }
 

pkg/compute/models/guests.go

@@ -1149,6 +1149,7 @@ func (manager *SGuestManager) validateCreateData(
 		if len(imgProperties) == 0 {
 			imgProperties = map[string]string{"os_type": "Linux"}
 		}
+		input.DisableUsbKbd = imgProperties[imageapi.IMAGE_DISABLE_USB_KBD] == "true"
 
 		osType := input.OsType
 		osProf, err = osprofile.GetOSProfileFromImageProperties(imgProperties, hypervisor)
@@ -1707,6 +1708,9 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred
 	if osProfileJson != nil {
 		guest.setOSProfile(ctx, userCred, osProfileJson)
 	}
+	if jsonutils.QueryBoolean(data, imageapi.IMAGE_DISABLE_USB_KBD, false) {
+		guest.SetMetadata(ctx, imageapi.IMAGE_DISABLE_USB_KBD, "true", userCred)
+	}
 
 	userData, _ := data.GetString("user_data")
 	if len(userData) > 0 {

pkg/compute/models/storages.go

@@ -951,8 +951,6 @@ func (manager *SStorageManager) totalCapacityQ(
 		q = RangeObjectsFilter(q, rangeObjs, nil, storages.Field("zone_id"), storages.Field("manager_id"), nil, storages.Field("id"))
 	}
 
-	q = q.Distinct()
-
 	if storageOwnership {
 		switch scope {
 		case rbacutils.ScopeSystem:

pkg/compute/regiondrivers/aliyun.go

@@ -986,7 +986,7 @@ func (self *SAliyunRegionDriver) ValidateDBInstanceRecovery(ctx context.Context,
 		if !((utils.IsInStringArray(instance.EngineVersion, []string{"8.0", "5.7"}) &&
 			instance.StorageType == api.ALIYUN_DBINSTANCE_STORAGE_TYPE_LOCAL_SSD &&
 			instance.Category == api.ALIYUN_DBINSTANCE_CATEGORY_HA) || (instance.EngineVersion == "5.6" && instance.Category == api.ALIYUN_DBINSTANCE_CATEGORY_HA)) {
-			return httperrors.NewUnsupportOperationError("Aliyun %s only 8.0, 5.7 ha local_ssd or 5.6 ha support recovery from it self backups")
+			return httperrors.NewUnsupportOperationError("Aliyun %s only 8.0 and 5.7 high_availability local_ssd or 5.6 high_availability support recovery from it self backups", instance.Engine)
 		}
 	}
 	if len(input.Databases) == 0 {

pkg/compute/tasks/guest_detach_disk_task.go

@@ -99,7 +99,7 @@ func (self *GuestDetachDiskTask) OnDetachDiskComplete(ctx context.Context, guest
 			return
 		}
 	}
-	models.StartResourceSyncStatusTask(ctx, self.GetUserCred(), disk, "DiskSyncstatusTask", "")
+	disk.SetStatus(self.UserCred, api.DISK_READY, "on detach disk complete")
 	keepDisk := jsonutils.QueryBoolean(self.Params, "keep_disk", true)
 	host := guest.GetHost()
 	purge := false

pkg/hostman/guestfs/kvmpart.go

@@ -103,7 +103,7 @@ func (p *SKVMGuestDiskPartition) MountPartReadOnly() bool {
 
 func (p *SKVMGuestDiskPartition) Mount() bool {
 	if len(p.fs) == 0 || utils.IsInStringArray(p.fs, []string{"swap", "btrfs"}) {
-		log.Errorf("Mount fs failed: %s", p.fs)
+		log.Errorf("Mount fs failed: unsupport fs %s on %s", p.fs, p.partDev)
 		return false
 	}
 	err := p.fsck()
@@ -127,6 +127,7 @@ func (p *SKVMGuestDiskPartition) Mount() bool {
 			p.readonly = true
 		}
 	}
+	log.Infof("mount fs %s on %s success", p.fs, p.partDev)
 	return true
 }
 
@@ -154,7 +155,21 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error {
 		cmds = append(cmds, "-o", opt)
 	}
 	cmds = append(cmds, p.partDev, p.mountPath)
-	_, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output()
+
+	var err error
+	if fsType == "xfs" {
+		uuids := fileutils2.GetDevUuid(p.partDev)
+		uuid := uuids["UUID"]
+		if len(uuid) > 0 {
+			LockXfsPartition(uuid)
+			defer func() {
+				if err != nil {
+					UnlockXfsPartition(uuid)
+				}
+			}()
+		}
+	}
+	_, err = procutils.NewCommand(cmds[0], cmds[1:]...).Output()
 	return err
 }
 
@@ -217,6 +232,13 @@ func (p *SKVMGuestDiskPartition) Umount() bool {
 			tries += 1
 			_, err := procutils.NewCommand("umount", p.mountPath).Output()
 			if err == nil {
+				if p.fs == "xfs" {
+					uuids := fileutils2.GetDevUuid(p.partDev)
+					uuid := uuids["UUID"]
+					if len(uuid) > 0 {
+						UnlockXfsPartition(uuid)
+					}
+				}
 				procutils.NewCommand("blockdev", "--flushbufs", p.partDev).Output()
 				os.Remove(p.mountPath)
 				return true

pkg/hostman/guestfs/utils.go

@@ -0,0 +1,40 @@
+package guestfs
+
+import (
+	"sync"
+
+	"yunion.io/x/log"
+)
+
+func LockXfsPartition(uuid string) {
+	log.Infof("xfs lock %s", uuid)
+
+	var (
+		xfsLock *sync.Mutex
+		ok      bool
+	)
+
+	mapLock.Lock()
+	xfsLock, ok = xfsMountUniqueTool[uuid]
+	if !ok {
+		xfsLock = new(sync.Mutex)
+		xfsMountUniqueTool[uuid] = xfsLock
+	}
+	mapLock.Unlock()
+
+	xfsLock.Lock()
+}
+
+func UnlockXfsPartition(uuid string) {
+	log.Infof("xfs unlock %s", uuid)
+	mapLock.Lock()
+	xfsLock := xfsMountUniqueTool[uuid]
+	mapLock.Unlock()
+
+	xfsLock.Unlock()
+}
+
+var (
+	mapLock            = sync.Mutex{}
+	xfsMountUniqueTool = map[string]*sync.Mutex{}
+)

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -79,6 +79,11 @@ func (s *SKVMGuestInstance) getOsname() string {
 	return osName
 }
 
+func (s *SKVMGuestInstance) disableUsbKbd() bool {
+	val, _ := s.Desc.GetString("metadata", "disable_usb_kbd")
+	return val == "true"
+}
+
 func (s *SKVMGuestInstance) getOsDistribution() string {
 	osDis, _ := s.Desc.GetString("metadata", "os_distribution")
 	return osDis
@@ -499,7 +504,7 @@ func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (stri
 
 	cmd += " -device virtio-serial"
 	cmd += " -usb"
-	if !utils.IsInStringArray(s.getOsDistribution(), []string{OS_NAME_OPENWRT, OS_NAME_CIRROS}) {
+	if !utils.IsInStringArray(s.getOsDistribution(), []string{OS_NAME_OPENWRT, OS_NAME_CIRROS}) && !s.disableUsbKbd() {
 		cmd += " -device usb-kbd"
 	}
 	// # if osname == self.OS_NAME_ANDROID:

pkg/hostman/hostinfo/hostinfo.go

@@ -1383,7 +1383,7 @@ func (h *SHostInfo) deployAdminAuthorizedKeys() {
 	}
 
 	query := jsonutils.NewDict()
-	query.Add(jsonutils.NewString("system"), "scope")
+	query.Set("admin", jsonutils.JSONTrue)
 	ret, err := modules.Sshkeypairs.List(h.GetSession(), query)
 	if err != nil {
 		onErr("Get admin sshkey: %v", err)

pkg/monitor/models/nodealert.go

@@ -136,6 +136,9 @@ func (man *SNodeAlertManager) genName(ownerId mcclient.IIdentityProvider, resTyp
 	if err != nil {
 		return "", err
 	}
+	if name != nameHint {
+		return "", httperrors.NewDuplicateNameError(man.Keyword(), metric)
+	}
 	return name, nil
 }
 

pkg/multicloud/azure/classic_storage.go

@@ -62,7 +62,7 @@ func (self *SClassicStorage) GetGlobalId() string {
 }
 
 func (self *SClassicStorage) IsEmulated() bool {
-	return false
+	return true
 }
 
 func (self *SClassicStorage) GetIZone() cloudprovider.ICloudZone {

pkg/multicloud/huawei/client/modules/manager_base.go

@@ -146,17 +146,13 @@ func (ce *HuaweiClientError) Error() string {
 }
 
 func (ce *HuaweiClientError) ParseErrorFromJsonResponse(statusCode int, body jsonutils.JSONObject) error {
-	err := body.Unmarshal(ce)
-	if err != nil {
-		ce.err = errors.Wrapf(err, "body.Unmarshal(%s)", body.String())
-		ce.Code = statusCode
-		ce.Details = body.String()
-		return ce
+	if body != nil {
+		body.Unmarshal(ce)
 	}
 	if ce.Code == 0 {
 		ce.Code = statusCode
 	}
-	if len(ce.Details) == 0 {
+	if len(ce.Details) == 0 && body != nil {
 		ce.Details = body.String()
 	}
 	return ce
@@ -187,10 +183,6 @@ func (self *SBaseManager) jsonRequest(request requests.IRequest) (http.Header, j
 		}
 	}
 
-	if self.debug {
-		log.Debugf("url: %s", request.BuildUrl())
-	}
-
 	client := httputils.NewJsonClient(self.httpClient)
 	req := httputils.NewJsonRequest(httputils.THttpMethod(request.GetMethod()), request.BuildUrl(), jsonBody)
 	req.SetHeader(header)
@@ -201,12 +193,11 @@ func (self *SBaseManager) jsonRequest(request requests.IRequest) (http.Header, j
 	for {
 		h, b, e := client.Send(ctx, req, resp, self.debug)
 		if e == nil {
-			if self.debug {
-				log.Debugf("response: %s body: %s", h, b)
-			}
-			return h, b, e
+			return h, b, nil
 		}
 
+		log.Errorf("[%s] %s body: %v error: %v", req.GetHttpMethod(), req.GetUrl(), jsonBody, e)
+
 		switch err := e.(type) {
 		case *HuaweiClientError:
 			if (err.Code == 499 || err.Code == 429) && retry > 0 && request.GetMethod() == "GET" {

pkg/multicloud/huawei/region.go

@@ -537,16 +537,19 @@ func (self *SRegion) CreateISecurityGroup(conf *cloudprovider.SecurityGroupCreat
 
 // https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090608.html
 func (self *SRegion) CreateIVpc(name string, desc string, cidr string) (cloudprovider.ICloudVpc, error) {
-	params := jsonutils.NewDict()
-	vpcObj := jsonutils.NewDict()
-	vpcObj.Add(jsonutils.NewString(name), "name")
-	vpcObj.Add(jsonutils.NewString(cidr), "cidr")
-	params.Add(vpcObj, "vpc")
+	return self.CreateVpc(name, cidr, desc)
+}
 
-	vpc := SVpc{}
-	err := DoCreate(self.ecsClient.Vpcs.Create, params, &vpc)
-	vpc.region = self
-	return &vpc, err
+func (self *SRegion) CreateVpc(name, cidr, desc string) (*SVpc, error) {
+	params := map[string]interface{}{
+		"vpc": map[string]string{
+			"name":        name,
+			"cidr":        cidr,
+			"description": desc,
+		},
+	}
+	vpc := &SVpc{region: self}
+	return vpc, DoCreate(self.ecsClient.Vpcs.Create, jsonutils.Marshal(params), vpc)
 }
 
 // https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090596.html

pkg/multicloud/huawei/securitygroup.go

@@ -31,6 +31,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/secrules"
+	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -227,7 +228,7 @@ func (self *SRegion) GetSecurityGroupDetails(secGroupId string) (*SSecurityGroup
 // https://support.huaweicloud.com/api-vpc/zh-cn_topic_0020090617.html
 func (self *SRegion) GetSecurityGroups(vpcId string, name string) ([]SSecurityGroup, error) {
 	querys := map[string]string{}
-	if len(vpcId) > 0 {
+	if len(vpcId) > 0 && !utils.IsInStringArray(vpcId, []string{"default", api.NORMAL_VPC_ID}) { // vpc_id = default or normal 时报错 '{"code":"VPC.0601","message":"Query security groups error vpcId is invalid."}'
 		querys["vpc_id"] = vpcId
 	}
 

pkg/multicloud/huawei/shell/vpc.go

@@ -30,4 +30,28 @@ func init() {
 		printList(vpcs, 0, 0, 0, nil)
 		return nil
 	})
+
+	type VpcCreateOptions struct {
+		NAME string
+		CIDR string
+		Desc string
+	}
+
+	shellutils.R(&VpcCreateOptions{}, "vpc-create", "Create vpc", func(cli *huawei.SRegion, args *VpcCreateOptions) error {
+		vpc, err := cli.CreateVpc(args.NAME, args.CIDR, args.Desc)
+		if err != nil {
+			return err
+		}
+		printObject(vpc)
+		return nil
+	})
+
+	type VpcIdOption struct {
+		ID string
+	}
+
+	shellutils.R(&VpcIdOption{}, "vpc-delete", "Delete vpc", func(cli *huawei.SRegion, args *VpcIdOption) error {
+		return cli.DeleteVpc(args.ID)
+	})
+
 }

pkg/multicloud/huawei/vpc.go

@@ -225,6 +225,18 @@ func (self *SRegion) getVpc(vpcId string) (*SVpc, error) {
 }
 
 func (self *SRegion) DeleteVpc(vpcId string) error {
+	if vpcId != "default" {
+		secgroups, err := self.GetSecurityGroups(vpcId, "")
+		if err != nil {
+			return errors.Wrap(err, "GetSecurityGroups")
+		}
+		for _, secgroup := range secgroups {
+			err = self.DeleteSecurityGroup(secgroup.ID)
+			if err != nil {
+				return errors.Wrapf(err, "DeleteSecurityGroup(%s)", secgroup.ID)
+			}
+		}
+	}
 	return DoDelete(self.ecsClient.Vpcs.Delete, vpcId, nil, nil)
 }
 

pkg/multicloud/objectstore/objectstore.go

@@ -17,6 +17,7 @@ package objectstore
 import (
 	"net/url"
 	"os"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -504,7 +505,12 @@ func (cli *SObjectStoreClient) SetIBucketAcl(name string, cannedAcl cloudprovide
 	acl := s3cli.CannedAcl(cli.ownerId, cli.ownerName, string(cannedAcl))
 	err := cli.client.SetBucketAcl(name, acl)
 	if err != nil {
-		return errors.Wrap(err, "SetBucketAcl")
+		if strings.Contains(err.Error(), "not implemented") {
+			// ignore not implemented error
+			return nil // cloudprovider.ErrNotImplemented
+		} else {
+			return errors.Wrap(err, "SetBucketAcl")
+		}
 	}
 	return nil
 }

pkg/multicloud/openstack/hypervisor.go

@@ -100,7 +100,7 @@ func (host *SHypervisor) GetIWires() ([]cloudprovider.ICloudWire, error) {
 func (host *SHypervisor) GetIStorages() ([]cloudprovider.ICloudStorage, error) {
 	istorages := []cloudprovider.ICloudStorage{}
 	storages, err := host.zone.region.GetStorageTypes()
-	if err != nil {
+	if err != nil && errors.Cause(err) != ErrNoEndpoint {
 		return nil, errors.Wrap(err, "GetStorageTypes")
 	}
 	for i := range storages {
@@ -148,7 +148,12 @@ func (host *SHypervisor) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (c
 }
 
 func (host *SHypervisor) GetEnabled() bool {
-	return true
+	switch host.Status {
+	case "enabled", "":
+		return true
+	default:
+		return false
+	}
 }
 
 func (host *SHypervisor) GetAccessIp() string {
@@ -266,9 +271,6 @@ func (host *SHypervisor) GetHostType() string {
 }
 
 func (host *SHypervisor) GetHostStatus() string {
-	if host.Status == "disabled" {
-		return api.HOST_OFFLINE
-	}
 	switch host.State {
 	case "up", "":
 		return api.HOST_ONLINE
@@ -282,12 +284,7 @@ func (host *SHypervisor) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface,
 }
 
 func (host *SHypervisor) GetIsMaintenance() bool {
-	switch host.Status {
-	case "enabled", "":
-		return false
-	default:
-		return true
-	}
+	return false
 }
 
 func (host *SHypervisor) GetVersion() string {

pkg/multicloud/openstack/openstack.go

@@ -45,6 +45,8 @@ const (
 	OPENSTACK_SERVICE_VOLUMEV2 = "volumev2"
 	OPENSTACK_SERVICE_VOLUME   = "volume"
 	OPENSTACK_SERVICE_IMAGE    = "image"
+
+	ErrNoEndpoint = errors.Error("no valid endpoint")
 )
 
 type OpenstackClientConfig struct {
@@ -159,11 +161,13 @@ type OpenstackError struct {
 }
 
 func (ce *OpenstackError) ParseErrorFromJsonResponse(statusCode int, body jsonutils.JSONObject) error {
-	body.Unmarshal(ce)
+	if body != nil {
+		body.Unmarshal(ce)
+	}
 	if ce.Code == 0 {
 		ce.Code = statusCode
 	}
-	if len(ce.Details) == 0 {
+	if len(ce.Details) == 0 && body != nil {
 		ce.Details = body.String()
 	}
 	if len(ce.Class) == 0 {
@@ -250,12 +254,20 @@ func jsonReuest(token mcclient.TokenCredential, service, region, endpointType st
 		}
 	}
 
-	requestUrl := resource
-	if !strings.HasPrefix(resource, serviceUrl) {
-		requestUrl = fmt.Sprintf("%s/%s", serviceUrl, resource)
+	if service == OPENSTACK_SERVICE_IDENTITY {
+		if strings.HasSuffix(serviceUrl, "/v3/") {
+			serviceUrl = strings.TrimSuffix(serviceUrl, "/v3/")
+		} else if strings.HasSuffix(serviceUrl, "/v3") {
+			serviceUrl = strings.TrimSuffix(serviceUrl, "/v3")
+		}
 	}
 
-	if query != nil {
+	requestUrl := resource
+	if !strings.HasPrefix(resource, serviceUrl) {
+		requestUrl = fmt.Sprintf("%s/%s", strings.TrimSuffix(serviceUrl, "/"), strings.TrimPrefix(resource, "/"))
+	}
+
+	if query != nil && len(query) > 0 {
 		requestUrl = fmt.Sprintf("%s?%s", requestUrl, query.Encode())
 	}
 
@@ -327,7 +339,7 @@ func (cli *SOpenStackClient) bsRequest(region string, method httputils.THttpMeth
 			return jsonReuest(cli.tokenCredential, service, region, cli.endpointType, method, resource, query, body, cli.debug)
 		}
 	}
-	return nil, fmt.Errorf("no valid volume service endpoint")
+	return nil, errors.Wrap(ErrNoEndpoint, "cinder service")
 }
 
 func (cli *SOpenStackClient) bsCreate(projectId, region, resource string, body interface{}) (jsonutils.JSONObject, error) {
@@ -345,7 +357,7 @@ func (cli *SOpenStackClient) bsCreate(projectId, region, resource string, body i
 			return jsonReuest(token, service, region, cli.endpointType, httputils.POST, resource, nil, body, cli.debug)
 		}
 	}
-	return nil, fmt.Errorf("no valid volume service endpoint")
+	return nil, errors.Wrap(ErrNoEndpoint, "cinder service")
 }
 
 func (cli *SOpenStackClient) imageUpload(region, url string, body io.Reader) (*http.Response, error) {

pkg/multicloud/openstack/zone.go

@@ -112,7 +112,7 @@ func (zone *SZone) getStorageByCategory(category, host string) (*SStorage, error
 
 func (zone *SZone) GetIStorages() ([]cloudprovider.ICloudStorage, error) {
 	storages, err := zone.region.GetStorageTypes()
-	if err != nil {
+	if err != nil && errors.Cause(err) != ErrNoEndpoint {
 		return nil, errors.Wrap(err, "GetStorageTypes")
 	}
 	istorages := []cloudprovider.ICloudStorage{}

pkg/multicloud/zstack/zstack.go

@@ -59,7 +59,7 @@ type ZstackClientConfig struct {
 
 func NewZstackClientConfig(authURL, username, password string) *ZstackClientConfig {
 	cfg := &ZstackClientConfig{
-		authURL:  authURL,
+		authURL:  strings.TrimSuffix(authURL, "/"),
 		username: username,
 		password: password,
 	}