Merge pull request #5741 from yousong/automated-cherry-pick-of-#5740-upstream-release-3.1

Automated cherry pick of #5740: region: lbagents: fix returning error when glob has no match

.github/workflows/docker.yml

@@ -39,7 +39,7 @@ jobs:
         make cmd/cloudevent cmd/devtool
         make cmd/*cli
         make cmd/esxi-agent
-        make docker-alpine-build F='-j4 cmd/host cmd/vpcagent'
+        make docker-alpine-build F='-j4 cmd/host cmd/vpcagent cmd/region-dns'
 
     - name: Image baremetal-agent
       uses: elgohr/Publish-Docker-Github-Action@master
@@ -220,3 +220,13 @@ jobs:
         registry: registry.cn-beijing.aliyuncs.com
         snapshot: true
         dockerfile: build/docker/Dockerfile.devtool
+
+    - name: Image region DNS
+      uses: elgohr/Publish-Docker-Github-Action@master
+      with:
+        name: registry.cn-beijing.aliyuncs.com/yunionio/region-dns
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+        registry: registry.cn-beijing.aliyuncs.com
+        snapshot: true
+        dockerfile: build/docker/Dockerfile.region-dns

Makefile

@@ -211,6 +211,7 @@ docker-centos-build:
 		--name onecloud-ci-build \
 		--rm \
 		--volume $(CURDIR):/root/onecloud \
+		--volume $(CURDIR)/_output/_cache:/root/.cache \
 		registry.cn-beijing.aliyuncs.com/yunionio/centos-build:$(DOCKER_BUILD_IMAGE_VERSION) \
 		/bin/bash -c "$$dockerCentOSBuildCmd"
 	chown -R $$(id -u):$$(id -g) _output
@@ -241,6 +242,7 @@ docker-alpine-build:
 		--name onecloud-docker-alpine-build \
 		-v $(CURDIR):/root/go/src/yunion.io/x/onecloud \
 		-v $(CURDIR)/_output/alpine-build:/root/go/src/yunion.io/x/onecloud/_output \
+		-v $(CURDIR)/_output/alpine-build/_cache:/root/.cache \
 		registry.cn-beijing.aliyuncs.com/yunionio/alpine-build:1.0-1 \
 		/bin/sh -c "$$dockerAlpineBuildCmd"
 	ls -lh _output/alpine-build/bin

build/docker/Dockerfile.region-dns

@@ -1,3 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:latest
+FROM alpine:3.11.3
 
-ADD ./_output/bin/region-dns /opt/yunion/bin/region-dns
+RUN mkdir -p /opt/yunion/bin
+ADD ./_output/alpine-build/bin/region-dns /opt/yunion/bin/region-dns

cmd/aliyuncli/main.go

@@ -84,7 +84,11 @@ func newClient(options *BaseOptions) (*aliyun.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
-	cli, err := aliyun.NewAliyunClient("", "", options.AccessKey, options.Secret, options.Debug)
+	cli, err := aliyun.NewAliyunClient(
+		aliyun.NewAliyunClientConfig(
+			options.AccessKey, options.Secret,
+		).Debug(options.Debug),
+	)
 	if err != nil {
 		return nil, err
 	}

cmd/awscli/main.go

@@ -85,7 +85,13 @@ func newClient(options *BaseOptions) (*aws.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
-	cli, err := aws.NewAwsClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug)
+	cli, err := aws.NewAwsClient(
+		aws.NewAwsClientConfig(
+			options.AccessUrl,
+			options.AccessKey,
+			options.Secret,
+		).Debug(options.Debug),
+	)
 	if err != nil {
 		return nil, err
 	}

cmd/azurecli/main.go

@@ -100,11 +100,17 @@ func newClient(options *BaseOptions) (*azure.SRegion, error) {
 		return nil, fmt.Errorf("Missing Cloud Environment")
 	}
 
-	cli, err := azure.NewAzureClient("", "", options.CloudEnv,
-		options.DirectoryID,
-		options.ApplicationID, options.ApplicationKey,
-		options.SubscriptionID,
-		options.Debug)
+	cli, err := azure.NewAzureClient(
+		azure.NewAzureClientConfig(
+			options.CloudEnv,
+			options.DirectoryID,
+			options.ApplicationID,
+			options.ApplicationKey,
+		).
+			SubscriptionId(options.SubscriptionID).
+			Debug(options.Debug),
+	)
+
 	if err != nil {
 		return nil, err
 	}

cmd/climc/shell/bucket_options.go

@@ -0,0 +1,46 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	type BucketOptionsVerifyOptions struct {
+		options.BaseListOptions
+		CloudaccountId       string `help:"cloudaccount Id" required:"true"`
+		BillingReportBucket  string `help:"S3 bucket that stores account billing report"`
+		BillingFilePrefix    string `help:"prefix of billing file name"`
+		BillingBucketAccount string `help:"Id of cloudaccount that can access billing bucket"`
+		UsageReportBucket    string `help:"S3 bucket that stores account usage report"`
+		UsageFilePrefix      string `help:"prefix of usage file name"`
+		UsageBucketAccount   string `help:"Id of cloudaccount that can access usage bucket"`
+	}
+	R(&BucketOptionsVerifyOptions{}, "bucket-options-verify", "billing bucket connection test",
+		func(s *mcclient.ClientSession, args *BucketOptionsVerifyOptions) error {
+			params := jsonutils.Marshal(args)
+			result, err := modules.BucketOptions.PerformAction(s, args.CloudaccountId, "verify", params)
+			if err != nil {
+				return err
+			}
+			printObject(result)
+			return nil
+		})
+}

cmd/climc/shell/cloudaccounts.go

@@ -381,6 +381,19 @@ func init() {
 		return nil
 	})
 
+	R(&options.SGoogleCloudAccountUpdateOptions{}, "cloud-account-update-google", "update a google cloud account", func(s *mcclient.ClientSession, args *options.SGoogleCloudAccountUpdateOptions) error {
+		params := jsonutils.Marshal(args).(*jsonutils.JSONDict)
+		if params.Size() == 0 {
+			return InvalidUpdateError()
+		}
+		result, err := modules.Cloudaccounts.Update(s, args.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	R(&options.SAWSCloudAccountUpdateOptions{}, "cloud-account-update-aws", "update an AWS cloud account", func(s *mcclient.ClientSession, args *options.SAWSCloudAccountUpdateOptions) error {
 		params := jsonutils.Marshal(args).(*jsonutils.JSONDict)
 
@@ -422,7 +435,7 @@ func init() {
 		return nil
 	})
 
-	R(&options.SOpenStackCloudAccountUpdateOptions{}, "cloud-account-update-openstack", "update an AWS cloud account", func(s *mcclient.ClientSession, args *options.SOpenStackCloudAccountUpdateOptions) error {
+	R(&options.SOpenStackCloudAccountUpdateOptions{}, "cloud-account-update-openstack", "update an OpenStack cloud account", func(s *mcclient.ClientSession, args *options.SOpenStackCloudAccountUpdateOptions) error {
 		params := jsonutils.Marshal(args).(*jsonutils.JSONDict)
 		if params.Size() == 0 {
 			return InvalidUpdateError()

cmd/climc/shell/cloudnet/routes.go

@@ -22,7 +22,7 @@ import (
 )
 
 func init() {
-	R(&options.RouteCreateOptions{}, "router-route-create", "Create router", func(s *mcclient.ClientSession, opts *options.RouteCreateOptions) error {
+	R(&options.RouteCreateOptions{}, "router-route-create", "Create router route", func(s *mcclient.ClientSession, opts *options.RouteCreateOptions) error {
 		params, err := base_options.StructToParams(opts)
 		if err != nil {
 			return err
@@ -34,7 +34,7 @@ func init() {
 		printObject(router)
 		return nil
 	})
-	R(&options.RouteGetOptions{}, "router-route-show", "Show router", func(s *mcclient.ClientSession, opts *options.RouteGetOptions) error {
+	R(&options.RouteGetOptions{}, "router-route-show", "Show router route", func(s *mcclient.ClientSession, opts *options.RouteGetOptions) error {
 		router, err := modules.Routes.Get(s, opts.ID, nil)
 		if err != nil {
 			return err
@@ -42,7 +42,7 @@ func init() {
 		printObject(router)
 		return nil
 	})
-	R(&options.RouteListOptions{}, "router-route-list", "List routers", func(s *mcclient.ClientSession, opts *options.RouteListOptions) error {
+	R(&options.RouteListOptions{}, "router-route-list", "List router routes", func(s *mcclient.ClientSession, opts *options.RouteListOptions) error {
 		params, err := base_options.ListStructToParams(opts)
 		if err != nil {
 			return err
@@ -54,7 +54,7 @@ func init() {
 		printList(result, modules.Routes.GetColumns(s))
 		return nil
 	})
-	R(&options.RouteUpdateOptions{}, "router-route-update", "Update router", func(s *mcclient.ClientSession, opts *options.RouteUpdateOptions) error {
+	R(&options.RouteUpdateOptions{}, "router-route-update", "Update router route", func(s *mcclient.ClientSession, opts *options.RouteUpdateOptions) error {
 		params, err := base_options.StructToParams(opts)
 		router, err := modules.Routes.Update(s, opts.ID, params)
 		if err != nil {
@@ -63,7 +63,7 @@ func init() {
 		printObject(router)
 		return nil
 	})
-	R(&options.RouteDeleteOptions{}, "router-route-delete", "Delete router", func(s *mcclient.ClientSession, opts *options.RouteDeleteOptions) error {
+	R(&options.RouteDeleteOptions{}, "router-route-delete", "Delete router route", func(s *mcclient.ClientSession, opts *options.RouteDeleteOptions) error {
 		router, err := modules.Routes.Delete(s, opts.ID, nil)
 		if err != nil {
 			return err

cmd/climc/shell/contacts.go

@@ -29,11 +29,11 @@ func init() {
 	 * 操作用户的通信地址（如果用户的通信地址不存在则进行添加；如果已存在则进行修改；如果设置空则进行删除。）
 	 */
 	type ContactsUpdateOptions struct {
-		UID         string `help:"The user you wanna add contact to (Keystone User ID)"`
-		CONTACTTYPE string `help:"The contact type email|mobile" choices:"email|mobile|dingtalk"`
-		CONTACT     string `help:"The contacts details mobile number or email address or dingtalk's userid, if set it the empty str means delete"`
-		Status      string `help:"Enabled or disabled contact status" choices:"enable|disable"`
-		Pull        string `help:"pull some subcontacts(e.g., dingtalk, feishu, etc) related to mobile"`
+		UID         string   `help:"The user you wanna add contact to (Keystone User ID)"`
+		CONTACTTYPE string   `help:"The contact type email|mobile" choices:"email|mobile|dingtalk"`
+		CONTACT     string   `help:"The contacts details mobile number or email address or dingtalk's userid, if set it the empty str means delete"`
+		Status      string   `help:"Enabled or disabled contact status" choices:"enable|disable"`
+		Pull        []string `help:"pull some subcontacts(e.g., dingtalk, feishu, etc) related to mobile"`
 	}
 	R(&ContactsUpdateOptions{}, "contact-update", "Create, delete or update contact for user", func(s *mcclient.ClientSession, args *ContactsUpdateOptions) error {
 		arr := jsonutils.NewArray()
@@ -53,7 +53,13 @@ func init() {
 		params := jsonutils.NewDict()
 		params.Add(arr, "contacts")
 
-		contact, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "update-contact", args.Pull, params)
+		pulls := jsonutils.NewArray()
+		for _, pull := range args.Pull {
+			pulls.Add(jsonutils.NewString(pull))
+		}
+		params.Add(pulls, "pull")
+
+		contact, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "update-contact", params)
 		if err != nil {
 			return err
 		}
@@ -69,7 +75,8 @@ func init() {
 	R(&ContactsPullOptions{}, "contact-pull", "Pull contact", func(s *mcclient.ClientSession, args *ContactsPullOptions) error {
 		params := jsonutils.NewDict()
 		params.Set("contacts", jsonutils.NewArray())
-		contact, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "update-contact", args.CONTACTTYPE, params)
+		params.Set("pull", jsonutils.NewArray(jsonutils.NewString(args.CONTACTTYPE)))
+		contact, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "update-contact", params)
 		if err != nil {
 			return err
 		}
@@ -89,7 +96,7 @@ func init() {
 		arr.Add(tmpObj)
 		params := jsonutils.NewDict()
 		params.Add(arr, "contacts")
-		contact, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "update-contact", "", params)
+		contact, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "update-contact", params)
 		if err != nil {
 			return err
 		}
@@ -180,7 +187,7 @@ func init() {
 		tmpDict := jsonutils.NewDict()
 		tmpDict.Add(jsonutils.NewString(args.CONTACT_TYPE), "contact_type")
 		tmpDict.Add(jsonutils.NewString(args.CONTACT), "contact")
-		_, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "verify", "", tmpDict)
+		_, err := modules.Contacts.CustomizedPerformAction(s, args.UID, "verify", tmpDict)
 		if err != nil {
 			return err
 		}

cmd/climc/shell/k8s/clusters.go

@@ -205,4 +205,70 @@ func initKubeCluster() {
 		printObject(ret)
 		return nil
 	})
+
+	R(&o.IdentOptions{}, cmdN("components-status"), "Get cluster component status", func(s *mcclient.ClientSession, args *o.IdentOptions) error {
+		ret, err := k8s.KubeClusters.GetSpecific(s, args.ID, "components-status", nil)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterComponentTypeOptions{}, cmdN("component-setting"), "Get cluster component setting", func(s *mcclient.ClientSession, args *o.ClusterComponentTypeOptions) error {
+		q := jsonutils.NewDict()
+		q.Add(jsonutils.NewString(args.TYPE), "type")
+		ret, err := k8s.KubeClusters.GetSpecific(s, args.ID, "component-setting", q)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterEnableComponentCephCSIOpt{}, cmdN("component-enable-ceph-csi"), "Enable cluster component", func(s *mcclient.ClientSession, args *o.ClusterEnableComponentCephCSIOpt) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "enable-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterDisableComponent{}, cmdN("component-disable"), "Enable cluster component", func(s *mcclient.ClientSession, args *o.ClusterDisableComponent) error {
+		params := args.Params()
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "disable-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterDisableComponent{}, cmdN("component-delete"), "Delete cluster component", func(s *mcclient.ClientSession, args *o.ClusterDisableComponent) error {
+		params := args.Params()
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "delete-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+
+	R(&o.ClusterUpdateComponentCephCSIOpt{}, cmdN("component-update-ceph-csi"), "Update cluster component", func(s *mcclient.ClientSession, args *o.ClusterUpdateComponentCephCSIOpt) error {
+		params, err := args.Params()
+		if err != nil {
+			return err
+		}
+		ret, err := k8s.KubeClusters.PerformAction(s, args.ID, "update-component", params)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
 }

cmd/climc/shell/k8s/secret.go

@@ -40,22 +40,40 @@ func initSecret() {
 		})
 	secretCmd.AddR(listCmd)
 
-	registryCmd := initK8sNamespaceResource("registrysecret", k8s.RegistrySecrets)
-	createCmd := NewCommand(
+	registryCmd := NewCmdNameFactory("secret-registry")
+	registryCreateCmd := NewCommand(
 		&o.RegistrySecretCreateOptions{},
-		registryCmd.CommandNameFactory("create"),
-		"Create docker registry secret resource",
+		registryCmd.Do("create"),
+		"Create docker registry secret secret",
 		func(s *mcclient.ClientSession, args *o.RegistrySecretCreateOptions) error {
 			params, err := args.Params()
 			if err != nil {
 				return err
 			}
-			ret, err := k8s.RegistrySecrets.Create(s, params)
+			ret, err := k8s.Secrets.Create(s, params)
 			if err != nil {
 				return err
 			}
 			printObject(ret)
 			return nil
 		})
-	registryCmd.AddR(createCmd)
+
+	cephCSICmd := NewCmdNameFactory("secret-ceph-csi")
+	cephCSICreateCmd := NewCommand(
+		&o.CephCSISecretCreateOptions{},
+		cephCSICmd.Do("create"),
+		"Create ceph csi user secret",
+		func(s *mcclient.ClientSession, args *o.CephCSISecretCreateOptions) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			ret, err := k8s.Secrets.Create(s, params)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		})
+	secretCmd.AddR(cephCSICreateCmd, registryCreateCmd)
 }

cmd/climc/shell/k8s/storageclass.go

@@ -15,9 +15,70 @@
 package k8s
 
 import (
+	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+	o "yunion.io/x/onecloud/pkg/mcclient/options/k8s"
 )
 
 func initStorageClass() {
-	initK8sClusterResource("storageclass", k8s.Storageclass)
+	cmdN := NewCmdNameFactory("storageclass")
+	scCmd := initK8sClusterResource("storageclass", k8s.Storageclass)
+
+	setDefaultCmd := NewCommand(
+		&o.ClusterResourceBaseOptions{},
+		cmdN.Do("set-default"),
+		"Set storageclass as default",
+		func(s *mcclient.ClientSession, args *o.ClusterResourceBaseOptions) error {
+			ret, err := k8s.Storageclass.PerformAction(s, args.NAME, "set-default", args.Params())
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		},
+	)
+
+	scCmd.AddR(setDefaultCmd)
+
+	addStorageClassCephCSI(scCmd)
+}
+
+func addStorageClassCephCSI(cmd *ShellCommands) {
+	rbdN := NewCmdNameFactory("storageclass-ceph-csi-rbd")
+	rbdCreateCmd := NewCommand(
+		&o.StorageClassCephCSIRBDCreateOptions{},
+		rbdN.Do("create"),
+		"Create ceph csi rbd",
+		func(s *mcclient.ClientSession, args *o.StorageClassCephCSIRBDCreateOptions) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			ret, err := k8s.Storageclass.Create(s, params)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		},
+	)
+	testConnCmd := NewCommand(
+		&o.StorageClassCephCSIRBDTestOptions{},
+		rbdN.Do("connection-test"),
+		"Test storageclass connection",
+		func(s *mcclient.ClientSession, args *o.StorageClassCephCSIRBDTestOptions) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			ret, err := k8s.Storageclass.PerformClassAction(s, "connection-test", params)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		},
+	)
+
+	cmd.AddR(rbdCreateCmd, testConnCmd)
 }

cmd/climc/shell/monitor/alert.go

@@ -15,6 +15,9 @@
 package monitor
 
 import (
+	"fmt"
+
+	monitorapi "yunion.io/x/onecloud/pkg/apis/monitor"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
 	options "yunion.io/x/onecloud/pkg/mcclient/options/monitor"
@@ -36,6 +39,20 @@ func init() {
 			return nil
 		})
 
+	R(&options.AlertCreateOptions{}, aN("create"), "Create alert rule",
+		func(s *mcclient.ClientSession, args *options.AlertCreateOptions) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			ret, err := monitor.Alerts.DoCreate(s, params)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		})
+
 	R(&options.AlertShowOptions{}, aN("show"), "Show details of a alert rule",
 		func(s *mcclient.ClientSession, args *options.AlertShowOptions) error {
 			ret, err := monitor.Alerts.Get(s, args.ID, nil)
@@ -66,4 +83,34 @@ func init() {
 			printBatchResults(ret, monitor.Alerts.GetColumns(s))
 			return nil
 		})
+
+	R(&options.AlertTestRunOptions{}, aN("test-run"), "Test run alert",
+		func(s *mcclient.ClientSession, args *options.AlertTestRunOptions) error {
+			data := new(monitorapi.AlertTestRunInput)
+			if args.Debug {
+				data.IsDebug = true
+			}
+			ret, err := monitor.Alerts.DoTestRun(s, args.ID, data)
+			if err != nil {
+				return err
+			}
+			fmt.Println(ret.JSON(ret).YAMLString())
+			return nil
+		})
+
+	R(&options.AlertPauseOptions{}, aN("pause"), "Pause or unpause alert",
+		func(s *mcclient.ClientSession, args *options.AlertPauseOptions) error {
+			data := new(monitorapi.AlertPauseInput)
+			if args.UnPause {
+				data.Paused = false
+			} else {
+				data.Paused = true
+			}
+			ret, err := monitor.Alerts.PerformAction(s, args.ID, "pause", data.JSON(data))
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		})
 }

cmd/climc/shell/monitor/alertnotification.go

@@ -0,0 +1,60 @@
+package monitor
+
+import (
+	monitorapi "yunion.io/x/onecloud/pkg/apis/monitor"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/monitor"
+)
+
+func init() {
+	initAlertNotification()
+}
+
+func initAlertNotification() {
+	aN := cmdN("alert-notification")
+	R(&options.AlertNotificationListOptions{}, aN("list"), "List alert notification pairs",
+		func(s *mcclient.ClientSession, args *options.AlertNotificationListOptions) error {
+			params, err := args.Params()
+			if err != nil {
+				return err
+			}
+			var result *modulebase.ListResult
+			if len(args.Alert) > 0 {
+				result, err = monitor.Alertnotification.ListDescendent(s, args.Alert, params)
+			} else if len(args.Notification) > 0 {
+				result, err = monitor.Alertnotification.ListDescendent2(s, args.Notification, params)
+			} else {
+				result, err = monitor.Alertnotification.List(s, params)
+			}
+			if err != nil {
+				return err
+			}
+			printList(result, monitor.Alertnotification.GetColumns(s))
+			return nil
+		})
+
+	R(&options.AlertNotificationAttachOptions{}, aN("attach"), "Attach a notification to a alert",
+		func(s *mcclient.ClientSession, args *options.AlertNotificationAttachOptions) error {
+			input := &monitorapi.AlertnotificationCreateInput{
+				UsedBy: args.UsedBy,
+			}
+			ret, err := monitor.Alertnotification.Attach(s, args.ALERT, args.NOTIFICATION, input.JSON(input))
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		})
+
+	R(&options.AlertNotificationAttachOptions{}, aN("detach"), "Detach a notification to a alert",
+		func(s *mcclient.ClientSession, args *options.AlertNotificationAttachOptions) error {
+			ret, err := monitor.Alertnotification.Detach(s, args.ALERT, args.NOTIFICATION, nil)
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		})
+}

cmd/climc/shell/networks.go

@@ -346,77 +346,6 @@ func init() {
 		return nil
 	})
 
-	type NetworkAddDnsUpdateTargetOptions struct {
-		ID     string `help:"Network ID or name"`
-		DNS    string `help:"DNS server address" metavar:"DNS_SERVER"`
-		KEY    string `help:"DNS update key name" metavar:"DNS_UPDATE_KEY"`
-		SECRET string `help:"DNS update key secret" metavar:"DNS_UPDATE_SECRET"`
-	}
-	R(&NetworkAddDnsUpdateTargetOptions{}, "network-add-dns-update-target", "Add a dns update target to a network", func(s *mcclient.ClientSession, args *NetworkAddDnsUpdateTargetOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(args.DNS), "server")
-		params.Add(jsonutils.NewString(args.KEY), "key")
-		params.Add(jsonutils.NewString(args.SECRET), "secret")
-		result, err := modules.Networks.PerformAction(s, args.ID, "add-dns-update-target", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type NetworkRemoveDnsUpdateTargetOptions struct {
-		ID  string `help:"Network ID or name"`
-		DNS string `help:"DNS server address" metavar:"DNS_SERVER"`
-		KEY string `help:"DNS update key name" metavar:"DNS_UPDATE_KEY"`
-	}
-	R(&NetworkRemoveDnsUpdateTargetOptions{}, "network-remove-dns-update-target", "Remove a dns update target from a network", func(s *mcclient.ClientSession, args *NetworkRemoveDnsUpdateTargetOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(args.DNS), "server")
-		params.Add(jsonutils.NewString(args.KEY), "key")
-		result, err := modules.Networks.PerformAction(s, args.ID, "remove-dns-update-target", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type NetworkSetDnsUpdateKeyOptions struct {
-		ID     string `help:"ID of virtual network to update"`
-		KEY    string `help:"Key name of secret" metavar:"KEYNAME"`
-		SECRET string `help:"Key secret"`
-		SERVER string `help:"Alternate DNS update server"`
-	}
-	R(&NetworkSetDnsUpdateKeyOptions{}, "network-set-dns-update-key", "Set DNS update key info for a virtual network", func(s *mcclient.ClientSession, args *NetworkSetDnsUpdateKeyOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.NewString(args.KEY), "dns_update_key_name")
-		params.Add(jsonutils.NewString(args.SECRET), "dns_update_key_secret")
-		params.Add(jsonutils.NewString(args.SERVER), "dns_update_server")
-		result, err := modules.Networks.PerformAction(s, args.ID, "metadata", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type NetworkRemoveDnsUpdateKeyOptions struct {
-		ID string `help:"ID of virtual network"`
-	}
-	R(&NetworkRemoveDnsUpdateKeyOptions{}, "network-remove-dns-update-key", "Set DNS update key info for a virtual network", func(s *mcclient.ClientSession, args *NetworkRemoveDnsUpdateKeyOptions) error {
-		params := jsonutils.NewDict()
-		params.Add(jsonutils.JSONNull, "dns_update_key_name")
-		params.Add(jsonutils.JSONNull, "dns_update_key_secret")
-		params.Add(jsonutils.JSONNull, "dns_update_server")
-		result, err := modules.Networks.PerformAction(s, args.ID, "metadata", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
 	type NetworkAddressOptions struct {
 		NETWORK string `help:"id or name of network to query"`
 	}

cmd/climc/shell/notification.go

@@ -41,7 +41,7 @@ func init() {
 		NotificationCreateOptions
 	}
 
-	R(&NotificationCreateSingleOptions{}, "notify", "Send a notification to someones", func(s *mcclient.ClientSession,
+	R(&NotificationCreateSingleOptions{}, "notify", "Send a notification to someone", func(s *mcclient.ClientSession,
 		args *NotificationCreateSingleOptions) error {
 
 		msg := notify.SNotifyMessage{}

cmd/climc/shell/policies.go

@@ -134,7 +134,7 @@ func init() {
 		return nil
 	}
 	R(&PolicyPatchOptions{}, "policy-patch", "Patch policy", updateFunc)
-	R(&PolicyPatchOptions{}, "policy-update", "Patch policy", updateFunc)
+	R(&PolicyPatchOptions{}, "policy-update", "Update policy", updateFunc)
 
 	type PolicyPerformOptions struct {
 		ID string `help:"ID of policy to update"`

cmd/climc/shell/proxysettings.go

@@ -0,0 +1,81 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	R(&options.ProxySettingCreateOptions{}, "proxysetting-create", "Create proxysetting", func(s *mcclient.ClientSession, opts *options.ProxySettingCreateOptions) error {
+		params, err := options.StructToParams(opts)
+		if err != nil {
+			return err
+		}
+		proxysetting, err := modules.ProxySettings.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(proxysetting)
+		return nil
+	})
+	R(&options.ProxySettingGetOptions{}, "proxysetting-show", "Show proxysetting", func(s *mcclient.ClientSession, opts *options.ProxySettingGetOptions) error {
+		proxysetting, err := modules.ProxySettings.Get(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(proxysetting)
+		return nil
+	})
+	R(&options.ProxySettingListOptions{}, "proxysetting-list", "List proxysettings", func(s *mcclient.ClientSession, opts *options.ProxySettingListOptions) error {
+		params, err := options.ListStructToParams(opts)
+		if err != nil {
+			return err
+		}
+		result, err := modules.ProxySettings.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.ProxySettings.GetColumns(s))
+		return nil
+	})
+	R(&options.ProxySettingUpdateOptions{}, "proxysetting-update", "Update proxysetting", func(s *mcclient.ClientSession, opts *options.ProxySettingUpdateOptions) error {
+		params, err := options.StructToParams(opts)
+		proxysetting, err := modules.ProxySettings.Update(s, opts.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(proxysetting)
+		return nil
+	})
+	R(&options.ProxySettingDeleteOptions{}, "proxysetting-delete", "Delete proxysetting", func(s *mcclient.ClientSession, opts *options.ProxySettingDeleteOptions) error {
+		proxysetting, err := modules.ProxySettings.Delete(s, opts.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(proxysetting)
+		return nil
+	})
+	R(&options.ProxySettingTestOptions{}, "proxysetting-test", "Test proxysetting", func(s *mcclient.ClientSession, opts *options.ProxySettingTestOptions) error {
+		proxysetting, err := modules.ProxySettings.PerformAction(s, opts.ID, "test", nil)
+		if err != nil {
+			return err
+		}
+		printObject(proxysetting)
+		return nil
+	})
+}

cmd/climc/shell/servers.go

@@ -179,6 +179,19 @@ func init() {
 		},
 	)
 
+	R(&options.ServerCreateOptions{}, "server-check-create-data", "Check create server data", func(s *mcclient.ClientSession, opts *options.ServerCreateOptions) error {
+		params, err := opts.Params()
+		if err != nil {
+			return err
+		}
+		server, err := modules.Servers.PerformClassAction(s, "check-create-data", params.JSON(params))
+		if err != nil {
+			return err
+		}
+		printObject(server)
+		return nil
+	})
+
 	R(&options.ServerCreateOptions{}, "server-create", "Create a server", func(s *mcclient.ClientSession, opts *options.ServerCreateOptions) error {
 		params, err := opts.Params()
 		if err != nil {

cmd/ctyuncli/main.go

@@ -84,8 +84,11 @@ func newClient(options *BaseOptions) (*ctyun.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
-	cli, err := ctyun.NewSCtyunClient("", "", "",
-		options.AccessKey, options.Secret, options.Debug)
+	cli, err := ctyun.NewSCtyunClient(
+		ctyun.NewSCtyunClientConfig(
+			options.AccessKey, options.Secret,
+		).Debug(options.Debug),
+	)
 	if err != nil {
 		return nil, err
 	}

cmd/esxicli/main.go

@@ -88,7 +88,14 @@ func newClient(options *BaseOptions) (*esxi.SESXiClient, error) {
 		return nil, fmt.Errorf("Missing password")
 	}
 
-	return esxi.NewESXiClient2("", "", options.Host, options.Port, options.Account, options.Password, false)
+	return esxi.NewESXiClient2(
+		esxi.NewESXiClientConfig(
+			options.Host,
+			options.Port,
+			options.Account,
+			options.Password,
+		),
+	)
 }
 
 func main() {

cmd/googlecli/main.go

@@ -94,7 +94,14 @@ func newClient(options *BaseOptions) (*google.SRegion, error) {
 		return nil, fmt.Errorf("Missing ProjectID")
 	}
 
-	cli, err := google.NewGoogleClient("", "", options.ProjectID, options.ClientEmail, options.PrivateKeyID, options.PrivateKey, options.Debug)
+	cli, err := google.NewGoogleClient(
+		google.NewGoogleClientConfig(
+			options.ProjectID,
+			options.ClientEmail,
+			options.PrivateKeyID,
+			options.PrivateKey,
+		).Debug(options.Debug),
+	)
 	if err != nil {
 		return nil, err
 	}

cmd/huaweicli/main.go

@@ -86,9 +86,14 @@ func newClient(options *BaseOptions) (*huawei.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
-	cli, err := huawei.NewHuaweiClient("", "", options.CloudEnv,
-		options.AccessKey, options.Secret, options.ProjectId,
-		options.Debug)
+	cli, err := huawei.NewHuaweiClient(
+		huawei.NewHuaweiClientConfig(
+			options.CloudEnv,
+			options.AccessKey,
+			options.Secret,
+			options.ProjectId,
+		).Debug(options.Debug),
+	)
 	if err != nil {
 		return nil, err
 	}

cmd/openstackcli/main.go

@@ -93,7 +93,18 @@ func newClient(options *BaseOptions) (*openstack.SRegion, error) {
 		return nil, fmt.Errorf("Missing Password")
 	}
 
-	cli, err := openstack.NewOpenStackClient("", "", options.AuthURL, options.Username, options.Password, options.Project, options.EndpointType, options.DomainName, options.ProjectDomain, options.Debug)
+	cli, err := openstack.NewOpenStackClient(
+		openstack.NewOpenstackClientConfig(
+			options.AuthURL,
+			options.Username,
+			options.Password,
+			options.Project,
+			options.ProjectDomain,
+		).
+			EndpointType(options.EndpointType).
+			DomainName(options.DomainName).
+			Debug(options.Debug),
+	)
 	if err != nil {
 		return nil, err
 	}

cmd/qcloudcli/main.go

@@ -85,11 +85,12 @@ func newClient(options *BaseOptions) (*qcloud.SRegion, error) {
 		return nil, fmt.Errorf("Missing SecretID")
 	}
 
-	if cli, err := qcloud.NewQcloudClient("", "",
-		options.SecretID,
-		options.SecretKey,
-		options.AppID,
-		options.Debug); err != nil {
+	if cli, err := qcloud.NewQcloudClient(
+		qcloud.NewQcloudClientConfig(
+			options.SecretID,
+			options.SecretKey,
+		).AppId(options.AppID).Debug(options.Debug),
+	); err != nil {
 		return nil, err
 	} else if region := cli.GetRegion(options.RegionId); region == nil {
 		return nil, fmt.Errorf("No such region %s", options.RegionId)

cmd/s3cli/main.go

@@ -94,11 +94,23 @@ func newClient(options *BaseOptions) (cloudprovider.ICloudRegion, error) {
 	}
 
 	if options.Backend == api.CLOUD_PROVIDER_CEPH {
-		return ceph.NewCephRados("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug)
+		return ceph.NewCephRados(
+			objectstore.NewObjectStoreClientConfig(
+				options.AccessUrl, options.AccessKey, options.Secret,
+			).Debug(options.Debug),
+		)
 	} else if options.Backend == api.CLOUD_PROVIDER_XSKY {
-		return xsky.NewXskyClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug)
+		return xsky.NewXskyClient(
+			objectstore.NewObjectStoreClientConfig(
+				options.AccessUrl, options.AccessKey, options.Secret,
+			).Debug(options.Debug),
+		)
 	}
-	return objectstore.NewObjectStoreClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug)
+	return objectstore.NewObjectStoreClient(
+		objectstore.NewObjectStoreClientConfig(
+			options.AccessUrl, options.AccessKey, options.Secret,
+		).Debug(options.Debug),
+	)
 }
 
 func main() {

cmd/ucloudcli/main.go

@@ -86,11 +86,12 @@ func newClient(options *BaseOptions) (*ucloud.SRegion, error) {
 		return nil, fmt.Errorf("Missing secret")
 	}
 
-	cli, err := ucloud.NewUcloudClient("", "",
-		options.AccessKey,
-		options.Secret,
-		options.ProjectId,
-		options.Debug)
+	cli, err := ucloud.NewUcloudClient(
+		ucloud.NewUcloudClientConfig(
+			options.AccessKey,
+			options.Secret,
+		).ProjectId(options.ProjectId).Debug(options.Debug),
+	)
 	if err != nil {
 		return nil, err
 	}

cmd/zstackcli/main.go

@@ -89,7 +89,13 @@ func newClient(options *BaseOptions) (*zstack.SRegion, error) {
 		return nil, fmt.Errorf("Missing Password")
 	}
 
-	cli, err := zstack.NewZStackClient("", "", options.AuthURL, options.Username, options.Password, options.Debug)
+	cli, err := zstack.NewZStackClient(
+		zstack.NewZstackClientConfig(
+			options.AuthURL,
+			options.Username,
+			options.Password,
+		).Debug(options.Debug),
+	)
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -6,7 +6,8 @@ require (
 	cloud.google.com/go v0.38.0
 	github.com/360EntSecGroup-Skylar/excelize v1.4.0
 	github.com/Azure/azure-sdk-for-go v36.1.0+incompatible
-	github.com/Azure/go-autorest v10.15.5+incompatible
+	github.com/Azure/go-autorest/autorest v0.9.3
+	github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
 	github.com/DataDog/dd-trace-go v0.6.1 // indirect
 	github.com/DataDog/zstd v1.3.4 // indirect
 	github.com/Microsoft/azure-vhd-utils v0.0.0-20181115010904-44cbada2ece3
@@ -29,23 +30,18 @@ require (
 	github.com/c-bata/go-prompt v0.2.1
 	github.com/ceph/go-ceph v0.0.0-20181217221554-e32f9f0f2e94
 	github.com/coredns/coredns v1.3.0
-	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
 	github.com/denisenkom/go-mssqldb v0.0.0-20190806190131-db2462fef53b // indirect
-	github.com/dimchansky/utfbom v1.1.0 // indirect
-	github.com/dnaeon/go-vcr v1.0.1 // indirect
 	github.com/dnstap/golang-dnstap v0.0.0-20170829151710-2cf77a2b5e11 // indirect
 	github.com/edsrzf/mmap-go v1.0.0 // indirect
 	github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect
 	github.com/farsightsec/golang-framestream v0.0.0-20181102145529-8a0cb8ba8710 // indirect
 	github.com/fatih/color v1.7.0
 	github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e
-	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/ghodss/yaml v1.0.0
-	github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 // indirect
-	github.com/gin-gonic/gin v1.3.0
+	github.com/gin-gonic/gin v1.5.0
 	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.2 // indirect
@@ -58,14 +54,12 @@ require (
 	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
 	github.com/golang/protobuf v1.3.2
 	github.com/google/gopacket v1.1.17
-	github.com/google/uuid v1.1.0 // indirect
 	github.com/googleapis/gnostic v0.2.0 // indirect
 	github.com/googollee/go-engine.io v0.0.0-20180829091931-e2f255711dcb // indirect
 	github.com/googollee/go-socket.io v0.0.0-20181214084611-0ad7206c347a
-	github.com/gorilla/mux v1.6.2
+	github.com/gorilla/mux v1.7.0
 	github.com/gorilla/websocket v1.4.0
 	github.com/gosuri/uitable v0.0.0-20160404203958-36ee7e946282
-	github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/hako/durafmt v0.0.0-20180520121703-7b7ae1e72ead
 	github.com/huandu/xstrings v1.2.0 // indirect
@@ -74,7 +68,7 @@ require (
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.0.1 // indirect
 	github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c
-	github.com/kr/pty v1.1.3
+	github.com/kr/pty v1.1.5
 	github.com/lib/pq v1.2.0 // indirect
 	github.com/libvirt/libvirt-go-xml v5.2.0+incompatible
 	github.com/mattn/go-runewidth v0.0.4 // indirect
@@ -84,7 +78,7 @@ require (
 	github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7
 	github.com/mdlayher/raw v0.0.0-20190606144222-a54781e5f38f
 	github.com/mholt/caddy v0.10.11
-	github.com/miekg/dns v1.1.1
+	github.com/miekg/dns v1.1.4
 	github.com/minio/minio-go v6.0.14+incompatible
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/moul/http2curl v1.0.0
@@ -92,61 +86,82 @@ require (
 	github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect
 	github.com/opentracing/opentracing-go v1.0.2 // indirect
 	github.com/openzipkin/zipkin-go-opentracing v0.3.4 // indirect
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4 v2.0.5+incompatible
 	github.com/pkg/errors v0.8.1
 	github.com/pkg/term v0.0.0-20181116001808-27bbf2edb814 // indirect
 	github.com/pquerna/otp v1.2.0
-	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/serialx/hashring v0.0.0-20180504054112-49a4782e9908
 	github.com/shirou/gopsutil v2.18.10+incompatible
 	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
 	github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.3 // indirect
 	github.com/stretchr/testify v1.4.0
-	github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e
+	github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible
 	github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20191108095731-8ca4b370cde4
 	github.com/tinylib/msgp v1.1.0 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	github.com/tredoe/osutil v0.0.0-20161130133508-7d3ee1afa71c
-	github.com/ugorji/go v0.0.0-20161130061742-9c7f9b7a2bc3 // indirect
 	github.com/vishvananda/netlink v1.0.0
 	github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect
 	github.com/vmihailenco/msgpack v4.0.4+incompatible
-	github.com/vmware/govmomi v0.19.0
+	github.com/vmware/govmomi v0.20.1
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
 	go.uber.org/atomic v1.4.0 // indirect
-	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc
-	golang.org/x/net v0.0.0-20191007182048-72f939374954
-	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20191008105621-543471e840be
+	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413
+	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553
+	golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987
 	google.golang.org/api v0.13.0 // indirect
 	google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873
 	google.golang.org/grpc v1.23.1
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
-	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
-	gopkg.in/go-playground/validator.v8 v8.18.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.44.0 // indirect
 	gopkg.in/ldap.v3 v3.0.3
-	gopkg.in/yaml.v2 v2.2.2
-	k8s.io/api v0.0.0-20181004124137-fd83cbc87e76
-	k8s.io/apimachinery v0.0.0-20181215012845-4d029f033399
+	gopkg.in/yaml.v2 v2.2.8
+	k8s.io/api v0.17.3
+	k8s.io/apimachinery v0.17.3
 	k8s.io/client-go v9.0.0+incompatible
-	k8s.io/klog v0.1.0 // indirect
-	k8s.io/kubernetes v1.12.3
+	k8s.io/cluster-bootstrap v0.17.3
+	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200113074440-9297fd00ba07
+	yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda
 	yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
-	yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070
+	yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20200221103553-6a98f7f8ab92
+	yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
 	yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
 )
 
-replace github.com/ceph/go-ceph v0.0.0-20181217221554-e32f9f0f2e94 => github.com/yunionio/go-ceph v0.0.0-20190912101231-6f05a06b3859
+replace (
+	github.com/ceph/go-ceph v0.0.0-20181217221554-e32f9f0f2e94 => github.com/yunionio/go-ceph v0.0.0-20190912101231-6f05a06b3859
+
+	k8s.io/api => k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783
+	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
+	k8s.io/apiserver => k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20190918162238-f783a3654da8
+	k8s.io/client-go => k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20190918163234-a9c1f33e9fb9
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.0.0-20190918163108-da9fdfce26bb
+	k8s.io/code-generator => k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269
+	k8s.io/component-base => k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090
+	k8s.io/cri-api => k8s.io/cri-api v0.0.0-20190828162817-608eb1dad4ac
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.0.0-20190918163402-db86a8c7bb21
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.0.0-20190918161219-8c8f079fddc3
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.0.0-20190918162944-7a93a0ddadd8
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.0.0-20190918162534-de037b596c1e
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.0.0-20190918162820-3b5c1246eb18
+	k8s.io/kubectl => k8s.io/kubectl v0.0.0-20190918164019-21692a0861df
+	k8s.io/kubelet => k8s.io/kubelet v0.0.0-20190918162654-250a1838aa2c
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.0.0-20190918163543-cfa506e53441
+	k8s.io/metrics => k8s.io/metrics v0.0.0-20190918162108-227c654b2546
+	k8s.io/node-api => k8s.io/node-api v0.0.0-20190918163711-2299658ad911
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.0.0-20190918161442-d4c9c65c82af
+	k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.0.0-20190918162410-e45c26d066f2
+	k8s.io/sample-controller => k8s.io/sample-controller v0.0.0-20190918161628-92eb3cb7496c
+)

pkg/apigateway/handler/auth.go

@@ -29,6 +29,7 @@ import (
 	"yunion.io/x/onecloud/pkg/apigateway/constants"
 	"yunion.io/x/onecloud/pkg/apigateway/options"
 	policytool "yunion.io/x/onecloud/pkg/apigateway/policy"
+	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
@@ -646,8 +647,11 @@ func getUserAuthCookie(ctx context.Context, s *mcclient.ClientSession, token mcc
 }
 
 func getLBAgentInfo(s *mcclient.ClientSession, token mcclient.TokenCredential) (*jsonutils.JSONDict, error) {
-
-	lbagents, err := modules.LoadbalancerAgents.List(s, nil)
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewString("hb_last_seen.isnotempty()"), "filter.0")
+	params.Add(jsonutils.NewInt(1), "limit")
+	params.Add(jsonutils.JSONFalse, "details")
+	lbagents, err := modules.LoadbalancerAgents.List(s, params)
 	if err != nil {
 		return nil, errors.Wrapf(err, "user %s get lbagent", token.GetUserName())
 	}
@@ -665,6 +669,7 @@ func getLBAgentInfo(s *mcclient.ClientSession, token mcclient.TokenCredential) (
 }
 
 func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.TokenCredential, req *http.Request) (*jsonutils.JSONDict, error) {
+	log.Infof("getUserInfo modules.UsersV3.Get")
 	usr, err := modules.UsersV3.Get(s, token.GetUserId(), nil)
 	if err != nil {
 		log.Errorf("modules.UsersV3.Get fail %s", err)
@@ -693,6 +698,7 @@ func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.
 	data.Add(jsonutils.NewString(token.GetProjectDomain()), "projectDomain")
 	data.Add(jsonutils.NewString(token.GetProjectDomainId()), "projectDomainId")
 
+	log.Infof("getUserInfo modules.RoleAssignments.List")
 	query := jsonutils.NewDict()
 	query.Add(jsonutils.JSONNull, "effective")
 	query.Add(jsonutils.JSONNull, "include_names")
@@ -772,6 +778,7 @@ func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.
 		log.Errorf("fail to find services????: %#v %s", adminToken, curReg)
 	}
 
+	log.Infof("getUserInfo getLBAgentInfo")
 	lb, err := getLBAgentInfo(s, adminToken)
 	if err != nil {
 		log.Errorf("getLBAgentInfo fail %s", err)
@@ -788,13 +795,23 @@ func getUserInfo(ctx context.Context, s *mcclient.ClientSession, token mcclient.
 		}
 	}
 
+	log.Infof("getUserInfo modules.Hosts.Get")
 	s2 := auth.GetSession(ctx, token, FetchRegion(req), "v2")
-	cap, err := modules.Capabilities.List(s2, nil)
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewString("host_type"), "field")
+	params.Add(jsonutils.NewString("system"), "scope")
+	params.Add(jsonutils.JSONTrue, "usable")
+	params.Add(jsonutils.JSONTrue, "show_emulated")
+	cap, err := modules.Hosts.Get(s2, "distinct-field", params)
 	if err != nil {
-		log.Errorf("modules.Capabilities.List fail %s", err)
+		log.Errorf("modules.Servers.Get distinct-field fail %s", err)
 	} else {
-		hypervisors, _ := cap.Data[0].Get("hypervisors")
-		data.Add(hypervisors, "hypervisors")
+		hostTypes, _ := jsonutils.GetStringArray(cap, "host_type")
+		hypervisors := make([]string, len(hostTypes))
+		for i, hostType := range hostTypes {
+			hypervisors[i] = compute.HOSTTYPE_HYPERVISOR[hostType]
+		}
+		data.Add(jsonutils.NewStringArray(hypervisors), "hypervisors")
 	}
 
 	data.Add(menus, "menus")

pkg/apigateway/options/options.go

@@ -29,7 +29,7 @@ type GatewayOptions struct {
 
 	SqlitePath string `help:"sqlite db path" default:"/etc/yunion/data/yunionapi.db"`
 
-	common_options.CommonOptions
+	common_options.CommonOptions `"request_worker_count->default":"32"`
 }
 
 var (
@@ -40,9 +40,10 @@ func OnOptionsChange(oldO, newO interface{}) bool {
 	oldOpts := oldO.(*GatewayOptions)
 	newOpts := newO.(*GatewayOptions)
 
+	changed := false
 	if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) {
-		return true
+		changed = true
 	}
 
-	return false
+	return changed
 }

pkg/apis/cloudcommon/proxy/doc.go

@@ -0,0 +1 @@
+package proxy // import "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"

pkg/apis/cloudcommon/proxy/proxy.go

@@ -0,0 +1,41 @@
+package proxy
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+const (
+	ProxySettingId_DIRECT = "DIRECT"
+)
+
+type ProxySettingCreateInput struct {
+	apis.StandaloneResourceCreateInput
+
+	ProxySetting
+}
+
+type ProxySettingUpdateInput struct {
+	// 资源名称
+	Name string `json:"name"`
+	// 资源描述
+	Description string `json:"description"`
+
+	ProxySetting
+}
+
+// String implements ISerializable interface
+func (ps *SProxySetting) String() string {
+	return jsonutils.Marshal(ps).String()
+}
+
+// IsZero implements ISerializable interface
+func (ps *SProxySetting) IsZero() bool {
+	if ps.HTTPProxy == "" &&
+		ps.HTTPSProxy == "" &&
+		ps.NoProxy == "" {
+		return true
+	}
+	return false
+}

pkg/apis/cloudcommon/proxy/setting.go

@@ -0,0 +1,111 @@
+package proxy
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+
+	"yunion.io/x/pkg/errors"
+)
+
+type ProxySetting struct {
+	HttpProxy  string
+	HttpsProxy string
+	NoProxy    string
+}
+
+func (v *ProxySetting) Sanitize() error {
+	v.HttpProxy = strings.TrimSpace(v.HttpProxy)
+	if u, err := parseProxy(v.HttpProxy); err != nil {
+		return errors.Wrap(err, "invalid https_proxy url")
+	} else if u != nil {
+		v.HttpProxy = u.String()
+	}
+
+	v.HttpsProxy = strings.TrimSpace(v.HttpsProxy)
+	if u, err := parseProxy(v.HttpsProxy); err != nil {
+		return errors.Wrap(err, "invalid http_proxy url")
+	} else if u != nil {
+		v.HttpsProxy = u.String()
+	}
+
+	if noProxy, err := parseNoProxy(v.NoProxy); err == nil {
+		v.NoProxy = strings.Join(noProxy, ",")
+	} else {
+		return errors.Wrap(err, "invalid no_proxy")
+	}
+	return nil
+}
+
+func parseProxy(proxy string) (*url.URL, error) {
+	if proxy == "" {
+		return nil, nil
+	}
+
+	proxyURL, err := url.Parse(proxy)
+	if err != nil ||
+		(proxyURL.Scheme != "http" &&
+			proxyURL.Scheme != "https" &&
+			proxyURL.Scheme != "socks5") {
+		// proxy was bogus. Try prepending "http://" to it and
+		// see if that parses correctly. If not, we fall
+		// through and complain about the original one.
+		if proxyURL, err := url.Parse("http://" + proxy); err == nil {
+			return proxyURL, nil
+		}
+	}
+	if err != nil {
+		return nil, errors.Wrapf(err, "invalid proxy address %q", proxy)
+	}
+	return proxyURL, nil
+}
+
+func parseNoProxy(noProxy string) ([]string, error) {
+	var r []string
+	for _, p := range strings.Split(noProxy, ",") {
+		p = strings.ToLower(strings.TrimSpace(p))
+		if len(p) == 0 {
+			continue
+		}
+
+		if p == "*" {
+			r = append(r, p)
+			// let it go
+			//return r, nil
+		}
+
+		// IPv4/CIDR, IPv6/CIDR
+		if _, _, err := net.ParseCIDR(p); err == nil {
+			r = append(r, p)
+			continue
+		}
+
+		// IPv4:port, [IPv6]:port
+		phost, _, err := net.SplitHostPort(p)
+		if err == nil {
+			if len(phost) == 0 {
+				// There is no host part, likely the entry is malformed; ignore.
+				return nil, fmt.Errorf("host part must not be empty: %s", p)
+			}
+			if phost[0] == '[' && phost[len(phost)-1] == ']' {
+				phost = phost[1 : len(phost)-1]
+			}
+		} else {
+			phost = p
+		}
+		// IPv4, IPv6
+		if pip := net.ParseIP(phost); pip != nil {
+			r = append(r, p)
+			continue
+		}
+
+		if len(phost) == 0 {
+			// There is no host part, likely the entry is malformed; ignore.
+			continue
+		}
+
+		r = append(r, p)
+	}
+	return r, nil
+}

pkg/apis/cloudcommon/proxy/zz_generated.model.go

@@ -0,0 +1,27 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by model-api-gen. DO NOT EDIT.
+
+package proxy
+
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+// SProxySetting is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db/proxy.SProxySetting.
+type SProxySetting struct {
+	apis.SStandaloneResourceBase
+	HTTPProxy  string `json:"http_proxy"`
+	HTTPSProxy string `json:"https_proxy"`
+	NoProxy    string `json:"no_proxy"`
+}

pkg/apis/compute/cloudaccount.go

@@ -19,6 +19,7 @@ import (
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 )
@@ -79,6 +80,9 @@ type CloudaccountCreateInput struct {
 	// 额外信息,例如账单的access key
 	Options *jsonutils.JSONObject `json:"options"`
 
+	// 代理配置
+	ProxySettingId string `json:"proxy_setting_id"`
+
 	cloudprovider.SCloudaccount
 	cloudprovider.SCloudaccountCredential
 }
@@ -177,4 +181,6 @@ type CloudaccountDetail struct {
 	// 存储缓存数量
 	// example: 10
 	StoragecacheCount int `json:"storagecache_count,allowempty"`
+
+	ProxySetting proxyapi.SProxySetting `json:"proxy_setting"`
 }

pkg/apis/compute/cloudprovider.go

@@ -18,6 +18,7 @@ import (
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 )
 
@@ -96,6 +97,8 @@ type CloudproviderDetails struct {
 	apis.StandaloneResourceDetails
 	SCloudprovider
 
+	ProxySetting proxyapi.SProxySetting `json:"proxy_setting"`
+
 	// 云账号名称
 	// example: google-account
 	Cloudaccount string `json:"cloudaccount"`

pkg/apis/compute/dbinstance_const.go

@@ -108,10 +108,10 @@ const (
 	ALIYUN_DBINSTANCE_STORAGE_TYPE_CLOUD_SSD  = "cloud_ssd"  //SSD云盘
 
 	//华为云存储类型
-	HUAWEI_DBINSTANCE_STORAGE_TYPE_SSD  = "SSD"  //超高IO云硬盘
-	HUAWEI_DBINSTANCE_STORAGE_TYPE_SAS  = "SAS"  //高IO云硬盘
-	HUAWEI_DBINSTANCE_STORAGE_TYPE_SATA = "SATA" //普通IO云硬盘
-
+	HUAWEI_DBINSTANCE_STORAGE_TYPE_ULTRAHIGH    = "ULTRAHIGH" //超高IO云硬盘
+	HUAWEI_DBINSTANCE_STORAGE_TYPE_ULTRAHIGHPRO = "ULTRAHIGHPRO"
+	HUAWEI_DBINSTANCE_STORAGE_TYPE_COMMON       = "COMMON"
+	HUAWEI_DBINSTANCE_STORAGE_TYPE_HIGH         = "HIGH"
 )
 
 var (

pkg/apis/compute/dbinstance_sku.go

@@ -0,0 +1,23 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import "yunion.io/x/onecloud/pkg/apis"
+
+type DBInstanceSkuDetails struct {
+	apis.StandaloneResourceDetails
+
+	CloudregionInfo
+}

pkg/apis/compute/elasticips_const.go

@@ -43,10 +43,6 @@ const (
 	EIP_CHARGE_TYPE_BY_TRAFFIC   = "traffic"
 	EIP_CHARGE_TYPE_BY_BANDWIDTH = "bandwidth"
 	EIP_CHARGE_TYPE_DEFAULT      = EIP_CHARGE_TYPE_BY_TRAFFIC
-
-	// EIP associate resource type
-	EIP_ASSOCIATE_TYPE_VM  = "server"
-	EIP_ASSOCIATE_TYPE_ELB = "elb"
 )
 
 var (

pkg/apis/compute/guest_const.go

@@ -150,6 +150,11 @@ const (
 	HYPERVISOR_DEFAULT = HYPERVISOR_KVM
 )
 
+const (
+	CPU_MODE_QEMU = "qemu"
+	CPU_MODE_HOST = "host"
+)
+
 var VM_RUNNING_STATUS = []string{VM_START_START, VM_STARTING, VM_RUNNING, VM_BLOCK_STREAM, VM_BLOCK_STREAM_FAIL}
 var VM_CREATING_STATUS = []string{VM_CREATE_NETWORK, VM_CREATE_DISK, VM_START_DEPLOY, VM_DEPLOYING}
 

pkg/apis/compute/host.go

@@ -124,3 +124,10 @@ type HostDetails struct {
 	// 标签
 	Metadata map[string]string `json:"metadata"`
 }
+
+type HostRegisterMetadata struct {
+	apis.Meta
+
+	OnKubernetes bool   `json:"on_kubernetes"`
+	Hostname     string `json:"hostname"`
+}

pkg/apis/compute/hoststorage.go

@@ -51,7 +51,7 @@ type HoststorageDetails struct {
 	// 镜像缓存路径
 	ImagecachePath string `json:"imagecache_path"`
 	// 存储缓存Id
-	StoragecacheId string `json:"storagecached_id"`
+	StoragecacheId string `json:"storagecache_id"`
 
 	GuestDiskCount int `json:"guest_disk_count,allowempty"`
 }

pkg/apis/compute/instance_snapshot.go

@@ -29,6 +29,10 @@ type SimpleSnapshot struct {
 	CloudregionId string `json:"cloudregion_id"`
 	// 快照大小
 	Size int `json:"size"`
+	// 快照状态
+	Status string `json:"status"`
+	// 存储类型
+	StorageType string `json:"storage_type"`
 }
 
 type InstnaceSnapshotDetails struct {

pkg/apis/compute/storage.go

@@ -112,22 +112,27 @@ type StorageCreateInput struct {
 	NfsSharedDir string `json:"nfs_shared_dir"`
 }
 
-type StorageDetails struct {
-	apis.StandaloneResourceDetails
-	SStorage
-
+type SStorageCapacityInfo struct {
 	// 容量大小, 单位Mb
 	Capacity int64 `json:"capacity"`
 	// 已使用容量大小
-	Used int64 `json:"used"`
+	UsedCapacity int64 `json:"used_capacity"`
 	// 浪费容量大小(异常磁盘大小总和)
-	Wasted int64 `json:"wasted"`
+	WasteCapacity int64 `json:"waste_capacity"`
 	// 虚拟容量大小
-	VCapacity int64 `json:"vcapacity"`
+	VirtualCapacity int64 `json:"virtual_capacity"`
 	// 超分率
 	CommitRate float64 `json:"commit_rate"`
 	// 可使用容量
 	FreeCapacity int64 `json:"free_capacity"`
+}
+
+type StorageDetails struct {
+	apis.StandaloneResourceDetails
+
+	SStorageCapacityInfo
+
+	SStorage
 
 	CloudproviderInfo
 	Schedtags []SchedtagShortDescDetails `json:"schedtags"`

pkg/apis/compute/zone.go

@@ -46,7 +46,7 @@ type ZoneDetails struct {
 
 	// 可用区底下启用的宿主机数量
 	// example: 2
-	HostsEnabled int `json:"host_enabled"`
+	HostsEnabled int `json:"hosts_enabled"`
 
 	// 可用区底下的裸金属服务器数量
 	// example: 1

pkg/apis/compute/zz_generated.model.go

@@ -195,6 +195,8 @@ type SCloudaccount struct {
 	IsPublic bool `json:"is_public"`
 	// add share_mode field to indicate the share range of this account
 	ShareMode string `json:"share_mode"`
+	// 默认值proxyapi.ProxySettingId_DIRECT
+	ProxySettingId string `json:"proxy_setting_id"`
 }
 
 // SCloudprovider is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SCloudprovider.
@@ -215,6 +217,7 @@ type SCloudprovider struct {
 type SCloudproviderCapability struct {
 	apis.SResourceBase
 	CloudproviderId string `json:"cloudprovider_id"`
+	CloudregionId   string `json:"cloudregion_id"`
 	Capability      string `json:"capability"`
 }
 

pkg/apis/image/guestimage.go

@@ -15,6 +15,8 @@
 package image
 
 import (
+	"time"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -27,13 +29,23 @@ type GuestImageDetails struct {
 	ImageIds jsonutils.JSONObject `json:"image_ids"`
 
 	//Status     string               `json:"status"`
-	Size       int64                `json:"size"`
-	MinRamMb   int32                `json:"min_ram_mb"`
-	DiskFormat string               `json:"disk_format"`
-	RootImage  jsonutils.JSONObject `json:"root_image"`
-	DataImages jsonutils.JSONObject `json:"data_images"`
+	Size       int64          `json:"size"`
+	MinRamMb   int32          `json:"min_ram_mb"`
+	DiskFormat string         `json:"disk_format"`
+	RootImage  SubImageInfo   `json:"root_image"`
+	DataImages []SubImageInfo `json:"data_images"`
 
 	Properties *jsonutils.JSONDict `json:"properties"`
 
 	DisableDelete bool `json:"disable_delete"`
 }
+
+type SubImageInfo struct {
+	ID         string    `json:"id"`
+	Name       string    `json:"name"`
+	MinDiskMB  int32     `json:"min_disk_mb"`
+	DiskFormat string    `json:"disk_format"`
+	Size       int64     `json:"size"`
+	Status     string    `json:"status"`
+	CreatedAt  time.Time `json:"created_at"`
+}

pkg/apis/monitor/alert.go

@@ -15,6 +15,8 @@
 package monitor
 
 import (
+	"time"
+
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -103,12 +105,18 @@ type AlertCreateInput struct {
 	Name string `json:"name"`
 	// 报警执行频率
 	Frequency int64 `json:"frequency"`
+	// 报警持续时间
+	For int64 `json:"for"`
 	// 报警设置
 	Settings AlertSetting `json:"settings"`
 	// 启用报警
 	Enabled *bool `json:"enabled"`
 	// 报警级别
 	Level string `json:"level"`
+	// 没有收到监控指标时将当前报警状态设置为对应的状态
+	NoDataState string `json:"no_data_state"`
+	// 报警执行错误将当前报警状态设置为对应的状态
+	ExecutionErrorState string `json:"execution_error_state"`
 }
 
 type AlertUpdateInput struct {
@@ -124,6 +132,10 @@ type AlertUpdateInput struct {
 	Enabled *bool `json:"enabled"`
 	// 报警级别
 	Level *string `json:"level"`
+	// 没有收到监控指标时将当前报警状态设置为对应的状态
+	NoDataState string `json:"no_data_state"`
+	// 报警执行错误将当前报警状态设置为对应的状态
+	ExecutionErrorState string `json:"execution_error_state"`
 }
 
 type AlertListInput struct {
@@ -135,3 +147,41 @@ type AlertListInput struct {
 	// 以报警是否启用/禁用过滤列表
 	Enabled *bool `json:"enabled"`
 }
+
+type AlertTestRunInput struct {
+	apis.Meta
+
+	IsDebug bool `json:"is_debug"`
+}
+
+// ResultLogEntry represents log data for the alert evaluation.
+type ResultLogEntry struct {
+	Message string      `json:"message"`
+	Data    interface{} `json:"data"`
+}
+
+// EvalMatch represents the series violating the threshold.
+type EvalMatch struct {
+	Condition string            `json:"condition"`
+	Value     *float64          `json:"value"`
+	Metric    string            `json:"metric"`
+	Tags      map[string]string `json:"tags"`
+}
+
+type AlertTestRunOutput struct {
+	apis.Meta
+
+	Firing         bool              `json:"firing"`
+	EvalMatches    []*EvalMatch      `json:"eval_matches"`
+	Logs           []*ResultLogEntry `json:"logs"`
+	Error          error             `json:"error"`
+	ConditionEvals string            `json:"condition_evals"`
+	StartTime      time.Time         `json:"start_time"`
+	EndTime        time.Time         `json:"end_time"`
+}
+
+type AlertPauseInput struct {
+	apis.Meta
+
+	Paused bool `json:"paused"`
+}

pkg/apis/monitor/alertnotification.go

@@ -0,0 +1,31 @@
+package monitor
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type AlertJointResourceBaseDetails struct {
+	SAlertJointsBase
+	Alert string `json:"alert"`
+}
+
+type AlertnotificationDetails struct {
+	AlertJointResourceBaseDetails
+	Notification string `json:"notification"`
+}
+
+type AlertJointCreateInput struct {
+	apis.Meta
+
+	AlertId string `json:"alert_id"`
+}
+
+type AlertnotificationCreateInput struct {
+	AlertJointCreateInput
+
+	NotificationId string               `json:"notification_id"`
+	UsedBy         string               `json:"used_by"`
+	Params         jsonutils.JSONObject `json:"params"`
+}

pkg/apis/monitor/meteralert.go

@@ -70,4 +70,5 @@ type MeterAlertUpdateInput struct {
 	ProjectId *string `json:"project_id"`
 	// 通知方式, 比如: email, mobile
 	Channel *string `json:"channel"`
+	Status  *string `json:"status"`
 }

pkg/apis/monitor/nodealert.go

@@ -171,6 +171,7 @@ type AlertV1Details struct {
 	Measurement string  `json:"measurement"`
 	Field       string  `json:"field"`
 	NotifierId  string  `json:"notifier_id"`
+	Status      string  `json:"status"`
 }
 
 type NodeAlertDetails struct {

pkg/apis/monitor/template.go

@@ -32,11 +32,3 @@ type NotificationTemplateConfig struct {
 	Level       string `json:"level"`
 	IsRecovery  bool   `json:"is_recovery"`
 }
-
-// EvalMatch represents the series violating the threshold.
-type EvalMatch struct {
-	Condition string            `json:"condition"`
-	Value     *float64          `json:"value"`
-	Metric    string            `json:"metric"`
-	Tags      map[string]string `json:"tags"`
-}

pkg/apis/monitor/zz_generated.model.go

@@ -0,0 +1,77 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Code generated by model-api-gen. DO NOT EDIT.
+
+package monitor
+
+import (
+	time "time"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+// SAlert is an autogenerated struct via yunion.io/x/onecloud/pkg/monitor/models.SAlert.
+type SAlert struct {
+	apis.SVirtualResourceBase
+	Frequency int64       `json:"frequency"`
+	Settings  interface{} `json:"settings"`
+	Level     string      `json:"level"`
+	Message   string      `json:"message"`
+	UsedBy    string      `json:"used_by"`
+	// Silenced       bool
+	ExecutionError      string      `json:"execution_error"`
+	For                 int64       `json:"for"`
+	EvalData            interface{} `json:"eval_data"`
+	State               string      `json:"state"`
+	NoDataState         string      `json:"no_data_state"`
+	ExecutionErrorState string      `json:"execution_error_state"`
+	LastStateChange     time.Time   `json:"last_state_change"`
+	StateChanges        int         `json:"state_changes"`
+}
+
+// SAlertJointsBase is an autogenerated struct via yunion.io/x/onecloud/pkg/monitor/models.SAlertJointsBase.
+type SAlertJointsBase struct {
+	apis.SVirtualJointResourceBase
+	AlertId string `json:"alert_id"`
+}
+
+// SAlertnotification is an autogenerated struct via yunion.io/x/onecloud/pkg/monitor/models.SAlertnotification.
+type SAlertnotification struct {
+	SAlertJointsBase
+	NotificationId string `json:"notification_id"`
+	State          string `json:"state"`
+	Index          byte   `json:"index"`
+	UsedBy         string `json:"used_by"`
+}
+
+// SDataSource is an autogenerated struct via yunion.io/x/onecloud/pkg/monitor/models.SDataSource.
+type SDataSource struct {
+	apis.SStandaloneResourceBase
+	Type      string `json:"type"`
+	Url       string `json:"url"`
+	User      string `json:"user"`
+	Password  string `json:"password"`
+	Database  string `json:"database"`
+	IsDefault *bool  `json:"is_default,omitempty"`
+}
+
+// SNotification is an autogenerated struct via yunion.io/x/onecloud/pkg/monitor/models.SNotification.
+type SNotification struct {
+	apis.SVirtualResourceBase
+	Type                  string      `json:"type"`
+	IsDefault             bool        `json:"is_default"`
+	SendReminder          bool        `json:"send_reminder"`
+	DisableResolveMessage bool        `json:"disable_resolve_message"`
+	Frequency             int64       `json:"frequency"`
+	Settings              interface{} `json:"settings"`
+}

pkg/apis/notify/contact.go

@@ -19,7 +19,7 @@ import "yunion.io/x/onecloud/pkg/apis"
 type ContactDetails struct {
 	apis.ModelBaseDetails
 
-	Id      string `json:"id"`
+	UID     string `json:"uid"`
 	Name    string `json:"name"`
 	Details string `json:"details"`
 }

pkg/apis/scheduler/api.go

@@ -77,6 +77,7 @@ type ScheduleInput struct {
 	LiveMigrate  bool   `json:"live_migrate"`
 	CpuDesc      string `json:"cpu_desc"`
 	CpuMicrocode string `json:"cpu_microcode"`
+	CpuMode      string `json:"cpu_mode"`
 
 	PendingUsages []jsonutils.JSONObject
 }
@@ -93,6 +94,17 @@ type CandidateDisk struct {
 	StorageIds []string `json:"storage_ids"`
 }
 
+type CandidateDiskV2 struct {
+	Index    int                 `json:"index"`
+	Storages []*CandidateStorage `json:"storages"`
+}
+
+type CandidateStorage struct {
+	Id           string
+	Name         string
+	FreeCapacity int64
+}
+
 type CandidateNet struct {
 	Index      int      `json:"index"`
 	NetworkIds []string `json:"network_ids"`

pkg/appsrv/appsrv.go

@@ -268,7 +268,7 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 		hand, ok := handler.(*SHandlerInfo)
 		if ok {
 			fw := newResponseWriterChannel(w)
-			worker := make(chan *SWorker)
+			currentWorker := make(chan *SWorker, 1) // make it a buffered channel
 			to := hand.FetchProcessTimeout(r)
 			if to == 0 {
 				to = app.processTimeout
@@ -320,13 +320,13 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 					} // otherwise, the task has been timeout
 					fw.closeChannels()
 				},
-				worker,
+				currentWorker,
 				func(err error) {
 					httperrors.InternalServerError(&fw, "Internal server error: %s", err)
 					fw.closeChannels()
 				},
 			)
-			runErr := fw.wait(ctx, worker)
+			runErr := fw.wait(ctx, currentWorker)
 			if runErr != nil {
 				switch runErr.(type) {
 				case *httputils.JSONClientError:

pkg/appsrv/response.go

@@ -100,7 +100,12 @@ func (w *responseWriterChannel) wait(ctx context.Context, workerChan chan *SWork
 	stop := false
 	for !stop {
 		select {
-		case worker = <-workerChan:
+		case curWorker, more := <-workerChan:
+			if more {
+				worker = curWorker
+			} else {
+				// ignore, worker is responsible for close the channel
+			}
 		case <-ctx.Done():
 			// ctx deadline reached, timeout
 			if worker != nil {

pkg/appsrv/workers.go

@@ -83,6 +83,9 @@ func (worker *SWorker) run() {
 			task := req.(*sWorkerTask)
 			if task.worker != nil {
 				task.worker <- worker
+				// worker channel is buffered
+				// close the worker channel
+				close(task.worker)
 			}
 			execCallback(task)
 		} else {

pkg/cloudcommon/app/app.go

@@ -28,6 +28,7 @@ import (
 
 func InitApp(options *common_options.BaseOptions, dbAccess bool) *appsrv.Application {
 	// cache := appsrv.NewCache(options.AuthTokenCacheSize)
+	log.Infof("RequestWorkerCount: %d", options.RequestWorkerCount)
 	app := appsrv.NewApplication(options.ApplicationID, options.RequestWorkerCount, dbAccess)
 	app.CORSAllowHosts(options.CorsHosts)
 

pkg/cloudcommon/consts/consts.go

@@ -16,6 +16,8 @@ package consts
 
 import (
 	"time"
+
+	"yunion.io/x/log"
 )
 
 var (
@@ -58,6 +60,7 @@ func GetTenantCacheExpireSeconds() time.Duration {
 }
 
 func SetNonDefaultDomainProjects(val bool) {
+	log.Infof("set non_default_domain_projects to %v", val)
 	nonDefaultDomainProjects = val
 }
 

pkg/cloudcommon/db/caller.go

@@ -57,11 +57,10 @@ func call(obj interface{}, fName string, inputs ...interface{}) ([]reflect.Value
 
 func callObject(modelVal reflect.Value, fName string, inputs ...interface{}) ([]reflect.Value, error) {
 	funcVal := modelVal.MethodByName(fName)
-	return callFunc(funcVal, inputs...)
+	return callFunc(funcVal, fName, inputs...)
 }
 
-func callFunc(funcVal reflect.Value, inputs ...interface{}) ([]reflect.Value, error) {
-	fName := funcVal.String()
+func callFunc(funcVal reflect.Value, fName string, inputs ...interface{}) ([]reflect.Value, error) {
 	if !funcVal.IsValid() || funcVal.IsNil() {
 		return nil, httperrors.NewActionNotFoundError(fmt.Sprintf("%s method not found", fName))
 	}

pkg/cloudcommon/db/db_dispatcher.go

@@ -40,6 +40,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
@@ -792,12 +793,11 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr
 		return nil, nil
 	}
 	userCred := fetchUserCredential(ctx)
-	params := []reflect.Value{
-		reflect.ValueOf(ctx),
-		reflect.ValueOf(userCred),
-		reflect.ValueOf(query),
+	params := []interface{}{ctx, userCred, query}
+	outs, err := callFunc(funcValue, allowFuncName, params...)
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("reflect call %s fail %s", allowFuncName, err)
 	}
-	outs := funcValue.Call(params)
 	if len(outs) != 1 {
 		return nil, httperrors.NewInternalServerError("Invald %s return value", funcName)
 	}
@@ -806,7 +806,10 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr
 	}
 
 	funcValue = modelValue.MethodByName(funcName)
-	outs = funcValue.Call(params)
+	outs, err = callFunc(funcValue, funcName, params...)
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("reflect call %s fail %s", funcName, err)
+	}
 	if len(outs) != 2 {
 		return nil, httperrors.NewInternalServerError("Invald %s return value", funcName)
 	}
@@ -887,7 +890,7 @@ func (dispatcher *DBModelDispatcher) GetSpecific(ctx context.Context, idStr stri
 			return nil, httperrors.NewSpecNotFoundError("%s %s %s not found", dispatcher.Keyword(), idStr, spec)
 		}
 
-		outs, err := callFunc(funcValue, params...)
+		outs, err := callFunc(funcValue, funcName, params...)
 		if err != nil {
 			return nil, err
 		}
@@ -905,7 +908,7 @@ func (dispatcher *DBModelDispatcher) GetSpecific(ctx context.Context, idStr stri
 		return nil, httperrors.NewSpecNotFoundError("%s %s %s not found", dispatcher.Keyword(), idStr, spec)
 	}
 
-	outs, err := callFunc(funcValue, params...)
+	outs, err := callFunc(funcValue, funcName, params...)
 	if err != nil {
 		return nil, err
 	}
@@ -1166,7 +1169,11 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 		model.PostCreate(ctx, userCred, ownerId, query, data)
 	}()
 
-	OpsLog.LogEvent(model, ACT_CREATE, model.GetShortDesc(ctx), userCred)
+	{
+		notes := model.GetShortDesc(ctx)
+		OpsLog.LogEvent(model, ACT_CREATE, notes, userCred)
+		logclient.AddActionLogWithContext(ctx, model, logclient.ACT_CREATE, notes, userCred, true)
+	}
 	dispatcher.modelManager.OnCreateComplete(ctx, []IModel{model}, userCred, ownerId, query, data)
 	return getItemDetails(dispatcher.modelManager, model, ctx, userCred, query)
 }
@@ -1329,11 +1336,7 @@ func managerPerformCheckCreateData(
 	query jsonutils.JSONObject,
 	data jsonutils.JSONObject,
 ) (jsonutils.JSONObject, error) {
-	body, err := data.(*jsonutils.JSONDict).Get(manager.Keyword())
-	if err != nil {
-		return nil, httperrors.NewGeneralError(err)
-	}
-	bodyDict := body.(*jsonutils.JSONDict)
+	bodyDict := data.(*jsonutils.JSONDict)
 
 	if consts.IsRbacEnabled() {
 		err := isClassRbacAllowed(manager, userCred, ownerId, policy.PolicyActionPerform, action)
@@ -1470,7 +1473,7 @@ func reflectDispatcherInternal(
 			return nil, httperrors.NewActionNotFoundError(msg)
 		}
 
-		outs, err := callFunc(allowFuncValue, params...)
+		outs, err := callFunc(allowFuncValue, allowFuncName, params...)
 		if err != nil {
 			return nil, err
 		}
@@ -1483,7 +1486,7 @@ func reflectDispatcherInternal(
 		}
 	}
 
-	outs, err := callFunc(funcValue, params...)
+	outs, err := callFunc(funcValue, funcName, params...)
 	if err != nil {
 		return nil, err
 	}
@@ -1551,6 +1554,7 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 		return nil, httperrors.NewGeneralError(err)
 	}
 	OpsLog.LogEvent(item, ACT_UPDATE, diff, userCred)
+	logclient.AddActionLogWithContext(ctx, item, logclient.ACT_UPDATE, diff, userCred, true)
 
 	item.PostUpdate(ctx, userCred, query, data)
 
@@ -1615,7 +1619,7 @@ func objectUpdateSpec(dispatcher *DBModelDispatcher, model IModel, modelValue re
 
 func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IModel) error {
 	// log.Debugf("Ready to delete %s %s %#v", jsonutils.Marshal(item), item, manager)
-	cleanModelUsages(ctx, userCred, item)
+	// cleanModelUsages(ctx, userCred, item)
 	_, err := Update(item, func() error {
 		return item.MarkDelete()
 	})

pkg/cloudcommon/db/proxy/doc.go

@@ -0,0 +1 @@
+package proxy // import "yunion.io/x/onecloud/pkg/cloudcommon/db/proxy"

pkg/cloudcommon/db/proxy/proxysetting.go

@@ -0,0 +1,195 @@
+package proxy
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+
+	"golang.org/x/net/http/httpproxy"
+
+	"yunion.io/x/jsonutils"
+
+	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/httputils"
+)
+
+type SProxySettingManager struct {
+	db.SStandaloneResourceBaseManager
+	db.SDomainizedResourceBaseManager
+}
+
+var ProxySettingManager *SProxySettingManager
+
+func init() {
+	ProxySettingManager = &SProxySettingManager{
+		SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager(
+			SProxySetting{},
+			"proxysettings_tbl",
+			"proxysetting",
+			"proxysettings",
+		),
+	}
+	ProxySettingManager.SetVirtualObject(ProxySettingManager)
+}
+
+type SProxySetting struct {
+	db.SStandaloneResourceBase
+	db.SDomainizedResourceBase
+
+	HTTPProxy  string `create:"admin_optional" list:"admin" update:"admin"`
+	HTTPSProxy string `create:"admin_optional" list:"admin" update:"admin"`
+	NoProxy    string `create:"admin_optional" list:"admin" update:"admin"`
+}
+
+func (man *SProxySettingManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data proxyapi.ProxySettingCreateInput) (proxyapi.ProxySettingCreateInput, error) {
+	if err := data.ProxySetting.Sanitize(); err != nil {
+		return data, httperrors.NewInputParameterError("%s", err)
+	}
+	return data, nil
+}
+
+func (ps *SProxySetting) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	ps.DomainId = ownerId.GetProjectDomainId()
+	return ps.SStandaloneResourceBase.CustomizeCreate(ctx, userCred, ownerId, query, data)
+}
+
+func (ps *SProxySetting) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data proxyapi.ProxySettingUpdateInput) (proxyapi.ProxySettingUpdateInput, error) {
+	if ps.Id == proxyapi.ProxySettingId_DIRECT {
+		return data, httperrors.NewConflictError("DIRECT setting cannot be changed")
+	}
+	if err := data.ProxySetting.Sanitize(); err != nil {
+		return data, httperrors.NewInputParameterError("%s", err)
+	}
+	return data, nil
+}
+
+func (ps *SProxySetting) HttpTransportProxyFunc() httputils.TransportProxyFunc {
+	cfg := &httpproxy.Config{
+		HTTPProxy:  ps.HTTPProxy,
+		HTTPSProxy: ps.HTTPSProxy,
+		NoProxy:    ps.NoProxy,
+	}
+	proxyFunc := cfg.ProxyFunc()
+	return func(req *http.Request) (*url.URL, error) {
+		return proxyFunc(req.URL)
+	}
+}
+
+func (ps *SProxySetting) ValidateDeleteCondition(ctx context.Context) error {
+	if ps.Id == proxyapi.ProxySettingId_DIRECT {
+		return httperrors.NewConflictError("DIRECT setting cannot be deleted")
+	}
+	for _, man := range referrersMen {
+		t := man.TableSpec().Instance()
+		n, err := t.Query().
+			Equals("proxy_setting_id", ps.Id).
+			CountWithError()
+		if err != nil {
+			return httperrors.NewInternalServerError("get proxysetting refcount fail %s", err)
+		}
+		if n > 0 {
+			return httperrors.NewResourceBusyError("proxysetting %s is still referred to by %d %s",
+				ps.Id, n, man.KeywordPlural())
+		}
+	}
+	return nil
+}
+
+func (ps *SProxySetting) AllowPerformTest(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return db.IsAdminAllowPerform(userCred, ps, "test")
+}
+
+func (ps *SProxySetting) PerformTest(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	type TestURLResult struct {
+		Ok     bool   `json:"ok"`
+		Reason string `json:"reason"`
+	}
+	var (
+		r = map[string]TestURLResult{}
+		m = map[string]string{
+			"http_proxy":  ps.HTTPProxy,
+			"https_proxy": ps.HTTPSProxy,
+		}
+	)
+	for k, v := range m {
+		if v == "" {
+			r[k] = TestURLResult{Ok: true}
+			continue
+		}
+		u, err := url.Parse(v)
+		if err != nil {
+			r[k] = TestURLResult{
+				Reason: err.Error(),
+			}
+		} else if u == nil {
+			r[k] = TestURLResult{
+				Reason: fmt.Sprintf("bad url: %q", v),
+			}
+		} else {
+			host := u.Hostname()
+			port := u.Port()
+			if port == "" {
+				switch u.Scheme {
+				case "http":
+					port = "80"
+				case "https":
+					port = "443"
+				case "socks5":
+					port = "1080"
+				default:
+					r[k] = TestURLResult{
+						Reason: fmt.Sprintf("bad url scheme: %s", u.Scheme),
+					}
+					continue
+				}
+			}
+			addr := net.JoinHostPort(host, port)
+			conn, err := net.DialTimeout("tcp", addr, 7*time.Second)
+			if err != nil {
+				r[k] = TestURLResult{
+					Reason: err.Error(),
+				}
+			} else {
+				r[k] = TestURLResult{Ok: true}
+				conn.Close()
+			}
+		}
+	}
+	return jsonutils.Marshal(r), nil
+}
+
+func (man *SProxySettingManager) InitializeData() error {
+	_, err := man.FetchById(proxyapi.ProxySettingId_DIRECT)
+	if err == nil {
+		return nil
+	}
+	if err != sql.ErrNoRows {
+		return err
+	}
+
+	m, err := db.NewModelObject(man)
+	if err != nil {
+		return err
+	}
+	ps := m.(*SProxySetting)
+	ps.Id = proxyapi.ProxySettingId_DIRECT
+	ps.Name = proxyapi.ProxySettingId_DIRECT
+	ps.Description = "Connect directly"
+	if err := man.TableSpec().Insert(ps); err != nil {
+		return err
+	}
+	return nil
+}
+
+var referrersMen []db.IModelManager
+
+func RegisterReferrer(man db.IModelManager) {
+	referrersMen = append(referrersMen, man)
+}

pkg/cloudcommon/db/quotas/context.go

@@ -74,7 +74,8 @@ func cancelPendingUsagesInContext(ctx context.Context, userCred mcclient.TokenCr
 	}
 	errs := make([]error, 0)
 	for i := range quotas {
-		err := CancelPendingUsage(ctx, userCred, quotas[i], quotas[i])
+		// cancel and do not save pending usage
+		err := CancelPendingUsage(ctx, userCred, quotas[i], quotas[i], false)
 		if err != nil {
 			errs = append(errs, errors.Wrapf(err, "CancelPendingUsage %s", jsonutils.Marshal(quotas[i])))
 		}

pkg/cloudcommon/db/quotas/interface.go

@@ -69,7 +69,7 @@ type IQuotaManager interface {
 	db.IResourceModelManager
 
 	checkSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error
-	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error
+	cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error
 	cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage IQuota) error
 	getQuotaCount(ctx context.Context, request IQuota, pendingKey IQuotaKeys) (int, error)
 

pkg/cloudcommon/db/quotas/quotas.go

@@ -66,14 +66,14 @@ func (manager *SQuotaBaseManager) _cleanPendingUsage(ctx context.Context, userCr
 	return nil
 }
 
-func (manager *SQuotaBaseManager) cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error {
+func (manager *SQuotaBaseManager) cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
 	LockQuota(ctx, manager, localUsage)
 	defer ReleaseQuota(ctx, manager, localUsage)
 
-	return manager._cancelPendingUsage(ctx, userCred, localUsage, cancelUsage)
+	return manager._cancelPendingUsage(ctx, userCred, localUsage, cancelUsage, save)
 }
 
-func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error {
+func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
 	originKeys := localUsage.GetKeys()
 	// currentKeys := cancelUsage.GetKeys()
 
@@ -92,9 +92,11 @@ func (manager *SQuotaBaseManager) _cancelPendingUsage(ctx context.Context, userC
 
 	log.Debugf("cancelUsage: %s localUsage: %s pendingUsage: %s", jsonutils.Marshal(cancelUsage), jsonutils.Marshal(localUsage), jsonutils.Marshal(pendingUsage))
 
-	err = manager.changeUsage(ctx, userCred, pendingUsage, true)
-	if err != nil {
-		return errors.Wrap(err, "manager.changelUsage")
+	if save {
+		err = manager.changeUsage(ctx, userCred, pendingUsage, true)
+		if err != nil {
+			return errors.Wrap(err, "manager.changelUsage")
+		}
 	}
 
 	return nil

pkg/cloudcommon/db/quotas/register.go

@@ -55,9 +55,9 @@ func getQuotaManager(quota IQuota) IQuotaManager {
 	}
 }
 
-func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota) error {
+func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
 	manager := getQuotaManager(cancelUsage)
-	return manager.cancelPendingUsage(ctx, userCred, localUsage, cancelUsage)
+	return manager.cancelPendingUsage(ctx, userCred, localUsage, cancelUsage, save)
 }
 
 func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error {

pkg/cloudcommon/db/usercache.go

@@ -19,6 +19,7 @@ import (
 	"database/sql"
 	"fmt"
 	"runtime/debug"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -155,6 +156,7 @@ func (manager *SUserCacheManager) Save(ctx context.Context, idStr string, name s
 			obj.Name = name
 			obj.Domain = domain
 			obj.DomainId = domainId
+			obj.LastCheck = time.Now().UTC()
 			return nil
 		})
 		if err != nil {
@@ -169,7 +171,8 @@ func (manager *SUserCacheManager) Save(ctx context.Context, idStr string, name s
 		obj.Name = name
 		obj.Domain = domain
 		obj.DomainId = domainId
-		err = manager.TableSpec().Insert(obj)
+		obj.LastCheck = time.Now().UTC()
+		err = manager.TableSpec().InsertOrUpdate(obj)
 		if err != nil {
 			return nil, err
 		} else {

pkg/cloudcommon/options/changes.go

@@ -22,27 +22,31 @@ func OnBaseOptionsChange(oOpts, nOpts interface{}) bool {
 	oldOpts := oOpts.(*BaseOptions)
 	newOpts := nOpts.(*BaseOptions)
 
+	changed := false
 	if oldOpts.RequestWorkerCount != newOpts.RequestWorkerCount {
-		return true
+		changed = true
 	}
 	if oldOpts.TimeZone != newOpts.TimeZone {
-		return true
+		changed = true
 	}
 	if oldOpts.EnableRbac != newOpts.EnableRbac {
-		return true
+		changed = true
 	}
 	if oldOpts.NonDefaultDomainProjects != newOpts.NonDefaultDomainProjects {
 		consts.SetNonDefaultDomainProjects(newOpts.NonDefaultDomainProjects)
+		changed = true
 	}
-	return false
+	return changed
 }
 
 func OnCommonOptionsChange(oOpts, nOpts interface{}) bool {
 	oldOpts := oOpts.(*CommonOptions)
 	newOpts := nOpts.(*CommonOptions)
 
+	changed := false
 	if OnBaseOptionsChange(&oldOpts.BaseOptions, &newOpts.BaseOptions) {
-		return true
+		changed = true
 	}
-	return false
+
+	return changed
 }

pkg/cloudcommon/options/options.go

@@ -58,7 +58,7 @@ type BaseOptions struct {
 	TempPath  string   `help:"Path for store temp file, at least 40G space" default:"/opt/yunion/tmp"`
 
 	ApplicationID      string `help:"Application ID"`
-	RequestWorkerCount int    `default:"4" help:"Request worker thread count, default is 4"`
+	RequestWorkerCount int    `default:"8" help:"Request worker thread count, default is 8"`
 
 	EnableSsl   bool   `help:"Enable https"`
 	SslCaCerts  string `help:"ssl certificate ca root file, separating ca and cert file is not encouraged" alias:"ca-file"`

pkg/cloudcommon/policy/defaults.go

@@ -162,6 +162,12 @@ var (
 					Action:   PolicyActionList,
 					Result:   rbacutils.Allow,
 				},
+				{
+					Service:  "compute",
+					Resource: "loadbalancerclusters",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
 				{
 					Service:  "yunionagent",
 					Resource: "notices",
@@ -444,18 +450,6 @@ var (
 					Action:   PolicyActionGet,
 					Result:   rbacutils.Allow,
 				},
-				{
-					Service:  "compute",
-					Resource: "cloudproviders",
-					Action:   PolicyActionList,
-					Result:   rbacutils.Allow,
-				},
-				{
-					Service:  "compute",
-					Resource: "cloudproviders",
-					Action:   PolicyActionGet,
-					Result:   rbacutils.Allow,
-				},
 				{
 					Service:  identityapi.SERVICE_TYPE,
 					Resource: "users",
@@ -535,6 +529,30 @@ var (
 					Action:   PolicyActionGet,
 					Result:   rbacutils.Allow,
 				},
+				{
+					Service:  "compute",
+					Resource: "cloudaccounts",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  "compute",
+					Resource: "cloudaccounts",
+					Action:   PolicyActionGet,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  "compute",
+					Resource: "cloudproviders",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  "compute",
+					Resource: "cloudproviders",
+					Action:   PolicyActionGet,
+					Result:   rbacutils.Allow,
+				},
 			},
 		},
 		{

pkg/cloudcommon/policy/policy.go

@@ -16,6 +16,7 @@ package policy
 
 import (
 	"context"
+	"runtime/debug"
 	"sort"
 	"strings"
 	"sync"
@@ -168,11 +169,6 @@ func (manager *SPolicyManager) start(refreshInterval time.Duration, retryInterva
 	}
 
 	manager.cache = hashcache.NewCache(2048, manager.refreshInterval/2)
-	err := manager.doSync()
-	if err != nil {
-		log.Errorf("doSync error %s", err)
-		return
-	}
 
 	manager.SyncOnce()
 }
@@ -182,9 +178,16 @@ func (manager *SPolicyManager) SyncOnce() {
 }
 
 func (manager *SPolicyManager) doSync() error {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Errorf("policyManager doSync error %s", r)
+			debug.PrintStack()
+		}
+	}()
+
 	policies, err := DefaultPolicyFetcher()
 	if err != nil {
-		// log.Errorf("sync rbac policy failed: %s", err)
+		log.Errorf("sync rbac policy failed: %s", err)
 		return errors.Wrap(err, "DefaultPolicyFetcher")
 	}
 

pkg/cloudevent/models/cloudproviders.go

@@ -16,8 +16,12 @@ package models
 
 import (
 	"context"
+	"net/http"
+	"net/url"
 	"time"
 
+	"golang.org/x/net/http/httpproxy"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
@@ -25,6 +29,7 @@ import (
 	"yunion.io/x/pkg/util/timeutils"
 	"yunion.io/x/pkg/utils"
 
+	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
@@ -68,6 +73,8 @@ type SCloudprovider struct {
 
 	Provider string `width:"64" charset:"ascii" list:"domain"`
 	Brand    string `width:"64" charset:"ascii" list:"domain"`
+
+	ProxySetting *proxyapi.SProxySetting
 }
 
 func (manager *SCloudproviderManager) GetRegionCloudproviders(ctx context.Context, userCred mcclient.TokenCredential) ([]SCloudprovider, error) {
@@ -299,7 +306,29 @@ func (provider *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, err
 	if err != nil {
 		return nil, err
 	}
-	return cloudprovider.GetProvider(provider.Id, provider.Name, accessUrl, provider.Account, passwd, provider.Provider)
+
+	ps := provider.ProxySetting
+	cfg := &httpproxy.Config{
+		HTTPProxy:  ps.HTTPProxy,
+		HTTPSProxy: ps.HTTPSProxy,
+		NoProxy:    ps.NoProxy,
+	}
+	cfgProxyFunc := cfg.ProxyFunc()
+	proxyFunc := func(req *http.Request) (*url.URL, error) {
+		return cfgProxyFunc(req.URL)
+	}
+	return cloudprovider.GetProvider(
+		cloudprovider.ProviderConfig{
+			Id:      provider.Id,
+			Name:    provider.Name,
+			Vendor:  provider.Provider,
+			URL:     accessUrl,
+			Account: provider.Account,
+			Secret:  passwd,
+
+			ProxyFunc: proxyFunc,
+		},
+	)
 }
 
 func (manager *SCloudproviderManager) InitializeData() error {

pkg/cloudevent/models/initdb.go

@@ -18,6 +18,7 @@ import (
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/proxy"
 )
 
 func InitDB() error {
@@ -28,6 +29,8 @@ func InitDB() error {
 		 */
 		db.TenantCacheManager,
 
+		proxy.ProxySettingManager,
+
 		CloudproviderManager,
 		CloudeventManager,
 	} {

pkg/cloudevent/service/handlers.go

@@ -32,6 +32,7 @@ import (
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/proxy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudevent/models"
 )
@@ -54,6 +55,7 @@ func InitHandlers(app *appsrv.Application) {
 
 	for _, manager := range []db.IModelManager{
 		db.OpsLog,
+		proxy.ProxySettingManager,
 		models.CloudeventManager,
 	} {
 		db.RegisterModelManager(manager)

pkg/cloudprovider/cloudprovider.go

@@ -25,6 +25,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 const (
@@ -140,8 +141,22 @@ type SCloudaccount struct {
 	AccessUrl string `json:"access_url"`
 }
 
+type ProviderConfig struct {
+	// Id, Name are properties of Cloudprovider object
+	Id   string
+	Name string
+
+	// Vendor are names like Aliyun, OpenStack, etc.
+	Vendor  string
+	URL     string
+	Account string
+	Secret  string
+
+	ProxyFunc httputils.TransportProxyFunc
+}
+
 type ICloudProviderFactory interface {
-	GetProvider(providerId, providerName, url, account, secret string) (ICloudProvider, error)
+	GetProvider(cfg ProviderConfig) (ICloudProvider, error)
 
 	GetClientRC(url, account, secret string) (map[string]string, error)
 
@@ -240,12 +255,12 @@ func GetRegistedProviderIds() []string {
 	return providers
 }
 
-func GetProvider(providerId, providerName, accessUrl, account, secret, provider string) (ICloudProvider, error) {
-	driver, err := GetProviderFactory(provider)
+func GetProvider(cfg ProviderConfig) (ICloudProvider, error) {
+	driver, err := GetProviderFactory(cfg.Vendor)
 	if err != nil {
 		return nil, errors.Wrap(err, "GetProviderFactory")
 	}
-	return driver.GetProvider(providerId, providerName, accessUrl, account, secret)
+	return driver.GetProvider(cfg)
 }
 
 func GetClientRC(accessUrl, account, secret, provider string) (map[string]string, error) {
@@ -261,10 +276,10 @@ func IsSupported(provider string) bool {
 	return ok
 }
 
-func IsValidCloudAccount(accessUrl, account, secret, provider string) (string, error) {
-	factory, ok := providerTable[provider]
+func IsValidCloudAccount(cfg ProviderConfig) (string, error) {
+	factory, ok := providerTable[cfg.Vendor]
 	if ok {
-		provider, err := factory.GetProvider("", "", accessUrl, account, secret)
+		provider, err := factory.GetProvider(cfg)
 		if err != nil {
 			return "", err
 		}

pkg/cloudprovider/resources.go

@@ -490,7 +490,7 @@ type ICloudLoadbalancer interface {
 
 	GetIEIP() (ICloudEIP, error)
 
-	Delete() error
+	Delete(ctx context.Context) error
 
 	Start() error
 	Stop() error
@@ -501,7 +501,7 @@ type ICloudLoadbalancer interface {
 	CreateILoadBalancerBackendGroup(group *SLoadbalancerBackendGroup) (ICloudLoadbalancerBackendGroup, error)
 	GetILoadBalancerBackendGroupById(groupId string) (ICloudLoadbalancerBackendGroup, error)
 
-	CreateILoadBalancerListener(listener *SLoadbalancerListener) (ICloudLoadbalancerListener, error)
+	CreateILoadBalancerListener(ctx context.Context, listener *SLoadbalancerListener) (ICloudLoadbalancerListener, error)
 	GetILoadBalancerListenerById(listenerId string) (ICloudLoadbalancerListener, error)
 }
 
@@ -551,9 +551,9 @@ type ICloudLoadbalancerListener interface {
 
 	Start() error
 	Stop() error
-	Sync(listener *SLoadbalancerListener) error
+	Sync(ctx context.Context, listener *SLoadbalancerListener) error
 
-	Delete() error
+	Delete(ctx context.Context) error
 }
 
 type ICloudLoadbalancerListenerRule interface {
@@ -565,7 +565,7 @@ type ICloudLoadbalancerListenerRule interface {
 	GetCondition() string
 	GetBackendGroupId() string
 
-	Delete() error
+	Delete(ctx context.Context) error
 }
 
 type ICloudLoadbalancerBackendGroup interface {
@@ -583,8 +583,8 @@ type ICloudLoadbalancerBackendGroup interface {
 	AddBackendServer(serverId string, weight int, port int) (ICloudLoadbalancerBackend, error)
 	RemoveBackendServer(serverId string, weight int, port int) error
 
-	Delete() error
-	Sync(group *SLoadbalancerBackendGroup) error
+	Delete(ctx context.Context) error
+	Sync(ctx context.Context, group *SLoadbalancerBackendGroup) error
 }
 
 type ICloudLoadbalancerBackend interface {
@@ -595,7 +595,7 @@ type ICloudLoadbalancerBackend interface {
 	GetBackendType() string
 	GetBackendRole() string
 	GetBackendId() string
-	SyncConf(port, weight int) error
+	SyncConf(ctx context.Context, port, weight int) error
 }
 
 type ICloudLoadbalancerCertificate interface {

pkg/compute/hostdrivers/esxi.go

@@ -62,10 +62,17 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mo
 		return err
 	}
 	cacheImage := obj.(*models.SCachedimage)
-	srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange(api.HOST_TYPE_ESXI, []string{host.Id},
-		[]interface{}{host.GetZone(), host.GetCloudprovider()})
-	if err != nil {
-		return err
+	var srcHostCacheImage *models.SStoragecachedimage
+	// Check if storageCache has this cacheImage.
+	// If no, choose source storage cache
+	// else, use it
+	hostCacheImage := models.StoragecachedimageManager.GetStoragecachedimage(storageCache.GetId(), cacheImage.GetId())
+	if hostCacheImage == nil {
+		srcHostCacheImage, err = cacheImage.ChooseSourceStoragecacheInRange(api.HOST_TYPE_ESXI, []string{host.Id},
+			[]interface{}{host.GetZone(), host.GetCloudprovider()})
+		if err != nil {
+			return err
+		}
 	}
 
 	type contentStruct struct {
@@ -136,7 +143,7 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mo
 		if err != nil {
 			return errors.Wrap(err, "StorageCacheImage.GetStoragecaceh")
 		}
-		srcStorage := host.GetStorageByFilePath(srcStorageCache.Path)
+		srcStorage := srcHost.GetStorageByFilePath(srcStorageCache.Path)
 		accessInfo, err := srcHost.GetCloudaccount().GetVCenterAccessInfo(srcStorage.ExternalId)
 		if err != nil {
 			return err

pkg/compute/hostdrivers/kvm.go

@@ -19,9 +19,13 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -33,6 +37,7 @@ import (
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/onecloud/pkg/util/k8s/tokens"
 )
 
 type SKVMHostDriver struct {
@@ -454,6 +459,16 @@ func (self *SKVMHostDriver) PrepareConvert(host *models.SHost, image, raid strin
 		nets = append(nets, net)
 	}
 	params.Networks = nets
+
+	deployConfigs, err := self.getDeployConfig(host)
+	if err != nil {
+		return nil, err
+	}
+	params.DeployConfigs = deployConfigs
+	return params, nil
+}
+
+func (self *SKVMHostDriver) getDeployConfig(host *models.SHost) ([]*api.DeployConfig, error) {
 	deployConf := &api.DeployConfig{
 		Action: "create",
 		Path:   "/etc/sysconfig/yunionauth",
@@ -484,10 +499,18 @@ func (self *SKVMHostDriver) PrepareConvert(host *models.SHost, image, raid strin
 	authInfo += fmt.Sprintf("YUNION_HOST_PASSWORD=%s\n", options.Options.AdminPassword)
 	authInfo += fmt.Sprintf("YUNION_HOST_PROJECT=%s\n", options.Options.AdminProject)
 	authInfo += fmt.Sprintf("YUNION_START=yes\n")
+	apiServer, err := tokens.GetControlPlaneEndpoint()
+	if err != nil {
+		log.Errorf("Failed to get kubernetes controlplane endpoint: %v", err)
+	}
+	joinToken, err := tokens.GetNodeJoinToken()
+	if err != nil {
+		log.Errorf("Failed to get kubernetes node join token: %v", err)
+	}
+	authInfo += fmt.Sprintf("API_SERVER=%s\n", apiServer)
+	authInfo += fmt.Sprintf("JOIN_TOKEN=%s\n", joinToken)
 	deployConf.Content = authInfo
-	params.DeployConfigs = []*api.DeployConfig{deployConf}
-	log.Infof("%v", params)
-	return params, nil
+	return []*api.DeployConfig{deployConf}, nil
 }
 
 func (self *SKVMHostDriver) PrepareUnconvert(host *models.SHost) error {
@@ -523,11 +546,31 @@ func (self *SKVMHostDriver) FinishUnconvert(ctx context.Context, userCred mcclie
 			}
 		}
 	}
+	onK8s := host.GetMetadata("on_kubernetes", userCred)
+	hostname := host.GetMetadata("hostname", userCred)
+	if strings.ToLower(onK8s) == "true" {
+		if err := self.tryCleanKubernetesData(host, hostname); err != nil {
+			log.Errorf("try clean kubernetes data: %v", err)
+		}
+	}
 	kwargs := make(map[string]interface{}, 0)
-	for _, k := range []string{"kernel_version", "nest", "os_distribution", "os_version",
-		"ovs_version", "qemu_version", "storage_type"} {
+	for _, k := range []string{
+		"kernel_version", "nest", "os_distribution", "os_version",
+		"ovs_version", "qemu_version", "storage_type", "on_kubernetes",
+	} {
 		kwargs[k] = "None"
 	}
 	host.SetAllMetadata(ctx, kwargs, userCred)
 	return self.SBaseHostDriver.FinishUnconvert(ctx, userCred, host)
 }
+
+func (self *SKVMHostDriver) tryCleanKubernetesData(host *models.SHost, hostname string) error {
+	cli, err := tokens.GetCoreClient()
+	if err != nil {
+		return errors.Wrap(err, "get k8s client")
+	}
+	if hostname == "" {
+		hostname = host.GetName()
+	}
+	return cli.Nodes().Delete(hostname, &metav1.DeleteOptions{})
+}

pkg/compute/models/buckets.go

@@ -463,7 +463,7 @@ func (bucket *SBucket) PostCreate(
 		log.Errorf("bucket.GetQuotaKeys fail %s", err)
 	} else {
 		pendingUsage.SetKeys(keys)
-		err = quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+		err = quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, true)
 		if err != nil {
 			log.Errorf("CancelPendingUsage error %s", err)
 		}
@@ -780,7 +780,7 @@ func (bucket *SBucket) PerformMakedir(
 
 	bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, true)
 
-	quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+	quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, true)
 
 	return nil, nil
 }
@@ -940,12 +940,13 @@ func (bucket *SBucket) PerformUpload(
 			return nil, httperrors.NewOutOfQuotaError("%s", err)
 		}
 
-		// always cancel pending usage
-		defer quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
 	}
 
 	err = cloudprovider.UploadObject(ctx, iBucket, key, 0, appParams.Request.Body, sizeBytes, cloudprovider.TBucketACLType(aclStr), storageClass, meta, false)
 	if err != nil {
+		if !pendingUsage.IsEmpty() {
+			quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
+		}
 		return nil, httperrors.NewInternalServerError("put object error %s", err)
 	}
 
@@ -954,6 +955,10 @@ func (bucket *SBucket) PerformUpload(
 
 	bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, true)
 
+	if !pendingUsage.IsEmpty() {
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, true)
+	}
+
 	return nil, nil
 }
 

pkg/compute/models/cloudaccounts.go

@@ -32,10 +32,12 @@ import (
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
+	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/proxy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
@@ -46,6 +48,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/choices"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
@@ -68,6 +71,8 @@ func init() {
 		),
 	}
 	CloudaccountManager.SetVirtualObject(CloudaccountManager)
+
+	proxy.RegisterReferrer(CloudaccountManager)
 }
 
 type SCloudaccount struct {
@@ -148,6 +153,9 @@ type SCloudaccount struct {
 	IsPublic bool `default:"false" nullable:"false"`
 	// add share_mode field to indicate the share range of this account
 	ShareMode string `width:"32" charset:"ascii" nullable:"true" list:"domain"`
+
+	// 默认值proxyapi.ProxySettingId_DIRECT
+	ProxySettingId string `width:"36" charset:"ascii" nullable:"false" list:"domain" create:"optional" update:"domain" default:"DIRECT"`
 }
 
 func (self *SCloudaccountManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
@@ -305,6 +313,16 @@ func (self *SCloudaccount) ValidateUpdateData(ctx context.Context, userCred mccl
 		}
 		data.Set("options", optionsJson)
 	}
+
+	v := validators.NewModelIdOrNameValidator(
+		"proxy_setting",
+		proxy.ProxySettingManager.Keyword(),
+		userCred,
+	).Optional(true)
+	if err := v.Validate(data); err != nil {
+		return nil, err
+	}
+
 	return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data)
 }
 
@@ -362,7 +380,27 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 		return input, httperrors.NewConflictError("The account has been registered")
 	}
 
-	accountId, err := cloudprovider.IsValidCloudAccount(input.AccessUrl, input.Account, input.Secret, input.Provider)
+	var proxyFunc httputils.TransportProxyFunc
+	{
+		if input.ProxySettingId == "" {
+			input.ProxySettingId = proxyapi.ProxySettingId_DIRECT
+		}
+		m, err := proxy.ProxySettingManager.FetchByIdOrName(userCred, input.ProxySettingId)
+		if err != nil {
+			return input, httperrors.NewInputParameterError("fetch proxysetting %s: %s",
+				input.ProxySettingId, err)
+		}
+		proxySetting := m.(*proxy.SProxySetting)
+		input.ProxySettingId = proxySetting.Id
+		proxyFunc = proxySetting.HttpTransportProxyFunc()
+	}
+	accountId, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{
+		Vendor:    input.Provider,
+		URL:       input.AccessUrl,
+		Account:   input.Account,
+		Secret:    input.Secret,
+		ProxyFunc: proxyFunc,
+	})
 	if err != nil {
 		if err == cloudprovider.ErrNoSuchProvder {
 			return input, httperrors.NewResourceNotFoundError("no such provider %s", input.Provider)
@@ -516,7 +554,13 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred
 
 	originSecret, _ := self.getPassword()
 
-	accountId, err := cloudprovider.IsValidCloudAccount(self.AccessUrl, account.Account, account.Secret, self.Provider)
+	accountId, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{
+		Vendor:    self.Provider,
+		URL:       self.AccessUrl,
+		Account:   account.Account,
+		Secret:    account.Secret,
+		ProxyFunc: self.proxyFunc(),
+	})
 	if err != nil {
 		return nil, httperrors.NewInputParameterError("invalid cloud account info error: %s", err.Error())
 	}
@@ -668,12 +712,40 @@ func (self *SCloudaccount) GetProvider() (cloudprovider.ICloudProvider, error) {
 	return self.getProviderInternal()
 }
 
+func (self *SCloudaccount) proxySetting() *proxy.SProxySetting {
+	m, err := proxy.ProxySettingManager.FetchById(self.ProxySettingId)
+	if err != nil {
+		log.Errorf("cloudaccount %s(%s): get proxysetting %s: %v",
+			self.Name, self.Id, self.ProxySettingId, err)
+		return nil
+	}
+	ps := m.(*proxy.SProxySetting)
+	return ps
+}
+
+func (self *SCloudaccount) proxyFunc() httputils.TransportProxyFunc {
+	ps := self.proxySetting()
+	if ps != nil {
+		return ps.HttpTransportProxyFunc()
+	}
+	return nil
+}
+
 func (self *SCloudaccount) getProviderInternal() (cloudprovider.ICloudProvider, error) {
 	secret, err := self.getPassword()
 	if err != nil {
 		return nil, fmt.Errorf("Invalid password %s", err)
 	}
-	return cloudprovider.GetProvider(self.Id, self.Name, self.AccessUrl, self.Account, secret, self.Provider)
+	return cloudprovider.GetProvider(cloudprovider.ProviderConfig{
+		Id:      self.Id,
+		Name:    self.Name,
+		Vendor:  self.Provider,
+		URL:     self.AccessUrl,
+		Account: self.Account,
+		Secret:  secret,
+
+		ProxyFunc: self.proxyFunc(),
+	})
 }
 
 func (self *SCloudaccount) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
@@ -1642,6 +1714,33 @@ func (manager *SCloudaccountManager) FetchCustomizeColumns(ctx context.Context,
 			}
 		}
 	}
+	proxySettings := make(map[string]proxy.SProxySetting)
+	{
+		proxySettingIds := make([]string, len(objs))
+		for i := range objs {
+			proxySettingId := objs[i].(*SCloudaccount).ProxySettingId
+			if !utils.IsInStringArray(proxySettingId, proxySettingIds) {
+				proxySettingIds = append(proxySettingIds, proxySettingId)
+			}
+		}
+		q := proxy.ProxySettingManager.Query().In("id", proxySettingIds)
+		r := []proxy.SProxySetting{}
+		if err := db.FetchModelObjects(proxy.ProxySettingManager, q, &r); err != nil {
+			log.Errorf("FetchModelObjects (%s) fail %s",
+				proxy.ProxySettingManager.KeywordPlural(), err)
+			return rows
+		}
+		for i := range r {
+			proxySetting := r[i]
+			proxySettings[proxySetting.Id] = proxySetting
+		}
+	}
+	for i := range rows {
+		account := objs[i].(*SCloudaccount)
+		if proxySetting, ok := proxySettings[account.ProxySettingId]; ok {
+			rows[i].Set("proxy_setting", jsonutils.Marshal(proxySetting))
+		}
+	}
 	return rows
 }
 

pkg/compute/models/cloudproviders.go

@@ -737,7 +737,18 @@ func (self *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error)
 	if err != nil {
 		return nil, err
 	}
-	return cloudprovider.GetProvider(self.Id, self.Name, accessUrl, self.Account, passwd, self.Provider)
+
+	account := self.GetCloudaccount()
+
+	return cloudprovider.GetProvider(cloudprovider.ProviderConfig{
+		Id:        self.Id,
+		Name:      self.Name,
+		Vendor:    self.Provider,
+		URL:       accessUrl,
+		Account:   self.Account,
+		Secret:    passwd,
+		ProxyFunc: account.proxyFunc(),
+	})
 }
 
 func (self *SCloudprovider) savePassword(secret string) error {
@@ -871,7 +882,16 @@ func (self *SCloudprovider) getMoreDetails(ctx context.Context, out api.Cloudpro
 		// 此字段不能删除，公有云日志同步需要这个字段
 		out.Brand = account.Brand
 		out.Cloudaccount = account.GetName()
+		ps := account.proxySetting()
+		if ps != nil {
+			out.ProxySetting.Id = ps.Id
+			out.ProxySetting.Name = ps.Name
+			out.ProxySetting.HTTPProxy = ps.HTTPProxy
+			out.ProxySetting.HTTPSProxy = ps.HTTPSProxy
+			out.ProxySetting.NoProxy = ps.NoProxy
+		}
 	}
+
 	out.SyncStatus2 = self.getSyncStatus2()
 	capabilities, _ := CloudproviderCapabilityManager.getCapabilities(self.Id)
 	if len(capabilities) > 0 {
@@ -987,29 +1007,24 @@ func (manager *SCloudproviderManager) ListItemFilter(ctx context.Context, q *sql
 		networks := NetworkManager.Query().SubQuery()
 		wires := WireManager.Query().SubQuery()
 		vpcs := VpcManager.Query().SubQuery()
+		providerRegions := CloudproviderRegionManager.Query().SubQuery()
 
 		sq := providers.Query(sqlchemy.DISTINCT("id", providers.Field("id")))
-		sq = sq.Join(vpcs, sqlchemy.Equals(vpcs.Field("manager_id"), providers.Field("id")))
+		sq = sq.Join(providerRegions, sqlchemy.Equals(providers.Field("id"), providerRegions.Field("cloudprovider_id")))
+		sq = sq.Join(vpcs, sqlchemy.Equals(providerRegions.Field("cloudregion_id"), vpcs.Field("cloudregion_id")))
 		sq = sq.Join(wires, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
 		sq = sq.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id")))
-		sq = sq.Filter(sqlchemy.Equals(networks.Field("status"), api.NETWORK_STATUS_AVAILABLE))
 		sq = sq.Filter(sqlchemy.IsTrue(providers.Field("enabled")))
 		sq = sq.Filter(sqlchemy.In(providers.Field("status"), api.CLOUD_PROVIDER_VALID_STATUS))
 		sq = sq.Filter(sqlchemy.In(providers.Field("health_status"), api.CLOUD_PROVIDER_VALID_HEALTH_STATUS))
 		sq = sq.Filter(sqlchemy.Equals(vpcs.Field("status"), api.VPC_STATUS_AVAILABLE))
-
-		sq2 := providers.Query(sqlchemy.DISTINCT("id", providers.Field("id")))
-		sq2 = sq2.Join(vpcs, sqlchemy.Equals(vpcs.Field("manager_id"), providers.Field("id")))
-		sq2 = sq2.Join(wires, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
-		sq2 = sq2.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id")))
-		sq2 = sq2.Filter(sqlchemy.Equals(networks.Field("status"), api.NETWORK_STATUS_AVAILABLE))
-		sq2 = sq2.Filter(sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id")))
-		sq2 = sq2.Filter(sqlchemy.Equals(vpcs.Field("status"), api.VPC_STATUS_AVAILABLE))
-
-		q = q.Filter(sqlchemy.OR(
-			sqlchemy.In(q.Field("id"), sq.SubQuery()),
-			sqlchemy.In(q.Field("id"), sq2.SubQuery()),
+		sq = sq.Filter(sqlchemy.Equals(networks.Field("status"), api.NETWORK_STATUS_AVAILABLE))
+		sq = sq.Filter(sqlchemy.OR(
+			sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id")),
+			sqlchemy.Equals(vpcs.Field("manager_id"), providers.Field("id")),
 		))
+
+		q = q.Filter(sqlchemy.In(q.Field("id"), sq.SubQuery()))
 	}
 
 	q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledStatusStandaloneResourceListInput)

pkg/compute/models/cloudregionresource.go

@@ -37,7 +37,7 @@ func (self *SCloudregionResourceBase) GetRegion() *SCloudregion {
 	return region.(*SCloudregion)
 }
 
-func (self *SCloudregionResourceBase) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) api.CloudregionInfo {
+func (self *SCloudregionResourceBase) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, isList bool) api.CloudregionInfo {
 	out := api.CloudregionInfo{}
 	region := self.GetRegion()
 	if region == nil {

pkg/compute/models/dbinstance_skus.go

@@ -136,6 +136,14 @@ func (manager *SDBInstanceSkuManager) ListItemFilter(ctx context.Context, q *sql
 	return q, nil
 }
 
+func (self *SDBInstanceSku) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, isList bool) (api.DBInstanceSkuDetails, error) {
+	var err error
+	out := api.DBInstanceSkuDetails{}
+	out.StandaloneResourceDetails, err = self.SEnabledStatusStandaloneResourceBase.GetExtraDetails(ctx, userCred, query, isList)
+	out.CloudregionInfo = self.SCloudregionResourceBase.GetExtraDetails(ctx, userCred, query, isList)
+	return out, err
+}
+
 func (manager *SDBInstanceSkuManager) GetDBStringArray(q *sqlchemy.SQuery) ([]string, error) {
 	array := []string{}
 	rows, err := q.Rows()

pkg/compute/models/disks.go

@@ -984,6 +984,13 @@ func (self *SDisk) ValidatePurgeCondition(ctx context.Context) error {
 }
 
 func (self *SDisk) validateDeleteCondition(ctx context.Context, isPurge bool) error {
+	if !isPurge {
+		storage := self.GetStorage()
+		host := storage.GetMasterHost()
+		if host == nil {
+			return httperrors.NewBadRequestError("storage of disk no valid host")
+		}
+	}
 	cnt, err := self.GetGuestDiskCount()
 	if err != nil {
 		return httperrors.NewInternalServerError("GetGuestDiskCount fail %s", err)

pkg/compute/models/elasticips.go

@@ -776,7 +776,7 @@ func (self *SElasticip) PostCreate(ctx context.Context, userCred mcclient.TokenC
 		log.Errorf("GetQuotaKeys fail %s", err)
 	} else {
 		eipPendingUsage.SetKeys(keys)
-		err := quotas.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage)
+		err := quotas.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage, true)
 		if err != nil {
 			log.Errorf("SElasticip CancelPendingUsage error: %s", err)
 		}
@@ -1140,7 +1140,7 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre
 		host.GetCloudprovider(),
 	)
 	eipPendingUsage.SetKeys(keys)
-	quotas.CancelPendingUsage(ctx, userCred, pendingUsage, eipPendingUsage)
+	quotas.CancelPendingUsage(ctx, userCred, pendingUsage, eipPendingUsage, true)
 
 	return &eip, nil
 }

pkg/compute/models/guest_actions.go

@@ -1556,7 +1556,7 @@ func (self *SGuest) PerformCreatedisk(ctx context.Context, userCred mcclient.Tok
 
 	err = self.CreateDisksOnHost(ctx, userCred, host, disksConf, pendingUsage, false, false, nil, nil, false)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false)
 		logclient.AddActionLogWithContext(ctx, self, logclient.ACT_CREATE, err.Error(), userCred, false)
 		return nil, httperrors.NewBadRequestError(err.Error())
 	}
@@ -2030,12 +2030,16 @@ func (self *SGuest) AllowPerformDetachnetwork(ctx context.Context, userCred mccl
 }
 
 func (self *SGuest) PerformDetachnetwork(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.Status != api.VM_READY {
+	if !utils.IsInStringArray(self.Status, []string{api.VM_READY, api.VM_RUNNING}) {
 		return nil, httperrors.NewInvalidStatusError("Cannot detach network in status %s", self.Status)
 	}
-	reserve := jsonutils.QueryBoolean(data, "reserve", false)
+	var (
+		reserve   = jsonutils.QueryBoolean(data, "reserve", false)
+		netStr, _ = data.GetString("net_id")
+		gns       []SGuestnetwork
+		err       error
+	)
 
-	netStr, _ := data.GetString("net_id")
 	if len(netStr) > 0 {
 		netObj, err := NetworkManager.FetchByIdOrName(userCred, netStr)
 		if err != nil {
@@ -2044,12 +2048,10 @@ func (self *SGuest) PerformDetachnetwork(ctx context.Context, userCred mcclient.
 			}
 			return nil, httperrors.NewGeneralError(err)
 		}
-		gns, err := self.GetNetworks(netObj.GetId())
+		gns, err = self.GetNetworks(netObj.GetId())
 		if err != nil {
 			return nil, httperrors.NewGeneralError(err)
 		}
-		err = self.detachNetworks(ctx, userCred, gns, reserve, true)
-		return nil, err
 	}
 	ipStr, _ := data.GetString("ip_addr")
 	if len(ipStr) > 0 {
@@ -2060,8 +2062,7 @@ func (self *SGuest) PerformDetachnetwork(ctx context.Context, userCred mcclient.
 			}
 			return nil, httperrors.NewGeneralError(err)
 		}
-		err = self.detachNetworks(ctx, userCred, []SGuestnetwork{*gn}, reserve, true)
-		return nil, err
+		gns = []SGuestnetwork{*gn}
 	}
 	macStr, _ := data.GetString("mac")
 	if len(macStr) > 0 {
@@ -2072,10 +2073,18 @@ func (self *SGuest) PerformDetachnetwork(ctx context.Context, userCred mcclient.
 			}
 			return nil, httperrors.NewGeneralError(err)
 		}
-		err = self.detachNetworks(ctx, userCred, []SGuestnetwork{*gn}, reserve, true)
-		return nil, err
+		gns = []SGuestnetwork{*gn}
+	}
+	if self.Status == api.VM_READY {
+		err = self.detachNetworks(ctx, userCred, gns, reserve, true)
+		return nil, err
+	} else {
+		err = self.detachNetworks(ctx, userCred, gns, reserve, false)
+		if err != nil {
+			return nil, err
+		}
+		return nil, self.StartSyncTask(ctx, userCred, false, "")
 	}
-	return nil, httperrors.NewInputParameterError("no either ip_addr, mac or network specified")
 }
 
 func (self *SGuest) AllowPerformAttachnetwork(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
@@ -2126,7 +2135,7 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient.
 		_, err = self.attach2NetworkDesc(ctx, userCred, host, input.Nets[i], pendingUsage, nil)
 		logclient.AddSimpleActionLog(self, logclient.ACT_ATTACH_NETWORK, input.Nets[i], userCred, err == nil)
 		if err != nil {
-			quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage)
+			quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false)
 			return nil, httperrors.NewBadRequestError(err.Error())
 		}
 	}
@@ -2137,7 +2146,7 @@ func (self *SGuest) PerformAttachnetwork(ctx context.Context, userCred mcclient.
 		err = self.StartSyncTask(ctx, userCred, false, "")
 	}
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false)
 	}
 	return nil, err
 }
@@ -2474,7 +2483,7 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T
 	if len(newDisks) > 0 {
 		err := self.CreateDisksOnHost(ctx, userCred, host, newDisks, pendingUsage, false, false, nil, nil, false)
 		if err != nil {
-			quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage)
+			quotas.CancelPendingUsage(ctx, userCred, pendingUsage, pendingUsage, false)
 			return nil, httperrors.NewBadRequestError("Create disk on host error: %s", err)
 		}
 		confs.Add(jsonutils.Marshal(newDisks), "create")
@@ -2959,7 +2968,7 @@ func (self *SGuest) PerformCreateEip(ctx context.Context, userCred mcclient.Toke
 
 	eip, err := ElasticipManager.NewEipForVMOnHost(ctx, userCred, self, host, int(bw), chargeType, eipPendingUsage)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, eipPendingUsage, eipPendingUsage, false)
 		return nil, httperrors.NewGeneralError(err)
 	}
 
@@ -3309,11 +3318,20 @@ func (self *SGuest) AllowPerformCreateBackup(ctx context.Context, userCred mccli
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "create-backup")
 }
 
+func (self *SGuest) guestDisksStorageTypeIsLocal() bool {
+	for _, gd := range self.GetDisks() {
+		if gd.GetDisk().GetStorage().StorageType != api.STORAGE_LOCAL {
+			return false
+		}
+	}
+	return true
+}
+
 func (self *SGuest) PerformCreateBackup(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	if len(self.BackupHostId) > 0 {
 		return nil, httperrors.NewBadRequestError("Already have backup server")
 	}
-	if self.getDefaultStorageType() != api.STORAGE_LOCAL {
+	if !self.guestDisksStorageTypeIsLocal() {
 		return nil, httperrors.NewBadRequestError("Cannot create backup with shared storage")
 	}
 	if self.Hypervisor != api.HYPERVISOR_KVM {
@@ -3345,7 +3363,7 @@ func (self *SGuest) PerformCreateBackup(ctx context.Context, userCred mcclient.T
 	params.Set("guest_status", jsonutils.NewString(self.Status))
 	task, err := taskman.TaskManager.NewTask(ctx, "GuestCreateBackupTask", self, userCred, params, "", "", &req)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &req, &req)
+		quotas.CancelPendingUsage(ctx, userCred, &req, &req, false)
 		log.Errorln(err)
 		return nil, err
 	} else {
@@ -4300,13 +4318,13 @@ func (self *SGuest) PerformInstanceSnapshot(
 	instanceSnapshot, err := InstanceSnapshotManager.CreateInstanceSnapshot(ctx, userCred, self, name, false)
 	if err != nil {
 		quotas.CancelPendingUsage(
-			ctx, userCred, pendingUsage, pendingUsage)
+			ctx, userCred, pendingUsage, pendingUsage, false)
 		return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err)
 	}
 	err = self.InstaceCreateSnapshot(ctx, userCred, instanceSnapshot, pendingUsage)
 	if err != nil {
 		quotas.CancelPendingUsage(
-			ctx, userCred, pendingUsage, pendingUsage)
+			ctx, userCred, pendingUsage, pendingUsage, false)
 		return nil, httperrors.NewInternalServerError("start create snapshot task failed: %s", err)
 	}
 	return nil, nil
@@ -4404,49 +4422,52 @@ func (self *SGuest) PerformSnapshotAndClone(
 	pendingUsage, pendingRegionUsage, err := self.getGuestUsage(int(count))
 	keys, err := self.GetQuotaKeys()
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage)
+		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage, false)
 		return nil, err
 	}
 	pendingUsage.SetKeys(keys)
 	err = quotas.CheckSetPendingQuota(ctx, userCred, &pendingUsage)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage)
+		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage, false)
 		return nil, httperrors.NewOutOfQuotaError("Check set pending quota error %s", err)
 	}
 	regionKeys, err := self.GetRegionalQuotaKeys()
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
 		return nil, err
 	}
 	pendingRegionUsage.SetKeys(regionKeys)
 	err = quotas.CheckSetPendingQuota(ctx, userCred, &pendingRegionUsage)
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
+		quotas.CancelPendingUsage(ctx, userCred, snapshotUsage, snapshotUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
 		return nil, err
 	}
+	// migrate snapshotUsage into regionUsage, then discard snapshotUsage
 	pendingRegionUsage.Snapshot = snapshotUsage.Snapshot
 
 	instanceSnapshotName, err := db.GenerateName(InstanceSnapshotManager, self.GetOwnerId(),
 		fmt.Sprintf("%s-%s", newlyGuestName, rand.String(8)))
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
-		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage, false)
 		return nil, httperrors.NewInternalServerError("Generate snapshot name failed %s", err)
 	}
 	instanceSnapshot, err := InstanceSnapshotManager.CreateInstanceSnapshot(
 		ctx, userCred, self, instanceSnapshotName,
 		jsonutils.QueryBoolean(data, "auto_delete_instance_snapshot", false))
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
-		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage, false)
 		return nil, httperrors.NewInternalServerError("create instance snapshot failed: %s", err)
 	}
 
 	err = self.StartInstanceSnapshotAndCloneTask(
 		ctx, userCred, newlyGuestName, &pendingUsage, &pendingRegionUsage, instanceSnapshot, data.(*jsonutils.JSONDict))
 	if err != nil {
-		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage)
-		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingUsage, &pendingUsage, false)
+		quotas.CancelPendingUsage(ctx, userCred, &pendingRegionUsage, &pendingRegionUsage, false)
 		return nil, err
 	}
 	return nil, nil

pkg/compute/models/guests.go

@@ -989,9 +989,6 @@ func (manager *SGuestManager) validateCreateData(
 		if err != nil {
 			return nil, httperrors.NewInputParameterError("Invalid root image: %s", err)
 		}
-		if len(diskConfig.SnapshotId) > 0 && diskConfig.DiskType != api.DISK_TYPE_SYS {
-			return nil, httperrors.NewBadRequestError("Snapshot error: disk index 0 but disk type is %s", diskConfig.DiskType)
-		}
 
 		// if len(diskConfig.Backend) == 0 {
 		// 	diskConfig.Backend = STORAGE_LOCAL
@@ -1363,7 +1360,7 @@ func (self *SGuest) PostUpdate(ctx context.Context, userCred mcclient.TokenCrede
 	if data.Contains("pending_usage") {
 		quota := SQuota{}
 		data.Unmarshal(&quota, "pending_usage")
-		quotas.CancelPendingUsage(ctx, userCred, &quota, &quota)
+		quotas.CancelPendingUsage(ctx, userCred, &quota, &quota, true)
 	}
 
 	self.StartSyncTask(ctx, userCred, true, "")
@@ -2583,7 +2580,7 @@ func (self *SGuest) attach2NetworkOnce(
 			log.Warningf("self.GetRegionalQuotaKeys fail %s", err)
 		}
 		cancelUsage.SetKeys(keys)
-		err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage)
+		err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true)
 		if err != nil {
 			log.Warningf("QuotaManager.CancelPendingUsage fail %s", err)
 		}
@@ -3168,7 +3165,7 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T
 	lockman.LockClass(ctx, QuotaManager, self.ProjectId)
 	defer lockman.ReleaseClass(ctx, QuotaManager, self.ProjectId)
 
-	diskName := fmt.Sprintf("vdisk_%s_%d", self.Name, time.Now().UnixNano())
+	diskName := fmt.Sprintf("vdisk-%s-%d", self.Name, time.Now().UnixNano())
 
 	billingType := billing_api.BILLING_TYPE_POSTPAID
 	billingCycle := ""
@@ -3195,7 +3192,7 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T
 		return nil, err
 	}
 	cancelUsage.SetKeys(keys)
-	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage)
+	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true)
 
 	return disk, nil
 }
@@ -3284,7 +3281,7 @@ func (self *SGuest) createIsolatedDeviceOnHost(ctx context.Context, userCred mcc
 		return err
 	}
 	cancelUsage.SetKeys(keys)
-	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage)
+	err = quotas.CancelPendingUsage(ctx, userCred, pendingUsage, &cancelUsage, true) // success
 	return err
 }
 
@@ -3440,11 +3437,14 @@ func (self *SGuest) DeleteAllDisksInDB(ctx context.Context, userCred mcclient.To
 		if err != nil {
 			return err
 		}
-		db.OpsLog.LogEvent(disk, db.ACT_DELETE, nil, userCred)
-		db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, nil, userCred)
-		err = disk.RealDelete(ctx, userCred)
-		if err != nil {
-			return err
+
+		if disk != nil {
+			db.OpsLog.LogEvent(disk, db.ACT_DELETE, nil, userCred)
+			db.OpsLog.LogEvent(disk, db.ACT_DELOCATE, nil, userCred)
+			err = disk.RealDelete(ctx, userCred)
+			if err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -4737,8 +4737,12 @@ func (self *SGuest) ToCreateInput(userCred mcclient.TokenCredential) *api.Server
 	if genInput.PreferZone != "" {
 		userInput.PreferZone = genInput.PreferZone
 	}
-	// clean GenerateName
+	// clean some of user input
 	userInput.GenerateName = ""
+	userInput.Description = ""
+	userInput.BillingType = ""
+	userInput.BillingCycle = ""
+	userInput.Duration = ""
 	return userInput
 }
 

pkg/compute/models/hosts.go

@@ -1066,11 +1066,15 @@ func (cap *SStorageCapacity) Add(cap2 SStorageCapacity) {
 	cap.VCapacity += cap2.VCapacity
 }
 
-func (cap *SStorageCapacity) ToJson() *jsonutils.JSONDict {
-	ret := jsonutils.Marshal(cap).(*jsonutils.JSONDict)
-	ret.Add(jsonutils.NewFloat(cap.GetCommitRate()), "commit_rate")
-	ret.Add(jsonutils.NewInt(int64(cap.GetFree())), "free_capacity")
-	return ret
+func (cap *SStorageCapacity) toCapacityInfo() api.SStorageCapacityInfo {
+	info := api.SStorageCapacityInfo{}
+	info.Capacity = cap.Capacity
+	info.UsedCapacity = cap.Used
+	info.WasteCapacity = cap.Wasted
+	info.VirtualCapacity = cap.VCapacity
+	info.CommitRate = cap.GetCommitRate()
+	info.FreeCapacity = cap.GetFree()
+	return info
 }
 
 func (self *SHost) GetAttachedLocalStorageCapacity() SStorageCapacity {

pkg/compute/models/initdb.go

@@ -18,6 +18,7 @@ import (
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/proxy"
 )
 
 func InitDB() error {
@@ -28,6 +29,8 @@ func InitDB() error {
 		 */
 		db.TenantCacheManager,
 
+		proxy.ProxySettingManager,
+
 		QuotaManager,
 
 		CloudproviderManager,

pkg/compute/models/instance_snapshots.go

@@ -65,6 +65,8 @@ type SInstanceSnapshot struct {
 	KeypairId string `width:"36" charset:"ascii" nullable:"true" list:"user"`
 	// 操作系统类型
 	OsType string `width:"36" charset:"ascii" nullable:"true" list:"user"`
+	// 套餐名称
+	InstanceType string `width:"64" charset:"utf8" nullable:"true" list:"user" create:"optional"`
 }
 
 type SInstanceSnapshotManager struct {
@@ -125,13 +127,12 @@ func (self *SInstanceSnapshot) getMoreDetails(userCred mcclient.TokenCredential,
 			DiskType:      snapshots[i].DiskType,
 			CloudregionId: snapshots[i].CloudregionId,
 			Size:          snapshots[i].Size,
+			Status:        snapshots[i].Status,
+			StorageType:   snapshots[i].GetStorageType(),
 		})
 
-		if len(snapshots[i].StorageId) > 0 {
-			storage := snapshots[i].GetStorage()
-			if storage != nil {
-				out.StorageType = storage.StorageType
-			}
+		if len(snapshots[i].StorageId) > 0 && out.StorageType == "" {
+			out.StorageType = snapshots[i].GetStorageType()
 		}
 	}
 	if len(osType) > 0 {
@@ -229,6 +230,7 @@ func (manager *SInstanceSnapshotManager) CreateInstanceSnapshot(
 	}
 	instanceSnapshot.OsType = guest.OsType
 	instanceSnapshot.ServerMetadata = serverMetadata
+	instanceSnapshot.InstanceType = guest.InstanceType
 	err := manager.TableSpec().Insert(instanceSnapshot)
 	if err != nil {
 		return nil, err
@@ -274,7 +276,8 @@ func (self *SInstanceSnapshot) ToInstanceCreateInput(
 		sourceInput.Secgroups = inputSecgs
 	}
 	sourceInput.OsType = self.OsType
-	// sourceInput.Networks = serverConfig.Networks
+	sourceInput.InstanceType = self.InstanceType
+	sourceInput.Networks = serverConfig.Networks
 	return sourceInput, nil
 }
 

pkg/compute/models/loadbalanceragents_deploy.go

@@ -29,6 +29,7 @@ import (
 
 	ansible_apis "yunion.io/x/onecloud/pkg/apis/ansible"
 	compute_apis "yunion.io/x/onecloud/pkg/apis/compute"
+	identity_apis "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/compute/options"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -159,7 +160,7 @@ func (lbagent *SLoadbalancerAgent) deploy(ctx context.Context, userCred mcclient
 				return nil, errors.WithMessagef(err, "glob error %s", pattern)
 			}
 			if len(matches) == 0 {
-				return nil, errors.WithMessagef(err, "glob nomatch %s", pattern)
+				return nil, errors.Errorf("no match for %q", pattern)
 			}
 			path := matches[len(matches)-1]
 			name := filepath.Base(path)
@@ -195,14 +196,6 @@ func (lbagent *SLoadbalancerAgent) deploy(ctx context.Context, userCred mcclient
 	}
 
 	pb.Modules = append(pb.Modules,
-		ansible.Module{
-			Name: "copy",
-			Args: []string{
-				"remote_src=yes",
-				"src=/opt/yunion/share/lbagent/yunion-lbagent.service",
-				"dest=/etc/systemd/system/yunion-lbagent.service",
-			},
-		},
 		ansible.Module{
 			Name: "systemd",
 			Args: []string{
@@ -302,6 +295,7 @@ func (lbagent *SLoadbalancerAgent) PerformDeploy(ctx context.Context, userCred m
 			return nil, httperrors.NewBadRequestError("empty host %s field", k)
 		}
 	}
+	authURL := options.Options.AuthURL
 	{
 		cli := mcclient.NewClient(options.Options.AuthURL, 10, false, true, "", "")
 		token, err := cli.Authenticate(host.Vars["user"], host.Vars["pass"], "", host.Vars["proj"], "")
@@ -311,12 +305,23 @@ func (lbagent *SLoadbalancerAgent) PerformDeploy(ctx context.Context, userCred m
 		if !token.HasSystemAdminPrivilege() {
 			return nil, httperrors.NewBadRequestError("user must have system admin privileges")
 		}
+		authURL, err = token.GetServiceURL(
+			identity_apis.SERVICE_TYPE,
+			options.Options.Region,
+			"",
+			identity_apis.EndpointInterfacePublic)
+		if err != nil {
+			return nil, httperrors.NewClientError("get %s service %s url: %v",
+				identity_apis.SERVICE_TYPE,
+				identity_apis.EndpointInterfacePublic,
+				err)
+		}
 	}
 	if err := lbagent.validateHost(ctx, userCred, &host); err != nil {
 		return nil, err
 	}
 	host.SetVar("region", options.Options.Region)
-	host.SetVar("auth_uri", options.Options.AuthURL)
+	host.SetVar("auth_uri", authURL)
 	host.SetVar("id", lbagent.Id)
 	host.SetVar("ansible_become", "yes")
 

pkg/compute/models/loadbalancerbackendgroups.go

@@ -400,7 +400,7 @@ func (lbbg *SLoadbalancerBackendGroup) GetExtraDetails(ctx context.Context, user
 		return out, err
 	}
 	out.Loadbalancer = lb.GetName()
-	out.CloudregionInfo = lbbg.SCloudregionResourceBase.GetExtraDetails(ctx, userCred, query)
+	out.CloudregionInfo = lbbg.SCloudregionResourceBase.GetExtraDetails(ctx, userCred, query, isList)
 	return out, nil
 }
 

pkg/compute/models/loadbalancercertificates.go

@@ -131,6 +131,7 @@ func (lbcert *SLoadbalancerCertificate) GetExtraDetails(ctx context.Context, use
 
 func (lbcert *SLoadbalancerCertificate) ValidateDeleteCondition(ctx context.Context) error {
 	men := []db.IModelManager{
+		LoadbalancerListenerManager,
 		CachedLoadbalancerCertificateManager,
 	}
 	lbcertId := lbcert.Id

pkg/compute/models/loadbalancerlistenerrules.go

@@ -505,7 +505,7 @@ func (lbr *SLoadbalancerListenerRule) GetExtraDetails(ctx context.Context, userC
 	if err != nil {
 		return out, err
 	}
-	out.CloudregionInfo = lbr.SCloudregionResourceBase.GetExtraDetails(ctx, userCred, query)
+	out.CloudregionInfo = lbr.SCloudregionResourceBase.GetExtraDetails(ctx, userCred, query, isList)
 	if lbr.BackendGroupId == "" {
 		log.Errorf("loadbalancer listener rule %s(%s): empty backend group field", lbr.Name, lbr.Id)
 		return out, nil