Merge pull request #8656 from lvyangyang/automated-cherry-pick-of-#8655-upstream-release-3.4

Automated cherry pick of #8655: fix:域名正则校验dnsrecord name，同步中文域名取punycode

.circleci/config.yml

@@ -38,7 +38,7 @@ jobs:
           command: |
             targets="$(circleci tests glob "cmd/*" | grep -v cmd/host-image | circleci tests split)"
             echo $targets | tr ' ' '\n'
-            make -j3 $targets
+            make -j2 $targets
             ls -lh _output/bin/
       - save_cache:
           key: onecloud-build-cache-{{ checksum "go.mod" }}-{{ checksum "Makefile" }}

Makefile

@@ -313,7 +313,7 @@ REGISTRY ?= "registry.cn-beijing.aliyuncs.com/yunionio"
 VERSION ?= $(shell git describe --exact-match 2> /dev/null || \
                 git describe --match=$(git rev-parse --short=8 HEAD) --always --dirty --abbrev=8)
 
-image: clean
+image:
 	mkdir -p $(ROOT_DIR)/_output
 	DEBUG=$(DEBUG) ARCH=$(ARCH) TAG=$(VERSION) REGISTRY=$(REGISTRY) $(ROOT_DIR)/scripts/docker_push.sh $(filter-out $@,$(MAKECMDGOALS))
 

build/docker/Dockerfile.telegraf-raid

@@ -0,0 +1,3 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/yunion-rootfs:v1.0
+
+ADD ./_output/bin/telegraf-raid-plugin /opt/yunion/bin/telegraf-raid-plugin

build/notify/root/opt/yunion/share/template/title/VERIFY.email

@@ -1 +1 @@
-Yunion Verify
+{{.copyright}} Verify

cmd/aliyuncli/main.go

@@ -28,6 +28,7 @@ import (
 type BaseOptions struct {
 	Debug      bool   `help:"debug mode"`
 	Help       bool   `help:"Show help"`
+	CloudEnv   string `help:"Cloud environment" default:"$ALIYUN_CLOUD_ENV" choices:"InternationalCloud|FinanceCloud" metavar:"ALIYUN_CLOUD_ENV"`
 	AccessKey  string `help:"Access key" default:"$ALIYUN_ACCESS_KEY" metavar:"ALIYUN_ACCESS_KEY"`
 	Secret     string `help:"Secret" default:"$ALIYUN_SECRET" metavar:"ALIYUN_SECRET"`
 	RegionId   string `help:"RegionId" default:"$ALIYUN_REGION" metavar:"ALIYUN_REGION"`
@@ -86,7 +87,9 @@ func newClient(options *BaseOptions) (*aliyun.SRegion, error) {
 
 	cli, err := aliyun.NewAliyunClient(
 		aliyun.NewAliyunClientConfig(
-			options.AccessKey, options.Secret,
+			options.CloudEnv,
+			options.AccessKey,
+			options.Secret,
 		).Debug(options.Debug),
 	)
 	if err != nil {

cmd/climc/shell/compute/cachedimages.go

@@ -29,6 +29,7 @@ func init() {
 		Zone   string `help:"show images cached at zone"`
 
 		HostSchedtagId string `help:"filter cached image with host schedtag"`
+		Valid          *bool  `help:"valid cachedimage"`
 	}
 	R(&CachedImageListOptions{}, "cached-image-list", "List cached images", func(s *mcclient.ClientSession, args *CachedImageListOptions) error {
 		params, err := options.ListStructToParams(args)

cmd/climc/shell/compute/servers.go

@@ -83,6 +83,7 @@ func init() {
 	cmd.Perform("publicip-to-eip", new(options.ServerPublicipToEip))
 	cmd.Perform("set-auto-renew", new(options.ServerSetAutoRenew))
 	cmd.Perform("save-template", new(options.ServerSaveImageOptions))
+	cmd.Perform("remote-update", new(options.ServerRemoteUpdateOptions))
 
 	cmd.Get("vnc", new(options.ServerIdOptions))
 	cmd.Get("desc", new(options.ServerIdOptions))

cmd/climc/shell/compute/storages.go

@@ -37,6 +37,7 @@ func init() {
 		Schedtag string `help:"filter storage by schedtag"`
 
 		HostSchedtagId string `help:"filter storage by host schedtag"`
+		ImageId        string `help:"filter storage by image"`
 	}
 	R(&StorageListOptions{}, "storage-list", "List storages", func(s *mcclient.ClientSession, opts *StorageListOptions) error {
 		params, err := options.ListStructToParams(opts)

cmd/climc/shell/helper.go

@@ -284,10 +284,7 @@ type IPerformOpt interface {
 func (cmd ResourceCmd) Perform(action string, args IPerformOpt) {
 	man := cmd.manager
 	callback := func(s *mcclient.ClientSession, args IPerformOpt) error {
-		params, err := args.Params()
-		if err != nil {
-			return err
-		}
+		params := jsonutils.Marshal(args) // .Params()
 		ret, err := man.(modulebase.Manager).PerformAction(s, args.GetId(), action, params)
 		if err != nil {
 			return err
@@ -488,7 +485,7 @@ func (cmd JointCmd) List(args IJointListOpt) {
 		if len(args.GetMasterOpt()) > 0 {
 			result, err = man.ListDescendent(s, args.GetMasterOpt(), params)
 		} else if len(args.GetSlaveOpt()) > 0 {
-			result, err = man.ListDescendent2(s, args.GetMasterOpt(), params)
+			result, err = man.ListDescendent2(s, args.GetSlaveOpt(), params)
 		} else {
 			result, err = man.List(s, params)
 		}

cmd/climc/shell/monitor/alertresource.go

@@ -0,0 +1,31 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package monitor
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
+	options "yunion.io/x/onecloud/pkg/mcclient/options/monitor"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(monitor.AlertResources)
+	cmd.List(new(options.AlertResourceListOptions))
+	cmd.Show(new(options.AlertResourceShowOptions))
+	cmd.BatchDelete(new(options.AlertResourceDeleteOptions))
+
+	jCmd := shell.NewJointCmd(monitor.AlertResourceAlert)
+	jCmd.List(new(options.AlertResourceAlertListOptions))
+}

cmd/climc/shell/monitor/commonalert.go

@@ -12,5 +12,6 @@ func init() {
 	cmd.Show(new(options.CommonAlertShowOptions))
 	cmd.Perform("enable", &options.CommonAlertShowOptions{})
 	cmd.Perform("disable", &options.CommonAlertShowOptions{})
-	cmd.Delete(new(options.CommonAlertDeleteOptions))
+	cmd.BatchDelete(new(options.CommonAlertDeleteOptions))
+	cmd.Perform("config", &options.CommonAlertUpdateOptions{})
 }

cmd/climc/shell/monitor/suggestsysrule.go

@@ -27,7 +27,7 @@ func init() {
 	cmd.Show(new(options.SuggestRuleShowOptions))
 	cmd.Perform("enable", new(options.SuggestRuleShowOptions))
 	cmd.Perform("disable", new(options.SuggestRuleShowOptions))
-	cmd.Delete(new(options.CommonAlertDeleteOptions))
+	cmd.BatchDelete(new(options.CommonAlertDeleteOptions))
 	cmd.Perform("config", new(options.SuggestRuleConfigOptions))
 
 }

cmd/raidcli/main.go

@@ -33,7 +33,8 @@ type BaseOptions struct {
 	Host       string `help:"SSH Host IP" default:"$RAID_HOST" metavar:"RAID_HOST"`
 	Username   string `help:"Username, usually root" default:"$RAID_USERNAME" metavar:"RAID_USERNAME"`
 	Password   string `help:"Password" default:"$RAID_PASSWORD" metavar:"RAID_PASSWORD"`
-	Driver     string `help:"Password" default:"$RAID_DRIVER" metavar:"RAID_DRIVER" choices:"MegaRaid|HPSARaid|Mpt2SAS|MarvelRaid"`
+	Driver     string `help:"Raid dirver" default:"$RAID_DRIVER" metavar:"RAID_DRIVER" choices:"MegaRaid|HPSARaid|Mpt2SAS|MarvelRaid"`
+	LocalHost  bool   `help:"Run raidcli in localhost"`
 	SUBCOMMAND string `help:"s3cli subcommand" subcommand:"true"`
 }
 
@@ -83,43 +84,48 @@ func showErrorAndExit(e error) {
 }
 
 func newClient() (raid.IRaidDriver, error) {
-	if len(options.Host) == 0 {
-		return nil, fmt.Errorf("Missing host")
-	}
-
-	if len(options.Username) == 0 {
-		return nil, fmt.Errorf("Missing username")
-	}
-
-	if len(options.Password) == 0 {
-		return nil, fmt.Errorf("Missing password")
+	if options.Debug {
+		raid.Debug = true
 	}
 
 	if len(options.Driver) == 0 {
 		return nil, fmt.Errorf("Missing driver")
 	}
 
-	if options.Debug {
-		raid.Debug = true
+	var drv raid.IRaidDriver
+	if !options.LocalHost {
+		if len(options.Host) == 0 {
+			return nil, fmt.Errorf("Missing host")
+		}
+
+		if len(options.Username) == 0 {
+			return nil, fmt.Errorf("Missing username")
+		}
+
+		if len(options.Password) == 0 {
+			return nil, fmt.Errorf("Missing password")
+		}
+
+		sshClient, err := ssh.NewClient(
+			options.Host,
+			22,
+			options.Username,
+			options.Password,
+			"",
+		)
+		if err != nil {
+			return nil, fmt.Errorf("ssh client init fail: %s", err)
+		}
+
+		drv = drivers.GetDriver(options.Driver, sshClient)
+		if drv == nil {
+			return nil, fmt.Errorf("not supported driver %s", options.Driver)
+		}
+	} else {
+		drv = drivers.GetLocalDriver(options.Driver)
 	}
 
-	sshClient, err := ssh.NewClient(
-		options.Host,
-		22,
-		options.Username,
-		options.Password,
-		"",
-	)
-	if err != nil {
-		return nil, fmt.Errorf("ssh client init fail: %s", err)
-	}
-
-	drv := drivers.GetDriver(options.Driver, sshClient)
-	if drv == nil {
-		return nil, fmt.Errorf("not supported driver %s", options.Driver)
-	}
-
-	err = drv.ParsePhyDevs()
+	err := drv.ParsePhyDevs()
 	if err != nil {
 		return nil, fmt.Errorf("parse phyical devices error %s", err)
 	}

cmd/telegraf-raid-plugin/main.go

@@ -0,0 +1,193 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/baremetal/utils/detect_storages"
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
+	"yunion.io/x/onecloud/pkg/compute/baremetal"
+	"yunion.io/x/onecloud/pkg/util/httputils"
+)
+
+const (
+	INTERVAL_SECOND = 300
+	TelegrafServer  = "http://localhost:8087/write"
+)
+
+// Failed, Offline, Degraded, Rebuilding, Out of Sync (OSY)
+
+func main() {
+	// creates the in-cluster config
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	// creates the clientset
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	nodeName := os.Getenv("NODENAME")
+	if len(nodeName) == 0 {
+		log.Fatalf("Missing env nodename")
+	}
+
+	node, err := clientset.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+	if err != nil {
+		log.Fatalln(err)
+	}
+	var masterAddress string
+	if length := len(node.Status.Conditions); length > 0 {
+		if node.Status.Conditions[length-1].Type == v1.NodeReady &&
+			node.Status.Conditions[length-1].Status == v1.ConditionTrue {
+			for _, addr := range node.Status.Addresses {
+				if addr.Type == v1.NodeInternalIP {
+					masterAddress = addr.Address
+					break
+				}
+			}
+		}
+	}
+
+	log.Infof("Start Colloct Raid Info And Send To Telegraf ...")
+	c := NewRaidInfoCollector(nodeName, masterAddress, INTERVAL_SECOND)
+	c.Start()
+}
+
+type RaidInfoCollector struct {
+	waitingReportData []string
+	LastCollectTime   time.Time
+	ReportInterval    int // seconds
+	Hostname          string
+	HostIp            string
+}
+
+func NewRaidInfoCollector(hostname, hostIp string, interval int) *RaidInfoCollector {
+	return &RaidInfoCollector{
+		waitingReportData: make([]string, 0),
+		ReportInterval:    interval,
+		Hostname:          hostname,
+		HostIp:            hostIp,
+	}
+}
+
+func (c *RaidInfoCollector) runMain() {
+	timeBegin := time.Now()
+	elapse := timeBegin.Sub(c.LastCollectTime)
+	if elapse < time.Second*time.Duration(c.ReportInterval) {
+		return
+	} else {
+		c.LastCollectTime = timeBegin
+	}
+	c.runMonitor()
+}
+
+func (c *RaidInfoCollector) runMonitor() {
+	reportData := c.collectReportData()
+	if len(reportData) > 0 {
+		c.reportRaidInfoToTelegraf(reportData)
+	}
+}
+
+func (c *RaidInfoCollector) collectReportData() string {
+	if len(c.waitingReportData) > 60 {
+		c.waitingReportData = c.waitingReportData[1:]
+	}
+	return c.CollectReportData()
+}
+
+func (c *RaidInfoCollector) CollectReportData() string {
+	raidDiskInfo := make([]*baremetal.BaremetalStorage, 0)
+	// raidDrivers := []string{}
+	for _, drv := range drivers.GetDrivers(drivers.NewExecutor()) {
+		if err := drv.ParsePhyDevs(); err != nil {
+			log.Warningf("Raid driver %s ParsePhyDevs failed: %s", drv.GetName(), err)
+			continue
+		}
+		raidDiskInfo = append(raidDiskInfo, detect_storages.GetRaidDevices(drv)...)
+		// raidDrivers = append(raidDrivers, drv.GetName())
+	}
+	if len(raidDiskInfo) > 0 {
+		ret := c.toTelegrafReportData(raidDiskInfo)
+		return ret
+	}
+	return ""
+}
+
+func (c *RaidInfoCollector) Start() {
+	for {
+		c.runMain()
+		time.Sleep(time.Second * 1)
+	}
+}
+
+const MEASUREMENT = "host_raid"
+
+func (c *RaidInfoCollector) toTelegrafReportData(raidDiskInfo []*baremetal.BaremetalStorage) string {
+	tag := fmt.Sprintf("%s=%s,%s=%s", "hostname", c.Hostname, "host_ip", c.HostIp)
+	ret := []string{}
+	for i := 0; i < len(raidDiskInfo); i++ {
+		statArr := []string{}
+		raidDiskInfo[i].Status = strings.ToLower(raidDiskInfo[i].Status)
+		jStat := jsonutils.Marshal(raidDiskInfo[i])
+		jMap, _ := jStat.GetMap()
+		for k, v := range jMap {
+			statArr = append(statArr, fmt.Sprintf("%s=%s", k, v.String()))
+		}
+		stat := strings.Join(statArr, ",")
+		diskTag := fmt.Sprintf(
+			"%s,%s=%s,%s=%d,%s=%d", tag, "driver", raidDiskInfo[i].Driver,
+			"adapter", raidDiskInfo[i].Adapter, "slot", raidDiskInfo[i].Slot,
+		)
+		line := fmt.Sprintf("%s,%s %s", MEASUREMENT, diskTag, stat)
+		ret = append(ret, line)
+	}
+	return strings.Join(ret, "\n")
+}
+
+func (c *RaidInfoCollector) reportRaidInfoToTelegraf(data string) {
+	body := strings.NewReader(data)
+	res, err := httputils.Request(
+		httputils.GetDefaultClient(), context.Background(), "POST", TelegrafServer, nil, body, false)
+	if err != nil {
+		log.Errorf("Upload guest metric failed: %s", err)
+		return
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 204 {
+		log.Errorf("upload guest metric failed %d", res.StatusCode)
+		timestamp := time.Now().UnixNano()
+		for _, line := range strings.Split(data, "\n") {
+			c.waitingReportData = append(c.waitingReportData,
+				fmt.Sprintf("%s %d", line, timestamp))
+		}
+	} else {
+		if len(c.waitingReportData) > 0 {
+			oldDatas := strings.Join(c.waitingReportData, "\n")
+			body = strings.NewReader(oldDatas)
+			res, err = httputils.Request(
+				httputils.GetDefaultClient(), context.Background(), "POST", TelegrafServer, nil, body, false)
+			if err == nil {
+				defer res.Body.Close()
+			}
+			if res.StatusCode == 204 {
+				c.waitingReportData = c.waitingReportData[len(c.waitingReportData):]
+			} else {
+				log.Errorf("upload guest metric failed code: %d", res.StatusCode)
+			}
+		}
+	}
+}

go.mod

@@ -144,9 +144,9 @@ require (
 	yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
-	yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6
+	yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20201009235752-2ef1a29656b0
+	yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -1137,9 +1137,11 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZV
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6 h1:UarEDTBGkgcgc+nc+PZ75uo9M9+jiOGd5P2B90TxDNw=
 yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc h1:cgwWI6k9LsXpq6yiOuEIprsXmPyPGUa60sOZu57e8Jo=
+yunion.io/x/pkg v0.0.0-20201028134817-3ed15ee169bc/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20201009235752-2ef1a29656b0 h1:ETDt3Xc3a6WnbHvhrMUemXVsnWY/RgHrsNRCVFWkCYo=
-yunion.io/x/sqlchemy v0.0.0-20201009235752-2ef1a29656b0/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b h1:xBAzDPvvmsOBjxAj0PLHMNLVdPNOlYFHAm7T3IZ2ehs=
+yunion.io/x/sqlchemy v0.0.0-20201029091740-cd5e77b56d4b/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/compute/cloudaccount_const.go

@@ -71,6 +71,8 @@ const (
 	CLOUD_ACCESS_ENV_AZURE_CHINA         = CLOUD_PROVIDER_AZURE
 	CLOUD_ACCESS_ENV_HUAWEI_GLOBAL       = CLOUD_PROVIDER_HUAWEI + "-int"
 	CLOUD_ACCESS_ENV_HUAWEI_CHINA        = CLOUD_PROVIDER_HUAWEI
+	CLOUD_ACCESS_ENV_ALIYUN_GLOBAL       = CLOUD_PROVIDER_ALIYUN
+	CLOUD_ACCESS_ENV_ALIYUN_FINANCE      = CLOUD_PROVIDER_ALIYUN + "-fin"
 	CLOUD_ACCESS_ENV_CTYUN_CHINA         = CLOUD_PROVIDER_CTYUN
 )
 

pkg/apis/compute/guests.go

@@ -98,6 +98,9 @@ type ServerListInput struct {
 	SrcMacCheck *bool `json:"src_mac_check"`
 
 	InstanceType []string `json:"instance_type"`
+
+	// 是否调度到宿主机上
+	WithHost *bool `json:"with_host"`
 }
 
 func (input *ServerListInput) AfterUnmarshal() {
@@ -409,3 +412,21 @@ type GuestAddSecgroupInput struct {
 	// | 其他	     | 5					|
 	SecgroupIds []string `json:"secgroup_ids"`
 }
+
+type ServerRemoteUpdateInput struct {
+	// 是否覆盖替换所有标签
+	ReplaceTags *bool `json:"replace_tags" help:"replace all remote tags"`
+}
+
+type ServerAssociateEipInput struct {
+	// swagger:ignore
+	// Deprecated
+	Eip string `json:"eip" yunion-deprecated-by:"eip_id"`
+	// 弹性公网IP的ID
+	EipId string `json:"eip_id"`
+}
+
+type ServerDissociateEipInput struct {
+	// 是否自动释放
+	AudoDelete *bool `json:"auto_delete"`
+}

pkg/apis/compute/input.go

@@ -61,6 +61,9 @@ type CachedimageListInput struct {
 
 	// filter by host schedtag
 	HostSchedtagId string `json:"host_schedtag_id"`
+
+	// valid cachedimage
+	Valid *bool `json:"valid"`
 }
 
 type ExternalProjectListInput struct {

pkg/apis/compute/secgroupcache.go

@@ -18,6 +18,7 @@ import "yunion.io/x/onecloud/pkg/apis"
 
 type SecurityGroupCacheDetails struct {
 	apis.StatusStandaloneResourceDetails
+	apis.ProjectizedResourceInfo
 	ManagedResourceInfo
 	CloudregionResourceInfo
 
@@ -25,4 +26,6 @@ type SecurityGroupCacheDetails struct {
 
 	// 虚拟私有网络名称
 	Vpc string `json:"vpc"`
+
+	ProjectId string `json:"tenant_id"`
 }

pkg/apis/compute/snapshotpolicy.go

@@ -21,10 +21,10 @@ type SnapshotPolicyDetails struct {
 
 	SSnapshotPolicy
 
-	RetentionDays  int   `json:"retention_days"`
-	RepeatWeekdays []int `json:"repeat_weekdays"`
-	TimePoints     []int `json:"time_points"`
-	IsActivated    *bool `json:"is_activated,omitempty"`
+	RetentionDays         int   `json:"retention_days"`
+	RepeatWeekdaysDisplay []int `json:"repeat_weekdays_display"`
+	TimePointsDisplay     []int `json:"time_points_display"`
+	IsActivated           *bool `json:"is_activated,omitempty"`
 
 	BindingDiskCount int `json:"binding_disk_count"`
 }

pkg/apis/compute/storage_const.go

@@ -55,7 +55,8 @@ const (
 	STORAGE_LOCAL_SSD     = "local_ssd"
 	STORAGE_LOCAL_PRO     = "local_pro"
 	STORAGE_CLOUD_BASIC   = "cloud_basic"
-	STORAGE_CLOUD_PREMIUM = "cloud_premium"
+	STORAGE_CLOUD_PREMIUM = "cloud_premium" //高性能云硬盘
+	STORAGE_CLOUD_HSSD    = "cloud_hssd"    //增强型SSD云硬盘
 
 	// huawei storage type
 	STORAGE_HUAWEI_SSD  = "SSD"  // 超高IO云硬盘
@@ -81,6 +82,7 @@ const (
 	STORAGE_GOOGLE_LOCAL_SSD   = "local-ssd"   //本地SSD暂存盘 (最多8个)
 	STORAGE_GOOGLE_PD_STANDARD = "pd-standard" //标准永久性磁盘
 	STORAGE_GOOGLE_PD_SSD      = "pd-ssd"      //SSD永久性磁盘
+	STORAGE_GOOGLE_PD_BALANCED = "pd-balanced" //平衡永久性磁盘
 
 	// ctyun storage type
 	STORAGE_CTYUN_SSD  = "SSD"  // 超高IO云硬盘
@@ -186,4 +188,7 @@ type StorageListInput struct {
 
 	// filter by host schedtag
 	HostSchedtagId string `json:"host_schedtag_id"`
+
+	// filter by cachedimage
+	ImageId string `json:"image_id"`
 }

pkg/apis/identity/consts.go

@@ -202,3 +202,19 @@ var (
 		},
 	}
 )
+
+func mergeConfigOptionsFrom(opt1, opt2 map[string][]string) map[string][]string {
+	for opt, values := range opt2 {
+		ovalues, _ := opt1[opt]
+		opt1[opt] = append(ovalues, values...)
+	}
+	return opt1
+}
+
+func MergeServiceConfigOptions(opts ...map[string][]string) map[string][]string {
+	ret := make(map[string][]string)
+	for i := range opts {
+		ret = mergeConfigOptionsFrom(ret, opts[i])
+	}
+	return ret
+}

pkg/apis/list.go

@@ -199,12 +199,15 @@ type MetadataResourceListInput struct {
 	// 返回资源的标签不包含用户标签
 	WithoutUserMeta *bool `json:"without_user_meta"`
 
-	// 返回资源的标签包含用户标签
+	// 返回包含用户标签的资源
 	WithUserMeta *bool `json:"with_user_meta"`
 
-	// 返回资源的标签包含外部标签
+	// 返回包含外部标签的资源
 	WithCloudMeta *bool `json:"with_cloud_meta"`
 
+	// 返回包含任意标签的资源
+	WithAnyMeta *bool `json:"with_any_meta"`
+
 	// 返回列表数据中包含资源的标签数据（Metadata）
 	WithMeta *bool `json:"with_meta"`
 }

pkg/apis/monitor/alertrecord.go

@@ -39,6 +39,6 @@ type AlertRecordRule struct {
 	// 比较运算符, 比如: >, <, >=, <=
 	Comparator string `json:"comparator"`
 	// 报警阀值
-	Threshold float64 `json:"threshold"`
-	Period    string  `json:"period"`
+	Threshold string `json:"threshold"`
+	Period    string `json:"period"`
 }

pkg/apis/monitor/alertresource.go

@@ -0,0 +1,77 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package monitor
+
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type AlertResourceType string
+
+const (
+	// AlertResourceTypeNode means onecloud system infrastructure controller or host node
+	AlertResourceTypeNode AlertResourceType = "node"
+	// AlertResourceTypeCloudaccount means cloudaccount resource
+	AlertResourceTypeCloudaccount AlertResourceType = "cloudaccount"
+	// AlertResourceTypeVM means virtual machine guest resource
+	// AlertResourceTypeVM AlertResourceType = "vm"
+)
+
+type AlertResourceCreateInput struct {
+	apis.StandaloneResourceCreateInput
+
+	Type AlertResourceType `json:"type"`
+}
+
+type AlertResourceListInput struct {
+	apis.StandaloneResourceListInput
+	Type string `json:"type"`
+}
+
+type AlertResourceAttachInput struct {
+	apis.Meta
+
+	AlertResourceId string    `json:"alert_resource_id"`
+	AlertId         string    `json:"alert_id"`
+	AlertRecordId   string    `json:"alert_record_id"`
+	Data            EvalMatch `json:"data"`
+}
+
+type AlertResourceDetails struct {
+	apis.StandaloneResourceDetails
+	Count int               `json:"count"`
+	Tags  map[string]string `json:"tags"`
+}
+
+type AlertResourceAlertListInput struct {
+	apis.JointResourceBaseListInput
+	AlertResourceId string `json:"alert_resource_id"`
+	AlertId         string `json:"alert_id"`
+}
+
+type AlertResourceJointBaseDetails struct {
+	apis.JointResourceBaseDetails
+
+	AlertResource string            `json:"alert_resource"`
+	Type          AlertResourceType `json:"type"`
+}
+
+type AlertResourceAlertDetails struct {
+	AlertResourceJointBaseDetails
+	Alert                    string                      `json:"alert"`
+	AlertType                string                      `json:"alert_type"`
+	Level                    string                      `json:"level"`
+	CommonAlertMetricDetails []*CommonAlertMetricDetails `json:"common_alert_metric_details"`
+}

pkg/apis/monitor/commalert.go

@@ -86,6 +86,8 @@ type CommonAlertUpdateInput struct {
 	Channel []string `json:"channel"`
 	// 通知接受者
 	Recipients []string `json:"recipients"`
+	// systemalert policy may need update through operator
+	ForceUpdate bool `json:"force_update"`
 }
 
 type CommonAlertDetails struct {
@@ -105,13 +107,14 @@ type CommonAlertMetricDetails struct {
 	Comparator string  `json:"comparator"`
 	Threshold  float64 `json:"threshold"`
 	// metric points'value的运算方式
-	Reduce                 string `json:"reduce"`
-	DB                     string `json:"db"`
-	Measurement            string `json:"measurement"`
-	MeasurementDisplayName string `json:"measurement_display_name"`
-	ResType                string `json:"res_type"`
-	Field                  string `json:"field"`
-	Groupby                string `json:"groupby"`
+	Reduce                 string           `json:"reduce"`
+	DB                     string           `json:"db"`
+	Measurement            string           `json:"measurement"`
+	MeasurementDisplayName string           `json:"measurement_display_name"`
+	ResType                string           `json:"res_type"`
+	Field                  string           `json:"field"`
+	Groupby                string           `json:"groupby"`
+	Filters                []MetricQueryTag `json:"filters"`
 	FieldDescription       MetricFieldDetail
 	FieldOpt               string `json:"field_opt"`
 }

pkg/apis/monitor/unifiedmonitor_const.go

@@ -41,8 +41,11 @@ type MetricFunc struct {
 }
 
 type MetricInputQuery struct {
-	From        string        `json:"from"`
-	To          string        `json:"to"`
+	From  string `json:"from"`
+	To    string `json:"to"`
+	Scope string `json:"scope"`
+	//default group by
+	Unit        bool          `json:"unit"`
 	Interval    string        `json:"interval"`
 	DomainId    string        `json:"domain_id"`
 	ProjectId   string        `json:"project_id"`

pkg/apis/notify/const.go

@@ -29,6 +29,7 @@ const (
 	FEISHU_ROBOT   = "feishu-robot"
 	DINGTALK_ROBOT = "dingtalk-robot"
 	WORKWX_ROBOT   = "workwx-robot"
+	WEBHOOK        = "webhook"
 
 	ROBOT = "robot"
 
@@ -60,4 +61,7 @@ const (
 	TEMPLATE_TYPE_TITLE   = "title"
 	TEMPLATE_TYPE_CONTENT = "content"
 	TEMPLATE_TYPE_REMOTE  = "remote"
+
+	CTYPE_ROBOT_YES  = "yes"
+	CTYPE_ROBOT_ONLY = "only"
 )

pkg/baremetal/utils/detect_storages/detect_storages.go

@@ -24,7 +24,6 @@ import (
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
 	"yunion.io/x/onecloud/pkg/cloudcommon/types"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
-	"yunion.io/x/onecloud/pkg/util/ssh"
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
 
@@ -48,7 +47,7 @@ func GetRaidLogicVolumes(drv raid.IRaidDriver) ([]*raid.RaidLogicalVolume, error
 	return lvs, nil
 }
 
-func DetectStorageInfo(term *ssh.Client, wait bool) ([]*baremetal.BaremetalStorage, []*baremetal.BaremetalStorage, []*baremetal.BaremetalStorage, error) {
+func DetectStorageInfo(term raid.IExecTerm, wait bool) ([]*baremetal.BaremetalStorage, []*baremetal.BaremetalStorage, []*baremetal.BaremetalStorage, error) {
 	raidDiskInfo := make([]*baremetal.BaremetalStorage, 0)
 	lvDiskInfo := make([]*raid.RaidLogicalVolume, 0)
 

pkg/baremetal/utils/raid/drivers/drivers.go

@@ -29,10 +29,9 @@ import (
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/mvcli"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/sas2iru"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
-	"yunion.io/x/onecloud/pkg/util/ssh"
 )
 
-func GetDriver(name string, term *ssh.Client) raid.IRaidDriver {
+func GetDriver(name string, term raid.IExecTerm) raid.IRaidDriver {
 	factory := raid.RaidDrivers[name]
 	if factory == nil {
 		return nil
@@ -40,7 +39,15 @@ func GetDriver(name string, term *ssh.Client) raid.IRaidDriver {
 	return factory(term)
 }
 
-func GetDriverWithInit(name string, term *ssh.Client) (raid.IRaidDriver, error) {
+func GetLocalDriver(name string) raid.IRaidDriver {
+	factory := raid.RaidDrivers[name]
+	if factory == nil {
+		return nil
+	}
+	return factory(NewExecutor())
+}
+
+func GetDriverWithInit(name string, term raid.IExecTerm) (raid.IRaidDriver, error) {
 	drv := GetDriver(name, term)
 	if drv == nil {
 		return nil, errors.Errorf("Not found raid driver %q", name)
@@ -48,7 +55,7 @@ func GetDriverWithInit(name string, term *ssh.Client) (raid.IRaidDriver, error)
 	return drv, drv.ParsePhyDevs()
 }
 
-func GetDriverByKernelModule(module string, term *ssh.Client) (raid.IRaidDriver, error) {
+func GetDriverByKernelModule(module string, term raid.IExecTerm) (raid.IRaidDriver, error) {
 	name := ""
 	switch module {
 	case raid.MODULE_MEGARAID:
@@ -64,7 +71,7 @@ func GetDriverByKernelModule(module string, term *ssh.Client) (raid.IRaidDriver,
 	return GetDriverWithInit(name, term)
 }
 
-func GetDrivers(term *ssh.Client) []raid.IRaidDriver {
+func GetDrivers(term raid.IExecTerm) []raid.IRaidDriver {
 	ret := []raid.IRaidDriver{}
 	for _, factory := range raid.RaidDrivers {
 		ret = append(ret, factory(term))

pkg/baremetal/utils/raid/drivers/executor.go

@@ -0,0 +1,56 @@
+package drivers
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os/exec"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/util/ssh"
+)
+
+type Executor struct{}
+
+func (e *Executor) Run(cmds ...string) ([]string, error) {
+	return e.run(true, cmds, nil)
+}
+
+func (e *Executor) RunWithInput(input io.Reader, cmds ...string) ([]string, error) {
+	return e.run(true, cmds, input)
+}
+
+func (e *Executor) run(parseOutput bool, cmds []string, input io.Reader) ([]string, error) {
+	ret := []string{}
+	for _, cmd := range cmds {
+		log.Debugf("Run command: %s", cmd)
+		proc := exec.Command("sh", "-c", cmd)
+		var stdOut bytes.Buffer
+		var stdErr bytes.Buffer
+		proc.Stdout = &stdOut
+		proc.Stderr = &stdErr
+		proc.Stdin = input
+		if err := proc.Run(); err != nil {
+			var outputErr error
+			errMsg := stdErr.String()
+			if len(stdOut.String()) != 0 {
+				errMsg = fmt.Sprintf("%s %s", errMsg, stdOut.String())
+			}
+			outputErr = errors.Error(errMsg)
+			err = errors.Wrapf(outputErr, "%q error: %v, cmd error", cmd, err)
+			return nil, err
+		}
+		if parseOutput {
+			ret = append(ret, ssh.ParseOutput(stdOut.Bytes())...)
+		} else {
+			ret = append(ret, stdOut.String())
+		}
+	}
+	return ret, nil
+}
+
+func NewExecutor() *Executor {
+	return new(Executor)
+}

pkg/baremetal/utils/raid/hpssactl/hpssactl.go

@@ -31,7 +31,6 @@ import (
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/util/regutils2"
-	"yunion.io/x/onecloud/pkg/util/ssh"
 )
 
 type HPSARaidPhyDev struct {
@@ -347,11 +346,11 @@ func (adapter *HPSARaidAdaptor) RemoveLogicVolumes() error {
 }
 
 type HPSARaid struct {
-	term     *ssh.Client
+	term     raid.IExecTerm
 	adapters []*HPSARaidAdaptor
 }
 
-func NewHPSARaid(term *ssh.Client) raid.IRaidDriver {
+func NewHPSARaid(term raid.IExecTerm) raid.IRaidDriver {
 	return &HPSARaid{
 		term:     term,
 		adapters: make([]*HPSARaidAdaptor, 0),

pkg/baremetal/utils/raid/interface.go

@@ -15,6 +15,8 @@
 package raid
 
 import (
+	"io"
+
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 )
@@ -41,3 +43,8 @@ type IRaidAdapter interface {
 	BuildRaid10(devs []*baremetal.BaremetalStorage, conf *api.BaremetalDiskConfig) error
 	BuildNoneRaid(devs []*baremetal.BaremetalStorage) error
 }
+
+type IExecTerm interface {
+	Run(cmds ...string) ([]string, error)
+	RunWithInput(input io.Reader, cmds ...string) ([]string, error)
+}

pkg/baremetal/utils/raid/megactl/megactl.go

@@ -27,10 +27,10 @@ import (
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
 	raiddrivers "yunion.io/x/onecloud/pkg/baremetal/utils/raid"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
 	"yunion.io/x/onecloud/pkg/util/regutils2"
-	"yunion.io/x/onecloud/pkg/util/ssh"
 )
 
 var (
@@ -320,7 +320,7 @@ func (adapter *MegaRaidAdaptor) GetIndex() int {
 	return adapter.index
 }
 
-func (adapter *MegaRaidAdaptor) getTerm() *ssh.Client {
+func (adapter *MegaRaidAdaptor) getTerm() raid.IExecTerm {
 	return adapter.raid.term
 }
 
@@ -376,7 +376,7 @@ func (adapter *MegaRaidAdaptor) parseLogicVolumes(lines []string) ([]*raiddriver
 	return lvs, nil
 }
 
-func getLogicVolumeDeviceById(hostNum, scsiId int, term *ssh.Client) (string, error) {
+func getLogicVolumeDeviceById(hostNum, scsiId int, term raid.IExecTerm) (string, error) {
 	items, err := raiddrivers.SGMap(term)
 	if err != nil {
 		return "", err
@@ -892,13 +892,13 @@ func (adapter *MegaRaidAdaptor) clearJBODDisks() {
 }
 
 type MegaRaid struct {
-	term       *ssh.Client
+	term       raid.IExecTerm
 	adapters   []*MegaRaidAdaptor
 	PhyDevsCnt int
 	Capacity   int64
 }
 
-func NewMegaRaid(term *ssh.Client) raiddrivers.IRaidDriver {
+func NewMegaRaid(term raid.IExecTerm) raiddrivers.IRaidDriver {
 	return &MegaRaid{
 		term:     term,
 		adapters: make([]*MegaRaidAdaptor, 0),

pkg/baremetal/utils/raid/mvcli/mvcli.go

@@ -25,7 +25,6 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
-	"yunion.io/x/onecloud/pkg/util/ssh"
 )
 
 type MarvelRaidPhyDev struct {
@@ -243,11 +242,11 @@ func (adapter *MarvelRaidAdaptor) BuildNoneRaid(devs []*baremetal.BaremetalStora
 }
 
 type MarvelRaid struct {
-	term     *ssh.Client
+	term     raid.IExecTerm
 	adapters []*MarvelRaidAdaptor
 }
 
-func NewMarvelRaid(term *ssh.Client) raid.IRaidDriver {
+func NewMarvelRaid(term raid.IExecTerm) raid.IRaidDriver {
 	return &MarvelRaid{
 		term:     term,
 		adapters: make([]*MarvelRaidAdaptor, 0),

pkg/baremetal/utils/raid/raid.go

@@ -24,7 +24,6 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
-	"yunion.io/x/onecloud/pkg/util/ssh"
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
 
@@ -45,7 +44,7 @@ const (
 	UnknownLogicalVolumeIndex = MaxInt
 )
 
-type RaidDriverFactory func(term *ssh.Client) IRaidDriver
+type RaidDriverFactory func(term IExecTerm) IRaidDriver
 
 type sRaidDrivers map[string]RaidDriverFactory
 
@@ -106,7 +105,7 @@ func (dev *RaidBasePhyDev) ToBaremetalStorage(index int) *baremetal.BaremetalSto
 	}
 }
 
-func GetModules(term *ssh.Client) []string {
+func GetModules(term IExecTerm) []string {
 	ret := []string{}
 	lines, err := term.Run("/sbin/lsmod")
 	if err != nil {
@@ -142,7 +141,7 @@ type RaidLogicalVolume struct {
 	BlockDev string
 }
 
-func SGMap(term *ssh.Client) ([]compute.SGMapItem, error) {
+func SGMap(term IExecTerm) ([]compute.SGMapItem, error) {
 	lines, err := term.Run("/usr/bin/sg_map -x")
 	if err != nil {
 		return nil, errors.Wrap(err, "run sg_map")

pkg/baremetal/utils/raid/sas2iru/sas2iru.go

@@ -26,7 +26,6 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
-	"yunion.io/x/onecloud/pkg/util/ssh"
 )
 
 type Mpt2SASRaidPhyDev struct {
@@ -307,12 +306,12 @@ func (adapter *Mpt2SASRaidAdaptor) RemoveLogicVolumes() error {
 }
 
 type Mpt2SASRaid struct {
-	term     *ssh.Client
+	term     raid.IExecTerm
 	utility  string
 	adapters []*Mpt2SASRaidAdaptor
 }
 
-func NewMpt2SASRaid(term *ssh.Client) raid.IRaidDriver {
+func NewMpt2SASRaid(term raid.IExecTerm) raid.IRaidDriver {
 	return &Mpt2SASRaid{
 		term:     term,
 		adapters: make([]*Mpt2SASRaidAdaptor, 0),

pkg/cloudcommon/app/endpoint.go

@@ -30,6 +30,7 @@ type SEndpointChangeManager struct {
 func newEndpointChangeManager() *SEndpointChangeManager {
 	man := &SEndpointChangeManager{}
 	man.InitSync(man)
+	man.FirstSync()
 	return man
 }
 

pkg/cloudcommon/db/caller.go

@@ -301,6 +301,7 @@ func FetchCustomizeColumns(
 	retVal := make([]*jsonutils.JSONDict, ret[0].Len())
 	for i := 0; i < ret[0].Len(); i += 1 {
 		jsonDict := ValueToJSONDict(ret[0].Index(i))
+		// NOTE: don't use obj update jsonDict as retval
 		jsonDict.Update(jsonutils.Marshal(objs[i]).(*jsonutils.JSONDict))
 		retVal[i] = jsonDict
 	}

pkg/cloudcommon/db/db_dispatcher.go

@@ -928,6 +928,12 @@ func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, quer
 	if err != nil {
 		return nil, err
 	} else if data != nil {
+		if dataDict, ok := data.(*jsonutils.JSONDict); ok {
+			i18nDict := dispatcher.modelManager.GetI18N(ctx, idStr, data)
+			if i18nDict != nil {
+				dataDict.Set("_i18n", i18nDict)
+			}
+		}
 		return data, nil
 	}
 

pkg/cloudcommon/db/domain.go

@@ -107,12 +107,7 @@ func (manager *SDomainizedResourceBaseManager) ListItemFilter(
 ) (*sqlchemy.SQuery, error) {
 	if len(query.ProjectDomainIds) > 0 {
 		// make sure ids are not utf8 string
-		idList := make([]string, 0)
-		for _, pid := range query.ProjectDomainIds {
-			if !stringutils2.IsUtf8(pid) {
-				idList = append(idList, pid)
-			}
-		}
+		idList := stringutils2.RemoveUtf8Strings(query.ProjectDomainIds)
 		tenants := TenantCacheManager.GetDomainQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
 			sqlchemy.In(tenants.Field("id"), idList),

pkg/cloudcommon/db/interface.go

@@ -127,6 +127,8 @@ type IModelManager interface {
 	QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error)
 
 	GetPagingConfig() *SPagingConfig
+
+	GetI18N(ctx context.Context, idstr string, resObj jsonutils.JSONObject) *jsonutils.JSONDict
 }
 
 type IModel interface {
@@ -246,6 +248,10 @@ type IMetadataBaseModelManager interface {
 	GetMetadataHiddenKeys() []string
 }
 
+type IMetadataBaseModel interface {
+	OnMetadataUpdated(ctx context.Context, userCred mcclient.TokenCredential)
+}
+
 type IStandaloneModelManager interface {
 	IResourceModelManager
 
@@ -282,6 +288,8 @@ type IStandaloneModel interface {
 	GetAllMetadata(userCred mcclient.TokenCredential) (map[string]string, error)
 
 	IsShared() bool
+
+	IMetadataBaseModel
 }
 
 type IDomainLevelModelManager interface {

pkg/cloudcommon/db/lockman/etcd_test.go

@@ -26,7 +26,8 @@ func TestEctdLockManager(t *testing.T) {
 	shared := newSharedObject()
 	for i := 0; i < 4; i++ {
 		lockman, err := NewEtcdLockManager(&SEtcdLockManagerConfig{
-			Endpoints: []string{"localhost:2379"},
+			LockPrefix: "test-etcd-lock-manager",
+			Endpoints:  []string{"localhost:2379"},
 		})
 		if err != nil {
 			t.Skipf("new etcd lockman: %v", err)

pkg/cloudcommon/db/metadata.go

@@ -492,6 +492,7 @@ func (manager *SMetadataManager) SetValues(ctx context.Context, obj IModel, stor
 		}
 
 		newRecord := SMetadata{}
+		newRecord.SetModelManager(manager, &newRecord)
 
 		newRecord.ObjId = obj.GetId()
 		newRecord.ObjType = obj.GetModelManager().Keyword()

pkg/cloudcommon/db/metadataresource.go

@@ -90,10 +90,10 @@ func (meta *SMetadataResourceBaseModelManager) ListItemFilter(
 		}
 	}
 
-	if input.WithMeta != nil {
+	if input.WithAnyMeta != nil {
 		metadatas := Metadata.Query().Equals("obj_type", manager.Keyword()).SubQuery()
 		sq := metadatas.Query(metadatas.Field("obj_id")).Distinct().SubQuery()
-		if *input.WithMeta {
+		if *input.WithAnyMeta {
 			q = q.Filter(sqlchemy.In(q.Field("id"), sq))
 		} else {
 			q = q.Filter(sqlchemy.NotIn(q.Field("id"), sq))

pkg/cloudcommon/db/modelbase.go

@@ -359,14 +359,15 @@ func (manager *SModelBaseManager) GetPropertyDistinctField(ctx context.Context,
 		}
 	}
 
-	var res = jsonutils.NewDict()
 	q := im.Query()
 	q, err = ListItemQueryFilters(im, ctx, q, userCred, query, policy.PolicyActionList)
 	if err != nil {
 		return nil, err
 	}
-	var backupQuery = *q
-
+	var (
+		backupQuery = *q
+		res         = jsonutils.NewDict()
+	)
 	// query field
 	for i := 0; i < len(fields); i++ {
 		var nq = backupQuery
@@ -424,6 +425,10 @@ func (manager *SModelBaseManager) OnCreateFailed(ctx context.Context, userCred m
 	return nil
 }
 
+func (manager *SModelBaseManager) GetI18N(ctx context.Context, idstr string, resObj jsonutils.JSONObject) *jsonutils.JSONDict {
+	return nil
+}
+
 func (model *SModelBase) GetId() string {
 	return ""
 }

pkg/cloudcommon/db/namevalidator.go

@@ -87,14 +87,17 @@ func alterNameValidator(model IModel, name string) error {
 	return nil
 }
 
-func GenerateAlertName(model IModel, hint string) (string, error) {
-	return GenerateName2(model.GetModelManager(), nil, hint, model, 1)
-}
-
 func GenerateName(manager IModelManager, ownerId mcclient.IIdentityProvider, hint string) (string, error) {
 	return GenerateName2(manager, ownerId, hint, nil, 1)
 }
 
+func GenerateAlterName(model IModel, hint string) (string, error) {
+	if hint == model.GetName() {
+		return hint, nil
+	}
+	return GenerateName2(nil, nil, hint, model, 1)
+}
+
 func GenerateName2(manager IModelManager, ownerId mcclient.IIdentityProvider, hint string, model IModel, baseIndex int) (string, error) {
 	_, pattern, patternLen := stringutils.ParseNamePattern(hint)
 	var name string

pkg/cloudcommon/db/opslog.go

@@ -45,31 +45,29 @@ type SOpsLogManager struct {
 type SOpsLog struct {
 	SModelBase
 
-	Id      int64  `primary:"true" auto_increment:"true" list:"user"`                                        // = Column(BigInteger, primary_key=True)
-	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required"`               // = Column(VARCHAR(40, charset='ascii'), nullable=False)
-	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` //  = Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False)
-	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`               //= Column(VARCHAR(128, charset='utf8'), nullable=False)
-	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`                //= Column(VARCHAR(32, charset='ascii'), nullable=False)
+	Id      int64  `primary:"true" auto_increment:"true" list:"user"`
+	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
+	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`
+	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`
 	Notes   string `charset:"utf8" list:"user" create:"required"`
 
-	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" list:"user" create:"required" index:"true"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
-	Project   string `name:"tenant" width:"128" charset:"utf8" list:"user" create:"required"`                  // tenant    = Column(VARCHAR(128, charset='utf8'))
+	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" list:"user" create:"optional" index:"true"`
+	Project   string `name:"tenant" width:"128" charset:"utf8" list:"user" create:"optional"`
 
-	ProjectDomainId string `name:"project_domain_id" default:"default" width:"128" charset:"ascii" list:"user" create:"required"`
-	ProjectDomain   string `name:"project_domain" default:"Default" width:"128" charset:"utf8" list:"user" create:"required"`
+	ProjectDomainId string `name:"project_domain_id" default:"default" width:"128" charset:"ascii" list:"user" create:"optional"`
+	ProjectDomain   string `name:"project_domain" default:"Default" width:"128" charset:"utf8" list:"user" create:"optional"`
 
-	UserId   string `width:"128" charset:"ascii" list:"user" create:"required"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
-	User     string `width:"128" charset:"utf8" list:"user" create:"required"`  // = Column(VARCHAR(128, charset='utf8'))
+	UserId   string `width:"128" charset:"ascii" list:"user" create:"required"`
+	User     string `width:"128" charset:"utf8" list:"user" create:"required"`
 	DomainId string `width:"128" charset:"ascii" list:"user" create:"optional"`
 	Domain   string `width:"128" charset:"utf8" list:"user" create:"optional"`
-	Roles    string `width:"64" charset:"ascii" list:"user" create:"optional"` // = Column(VARCHAR(64, charset='ascii'))
+	Roles    string `width:"64" charset:"ascii" list:"user" create:"optional"`
 
-	// BillingType    string    `width:"64" charset:"ascii" default:"postpaid" list:"user" create:"user"`      // billing_type = Column(VARCHAR(64, charset='ascii'), nullable=True)
-	OpsTime time.Time `nullable:"false" list:"user"` // = Column(DateTime, nullable=False)
+	OpsTime time.Time `nullable:"false" list:"user"`
 
 	OwnerDomainId  string `name:"owner_domain_id" default:"default" width:"128" charset:"ascii" list:"user" create:"optional"`
-	OwnerProjectId string `name:"owner_tenant_id" width:"128" charset:"ascii" list:"user" create:"optional"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
-	// owner_user_id   = Column(VARCHAR(ID_LENGTH, charset='ascii'))
+	OwnerProjectId string `name:"owner_tenant_id" width:"128" charset:"ascii" list:"user" create:"optional"`
 }
 
 var OpsLog *SOpsLogManager
@@ -434,6 +432,7 @@ func (manager *SOpsLogManager) ValidateCreateData(ctx context.Context,
 	query jsonutils.JSONObject,
 	data apis.OpsLogCreateInput,
 ) (apis.OpsLogCreateInput, error) {
+	data.User = ownerId.GetUserName()
 	return data, nil
 }
 

pkg/cloudcommon/db/project.go

@@ -102,12 +102,7 @@ func (manager *SProjectizedResourceBaseManager) ListItemFilter(
 	}
 	if len(query.ProjectIds) > 0 {
 		// make sure ids are not utf8 string
-		idList := make([]string, 0)
-		for _, pid := range query.ProjectIds {
-			if !stringutils2.IsUtf8(pid) {
-				idList = append(idList, pid)
-			}
-		}
+		idList := stringutils2.RemoveUtf8Strings(query.ProjectIds)
 		tenants := TenantCacheManager.GetTenantQuery().SubQuery()
 		subq := tenants.Query(tenants.Field("id")).Filter(sqlchemy.OR(
 			sqlchemy.In(tenants.Field("id"), idList),

pkg/cloudcommon/db/quotas/register.go

@@ -33,6 +33,7 @@ var (
 func init() {
 	quotaManagerTable = make(map[reflect.Type]IQuotaManager)
 
+	db.AddUsages = AddUsages
 	db.CancelUsages = CancelUsages
 	db.CancelPendingUsagesInContext = cancelPendingUsagesInContext
 	db.InitPendingUsagesInContext = initPendingUsagesInContext

pkg/cloudcommon/db/standalone.go

@@ -266,32 +266,81 @@ func (model *SStandaloneResourceBase) GetMetadataJson(key string, userCred mccli
 	return Metadata.GetJsonValue(model, key, userCred)
 }
 
+func isUserMetadata(key string) bool {
+	return strings.HasPrefix(key, USER_TAG_PREFIX)
+}
+
+func containsUserMetadata(dict map[string]interface{}) bool {
+	for k := range dict {
+		if isUserMetadata(k) {
+			return true
+		}
+	}
+	return false
+}
+
 func (model *SStandaloneResourceBase) SetMetadata(ctx context.Context, key string, value interface{}, userCred mcclient.TokenCredential) error {
-	return Metadata.SetValue(ctx, model, key, value, userCred)
+	err := Metadata.SetValue(ctx, model, key, value, userCred)
+	if err != nil {
+		return errors.Wrap(err, "SetValue")
+	}
+	if isUserMetadata(key) {
+		model.GetIStandaloneModel().OnMetadataUpdated(ctx, userCred)
+	}
+	return nil
 }
 
 func (model *SStandaloneResourceBase) SetAllMetadata(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error {
-	return Metadata.SetValuesWithLog(ctx, model, dictstore, userCred)
+	err := Metadata.SetValuesWithLog(ctx, model, dictstore, userCred)
+	if err != nil {
+		return errors.Wrap(err, "SetValuesWithLog")
+	}
+	if containsUserMetadata(dictstore) {
+		model.GetIStandaloneModel().OnMetadataUpdated(ctx, userCred)
+	}
+	return nil
 }
 
 func (model *SStandaloneResourceBase) SetUserMetadataValues(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error {
-	return Metadata.SetValuesWithLog(ctx, model, dictstore, userCred)
+	err := Metadata.SetValuesWithLog(ctx, model, dictstore, userCred)
+	if err != nil {
+		return errors.Wrap(err, "SetValuesWithLog")
+	}
+	model.GetIStandaloneModel().OnMetadataUpdated(ctx, userCred)
+	return nil
 }
 
 func (model *SStandaloneResourceBase) SetUserMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error {
-	return Metadata.SetAll(ctx, model, dictstore, userCred, USER_TAG_PREFIX)
+	err := Metadata.SetAll(ctx, model, dictstore, userCred, USER_TAG_PREFIX)
+	if err != nil {
+		return errors.Wrap(err, "SetAll")
+	}
+	model.GetIStandaloneModel().OnMetadataUpdated(ctx, userCred)
+	return nil
 }
 
 func (model *SStandaloneResourceBase) SetCloudMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error {
-	return Metadata.SetAll(ctx, model, dictstore, userCred, CLOUD_TAG_PREFIX)
+	err := Metadata.SetAll(ctx, model, dictstore, userCred, CLOUD_TAG_PREFIX)
+	if err != nil {
+		return errors.Wrap(err, "SetAll")
+	}
+	return nil
 }
 
 func (model *SStandaloneResourceBase) RemoveMetadata(ctx context.Context, key string, userCred mcclient.TokenCredential) error {
-	return Metadata.SetValue(ctx, model, key, "", userCred)
+	err := Metadata.SetValue(ctx, model, key, "", userCred)
+	if err != nil {
+		return errors.Wrap(err, "SetValue")
+	}
+	return nil
 }
 
 func (model *SStandaloneResourceBase) RemoveAllMetadata(ctx context.Context, userCred mcclient.TokenCredential) error {
-	return Metadata.RemoveAll(ctx, model, userCred)
+	err := Metadata.RemoveAll(ctx, model, userCred)
+	if err != nil {
+		return errors.Wrap(err, "RemoveAll")
+	}
+	return nil
 }
 
 func (model *SStandaloneResourceBase) GetAllMetadata(userCred mcclient.TokenCredential) (map[string]string, error) {
@@ -530,3 +579,7 @@ func (model *SStandaloneResourceBase) ValidateUpdateData(ctx context.Context, us
 func (model *SStandaloneResourceBase) IsShared() bool {
 	return false
 }
+
+func (model *SStandaloneResourceBase) OnMetadataUpdated(ctx context.Context, userCred mcclient.TokenCredential) {
+	// noop
+}

pkg/cloudcommon/db/usages.go

@@ -28,6 +28,8 @@ type IUsage interface {
 var (
 	CancelUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
 
+	AddUsages func(ctx context.Context, userCred mcclient.TokenCredential, usages []IUsage)
+
 	CancelPendingUsagesInContext func(ctx context.Context, userCred mcclient.TokenCredential) error
 
 	InitPendingUsagesInContext func(ctx context.Context) context.Context

pkg/cloudcommon/db/virtualresource.go

@@ -449,11 +449,21 @@ func (model *SVirtualResourceBase) AllowPerformCancelDelete(ctx context.Context,
 func (model *SVirtualResourceBase) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	if model.PendingDeleted && !model.Deleted {
 		err := model.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, errors.Wrap(err, "model.DoCancelPendingDelete")
+		}
+		model.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }
 
+func (model *SVirtualResourceBase) RecoverUsages(ctx context.Context, userCred mcclient.TokenCredential) {
+	usages := model.GetIModel().GetUsages()
+	if AddUsages != nil && len(usages) > 0 {
+		AddUsages(ctx, userCred, usages)
+	}
+}
+
 func (model *SVirtualResourceBase) DoCancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	err := model.CancelPendingDelete(ctx, userCred)
 	if err == nil {
@@ -472,7 +482,10 @@ func (model *SVirtualResourceBase) GetIVirtualModel() IVirtualModel {
 
 func (model *SVirtualResourceBase) CancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	if model.PendingDeleted && !model.Deleted {
-		return model.MarkCancelPendingDelete(ctx, userCred)
+		err := model.MarkCancelPendingDelete(ctx, userCred)
+		if err != nil {
+			return errors.Wrap(err, "MarkCancelPendingDelete")
+		}
 	}
 	return nil
 }

pkg/cloudcommon/informer/etcd.go

@@ -156,6 +156,10 @@ func (b *EtcdBackend) put(ctx context.Context, key, val string) error {
 	return b.PutWithLease(ctx, key, val, b.leaseTTL)
 }
 
+func (b *EtcdBackend) PutSession(ctx context.Context, key, val string) error {
+	return b.client.PutSession(ctx, key, val)
+}
+
 func (b *EtcdBackend) PutWithLease(ctx context.Context, key, val string, ttlSeconds int64) error {
 	return b.client.PutWithLease(ctx, key, val, ttlSeconds)
 }

pkg/cloudcommon/informer/etcd_client.go

@@ -99,7 +99,7 @@ func (b *EtcdBackendForClient) registerClientResource(ctx context.Context, key s
 	if err != nil {
 		return err
 	}
-	if err := b.PutWithLease(ctx, clientKey, "ok", 60); err != nil {
+	if err := b.PutSession(ctx, clientKey, "ok"); err != nil {
 		return err
 	}
 	return nil

pkg/cloudcommon/informer/informer.go

@@ -92,7 +92,7 @@ func Create(ctx context.Context, obj *ModelObject) error {
 	if !isResourceWatched(obj.KeywordPlural) {
 		return nil
 	}
-	return run(func(be IInformerBackend) error {
+	return run(ctx, func(ctx context.Context, be IInformerBackend) error {
 		return be.Create(ctx, obj)
 	})
 }
@@ -101,7 +101,7 @@ func Update(ctx context.Context, obj *ModelObject, oldObj *jsonutils.JSONDict) e
 	if !isResourceWatched(obj.KeywordPlural) {
 		return nil
 	}
-	return run(func(be IInformerBackend) error {
+	return run(ctx, func(ctx context.Context, be IInformerBackend) error {
 		return be.Update(ctx, obj, oldObj)
 	})
 }
@@ -110,7 +110,7 @@ func Delete(ctx context.Context, obj *ModelObject) error {
 	if !isResourceWatched(obj.KeywordPlural) {
 		return nil
 	}
-	return run(func(be IInformerBackend) error {
+	return run(ctx, func(ctx context.Context, be IInformerBackend) error {
 		return be.Delete(ctx, obj)
 	})
 }

pkg/cloudcommon/informer/worker.go

@@ -15,6 +15,8 @@
 package informer
 
 import (
+	"context"
+
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
@@ -29,14 +31,35 @@ func init() {
 	informerWorkerMan = appsrv.NewWorkerManager("InformerWorkerManager", 10, 10240, false)
 }
 
-func run(f func(be IInformerBackend) error) error {
+/*type noCancel struct {
+	ctx context.Context
+}
+
+func (c noCancel) Deadline() (time.Time, bool) {
+	return time.Time{}, false
+}
+
+func (c noCancel) Done() <-chan struct{} {
+	return nil
+}
+
+func (c noCancel) Err() error {
+	return nil
+}
+
+func (c noCancel) Value(key interface{}) interface{} {
+	return c.ctx.Value(key)
+}*/
+
+func run(ctx context.Context, f func(ctx context.Context, be IInformerBackend) error) error {
 	be := GetDefaultBackend()
 	if be == nil {
 		return ErrBackendNotInit
 	}
 	wf := func() {
 		nopanic.Run(func() {
-			if err := f(be); err != nil {
+			// outside context ignored cause of run in worker
+			if err := f(context.Background(), be); err != nil {
 				log.Errorf("run informer error: %v", err)
 			}
 		})

pkg/cloudcommon/notifyclient/events.go

@@ -14,6 +14,13 @@
 
 package notifyclient
 
+import (
+	"fmt"
+	"strings"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+)
+
 const (
 	SYSTEM_ERROR   = "SYSTEM_ERROR"
 	SYSTEM_WARNING = "SYSTEM_WARNING"
@@ -28,3 +35,42 @@ const (
 
 	IMAGE_ACTIVED = "IMAGE_ACTIVED"
 )
+
+type SAction string
+
+var (
+	Event SEvent
+
+	ActionCreate       SAction = "create"
+	ActionUpdate       SAction = "update"
+	ActionDelete       SAction = "delete"
+	ActionRebuildRoot  SAction = "rebuild_root"
+	ActionChangeConfig SAction = "change_config"
+)
+
+type SEvent struct {
+	resourceType string
+	action       SAction
+}
+
+func (se SEvent) WithResourceType(manager db.IModelManager) SEvent {
+	se.resourceType = manager.Keyword()
+	return se
+}
+
+func (se SEvent) WithAction(a SAction) SEvent {
+	se.action = a
+	return se
+}
+
+func (se SEvent) ResourceType() string {
+	return se.resourceType
+}
+
+func (se SEvent) Action() string {
+	return string(se.action)
+}
+
+func (se SEvent) String() string {
+	return strings.ToUpper(fmt.Sprintf("%s_%s", se.ResourceType(), se.Action()))
+}

pkg/cloudcommon/notifyclient/notify.go

@@ -20,6 +20,7 @@ import (
 	"html/template"
 	"io/ioutil"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
 	"time"
@@ -29,12 +30,15 @@ import (
 
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/i18n"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
+	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
 
 var (
@@ -97,6 +101,9 @@ func getTemplate(ctx context.Context, topic string, contType string, channel npk
 }
 
 func getContent(ctx context.Context, topic string, contType string, channel npk.TNotifyChannel, data jsonutils.JSONObject) (string, error) {
+	if channel == npk.NotifyByWebhook {
+		return "", nil
+	}
 	tmpl, err := getTemplate(ctx, topic, contType, channel)
 	if err != nil {
 		return "", err
@@ -110,6 +117,23 @@ func getContent(ctx context.Context, topic string, contType string, channel npk.
 	return buf.String(), nil
 }
 
+func NotifyWebhook(ctx context.Context, userCred mcclient.TokenCredential, obj db.IModel, action SAction) error {
+	ret, err := db.FetchCustomizeColumns(obj.GetModelManager(), ctx, userCred, jsonutils.NewDict(), []interface{}{obj}, stringutils2.SSortedStrings{}, false)
+	if err != nil {
+		return err
+	}
+	if len(ret) == 0 {
+		return fmt.Errorf("unable to get details for model %q", obj.GetId())
+	}
+	event := Event.WithAction(action).WithResourceType(obj.GetModelManager())
+	msg := jsonutils.NewDict()
+	msg.Set("resource_type", jsonutils.NewString(event.ResourceType()))
+	msg.Set("action", jsonutils.NewString(event.Action()))
+	msg.Set("resource_details", ret[0])
+	RawNotifyWithCtx(ctx, []string{}, false, npk.NotifyByWebhook, npk.NotifyPriorityNormal, event.String(), msg)
+	return nil
+}
+
 func NotifyWithCtx(ctx context.Context, recipientId []string, isGroup bool, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) {
 	notify(ctx, recipientId, isGroup, priority, event, data)
 }
@@ -137,7 +161,16 @@ func RawNotify(recipientId []string, isGroup bool, channel npk.TNotifyChannel, p
 	rawNotify(context.Background(), recipientId, isGroup, channel, priority, event, data)
 }
 
-func rawNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) {
+// IntelliNotify try to create receiver nonexistent if createReceiver is set to true
+func IntelliNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject, createReceiver bool) {
+	intelliNotify(ctx, recipientId, isGroup, channel, priority, event, data, createReceiver)
+}
+
+const noSuchReceiver = `no such receiver whose uid is '(.*)'`
+
+var noSuchReceiverRegexp = regexp.MustCompile(noSuchReceiver)
+
+func intelliNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject, createReceiver bool) {
 	log.Infof("notify %s event %s priority %s", recipientId, event, priority)
 	msg := npk.SNotifyMessage{}
 	if isGroup {
@@ -160,13 +193,46 @@ func rawNotify(ctx context.Context, recipientId []string, isGroup bool, channel
 	// log.Debugf("send notification %s %s", topic, body)
 	notifyClientWorkerMan.Run(func() {
 		s := auth.GetAdminSession(context.Background(), consts.GetRegion(), "")
-		err := npk.Notifications.Send(s, msg)
-		if err != nil {
-			log.Errorf("unable to send notification: %v", err)
+		for {
+			err := npk.Notifications.Send(s, msg)
+			if err == nil {
+				break
+			}
+			if !createReceiver {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			jerr, ok := err.(*httputils.JSONClientError)
+			if !ok {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			if jerr.Code > 500 {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			match := noSuchReceiverRegexp.FindStringSubmatch(jerr.Details)
+			if match == nil || len(match) <= 1 {
+				log.Errorf("unable to send notification: %v", err)
+				break
+			}
+			receiverId := match[1]
+			createData := jsonutils.NewDict()
+			createData.Set("uid", jsonutils.NewString(receiverId))
+			_, err = modules.NotifyReceiver.Create(s, createData)
+			if err != nil {
+				log.Errorf("try to create receiver %q, but failed: %v", receiverId, err)
+				break
+			}
+			log.Infof("create receiver %q successfully", receiverId)
 		}
 	}, nil, nil)
 }
 
+func rawNotify(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) {
+	intelliNotify(ctx, recipientId, isGroup, channel, priority, event, data, false)
+}
+
 func NotifyNormal(recipientId []string, isGroup bool, event string, data jsonutils.JSONObject) {
 	notifyNormal(context.Background(), recipientId, isGroup, event, data)
 }

pkg/cloudcommon/options/manager.go

@@ -124,7 +124,10 @@ func (manager *SOptionManager) DoSync(first bool) (time.Duration, error) {
 			appsrv.SetExitFlag()
 		}
 		copyOptions(manager.options, newOpts)
-		manager.session.Upload()
+		if first {
+			// upload config for the first time ONLY
+			manager.session.Upload()
+		}
 	}
 	return manager.refreshInterval, nil
 }

pkg/cloudprovider/cloudprovider.go

@@ -56,7 +56,7 @@ type SCloudaccountCredential struct {
 	// 秘钥key (Aliyun, Aws, huawei, ucloud, ctyun, zstack, s3)
 	AccessKeySecret string `json:"access_key_secret"`
 
-	// 环境 (Azure, Aws, huawei, ctyun)
+	// 环境 (Azure, Aws, huawei, ctyun, aliyun)
 	Environment string `json:"environment"`
 
 	// 目录ID (Azure)

pkg/cloudprovider/resources.go

@@ -317,6 +317,8 @@ type ICloudVM interface {
 	LiveMigrateVM(hostid string) error
 
 	GetError() error
+
+	SetMetadata(tags map[string]string, replace bool) error
 }
 
 type ICloudNic interface {

pkg/compute/guestdrivers/aliyun.go

@@ -77,8 +77,8 @@ func (self *SAliyunGuestDriver) GetStorageTypes() []string {
 	}
 }
 
-func (self *SAliyunGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
-	return self.chooseHostStorage(self, host, backend, storageIds)
+func (self *SAliyunGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
+	return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
 }
 
 func (self *SAliyunGuestDriver) GetDetachDiskStatus() ([]string, error) {

pkg/compute/guestdrivers/aws.go

@@ -131,8 +131,8 @@ func (self *SAwsGuestDriver) GetStorageTypes() []string {
 	}
 }
 
-func (self *SAwsGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
-	return self.chooseHostStorage(self, host, backend, storageIds)
+func (self *SAwsGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
+	return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
 }
 
 func (self *SAwsGuestDriver) GetDetachDiskStatus() ([]string, error) {

pkg/compute/guestdrivers/azure.go

@@ -79,8 +79,8 @@ func (self *SAzureGuestDriver) GetStorageTypes() []string {
 	}
 }
 
-func (self *SAzureGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
-	return self.chooseHostStorage(self, host, backend, storageIds)
+func (self *SAzureGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
+	return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
 }
 
 func (self *SAzureGuestDriver) GetMaxSecurityGroupCount() int {

pkg/compute/guestdrivers/baremetals.go

@@ -249,15 +249,15 @@ func (self *SBaremetalGuestDriver) GetStorageTypes() []string {
 	}
 }
 
-func (self *SBaremetalGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
+func (self *SBaremetalGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
 	if len(storageIds) != 0 {
-		return models.StorageManager.FetchStorageById(storageIds[0])
+		return models.StorageManager.FetchStorageById(storageIds[0]), nil
 	}
 	bs := host.GetBaremetalstorage()
 	if bs == nil {
-		return nil
+		return nil, nil
 	}
-	return bs.GetStorage()
+	return bs.GetStorage(), nil
 }
 
 func (self *SBaremetalGuestDriver) RequestGuestCreateAllDisks(ctx context.Context, guest *models.SGuest, task taskman.ITask) error {

pkg/compute/guestdrivers/base.go

@@ -393,3 +393,8 @@ func (self *SBaseGuestDriver) RequestMigrate(ctx context.Context, guest *models.
 func (self *SBaseGuestDriver) RequestLiveMigrate(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, task taskman.ITask) error {
 	return fmt.Errorf("Not Implement RequestLiveMigrate")
 }
+
+func (self *SBaseGuestDriver) RequestRemoteUpdate(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential, replaceTags bool) error {
+	// nil ops
+	return nil
+}

pkg/compute/guestdrivers/esxi.go

@@ -79,6 +79,46 @@ func (self *SESXiGuestDriver) GetDefaultSysDiskBackend() string {
 	return api.STORAGE_LOCAL
 }
 
+func (self *SESXiGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
+	if !options.Options.LockStorageFromCachedimage || len(diskConfig.ImageId) == 0 {
+		return self.SVirtualizedGuestDriver.ChooseHostStorage(host, diskConfig, storageIds)
+	}
+	var (
+		image *cloudprovider.SImage
+		err   error
+	)
+	obj, err := models.CachedimageManager.FetchById(diskConfig.ImageId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to fetch cachedimage %s", diskConfig.ImageId)
+	}
+	cachedimage := obj.(*models.SCachedimage)
+	if len(cachedimage.ExternalId) > 0 || cachedimage.ImageType != cloudprovider.CachedImageTypeSystem {
+		return self.SVirtualizedGuestDriver.ChooseHostStorage(host, diskConfig, storageIds)
+	}
+	storages, err := cachedimage.GetStorages()
+	if err != nil {
+		return nil, errors.Wrapf(err, "unable to GetStorages of cachedimage %s", diskConfig.ImageId)
+	}
+	if len(storages) == 0 {
+		log.Warningf("there no storage associated with cachedimage %q", image.Id)
+		return self.SVirtualizedGuestDriver.ChooseHostStorage(host, diskConfig, storageIds)
+	}
+	if len(storages) > 1 {
+		log.Warningf("there are multiple storageCache associated with caheimage %q", image.Id)
+	}
+	wantStorageIds := make([]string, len(storages))
+	for i := range wantStorageIds {
+		wantStorageIds[i] = storages[i].GetId()
+	}
+	for i := range wantStorageIds {
+		if utils.IsInStringArray(wantStorageIds[i], storageIds) {
+			log.Infof("use storage %q in where cachedimage %q", wantStorageIds[i], image.Id)
+			return &storages[i], nil
+		}
+	}
+	return self.SVirtualizedGuestDriver.ChooseHostStorage(host, diskConfig, storageIds)
+}
+
 func (self *SESXiGuestDriver) GetMinimalSysDiskSizeGb() int {
 	return options.Options.DefaultDiskSizeMB / 1024
 }
@@ -183,21 +223,34 @@ func (self *SESXiGuestDriver) GetJsonDescAtHost(ctx context.Context, userCred mc
 		return desc, errors.Errorf("no such storage cache associated with cacheimage %s", templateId)
 	}
 	if len(storageCaches) > 1 {
-		return desc, errors.Errorf("there are multiple storageCache associated with caheimage '%s' ??!!", templateId)
+		log.Warningf("there are multiple storageCache associated with caheimage '%s' ??!!", templateId)
 	}
 
-	var hostIp string
-	storageCacheHost, err := storageCaches[0].GetHost()
-	if err != nil {
-		log.Errorf("unable to GetHost of storageCache %s: %v", storageCaches[0].Id, err)
-		hostIp = storageCaches[0].ExternalId
-	} else if storageCacheHost == nil {
-		log.Errorf("unable to GetHost of storageCache %s: result is nil", storageCaches[0].Id)
-		hostIp = storageCaches[0].ExternalId
-	} else {
-		hostIp = storageCacheHost.AccessIp
+	var storageCacheHost *models.SHost
+	// select storagecacheHost
+	for i := range storageCaches {
+		hosts, err := storageCaches[i].GetHosts()
+		if err != nil {
+			return desc, errors.Wrap(err, "storageCaches.GetHosts")
+		}
+		for i := range hosts {
+			if host.GetId() == hosts[i].GetId() {
+				storageCacheHost = &hosts[i]
+			}
+		}
 	}
-	hostIp = storageCacheHost.AccessIp
+
+	if storageCacheHost == nil {
+		storageCacheHost, err = storageCaches[0].GetHost()
+		if err != nil {
+			return desc, errors.Wrapf(err, "unable to GetHost of storageCache %s", storageCaches[0].Id)
+		}
+		if storageCacheHost == nil {
+			return desc, fmt.Errorf("unable to GetHost of storageCache %s: result is nil", storageCaches[0].Id)
+		}
+	}
+
+	hostIp := storageCacheHost.AccessIp
 	imageInfo := SEsxiImageInfo{
 		ImageType:          img.ImageType,
 		ImageExternalId:    img.ExternalId,

pkg/compute/guestdrivers/google.go

@@ -77,12 +77,13 @@ func (self *SGoogleGuestDriver) GetStorageTypes() []string {
 	return []string{
 		api.STORAGE_GOOGLE_PD_SSD,
 		api.STORAGE_GOOGLE_PD_STANDARD,
+		api.STORAGE_GOOGLE_PD_BALANCED,
 		api.STORAGE_GOOGLE_LOCAL_SSD,
 	}
 }
 
-func (self *SGoogleGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
-	return self.chooseHostStorage(self, host, backend, storageIds)
+func (self *SGoogleGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
+	return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
 }
 
 func (self *SGoogleGuestDriver) GetGuestInitialStateAfterCreate() string {
@@ -113,7 +114,7 @@ func (self *SGoogleGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *m
 	if !utils.IsInStringArray(guest.Status, []string{api.VM_READY, api.VM_RUNNING}) {
 		return fmt.Errorf("Cannot resize disk when guest in status %s", guest.Status)
 	}
-	if !utils.IsInStringArray(storage.StorageType, []string{api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD}) {
+	if !utils.IsInStringArray(storage.StorageType, []string{api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_BALANCED}) {
 		return fmt.Errorf("Cannot resize %s disk", storage.StorageType)
 	}
 	return nil
@@ -132,13 +133,15 @@ func (self *SGoogleGuestDriver) ValidateCreateData(ctx context.Context, userCred
 		minGB := -1
 		maxGB := -1
 		switch disk.Backend {
-		case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD:
+		case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_BALANCED:
 			minGB = 10
 			maxGB = 65536
 		case api.STORAGE_GOOGLE_LOCAL_SSD:
 			minGB = 375
 			maxGB = 375
 			localDisk++
+		default:
+			return nil, httperrors.NewInputParameterError("Unknown google storage type %s", disk.Backend)
 		}
 		if i == 0 && disk.Backend == api.STORAGE_GOOGLE_LOCAL_SSD {
 			return nil, httperrors.NewInputParameterError("System disk does not support %s disk", disk.Backend)

pkg/compute/guestdrivers/huawei.go

@@ -67,8 +67,8 @@ func (self *SHuaweiGuestDriver) GetStorageTypes() []string {
 	return []string{api.STORAGE_HUAWEI_SATA, api.STORAGE_HUAWEI_SAS, api.STORAGE_HUAWEI_SSD}
 }
 
-func (self *SHuaweiGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
-	return self.chooseHostStorage(self, host, backend, storageIds)
+func (self *SHuaweiGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
+	return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
 }
 
 func (self *SHuaweiGuestDriver) GetDetachDiskStatus() ([]string, error) {

pkg/compute/guestdrivers/managedvirtual.go

@@ -1215,3 +1215,30 @@ func (self *SManagedVirtualizedGuestDriver) RequestSetAutoRenewInstance(ctx cont
 	})
 	return nil
 }
+
+func (self *SManagedVirtualizedGuestDriver) RequestRemoteUpdate(ctx context.Context, guest *models.SGuest, userCred mcclient.TokenCredential, replaceTags bool) error {
+	// nil ops
+	iVM, err := guest.GetIVM()
+	if err != nil {
+		return errors.Wrap(err, "guest.GetIVM")
+	}
+	tags, err := guest.GetAllUserMetadata()
+	if err != nil {
+		log.Errorf("GetAllUserMetadata fail %s", err)
+	} else {
+		err := iVM.SetMetadata(tags, replaceTags)
+		if err != nil {
+			return errors.Wrap(err, "iVM.SetMetadata")
+		}
+		// sync back cloud metadata
+		err = models.SyncVirtualResourceMetadata(ctx, userCred, guest, iVM)
+		if err != nil {
+			return errors.Wrap(err, "syncVirtualResourceMetadata")
+		}
+	}
+	err = iVM.UpdateVM(ctx, guest.Name)
+	if err != nil {
+		return errors.Wrap(err, "iVM.UpdateVM")
+	}
+	return nil
+}

pkg/compute/guestdrivers/openstack.go

@@ -91,8 +91,8 @@ func (self *SOpenStackGuestDriver) GetStorageTypes() []string {
 	return storages
 }
 
-func (self *SOpenStackGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
-	return self.chooseHostStorage(self, host, backend, storageIds)
+func (self *SOpenStackGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
+	return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
 }
 
 func (self *SOpenStackGuestDriver) GetDetachDiskStatus() ([]string, error) {
@@ -232,7 +232,7 @@ func (self *SOpenStackGuestDriver) RemoteDeployGuestForRebuildRoot(ctx context.C
 		}
 		defer self.attachDisks(ctx, ihost, instanceId, detachDisks)
 
-		eip, err := guest.GetEip()
+		eip, err := guest.GetElasticIp()
 		if err == nil && eip != nil {
 			ieip, err := eip.GetIEip()
 			if err != nil {

pkg/compute/guestdrivers/qcloud.go

@@ -72,11 +72,12 @@ func (self *SQcloudGuestDriver) GetStorageTypes() []string {
 		api.STORAGE_CLOUD_SSD,
 		api.STORAGE_LOCAL_BASIC,
 		api.STORAGE_LOCAL_SSD,
+		api.STORAGE_CLOUD_HSSD,
 	}
 }
 
-func (self *SQcloudGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
-	return self.chooseHostStorage(self, host, backend, storageIds)
+func (self *SQcloudGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
+	return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
 }
 
 func (self *SQcloudGuestDriver) GetDetachDiskStatus() ([]string, error) {
@@ -154,6 +155,10 @@ func (self *SQcloudGuestDriver) ValidateCreateData(ctx context.Context, userCred
 			if disk.SizeMb < 100*1024 || disk.SizeMb > 16000*1024 {
 				return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of 100GB ~ 16000GB", disk.Backend)
 			}
+		case api.STORAGE_CLOUD_HSSD:
+			if disk.SizeMb < 20*1024 || disk.SizeMb > 32000*1024 {
+				return nil, httperrors.NewInputParameterError("The %s disk size must be in the range of 20GB ~ 32000GB", disk.Backend)
+			}
 		case api.STORAGE_LOCAL_PRO:
 			return nil, httperrors.NewInputParameterError("storage %s can not be data disk", disk.Backend)
 		}
@@ -194,6 +199,10 @@ func (self *SQcloudGuestDriver) ValidateChangeConfig(ctx context.Context, userCr
 			if newDisk.SizeMb < 100*1024 || newDisk.SizeMb > 16000*1024 {
 				return httperrors.NewInputParameterError("The %s disk size must be in the range of 100GB ~ 16000GB", newDisk.Backend)
 			}
+		case api.STORAGE_CLOUD_HSSD:
+			if newDisk.SizeMb < 20*1024 || newDisk.SizeMb > 32000*1024 {
+				return httperrors.NewInputParameterError("The %s disk size must be in the range of 20GB ~ 32000GB", newDisk.Backend)
+			}
 		case api.STORAGE_LOCAL_BASIC, api.STORAGE_LOCAL_SSD, api.STORAGE_LOCAL_PRO:
 			return httperrors.NewUnsupportOperationError("Not support create local storage disks")
 		case "": //这里Backend为空有可能会导致创建出来还是local storage,依然会出错,需要用户显式指定

pkg/compute/guestdrivers/virtualization.go

@@ -174,11 +174,11 @@ func (self *SVirtualizedGuestDriver) GetStorageTypes() []string {
 	return nil
 }
 
-func (self *SVirtualizedGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
+func (self *SVirtualizedGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
 	if len(storageIds) == 0 {
-		return host.GetLeastUsedStorage(backend)
+		return host.GetLeastUsedStorage(diskConfig.Backend), nil
 	}
-	return models.StorageManager.FetchStorageById(storageIds[0])
+	return models.StorageManager.FetchStorageById(storageIds[0]), nil
 }
 
 func (self *SVirtualizedGuestDriver) RequestGuestCreateInsertIso(ctx context.Context, imageId string, guest *models.SGuest, task taskman.ITask) error {

pkg/compute/guestdrivers/zstack.go

@@ -85,8 +85,8 @@ func (self *SZStackGuestDriver) GetMaxSecurityGroupCount() int {
 	return 1
 }
 
-func (self *SZStackGuestDriver) ChooseHostStorage(host *models.SHost, backend string, storageIds []string) *models.SStorage {
-	return self.chooseHostStorage(self, host, backend, storageIds)
+func (self *SZStackGuestDriver) ChooseHostStorage(host *models.SHost, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
+	return self.chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
 }
 
 func (self *SZStackGuestDriver) GetDetachDiskStatus() ([]string, error) {

pkg/compute/hostdrivers/google.go

@@ -48,7 +48,7 @@ func (self *SGoogleHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb
 	minGB := 10
 	maxGB := -1
 	switch storage.StorageType {
-	case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD:
+	case api.STORAGE_GOOGLE_PD_SSD, api.STORAGE_GOOGLE_PD_STANDARD, api.STORAGE_GOOGLE_PD_BALANCED:
 		maxGB = 65536
 	default:
 		return fmt.Errorf("Not support resize %s disk", storage.StorageType)

pkg/compute/models/cachedimages.go

@@ -272,6 +272,18 @@ func (manager *SCachedimageManager) cacheGlanceImageInfo(ctx context.Context, us
 	}
 }
 
+func (image *SCachedimage) GetStorages() ([]SStorage, error) {
+	sq := StorageManager.Query()
+	storagecacheimageSubq := StoragecachedimageManager.Query("storagecache_id").Equals("cachedimage_id", image.GetId()).SubQuery()
+	sq.Join(storagecacheimageSubq, sqlchemy.Equals(sq.Field("storagecache_id"), storagecacheimageSubq.Field("storagecache_id")))
+	storages := make([]SStorage, 0, 1)
+	err := db.FetchModelObjects(StorageManager, sq, &storages)
+	if err != nil {
+		return nil, errors.Wrap(err, "FetchModelObjects")
+	}
+	return storages, nil
+}
+
 func (manager *SCachedimageManager) GetImageById(ctx context.Context, userCred mcclient.TokenCredential, imageId string, refresh bool) (*cloudprovider.SImage, error) {
 	imgObj, _ := manager.FetchById(imageId)
 	if imgObj != nil {
@@ -472,9 +484,9 @@ func (self *SCachedimage) canDeleteLastCache() bool {
 
 func (self *SCachedimage) syncWithCloudImage(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, image cloudprovider.ICloudImage, managerId string) error {
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
-		newName, err := db.GenerateAlertName(self, image.GetName())
+		newName, err := db.GenerateAlterName(self, image.GetName())
 		if err != nil {
-			return errors.Wrap(err, "GenerateAlertName")
+			return errors.Wrap(err, "GenerateAlterName")
 		}
 		self.Name = newName
 		self.Size = image.GetSizeByte()
@@ -672,20 +684,6 @@ func (manager *SCachedimageManager) ListItemFilter(
 		return nil, errors.Wrapf(err, "SSharableBaseResourceManager.ListItemFilter")
 	}
 
-	q, err = managedResourceFilterByAccount(q, query.ManagedResourceListInput, "id", func() *sqlchemy.SQuery {
-		cachedImages := CachedimageManager.Query().SubQuery()
-		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
-		storageCaches := StoragecacheManager.Query().SubQuery()
-
-		subq := cachedImages.Query(cachedImages.Field("id"))
-		subq = subq.Join(storagecachedImages, sqlchemy.Equals(cachedImages.Field("id"), storagecachedImages.Field("cachedimage_id")))
-		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
-		return subq
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "managedResourceFilterByAccount")
-	}
-
 	q, err = manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput)
 	if err != nil {
 		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter")
@@ -696,59 +694,74 @@ func (manager *SCachedimageManager) ListItemFilter(
 		return nil, errors.Wrap(err, "SExternalizedResourceBaseManager.ListItemFilter")
 	}
 
-	q, err = managedResourceFilterByRegion(q, query.RegionalFilterListInput, "id", func() *sqlchemy.SQuery {
+	{
+		var idFilter bool
 		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
 		storageCaches := StoragecacheManager.Query().SubQuery()
-		storages := StorageManager.Query().SubQuery()
+		var storages *sqlchemy.SSubQuery
+
+		if query.Valid == nil {
+			storages = StorageManager.Query().SubQuery()
+		} else if *query.Valid {
+			idFilter = true
+			storages = StorageManager.Query().In("status", []string{api.STORAGE_ENABLED, api.STORAGE_ONLINE}).IsTrue("enabled").SubQuery()
+		} else {
+			idFilter = true
+			stroage := StorageManager.Query()
+			storages = stroage.Filter(sqlchemy.OR(sqlchemy.NotIn(stroage.Field("status"), []string{}), sqlchemy.IsFalse(stroage.Field("enabled")))).SubQuery()
+		}
 		zones := ZoneManager.Query().SubQuery()
 
 		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
 		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
 		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
 		subq = subq.Join(zones, sqlchemy.Equals(storages.Field("zone_id"), zones.Field("id")))
-		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
-		return subq
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "managedResourceFilterByRegion")
-	}
 
-	q, err = managedResourceFilterByZone(q, query.ZonalFilterListInput, "id", func() *sqlchemy.SQuery {
-		storagecachedImages := StoragecachedimageManager.Query().SubQuery()
-		storageCaches := StoragecacheManager.Query().SubQuery()
-		storages := StorageManager.Query().SubQuery()
-
-		subq := storagecachedImages.Query(storagecachedImages.Field("cachedimage_id"))
-		subq = subq.Join(storageCaches, sqlchemy.Equals(storagecachedImages.Field("storagecache_id"), storageCaches.Field("id")))
-		subq = subq.Join(storages, sqlchemy.Equals(storageCaches.Field("id"), storages.Field("storagecache_id")))
+		if len(query.HostSchedtagId) > 0 {
+			idFilter = true
+			schedTagObj, err := SchedtagManager.FetchByIdOrName(userCred, query.HostSchedtagId)
+			if err != nil {
+				if errors.Cause(err) == sql.ErrNoRows {
+					return nil, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", SchedtagManager.Keyword(), query.HostSchedtagId)
+				} else {
+					return nil, errors.Wrap(err, "SchedtagManager.FetchByIdOrName")
+				}
+			}
+			hoststorages := HoststorageManager.Query("host_id", "storage_id").SubQuery()
+			hostschedtags := HostschedtagManager.Query().Equals("schedtag_id", schedTagObj.GetId()).SubQuery()
+			subq = subq.Join(hoststorages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id")))
+			subq = subq.Join(hostschedtags, sqlchemy.Equals(hostschedtags.Field("host_id"), hoststorages.Field("host_id")))
+		}
 		subq = subq.Filter(sqlchemy.Equals(storagecachedImages.Field("status"), api.CACHED_IMAGE_STATUS_ACTIVE))
-		return subq
-	})
-	if err != nil {
-		return nil, errors.Wrap(err, "managedResourceFilterByZone")
+
+		subq = subq.Snapshot()
+
+		subq, err = managedResourceFilterByAccount(subq, query.ManagedResourceListInput, "", nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "managedResourceFilterByAccount")
+		}
+
+		subq, err = managedResourceFilterByRegion(subq, query.RegionalFilterListInput, "", nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "_managedResourceFilterByRegion")
+		}
+
+		subq, err = managedResourceFilterByZone(subq, query.ZonalFilterListInput, "", nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "_managedResourceFilterByZone")
+		}
+
+		if subq.IsAltered() {
+			idFilter = true
+		}
+
+		if idFilter {
+			q = q.In("id", subq)
+		}
 	}
 
 	if len(query.ImageType) > 0 {
-		q = q.In("image_type", query.ImageType)
-	}
-
-	if len(query.HostSchedtagId) > 0 {
-		schedTagObj, err := SchedtagManager.FetchByIdOrName(userCred, query.HostSchedtagId)
-		if err != nil {
-			if errors.Cause(err) == sql.ErrNoRows {
-				return nil, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", SchedtagManager.Keyword(), query.HostSchedtagId)
-			} else {
-				return nil, errors.Wrap(err, "SchedtagManager.FetchByIdOrName")
-			}
-		}
-		subq := StoragecachedimageManager.Query("cachedimage_id")
-		storages := StorageManager.Query("id", "storagecache_id").SubQuery()
-		hoststorages := HoststorageManager.Query("host_id", "storage_id").SubQuery()
-		hostschedtags := HostschedtagManager.Query().Equals("schedtag_id", schedTagObj.GetId()).SubQuery()
-		subq = subq.Join(storages, sqlchemy.Equals(storages.Field("storagecache_id"), subq.Field("storagecache_id")))
-		subq = subq.Join(hoststorages, sqlchemy.Equals(hoststorages.Field("storage_id"), storages.Field("id")))
-		subq = subq.Join(hostschedtags, sqlchemy.Equals(hostschedtags.Field("host_id"), hoststorages.Field("host_id")))
-		q = q.In("id", subq.SubQuery())
+		q = q.Equals("image_type", query.ImageType)
 	}
 
 	return q, nil

pkg/compute/models/cloudregions.go

@@ -118,6 +118,22 @@ func (self *SCloudregion) ValidateDeleteCondition(ctx context.Context) error {
 	return self.SEnabledStatusStandaloneResourceBase.ValidateDeleteCondition(ctx)
 }
 
+func (self *SCloudregion) GetElasticIps(managerId, eipMode string) ([]SElasticip, error) {
+	q := ElasticipManager.Query().Equals("cloudregion_id", self.Id)
+	if len(managerId) > 0 {
+		q = q.Equals("manager_id", managerId)
+	}
+	if len(eipMode) > 0 {
+		q = q.Equals("mode", eipMode)
+	}
+	eips := []SElasticip{}
+	err := db.FetchModelObjects(ElasticipManager, q, &eips)
+	if err != nil {
+		return nil, errors.Wrapf(err, "db.FetchModelObjects")
+	}
+	return eips, nil
+}
+
 func (self *SCloudregion) GetZoneQuery() *sqlchemy.SQuery {
 	zones := ZoneManager.Query()
 	if self.Id == api.DEFAULT_REGION_ID {
@@ -464,6 +480,7 @@ func (self *SCloudregion) syncWithCloudRegion(ctx context.Context, userCred mccl
 		self.SGeographicInfo = cloudRegion.GetGeographicInfo()
 		self.Provider = cloudRegion.GetProvider()
 		self.Environment = cloudRegion.GetCloudEnv()
+		self.SetEnabled(true)
 
 		self.IsEmulated = cloudRegion.IsEmulated()
 

pkg/compute/models/dbinstances.go

@@ -1467,28 +1467,37 @@ func (self *SDBInstance) SetZoneInfo(ctx context.Context, userCred mcclient.Toke
 	return err
 }
 
-func (self *SDBInstance) SetZoneIds(extInstance cloudprovider.ICloudDBInstance) {
+func (self *SDBInstance) SetZoneIds(extInstance cloudprovider.ICloudDBInstance) error {
+	region := self.GetRegion()
+	if region == nil {
+		return fmt.Errorf("failed found region for dbinstance %s", self.Name)
+	}
+	zones, err := region.GetZones()
+	if err != nil {
+		return errors.Wrapf(err, "GetZones")
+	}
+	var setZoneId = func(input string, output *string) {
+		for _, zone := range zones {
+			if strings.HasSuffix(zone.ExternalId, input) {
+				*output = zone.Id
+				break
+			}
+		}
+		return
+	}
 	zone1 := extInstance.GetZone1Id()
 	if len(zone1) > 0 {
-		zone, _ := db.FetchByExternalId(ZoneManager, zone1)
-		if zone != nil {
-			self.Zone1 = zone.GetId()
-		}
+		setZoneId(zone1, &self.Zone1)
 	}
 	zone2 := extInstance.GetZone2Id()
 	if len(zone2) > 0 {
-		zone, _ := db.FetchByExternalId(ZoneManager, zone2)
-		if zone != nil {
-			self.Zone2 = zone.GetId()
-		}
+		setZoneId(zone2, &self.Zone2)
 	}
 	zone3 := extInstance.GetZone3Id()
 	if len(zone3) > 0 {
-		zone, _ := db.FetchByExternalId(ZoneManager, zone3)
-		if zone != nil {
-			self.Zone3 = zone.GetId()
-		}
+		setZoneId(zone3, &self.Zone3)
 	}
+	return nil
 }
 
 func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extInstance cloudprovider.ICloudDBInstance) error {
@@ -1536,7 +1545,9 @@ func (self *SDBInstance) SyncWithCloudDBInstance(ctx context.Context, userCred m
 
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extInstance.GetBillingType()
-			self.ExpiredAt = extInstance.GetExpiredAt()
+			if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extInstance.IsAutoRenew()
 		}
 
@@ -1626,7 +1637,9 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 
@@ -1642,17 +1655,31 @@ func (manager *SDBInstanceManager) newFromCloudDBInstance(ctx context.Context, u
 	return &instance, nil
 }
 
+type SRdsCountStat struct {
+	TotalRdsCount  int
+	TotalCpuCount  int
+	TotalMemSizeMb int
+}
+
 func (man *SDBInstanceManager) TotalCount(
 	scope rbacutils.TRbacScope,
 	ownerId mcclient.IIdentityProvider,
 	rangeObjs []db.IStandaloneModel,
 	providers []string, brands []string, cloudEnv string,
-) (int, error) {
-	q := man.Query()
+) (SRdsCountStat, error) {
+	sq := man.Query().SubQuery()
+	q := sq.Query(sqlchemy.COUNT("total_rds_count"),
+		sqlchemy.SUM("total_cpu_count", sq.Field("vcpu_count")),
+		sqlchemy.SUM("total_mem_size_mb", sq.Field("vmem_size_mb")))
+
 	q = scopeOwnerIdFilter(q, scope, ownerId)
 	q = CloudProviderFilter(q, q.Field("manager_id"), providers, brands, cloudEnv)
 	q = RangeObjectsFilter(q, rangeObjs, q.Field("cloudregion_id"), nil, q.Field("manager_id"), nil, nil)
-	return q.CountWithError()
+
+	stat := SRdsCountStat{}
+	row := q.Row()
+	err := q.Row2Struct(row, &stat)
+	return stat, err
 }
 
 func (dbinstance *SDBInstance) GetQuotaKeys() quotas.IQuotaKeys {

pkg/compute/models/disks.go

@@ -28,6 +28,7 @@ import (
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/pkg/util/fileutils"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -1542,7 +1543,9 @@ func (manager *SDiskManager) newFromCloudDisk(ctx context.Context, userCred mccl
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		disk.BillingType = extDisk.GetBillingType()
-		disk.ExpiredAt = extDisk.GetExpiredAt()
+		if expired := extDisk.GetExpiredAt(); !expired.IsZero() {
+			disk.ExpiredAt = expired
+		}
 		disk.AutoRenew = extDisk.IsAutoRenew()
 	}
 
@@ -1834,6 +1837,10 @@ func (self *SDisk) Delete(ctx context.Context, userCred mcclient.TokenCredential
 }
 
 func (self *SDisk) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	err := self.DetachAllSnapshotpolicies(ctx, userCred)
+	if err != nil {
+		log.Errorf("unable to DetachAllSnapshotpolicies: %v", err)
+	}
 	return self.SVirtualResourceBase.Delete(ctx, userCred)
 }
 
@@ -2147,9 +2154,12 @@ func (self *SDisk) AllowPerformCancelDelete(ctx context.Context, userCred mcclie
 }
 
 func (self *SDisk) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.PendingDeleted {
+	if self.PendingDeleted && !self.Deleted {
 		err := self.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, err
+		}
+		self.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }
@@ -2448,29 +2458,80 @@ func (self *SDisk) UpdataSnapshotsBackingDisk(backingDiskId string) error {
 	return nil
 }
 
-func (manager *SDiskManager) AutoSyncExtDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredential,
-	isStart bool) {
+func (manager *SDiskManager) AutoSyncExtDiskSnapshot(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
 
-	spds, err := manager.getAutoSnapshotDisksId(true)
-	if err != nil {
-		log.Errorf("Get auto snapshot ext disks id failed: %s", err)
-		return
+	now := time.Now()
+	log.Infof("AutoSyncExtDiskSnapshot starts: %s", now)
+
+	week := now.Weekday()
+	if week == 0 {
+		week += 7
 	}
-	if len(spds) == 0 {
-		log.Infof("CronJob AutoSyncExtDiskSnapshot: No external disk need sync snapshot")
-		return
+	timePoint := now.Hour()
+
+	q := SnapshotPolicyDiskManager.Query().LE("next_sync_time", now)
+	spds := make([]SSnapshotPolicyDisk, 0)
+	err := db.FetchModelObjects(SnapshotPolicyDiskManager, q, &spds)
+	if err != nil {
+		log.Errorf("unable to FetchModelObjects: %v", err)
+	}
+	// fetch all snapshotpolicy
+	spIdSet := sets.NewString()
+	for i := range spds {
+		spIdSet.Insert(spds[i].SnapshotpolicyId)
+	}
+	sps, err := SnapshotPolicyManager.FetchAllByIds(spIdSet.UnsortedList())
+	if err != nil {
+		log.Errorf("unable to FetchAllByIds: %v", err)
+	}
+	spMap := make(map[string]*SSnapshotPolicy, len(sps))
+	for i := range sps {
+		spMap[sps[i].GetId()] = &sps[i]
 	}
 
 	for i := 0; i < len(spds); i++ {
-		disk := manager.FetchDiskById(spds[i].DiskId)
-
+		spd := &spds[i]
+		obj, err := manager.FetchById(spd.DiskId)
+		if errors.Cause(err) == sql.ErrNoRows || errors.Cause(err) == errors.ErrNotFound {
+			err := spd.RealDetach(ctx, userCred)
+			if err != nil {
+				log.Errorf("unable to detach s %q, d %q: %v", spd.SnapshotpolicyId, spd.DiskId, err)
+			}
+			continue
+		}
+		disk := obj.(*SDisk)
 		syncResult := disk.syncSnapshots(ctx, userCred)
 		if syncResult.IsError() {
 			db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SYNC_SNAPSHOT_FAIL, syncResult.Result(), userCred)
 			continue
 		}
+		sp := spMap[spd.SnapshotpolicyId]
+		repeatWeekdays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
+		timePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
+		if isInInts(int(week), repeatWeekdays) && isInInts(timePoint, timePoints) && syncResult.AddCnt == 0 {
+			// should add one
+			continue
+		}
 		db.OpsLog.LogEvent(disk, db.ACT_DISK_AUTO_SYNC_SNAPSHOT, "disk auto sync snapshot successfully", userCred)
+		_, err = db.Update(spd, func() error {
+			newNextSyncTime := spMap[spd.SnapshotpolicyId].ComputeNextSyncTime(now)
+			spd.NextSyncTime = newNextSyncTime
+			return nil
+		})
+		if err != nil {
+			log.Errorf("unable to update NextSyncTime for snapshotpolicydisk %q %q", spd.SnapshotpolicyId, spd.DiskId)
+		}
 	}
+	log.Infof("AutoSyncExtDiskSnapshot ends: %s", time.Now())
+}
+
+func isInInts(a int, array []int) bool {
+	for _, i := range array {
+		if i == a {
+			return true
+		}
+	}
+	return false
 }
 
 func (self *SDisk) syncSnapshots(ctx context.Context, userCred mcclient.TokenCredential) compare.SyncResult {

pkg/compute/models/dns_recordsets.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
@@ -119,6 +120,11 @@ func (manager *SDnsRecordSetManager) ValidateCreateData(ctx context.Context, use
 		return input, err
 	}
 	input.Name = strings.ToLower(input.Name)
+	if input.Name != "*" && input.Name != "@" {
+		if !regutils.MatchDomainName(input.Name) {
+			return input, httperrors.NewInputParameterError("invalid domain name %s", input.Name)
+		}
+	}
 	if len(input.DnsZoneId) == 0 {
 		return input, httperrors.NewMissingParameterError("dns_zone_id")
 	}

pkg/compute/models/elasticcache_instances.go

@@ -649,7 +649,9 @@ func (manager *SElasticcacheManager) newFromCloudElasticcache(ctx context.Contex
 
 	if factory.IsSupportPrepaidResources() {
 		instance.BillingType = extInstance.GetBillingType()
-		instance.ExpiredAt = extInstance.GetExpiredAt()
+		if expired := extInstance.GetExpiredAt(); !expired.IsZero() {
+			instance.ExpiredAt = expired
+		}
 		instance.AutoRenew = extInstance.IsAutoRenew()
 	}
 

pkg/compute/models/elasticips.go

@@ -248,19 +248,6 @@ func (manager *SElasticipManager) QueryDistinctExtraField(q *sqlchemy.SQuery, fi
 	return q, httperrors.ErrNotFound
 }
 
-func (manager *SElasticipManager) getEipsByRegion(region *SCloudregion, provider *SCloudprovider) ([]SElasticip, error) {
-	eips := make([]SElasticip, 0)
-	q := manager.Query().Equals("cloudregion_id", region.Id)
-	if provider != nil {
-		q = q.Equals("manager_id", provider.Id)
-	}
-	err := db.FetchModelObjects(manager, q, &eips)
-	if err != nil {
-		return nil, err
-	}
-	return eips, nil
-}
-
 func (self *SElasticip) GetRegion() *SCloudregion {
 	return CloudregionManager.FetchRegionById(self.CloudregionId)
 }
@@ -329,7 +316,7 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien
 	// remoteEips := make([]cloudprovider.ICloudEIP, 0)
 	syncResult := compare.SyncResult{}
 
-	dbEips, err := manager.getEipsByRegion(region, provider)
+	dbEips, err := region.GetElasticIps(provider.Id, api.EIP_MODE_STANDALONE_EIP)
 	if err != nil {
 		syncResult.Error(err)
 		return syncResult
@@ -478,7 +465,9 @@ func (self *SElasticip) SyncWithCloudEip(ctx context.Context, userCred mcclient.
 		factory, _ := provider.GetProviderFactory()
 		if factory != nil && factory.IsSupportPrepaidResources() {
 			self.BillingType = ext.GetBillingType()
-			self.ExpiredAt = ext.GetExpiredAt()
+			if expired := ext.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = ext.IsAutoRenew()
 		}
 
@@ -981,7 +970,7 @@ func (self *SElasticip) PerformAssociate(ctx context.Context, userCred mcclient.
 
 	// IMPORTANT: this serves as a guard against a guest to have multiple
 	// associated elastic_ips
-	seip, _ := server.GetEip()
+	seip, _ := server.GetEipOrPublicIp()
 	if seip != nil {
 		return nil, httperrors.NewInvalidStatusError("instance is already associated with eip")
 	}

pkg/compute/models/guest_actions.go

@@ -2607,7 +2607,7 @@ func (self *SGuest) RevokeAllSecgroups(ctx context.Context, userCred mcclient.To
 }
 
 func (self *SGuest) DoPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) {
-	eip, _ := self.GetEip()
+	eip, _ := self.GetEipOrPublicIp()
 	if eip != nil {
 		eip.DoPendingDelete(ctx, userCred)
 	}
@@ -2638,14 +2638,22 @@ func (model *SGuest) AllowPerformCancelDelete(ctx context.Context, userCred mccl
 }
 
 func (self *SGuest) PerformCancelDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.PendingDeleted {
+	if self.PendingDeleted && !self.Deleted {
 		err := self.DoCancelPendingDelete(ctx, userCred)
-		return nil, err
+		if err != nil {
+			return nil, errors.Wrap(err, "DoCancelPendingDelete")
+		}
+		self.RecoverUsages(ctx, userCred)
 	}
 	return nil, nil
 }
 
 func (self *SGuest) DoCancelPendingDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	eip, _ := self.GetEipOrPublicIp()
+	if eip != nil {
+		eip.DoCancelPendingDelete(ctx, userCred)
+	}
+
 	for _, guestdisk := range self.GetDisks() {
 		disk := guestdisk.GetDisk()
 		disk.DoCancelPendingDelete(ctx, userCred)
@@ -2861,12 +2869,12 @@ func (self *SGuest) AllowPerformAssociateEip(ctx context.Context, userCred mccli
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "associate-eip")
 }
 
-func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ServerAssociateEipInput) (jsonutils.JSONObject, error) {
 	if !utils.IsInStringArray(self.Status, []string{api.VM_READY, api.VM_RUNNING}) {
 		return nil, httperrors.NewInvalidStatusError("cannot associate eip in status %s", self.Status)
 	}
 
-	eip, err := self.GetEip()
+	eip, err := self.GetEipOrPublicIp()
 	if err != nil {
 		log.Errorf("Fail to get Eip %s", err)
 		return nil, httperrors.NewGeneralError(err)
@@ -2874,7 +2882,7 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T
 	if eip != nil {
 		return nil, httperrors.NewInvalidStatusError("already associate with eip")
 	}
-	eipStr := jsonutils.GetAnyString(data, []string{"eip", "eip_id"})
+	eipStr := input.EipId
 	if len(eipStr) == 0 {
 		return nil, httperrors.NewMissingParameterError("eip_id")
 	}
@@ -2949,8 +2957,8 @@ func (self *SGuest) AllowPerformDissociateEip(ctx context.Context, userCred mccl
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "dissociate-eip")
 }
 
-func (self *SGuest) PerformDissociateEip(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	eip, err := self.GetEip()
+func (self *SGuest) PerformDissociateEip(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ServerDissociateEipInput) (jsonutils.JSONObject, error) {
+	eip, err := self.GetElasticIp()
 	if err != nil {
 		log.Errorf("Fail to get Eip %s", err)
 		return nil, httperrors.NewGeneralError(err)
@@ -2966,7 +2974,7 @@ func (self *SGuest) PerformDissociateEip(ctx context.Context, userCred mcclient.
 
 	self.SetStatus(userCred, api.VM_DISSOCIATE_EIP, "associate eip")
 
-	autoDelete := jsonutils.QueryBoolean(data, "auto_delete", false)
+	autoDelete := (input.AudoDelete != nil && *input.AudoDelete)
 
 	err = eip.StartEipDissociateTask(ctx, userCred, autoDelete, "")
 	if err != nil {
@@ -3794,7 +3802,7 @@ func (self *SGuest) AllowPerformStreamDisksComplete(ctx context.Context, userCre
 func (self *SGuest) PerformStreamDisksComplete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	for _, disk := range self.GetDisks() {
 		d := disk.GetDisk()
-		if len(d.SnapshotId) > 0 {
+		if len(d.SnapshotId) > 0 && d.GetMetadata("merge_snapshot", userCred) == "true" {
 			SnapshotManager.AddRefCount(d.SnapshotId, -1)
 			d.SetMetadata(ctx, "merge_snapshot", jsonutils.JSONFalse, userCred)
 		}
@@ -4310,7 +4318,7 @@ func (guest *SGuest) PerformChangeOwner(ctx context.Context, userCred mcclient.T
 		}
 	}
 
-	if eip, _ := guest.GetEip(); eip != nil {
+	if eip, _ := guest.GetEipOrPublicIp(); eip != nil {
 		_, err := eip.PerformChangeOwner(ctx, userCred, query, input)
 		if err != nil {
 			return nil, err
@@ -5055,3 +5063,15 @@ func (self *SGuest) StartSetAutoRenewTask(ctx context.Context, userCred mcclient
 	task.ScheduleRun(nil)
 	return nil
 }
+
+func (self *SGuest) AllowPerformRemoteUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {
+	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "remote-update")
+}
+
+func (self *SGuest) PerformRemoteUpdate(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ServerRemoteUpdateInput) (jsonutils.JSONObject, error) {
+	err := self.StartRemoteUpdateTask(ctx, userCred, (input.ReplaceTags != nil && *input.ReplaceTags), "")
+	if err != nil {
+		return nil, errors.Wrap(err, "StartRemoteUpdateTask")
+	}
+	return nil, nil
+}

pkg/compute/models/guestdrivers.go

@@ -73,7 +73,7 @@ type IGuestDriver interface {
 	GetRandomNetworkTypes() []string
 
 	GetStorageTypes() []string
-	ChooseHostStorage(host *SHost, backend string, storageIds []string) *SStorage
+	ChooseHostStorage(host *SHost, diskConfig *api.DiskConfig, storageIds []string) (*SStorage, error)
 
 	StartGuestCreateTask(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, params *jsonutils.JSONDict, pendingUsage quotas.IQuota, parentTaskId string) error
 
@@ -199,6 +199,8 @@ type IGuestDriver interface {
 	CheckLiveMigrate(guest *SGuest, userCred mcclient.TokenCredential, input api.GuestLiveMigrateInput) error
 	RequestMigrate(ctx context.Context, guest *SGuest, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, task taskman.ITask) error
 	RequestLiveMigrate(ctx context.Context, guest *SGuest, userCred mcclient.TokenCredential, data *jsonutils.JSONDict, task taskman.ITask) error
+
+	RequestRemoteUpdate(ctx context.Context, guest *SGuest, userCred mcclient.TokenCredential, replaceTags bool) error
 }
 
 var guestDrivers map[string]IGuestDriver

pkg/compute/models/guests.go

@@ -48,6 +48,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/userdata"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -493,6 +494,13 @@ func (manager *SGuestManager) ListItemFilter(
 	if len(query.InstanceType) > 0 {
 		q = q.In("instance_type", query.InstanceType)
 	}
+	if query.WithHost != nil {
+		if *query.WithHost {
+			q = q.IsNotEmpty("host_id")
+		} else {
+			q = q.IsNullOrEmpty("host_id")
+		}
+	}
 
 	return q, nil
 }
@@ -1592,6 +1600,18 @@ func (self *SGuest) PostUpdate(ctx context.Context, userCred mcclient.TokenCrede
 	}
 
 	self.StartSyncTask(ctx, userCred, true, "")
+
+	if data.Contains("name") || data.Contains("__meta__") {
+		err := self.StartRemoteUpdateTask(ctx, userCred, false, "")
+		if err != nil {
+			log.Errorf("StartRemoteUpdateTask fail: %s", err)
+		}
+	}
+	// notify webhook
+	err := notifyclient.NotifyWebhook(ctx, userCred, self, notifyclient.ActionUpdate)
+	if err != nil {
+		log.Errorf("unable to NotifyWebhook: %v", err)
+	}
 }
 
 func (manager *SGuestManager) checkCreateQuota(
@@ -2465,7 +2485,10 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		extVM.Refresh()
 		if options.NameSyncResources.Contains(self.Keyword()) && !recycle {
-			self.Name = extVM.GetName()
+			newName, _ := db.GenerateAlterName(self, extVM.GetName())
+			if len(newName) > 0 && newName != self.Name {
+				self.Name = newName
+			}
 		}
 		if !self.IsFailureStatus() {
 			self.Status = extVM.GetStatus()
@@ -2570,7 +2593,9 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient.
 
 	if provider.GetFactory().IsSupportPrepaidResources() {
 		guest.BillingType = extVM.GetBillingType()
-		guest.ExpiredAt = extVM.GetExpiredAt()
+		if expired := extVM.GetExpiredAt(); !expired.IsZero() {
+			guest.ExpiredAt = expired
+		}
 		if guest.GetDriver().IsSupportSetAutoRenew() {
 			guest.AutoRenew = extVM.IsAutoRenew()
 		}
@@ -3105,12 +3130,11 @@ func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCred
 
 func filterGuestByRange(q *sqlchemy.SQuery, rangeObjs []db.IStandaloneModel, hostTypes []string, resourceTypes []string, providers []string, brands []string, cloudEnv string) *sqlchemy.SQuery {
 	hosts := HostManager.Query().SubQuery()
+	subq := hosts.Query(hosts.Field("id"))
+	subq = AttachUsageQuery(subq, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs)
 
-	q = q.Join(hosts, sqlchemy.Equals(hosts.Field("id"), q.Field("host_id")))
-	//q = q.Filter(sqlchemy.IsTrue(hosts.Field("enabled")))
-	// q = q.Filter(sqlchemy.Equals(hosts.Field("host_status"), HOST_ONLINE))
+	q = q.In("host_id", subq.SubQuery())
 
-	q = AttachUsageQuery(q, hosts, hostTypes, resourceTypes, providers, brands, cloudEnv, rangeObjs)
 	return q
 }
 
@@ -3218,7 +3242,9 @@ func _guestResourceCountQuery(
 
 	q = q.LeftJoin(isoDevSubQuery, sqlchemy.Equals(isoDevSubQuery.Field("guest_id"), guests.Field("id")))
 
-	q = filterGuestByRange(q, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv)
+	if len(rangeObjs) > 0 || len(hostTypes) > 0 || len(resourceTypes) > 0 || len(providers) > 0 || len(brands) > 0 || len(cloudEnv) > 0 {
+		q = filterGuestByRange(q, rangeObjs, hostTypes, resourceTypes, providers, brands, cloudEnv)
+	}
 
 	switch scope {
 	case rbacutils.ScopeSystem:
@@ -3426,11 +3452,11 @@ func (self *SGuest) createDiskOnStorage(ctx context.Context, userCred mcclient.T
 	return disk, nil
 }
 
-func (self *SGuest) ChooseHostStorage(host *SHost, backend string, candidate *schedapi.CandidateDisk) *SStorage {
+func (self *SGuest) ChooseHostStorage(host *SHost, diskConfig *api.DiskConfig, candidate *schedapi.CandidateDisk) (*SStorage, error) {
 	if candidate == nil || len(candidate.StorageIds) == 0 {
-		return self.GetDriver().ChooseHostStorage(host, backend, nil)
+		return self.GetDriver().ChooseHostStorage(host, diskConfig, nil)
 	}
-	return self.GetDriver().ChooseHostStorage(host, backend, candidate.StorageIds)
+	return self.GetDriver().ChooseHostStorage(host, diskConfig, candidate.StorageIds)
 }
 
 func (self *SGuest) createDiskOnHost(
@@ -3445,7 +3471,10 @@ func (self *SGuest) createDiskOnHost(
 	backupCandidate *schedapi.CandidateDisk,
 	autoAttach bool,
 ) (*SDisk, error) {
-	var storage *SStorage
+	var (
+		storage *SStorage
+		err     error
+	)
 	if len(diskConfig.Storage) > 0 {
 		_storage, err := StorageManager.FetchByIdOrName(userCred, diskConfig.Storage)
 		if err != nil {
@@ -3456,7 +3485,10 @@ func (self *SGuest) createDiskOnHost(
 		}
 		storage = _storage.(*SStorage)
 	} else {
-		storage = self.ChooseHostStorage(host, diskConfig.Backend, candidate)
+		storage, err = self.ChooseHostStorage(host, diskConfig, candidate)
+		if err != nil {
+			return nil, errors.Wrap(err, "ChooseHostStorage")
+		}
 	}
 	if storage == nil {
 		return nil, fmt.Errorf("No storage on %s to create disk for %s", host.GetName(), diskConfig.Backend)
@@ -3468,7 +3500,10 @@ func (self *SGuest) createDiskOnHost(
 	}
 	if len(self.BackupHostId) > 0 {
 		backupHost := HostManager.FetchHostById(self.BackupHostId)
-		backupStorage := self.ChooseHostStorage(backupHost, diskConfig.Backend, backupCandidate)
+		backupStorage, err := self.ChooseHostStorage(backupHost, diskConfig, backupCandidate)
+		if err != nil {
+			return nil, errors.Wrap(err, "ChooseHostStorage")
+		}
 		diff, err := db.Update(disk, func() error {
 			disk.BackupStorageId = backupStorage.Id
 			return nil
@@ -4676,8 +4711,12 @@ func (self *SGuest) isInReconcile(userCred mcclient.TokenCredential) bool {
 	return false
 }
 
-func (self *SGuest) GetEip() (*SElasticip, error) {
-	return ElasticipManager.getEipForInstance(api.EIP_ASSOCIATE_TYPE_SERVER, self.Id)
+func (self *SGuest) GetEipOrPublicIp() (*SElasticip, error) {
+	return ElasticipManager.getEip(api.EIP_ASSOCIATE_TYPE_SERVER, self.Id, "")
+}
+
+func (self *SGuest) GetElasticIp() (*SElasticip, error) {
+	return ElasticipManager.getEip(api.EIP_ASSOCIATE_TYPE_SERVER, self.Id, api.EIP_MODE_STANDALONE_EIP)
 }
 
 func (self *SGuest) GetPublicIp() (*SElasticip, error) {
@@ -4687,7 +4726,7 @@ func (self *SGuest) GetPublicIp() (*SElasticip, error) {
 func (self *SGuest) SyncVMEip(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, extEip cloudprovider.ICloudEIP, syncOwnerId mcclient.IIdentityProvider) compare.SyncResult {
 	result := compare.SyncResult{}
 
-	eip, err := self.GetEip()
+	eip, err := self.GetEipOrPublicIp()
 	if err != nil {
 		result.Error(fmt.Errorf("getEip error %s", err))
 		return result
@@ -4831,7 +4870,7 @@ func (self *SGuest) DetachScheduledTask(ctx context.Context, userCred mcclient.T
 }
 
 func (self *SGuest) DeleteEip(ctx context.Context, userCred mcclient.TokenCredential) error {
-	eip, err := self.GetEip()
+	eip, err := self.GetEipOrPublicIp()
 	if err != nil {
 		log.Errorf("Delete eip fail for get Eip %s", err)
 		return err
@@ -5132,7 +5171,7 @@ func (self *SGuest) toCreateInput() *api.ServerCreateInput {
 	if host := self.GetHost(); host != nil {
 		r.ResourceType = host.ResourceType
 	}
-	if eip, _ := self.GetEip(); eip != nil && eip.Mode == api.EIP_MODE_STANDALONE_EIP {
+	if eip, _ := self.GetEipOrPublicIp(); eip != nil && eip.Mode == api.EIP_MODE_STANDALONE_EIP {
 		r.EipBw = eip.Bandwidth
 		r.EipChargeType = eip.ChargeType
 	}
@@ -5306,7 +5345,7 @@ func (self *SGuest) getGuestUsage(guestCount int) (SQuota, SRegionQuota, error)
 	}
 	regionUsage.Port = netCount
 	// regionUsage.Bw = self.getBandwidth(false)
-	eip, err := self.GetEip()
+	eip, err := self.GetEipOrPublicIp()
 	if err != nil && errors.Cause(err) != sql.ErrNoRows {
 		return usage, regionUsage, err
 	}
@@ -5466,3 +5505,24 @@ func (manager *SGuestManager) ValidateNameLoginAccount(name string) error {
 	}
 	return httperrors.NewInputParameterError("name starts with letter, and contains letter, number and - only")
 }
+
+func (guest *SGuest) StartRemoteUpdateTask(ctx context.Context, userCred mcclient.TokenCredential, replaceTags bool, parentTaskId string) error {
+	data := jsonutils.NewDict()
+	if replaceTags {
+		data.Add(jsonutils.JSONTrue, "replace_tags")
+	}
+	if task, err := taskman.TaskManager.NewTask(ctx, "GuestRemoteUpdateTask", guest, userCred, data, parentTaskId, "", nil); err != nil {
+		log.Errorln(err)
+		return errors.Wrap(err, "Start GuestRemoteUpdateTask")
+	} else {
+		task.ScheduleRun(nil)
+	}
+	return nil
+}
+
+func (guest *SGuest) OnMetadataUpdated(ctx context.Context, userCred mcclient.TokenCredential) {
+	err := guest.StartRemoteUpdateTask(ctx, userCred, false, "")
+	if err != nil {
+		log.Errorf("StartRemoteUpdateTask fail: %s", err)
+	}
+}

pkg/compute/models/helper.go

@@ -100,7 +100,10 @@ func ValidateScheduleCreateData(ctx context.Context, userCred mcclient.TokenCred
 			return nil, err
 		}
 
-		defaultStorage := GetDriver(hypervisor).ChooseHostStorage(baremetal, "", nil)
+		defaultStorage, err := GetDriver(hypervisor).ChooseHostStorage(baremetal, &api.DiskConfig{}, nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "ChooseHostStorage")
+		}
 		if defaultStorage == nil {
 			return nil, httperrors.NewInsufficientResourceError("no valid storage on host")
 		}

pkg/compute/models/initdb.go

@@ -67,6 +67,8 @@ func InitDB() error {
 		DBInstanceNetworkManager,
 		DBInstanceAccountManager,
 		DBInstanceDatabaseManager,
+
+		SnapshotPolicyDiskManager,
 	} {
 		err := manager.InitializeData()
 		if err != nil {

pkg/compute/models/loadbalancercachedcertificates.go

@@ -95,7 +95,7 @@ func (self *SCachedLoadbalancerCertificate) ValidateDeleteCondition(ctx context.
 	men := []db.IModelManager{
 		LoadbalancerListenerManager,
 	}
-	lbcertId := self.Id
+	lbcertId := self.CertificateId
 	for _, man := range men {
 		t := man.TableSpec().Instance()
 		pdF := t.Field("pending_deleted")

pkg/compute/models/loadbalancercertificates.go

@@ -151,7 +151,6 @@ func (manager *SLoadbalancerCertificateManager) FetchCustomizeColumns(
 func (lbcert *SLoadbalancerCertificate) ValidateDeleteCondition(ctx context.Context) error {
 	men := []db.IModelManager{
 		LoadbalancerListenerManager,
-		CachedLoadbalancerCertificateManager,
 	}
 	lbcertId := lbcert.Id
 	for _, man := range men {
@@ -169,6 +168,20 @@ func (lbcert *SLoadbalancerCertificate) ValidateDeleteCondition(ctx context.Cont
 				lbcertId, n, man.KeywordPlural())
 		}
 	}
+
+	// cache delete condition
+	caches, err := lbcert.GetCachedCerts()
+	if err != nil {
+		return errors.Wrap(err, "GetCachedCerts")
+	}
+
+	for i := range caches {
+		err := caches[i].ValidateDeleteCondition(ctx)
+		if err != nil {
+			return errors.Wrap(err, "ValidateDeleteCondition")
+		}
+	}
+
 	return nil
 }
 
@@ -182,6 +195,18 @@ func (lbcert *SLoadbalancerCertificate) PerformPurge(ctx context.Context, userCr
 
 func (lbcert *SLoadbalancerCertificate) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	if !lbcert.PendingDeleted {
+		caches, err := lbcert.GetCachedCerts()
+		if err != nil {
+			return errors.Wrap(err, "GetCachedCerts")
+		}
+
+		for i := range caches {
+			err := caches[i].MarkPendingDelete(userCred)
+			if err != nil {
+				return errors.Wrap(err, "MarkPendingDelete")
+			}
+		}
+
 		return lbcert.DoPendingDelete(ctx, userCred)
 	}
 

pkg/compute/models/managedresource.go

@@ -518,7 +518,7 @@ func _managedResourceFilterByAccount(managerIdFieldName string, q *sqlchemy.SQue
 	if len(cloudaccountArr) > 0 {
 		cpq := CloudaccountManager.Query().SubQuery()
 		subcpq := cpq.Query(cpq.Field("id")).Filter(sqlchemy.OR(
-			sqlchemy.In(cpq.Field("id"), cloudaccountArr),
+			sqlchemy.In(cpq.Field("id"), stringutils2.RemoveUtf8Strings(cloudaccountArr)),
 			sqlchemy.In(cpq.Field("name"), cloudaccountArr),
 		)).SubQuery()
 		subq := CloudproviderManager.Query("id").In("cloudaccount_id", subcpq).SubQuery()

pkg/compute/models/natgateways.go

@@ -363,7 +363,9 @@ func (self *SNatGateway) SyncWithCloudNatGateway(ctx context.Context, userCred m
 		factory, _ := provider.GetProviderFactory()
 		if factory.IsSupportPrepaidResources() {
 			self.BillingType = extNat.GetBillingType()
-			self.ExpiredAt = extNat.GetExpiredAt()
+			if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+				self.ExpiredAt = expired
+			}
 			self.AutoRenew = extNat.IsAutoRenew()
 		}
 
@@ -407,7 +409,9 @@ func (manager *SNatGatewayManager) newFromCloudNatGateway(ctx context.Context, u
 	factory, _ := provider.GetProviderFactory()
 	if factory.IsSupportPrepaidResources() {
 		nat.BillingType = extNat.GetBillingType()
-		nat.ExpiredAt = extNat.GetExpiredAt()
+		if expired := extNat.GetExpiredAt(); !expired.IsZero() {
+			nat.ExpiredAt = expired
+		}
 		nat.AutoRenew = extNat.IsAutoRenew()
 	}
 

pkg/compute/models/purge.go

@@ -150,12 +150,7 @@ func (disk *SDisk) purge(ctx context.Context, userCred mcclient.TokenCredential)
 	lockman.LockObject(ctx, disk)
 	defer lockman.ReleaseObject(ctx, disk)
 
-	err := disk.DetachAllSnapshotpolicies(ctx, userCred)
-	if err != nil {
-		return errors.Wrap(err, "disk.DetachAllSnapshotpolicies")
-	}
-
-	err = disk.ValidatePurgeCondition(ctx)
+	err := disk.ValidatePurgeCondition(ctx)
 	if err != nil {
 		return err
 	}

pkg/compute/models/regionquota.go

@@ -196,7 +196,8 @@ func (self *SRegionQuota) FetchUsage(ctx context.Context) error {
 	self.ObjectGB = int(bucketUsage.Bytes / 1000 / 1000 / 1000)
 	self.ObjectCnt = bucketUsage.Objects
 
-	self.Rds, _ = DBInstanceManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
+	rdsUsage, _ := DBInstanceManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
+	self.Rds = rdsUsage.TotalRdsCount
 	self.Cache, _ = ElasticcacheManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)
 
 	self.Loadbalancer, _ = LoadbalancerManager.TotalCount(scope, ownerId, rangeObjs, providers, brands, regionKeys.CloudEnv)

pkg/compute/models/secgroupcache.go

@@ -246,18 +246,45 @@ func (manager *SSecurityGroupCacheManager) FetchCustomizeColumns(
 	manRows := manager.SManagedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	regRows := manager.SCloudregionResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 
+	cacheIds := make([]string, len(objs))
+	secIds := make([]string, len(objs))
+	vpcIds := make([]string, len(objs))
 	for i := range rows {
 		rows[i] = api.SecurityGroupCacheDetails{
 			StatusStandaloneResourceDetails: stdRows[i],
 			ManagedResourceInfo:             manRows[i],
 			CloudregionResourceInfo:         regRows[i],
 		}
-		vpc, _ := objs[i].(*SSecurityGroupCache).GetVpc()
-		if vpc != nil {
-			rows[i].Vpc = vpc.Name
+		cache := objs[i].(*SSecurityGroupCache)
+		cacheIds[i] = cache.Id
+		vpcIds[i] = cache.VpcId
+		secIds[i] = cache.SecgroupId
+	}
+	vpcMaps, _ := db.FetchIdNameMap2(VpcManager, vpcIds)
+	for i := range rows {
+		rows[i].Vpc = vpcMaps[vpcIds[i]]
+	}
+
+	secgroups := make(map[string]SSecurityGroup)
+	err := db.FetchStandaloneObjectsByIds(SecurityGroupManager, secIds, &secgroups)
+	if err != nil {
+		log.Errorf("FetchStandaloneObjectsByIds fail: %v", err)
+		return rows
+	}
+
+	virObjs := make([]interface{}, len(objs))
+	for i := range rows {
+		if secgroup, ok := secgroups[secIds[i]]; ok {
+			virObjs[i] = &secgroup
+			rows[i].ProjectId = secgroup.ProjectId
 		}
 	}
 
+	projRows := SecurityGroupManager.SProjectizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, virObjs, fields, isList)
+	for i := range rows {
+		rows[i].ProjectizedResourceInfo = projRows[i]
+	}
+
 	return rows
 }
 

pkg/compute/models/secgroups.go

@@ -943,6 +943,7 @@ func (manager *SSecurityGroupManager) newFromCloudSecgroup(ctx context.Context,
 		return nil, err
 	}
 
+	secgroup.Status = api.SECGROUP_STATUS_READY
 	secgroup.Description = extSec.GetDescription()
 	secgroup.ProjectId = provider.ProjectId
 	secgroup.DomainId = provider.DomainId

pkg/compute/models/snapshotpolicy.go

@@ -17,12 +17,15 @@ package models
 import (
 	"context"
 	"fmt"
+	"sort"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -215,7 +218,10 @@ func (manager *SSnapshotPolicyManager) OnCreateComplete(ctx context.Context, ite
 	userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
 	for i := range items {
 		sp := items[i].(*SSnapshotPolicy)
-		sp.SetStatus(userCred, api.SNAPSHOT_POLICY_READY, "create complete")
+		db.Update(sp, func() error {
+			sp.Status = api.SNAPSHOT_POLICY_READY
+			return nil
+		})
 	}
 }
 
@@ -310,11 +316,11 @@ func (sp *SSnapshotPolicy) DetachAfterDelete(ctx context.Context, userCred mccli
 
 func (sp *SSnapshotPolicy) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
 	// check if sp bind to some disks
-	sds, err := SnapshotPolicyDiskManager.FetchAllBySnapshotpolicyID(ctx, userCred, sp.GetId())
+	count, err := SnapshotPolicyDiskManager.FetchDiskCountBySPID(sp.Id)
 	if err != nil {
-		return errors.Wrap(err, "fetch bind info failed")
+		return errors.Wrap(err, "unable to FetchDiskCountBySPID")
 	}
-	if len(sds) != 0 {
+	if count != 0 {
 		return httperrors.NewBadRequestError("Couldn't delete snapshot policy binding to disks")
 	}
 	sp.SetStatus(userCred, api.SNAPSHOT_POLICY_DELETING, "")
@@ -365,8 +371,8 @@ func (sp *SSnapshotPolicy) GetExtraDetails(
 }
 
 func (sp *SSnapshotPolicy) getMoreDetails(out api.SnapshotPolicyDetails) api.SnapshotPolicyDetails {
-	out.RepeatWeekdays = SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
-	out.TimePoints = SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
+	out.RepeatWeekdaysDisplay = SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
+	out.TimePointsDisplay = SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
 	out.BindingDiskCount, _ = SnapshotPolicyDiskManager.FetchDiskCountBySPID(sp.Id)
 	return out
 }
@@ -722,6 +728,63 @@ func (self *SSnapshotPolicyManager) TimePointsToIntArray(n uint32) []int {
 	return bitmap.Uint2IntArray(n)
 }
 
+func computeNextSyncTime(weekDays, timePoints []int, base time.Time) time.Time {
+	if base.IsZero() {
+		base = time.Now()
+	}
+	base = base.Truncate(time.Hour)
+
+	baseWeekday := int(base.Weekday())
+	if baseWeekday == 0 {
+		baseWeekday = 7
+	}
+	weekDays = append(weekDays, weekDays[0]+7)
+	index := sort.SearchInts(weekDays, baseWeekday)
+	addDay := weekDays[index] - baseWeekday
+	nextTime := base.AddDate(0, 0, addDay)
+
+	// find timePoint closest to the base
+	var newHour int
+	if addDay > 0 {
+		newHour = timePoints[0]
+	} else {
+		baseHour := base.Hour()
+		index := sort.SearchInts(timePoints, baseHour)
+		index = index % len(timePoints)
+		if timePoints[index] == baseHour {
+			index = (index + 1) % len(timePoints)
+			newHour = timePoints[index]
+		} else {
+			newHour = timePoints[index]
+		}
+	}
+	nextTime = time.Date(nextTime.Year(), nextTime.Month(), nextTime.Day(), newHour, 0, 0, 0, base.Location())
+
+	if !nextTime.After(base) {
+		// If the calculated NextSyncTime and base are equal, add 1 hour to base and recursive processing.
+		return computeNextSyncTime(weekDays, timePoints, base.Add(time.Hour))
+	}
+	return nextTime
+}
+
+func (sp *SSnapshotPolicy) ComputeNextSyncTime(base time.Time) time.Time {
+	weekDays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
+	timePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)
+	if sp.RetentionDays <= 0 {
+		return computeNextSyncTime(weekDays, timePoints, base)
+	}
+	// A snapshotpolicy takes effect every Monday with keeping snapshot 3 days. So, the snapshots should be synchronized every Monday (snapshots) and Thursdays (release snapshots).
+	set := sets.NewInt(weekDays...)
+	for _, day := range weekDays {
+		newDay := (day + sp.RetentionDays) % 7
+		if newDay == 0 {
+			newDay = 7
+		}
+		set.Insert(newDay)
+	}
+	return computeNextSyncTime(set.List(), timePoints, base)
+}
+
 func (sp *SSnapshotPolicy) GenerateCreateSpParams() *cloudprovider.SnapshotPolicyInput {
 	intWeekdays := SnapshotPolicyManager.RepeatWeekdaysToIntArray(sp.RepeatWeekdays)
 	intTimePoints := SnapshotPolicyManager.TimePointsToIntArray(sp.TimePoints)

pkg/compute/models/snapshotpolicy_test.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"testing"
+	"time"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/tristate"
@@ -70,3 +71,122 @@ func TestSSnapshotPolicy_Key(t *testing.T) {
 		}
 	}
 }
+
+func TestSSnapshotPolicy_ComputeNextSyncTime(t *testing.T) {
+	timeStr := "2006-01-02 15:04:05"
+	t.Run("base test", func(t *testing.T) {
+		cases := []struct {
+			in   *SSnapshotPolicy
+			base string
+			want string
+		}{
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{2}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{4}),
+				},
+				base: "2020-10-31 00:00:00",
+				want: "2020-11-03 04:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{5, 7}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{2, 6}),
+				},
+				base: "2020-10-31 00:00:00",
+				want: "2020-11-01 02:00:00",
+			},
+		}
+		for _, c := range cases {
+			base, _ := time.Parse(timeStr, c.base)
+			want, _ := time.Parse(timeStr, c.want)
+			real := c.in.ComputeNextSyncTime(base)
+			if want != real {
+				t.Fatalf("want: %s, real: %s", want, real)
+			}
+		}
+	})
+	t.Run("same day", func(t *testing.T) {
+		cases := []struct {
+			in   *SSnapshotPolicy
+			base string
+			want string
+		}{
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6, 7}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{2}),
+				},
+				base: "2020-10-31 00:00:00",
+				want: "2020-10-31 02:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6, 7}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{2}),
+				},
+				base: "2020-10-31 02:00:00",
+				want: "2020-11-01 02:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6, 7}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{2}),
+				},
+				base: "2020-10-31 01:00:00",
+				want: "2020-10-31 02:00:00",
+			},
+		}
+		for _, c := range cases {
+			base, _ := time.Parse(timeStr, c.base)
+			want, _ := time.Parse(timeStr, c.want)
+			real := c.in.ComputeNextSyncTime(base)
+			if want != real {
+				t.Fatalf("want: %s, real: %s", want, real)
+			}
+		}
+	})
+	t.Run("retentionday", func(t *testing.T) {
+		cases := []struct {
+			in   *SSnapshotPolicy
+			base string
+			want string
+		}{
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{5}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{4}),
+					RetentionDays:  2,
+				},
+				base: "2020-10-31 04:00:00",
+				want: "2020-11-01 04:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{6}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{4}),
+					RetentionDays:  8,
+				},
+				base: "2020-10-31 04:00:00",
+				want: "2020-11-01 04:00:00",
+			},
+			{
+				in: &SSnapshotPolicy{
+					RepeatWeekdays: SnapshotPolicyManager.RepeatWeekdaysParseIntArray([]int{1, 6}),
+					TimePoints:     SnapshotPolicyManager.TimePointsParseIntArray([]int{4}),
+					RetentionDays:  4,
+				},
+				base: "2020-10-31 04:00:00",
+				want: "2020-11-02 04:00:00",
+			},
+		}
+		for _, c := range cases {
+			base, _ := time.Parse(timeStr, c.base)
+			want, _ := time.Parse(timeStr, c.want)
+			real := c.in.ComputeNextSyncTime(base)
+			if want != real {
+				t.Fatalf("want: %s, real: %s", want, real)
+			}
+		}
+	})
+}

pkg/compute/models/snapshotpolicydisks.go

@@ -20,10 +20,12 @@ import (
 	"database/sql"
 	"fmt"
 	"strings"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -73,9 +75,8 @@ type SSnapshotPolicyDisk struct {
 
 	SSnapshotPolicyResourceBase `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
 	SDiskResourceBase           `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
-	// SnapshotpolicyId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
-	// DiskId           string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"`
-	Status string `width:"36" charset:"ascii" nullable:"false" default:"init" list:"user" create:"optional"`
+	Status                      string `width:"36" charset:"ascii" nullable:"false" default:"init" list:"user" create:"optional"`
+	NextSyncTime                time.Time
 }
 
 func (sd *SSnapshotPolicyDisk) SetStatus(userCred mcclient.TokenCredential, status string, reason string) error {
@@ -171,6 +172,59 @@ func (m *SSnapshotPolicyDiskManager) FetchBySnapshotPolicyDisk(spId, diskId stri
 	return &ret[0], nil
 }
 
+func (sdm *SSnapshotPolicyDiskManager) InitializeData() error {
+	diskQ := DiskManager.Query("id").SubQuery()
+	sdQ := sdm.Query().NotIn("disk_id", diskQ)
+
+	var sds []SSnapshotPolicyDisk
+	err := db.FetchModelObjects(sdm, sdQ, &sds)
+	if err != nil {
+		return errors.Wrap(err, "unable to FetchModelObjects")
+	}
+	for i := range sds {
+		sd := &sds[i]
+		_, err := db.Update(sd, func() error {
+			return sd.MarkDelete()
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	sds = make([]SSnapshotPolicyDisk, 0)
+	q := sdm.Query().IsNullOrEmpty("next_sync_time")
+	err = db.FetchModelObjects(sdm, q, &sds)
+	if err != nil {
+		return err
+	}
+
+	// fetch all snapshotpolicy
+	spIdSet := sets.NewString()
+	for i := range sds {
+		spIdSet.Insert(sds[i].SnapshotpolicyId)
+	}
+	sps, err := SnapshotPolicyManager.FetchAllByIds(spIdSet.UnsortedList())
+	if err != nil {
+		return errors.Wrap(err, "FetchAllByIds")
+	}
+	spMap := make(map[string]*SSnapshotPolicy, len(sps))
+	for i := range sps {
+		spMap[sps[i].GetId()] = &sps[i]
+	}
+	now := time.Now()
+	for i := range sds {
+		sd := &sds[i]
+		_, err := db.Update(sd, func() error {
+			sd.NextSyncTime = spMap[sd.SnapshotpolicyId].ComputeNextSyncTime(now)
+			return nil
+		})
+		if err != nil {
+			return errors.Wrap(err, "db.Update")
+		}
+	}
+	return nil
+}
+
 func (m *SSnapshotPolicyDiskManager) FetchAllByDiskID(ctx context.Context, userCred mcclient.TokenCredential,
 	diskID string) ([]SSnapshotPolicyDisk, error) {
 
@@ -420,6 +474,8 @@ func (self *SSnapshotPolicyDiskManager) newSnapshotpolicyDisk(ctx context.Contex
 	spd := SSnapshotPolicyDisk{}
 	spd.SnapshotpolicyId = sp.GetId()
 	spd.DiskId = disk.GetId()
+	now := time.Now()
+	spd.NextSyncTime = sp.ComputeNextSyncTime(now)
 	spd.SetModelManager(self, &spd)
 
 	lockman.LockJointObject(ctx, disk, sp)
@@ -456,6 +512,16 @@ func (self *SSnapshotPolicyDiskManager) ValidateCreateData(ctx context.Context,
 	return data, nil
 }
 
+func (sd *SSnapshotPolicyDisk) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	sp, err := SnapshotPolicyManager.FetchSnapshotPolicyById(sd.SnapshotpolicyId)
+	if err != nil {
+		return err
+	}
+	now := time.Now()
+	sd.NextSyncTime = sp.ComputeNextSyncTime(now)
+	return nil
+}
+
 func (sd *SSnapshotPolicyDisk) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.
 	IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {