Merge pull request #14953 from wanyaoqi/automated-cherry-pick-of-#14952-upstream-release-3.9

Automated cherry pick of #14952: fix(region): format and check existing path

Makefile

@@ -241,6 +241,21 @@ y18n-check:
 
 .PHONY: y18n-check
 
+define hostdeployer-grpc-gen
+	set -o errexit; \
+	set -o pipefail; \
+	protoc -I pkg/hostman/hostdeployer/apis \
+		--go_out=pkg/hostman/hostdeployer/apis \
+		--go_opt=paths=source_relative \
+		--go-grpc_out=pkg/hostman/hostdeployer/apis \
+		--go-grpc_opt=paths=source_relative \
+		pkg/hostman/hostdeployer/apis/deploy.proto
+
+endef
+
+hostdeployer-grpc-gen:
+	$(hostdeployer-grpc-gen)
+
 check: fmt-check
 check: gendocgo-check
 check: goimports-check

build/docker/Dockerfile.host-deployer

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/host-deployer-base:1.1
+FROM registry.cn-beijing.aliyuncs.com/yunionio/host-deployer-base:1.2
 
 MAINTAINER "Yaoqi Wan wanyaoqi@yunionyun.com"
 

cmd/climc/entry/climc.go

@@ -68,9 +68,9 @@ type BaseOptions struct {
 	OsRegionName   string `default:"$OS_REGION_NAME" help:"Defaults to env[OS_REGION_NAME]"`
 	OsZoneName     string `default:"$OS_ZONE_NAME" help:"Defaults to env[OS_ZONE_NAME]"`
 	OsEndpointType string `default:"$OS_ENDPOINT_TYPE|internalURL" help:"Defaults to env[OS_ENDPOINT_TYPE] or internalURL" choices:"publicURL|internalURL|adminURL"`
-	ApiVersion     string `default:"$API_VERSION" help:"override default modules service api version"`
-	OutputFormat   string `default:"$CLIMC_OUTPUT_FORMAT|table" choices:"table|kv|json|flatten-table|flatten-kv" help:"output format"`
-	SUBCOMMAND     string `help:"climc subcommand" subcommand:"true"`
+	// ApiVersion     string `default:"$API_VERSION" help:"override default modules service api version"`
+	OutputFormat string `default:"$CLIMC_OUTPUT_FORMAT|table" choices:"table|kv|json|flatten-table|flatten-kv" help:"output format"`
+	SUBCOMMAND   string `help:"climc subcommand" subcommand:"true"`
 }
 
 func getSubcommandsParser() (*structarg.ArgumentParser, error) {
@@ -222,16 +222,13 @@ func newClientSession(options *BaseOptions) (*mcclient.ClientSession, error) {
 		}
 	}
 
-	if options.ApiVersion != "" {
-		mcclient.DisableApiVersionByModule()
-	}
 	session := client.NewSession(
 		context.Background(),
 		options.OsRegionName,
 		options.OsZoneName,
 		options.OsEndpointType,
 		cacheToken,
-		options.ApiVersion)
+	)
 	return session, nil
 }
 

cmd/climc/shell/compute/disks.go

@@ -45,13 +45,17 @@ func init() {
 
 	type DiskListOptions struct {
 		options.BaseListOptions
-		Unused    *bool  `help:"Show unused disks"`
-		Share     *bool  `help:"Show Share storage disks"`
-		Local     *bool  `help:"Show Local storage disks"`
-		Guest     string `help:"Guest ID or name"`
-		Storage   string `help:"Storage ID or name"`
-		Type      string `help:"Disk type" choices:"sys|data|swap|volume"`
-		CloudType string `help:"Public cloud or private cloud" choices:"Public|Private"`
+		Unused        *bool  `help:"Show unused disks"`
+		Share         *bool  `help:"Show Share storage disks"`
+		Local         *bool  `help:"Show Local storage disks"`
+		Guest         string `help:"Guest ID or name"`
+		GuestStatus   string `help:"Guest Status"`
+		OrderByServer string `help:"Order By Server"`
+		Storage       string `help:"Storage ID or name"`
+		Type          string `help:"Disk type" choices:"sys|data|swap|volume"`
+		CloudType     string `help:"Public cloud or private cloud" choices:"Public|Private"`
+
+		OrderByGuestCount string `help:"Order By Guest Count"`
 
 		BillingType string `help:"billing type" choices:"postpaid|prepaid"`
 

cmd/climc/shell/compute/serverdisks.go

@@ -156,7 +156,9 @@ func init() {
 		params := jsonutils.NewDict()
 		params.Add(jsonutils.NewString(args.DISK), "disk_id")
 		if args.DeleteDisk {
-			params.Add(jsonutils.NewInt(0), "keep_disk")
+			params.Add(jsonutils.JSONFalse, "keep_disk")
+		} else {
+			params.Add(jsonutils.JSONTrue, "keep_disk")
 		}
 		srv, err := modules.Servers.PerformAction(s, args.SERVER, "detachdisk", params)
 		if err != nil {

cmd/climc/shell/compute/servers.go

@@ -105,6 +105,9 @@ func init() {
 	cmd.Perform("calculate-record-checksum", &options.ServerIdOptions{})
 	cmd.Perform("set-class-metadata", &baseoptions.ResourceMetadataOptions{})
 	cmd.BatchPerform("enable-memclean", new(options.ServerIdsOptions))
+	cmd.Perform("qga-set-password", &options.ServerQgaSetPassword{})
+	cmd.Perform("qga-command", &options.ServerQgaCommand{})
+	cmd.Perform("set-password", &options.ServerSetPasswordOptions{})
 
 	cmd.Get("vnc", new(options.ServerIdOptions))
 	cmd.Get("desc", new(options.ServerIdOptions))
@@ -700,10 +703,11 @@ func init() {
 	})
 
 	type ServerQemuParams struct {
-		ID               string `help:"ID or name of VM"`
-		DisableIsaSerial string `help:"disable isa serial device" choices:"true|false"`
-		DisablePvpanic   string `help:"disable pvpanic device" choices:"true|false"`
-		DisableUsbKbd    string `help:"disable usb kbd" choices:"true|false"`
+		ID                string `help:"ID or name of VM"`
+		DisableIsaSerial  string `help:"disable isa serial device" choices:"true|false"`
+		DisablePvpanic    string `help:"disable pvpanic device" choices:"true|false"`
+		DisableUsbKbd     string `help:"disable usb kbd" choices:"true|false"`
+		UsbControllerType string `help:"usb controller type" choices:"usb-ehci|qemu-xhci"`
 	}
 
 	R(&ServerQemuParams{}, "server-set-qemu-params", "config qemu params", func(s *mcclient.ClientSession,
@@ -718,6 +722,9 @@ func init() {
 		if len(opts.DisableUsbKbd) > 0 {
 			params.Set("disable_usb_kbd", jsonutils.NewString(opts.DisableUsbKbd))
 		}
+		if len(opts.UsbControllerType) > 0 {
+			params.Set("usb_controller_type", jsonutils.NewString(opts.UsbControllerType))
+		}
 		result, err := modules.Servers.PerformAction(s, opts.ID, "set-qemu-params", params)
 		if err != nil {
 			return err

cmd/climc/shell/events/events.go

@@ -74,6 +74,10 @@ func doMonitorEventList(s *mcclient.ClientSession, args *EventListOptions) error
 	return DoEventList(modules.MonitorLogs, s, args)
 }
 
+func doNotifyEventList(s *mcclient.ClientSession, args *EventListOptions) error {
+	return DoEventList(modules.NotifyLogs, s, args)
+}
+
 func DoEventList(man modulebase.ResourceManager, s *mcclient.ClientSession, args *EventListOptions) error {
 	params := jsonutils.NewDict()
 	if len(args.Type) > 0 {
@@ -262,4 +266,9 @@ func init() {
 		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"migrationalert"}}
 		return doMonitorEventList(s, &nargs)
 	})
+
+	R(&TypeEventListOptions{}, "notification-event", "Show operation event logs of a notification", func(s *mcclient.ClientSession, args *TypeEventListOptions) error {
+		nargs := EventListOptions{BaseEventListOptions: args.BaseEventListOptions, Id: args.ID, Type: []string{"notification"}}
+		return doNotifyEventList(s, &nargs)
+	})
 }

cmd/climc/shell/misc/pprof.go

@@ -58,7 +58,7 @@ func init() {
 			svcUrl string
 		)
 		if len(opts.Service) > 0 {
-			svcUrl, err = s.GetServiceURL(opts.Service, "")
+			svcUrl, err = s.GetServiceURL(opts.Service, "", "")
 			if err != nil {
 				return errors.Wrapf(err, "get service %s url", opts.Service)
 			}

cmd/climc/shell/watch.go

@@ -62,9 +62,7 @@ func init() {
 		if opts.All {
 			resSets := sets.NewString()
 			mods, _ := modulebase.GetRegisterdModules()
-			for _, ress := range mods {
-				resSets.Insert(ress...)
-			}
+			resSets.Insert(mods...)
 			resources = resSets.List()
 		}
 		if len(resources) == 0 {

cmd/cloudmon/main_exec.go

@@ -1,158 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"path"
-	"time"
-
-	"yunion.io/x/log"
-	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/util/version"
-
-	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors"
-	"yunion.io/x/onecloud/pkg/cloudmon/options"
-	"yunion.io/x/onecloud/pkg/mcclient"
-)
-
-func showErrorAndExit(e error) {
-	fmt.Fprintf(os.Stderr, "%s", e)
-	fmt.Fprintln(os.Stderr)
-	os.Exit(1)
-}
-
-func newClientSession(options *options.CloudMonOptions) (*mcclient.ClientSession, error) {
-	if len(options.AuthURL) == 0 {
-		return nil, errors.Error("empty auth_url")
-	}
-	if len(options.AdminUser) == 0 {
-		return nil, errors.Error("empty admin_user")
-	}
-	if len(options.AdminPassword) == 0 {
-		return nil, errors.Error("empty admin_password")
-	}
-	if len(options.AdminProject) == 0 {
-		return nil, errors.Error("empty admin_project")
-	}
-
-	client := mcclient.NewClient(
-		options.AuthURL,
-		options.ReqTimeout,
-		options.Debug,
-		options.Insecure,
-		options.CertFile,
-		options.KeyFile,
-	)
-
-	token, err := client.AuthenticateWithSource(
-		options.AdminUser,
-		options.AdminPassword,
-		options.AdminDomain,
-		options.AdminProject,
-		options.AdminProjectDomain,
-		mcclient.AuthSourceAPI)
-	if err != nil {
-		return nil, err
-	}
-
-	session := client.NewSession(
-		context.Background(),
-		options.Region,
-		"",
-		options.EndpointType,
-		token,
-		options.ApiVersion)
-
-	return session, nil
-}
-
-func TestCron() {
-	parser, err := options.GetArgumentParser()
-	if err != nil {
-		showErrorAndExit(err)
-	}
-
-	err = parser.ParseArgs2(os.Args[1:], false, false)
-
-	opts := parser.Options().(*options.SubCloudMonOptions)
-
-	if opts.Help {
-		fmt.Println(parser.HelpString())
-		os.Exit(0)
-	}
-
-	if opts.Version {
-		fmt.Println(version.GetJsonString())
-		os.Exit(0)
-	}
-
-	if len(opts.Config) == 0 {
-		for _, p := range []string{"./etc", "/etc/yunion"} {
-			confTmp := path.Join(p, "cloudmon.conf")
-			if _, err := os.Stat(confTmp); err == nil {
-				opts.Config = confTmp
-				break
-			}
-		}
-	}
-
-	if len(opts.Config) > 0 {
-		err := parser.ParseFile(opts.Config)
-		if err != nil {
-			showErrorAndExit(err)
-		}
-	}
-
-	if err != nil {
-		showErrorAndExit(err)
-	}
-
-	parser.SetDefault()
-
-	subcmd := parser.GetSubcommand()
-	subparser := subcmd.GetSubParser()
-	if err != nil {
-		if subparser != nil {
-			fmt.Print(subparser.Usage())
-		} else {
-			fmt.Print(parser.Usage())
-		}
-		showErrorAndExit(err)
-	}
-
-	suboptions := subparser.Options()
-	if opts.Subcommand == "help" {
-		err = subcmd.Invoke(suboptions)
-	} else {
-		timout := suboptions.(*options.ReportOptions).Timeout
-		endChan := make(chan int, 1)
-		go func() {
-			ticker := time.NewTicker(time.Duration(timout) * time.Second)
-			for {
-				select {
-				case <-ticker.C:
-					log.Errorf("cmd: %s,provider: %v,end due to timeout: %d s", opts.Subcommand,
-						suboptions.(*options.ReportOptions).Provider, suboptions.(*options.ReportOptions).Timeout)
-					os.Exit(3)
-				case <-endChan:
-				}
-			}
-		}()
-
-		var session *mcclient.ClientSession
-		session, err = newClientSession(&opts.CloudMonOptions)
-		if err != nil {
-			showErrorAndExit(err)
-		}
-		err = subcmd.Invoke(session, suboptions)
-		endChan <- 1
-	}
-	if err != nil {
-		showErrorAndExit(err)
-	}
-}
-
-//func main() {
-//	TestCron()
-//}

cmd/jdcloudcli/main.go

@@ -35,7 +35,7 @@ import (
 type Options struct {
 	Debug        bool   `help:"Show debug" default:"false"`
 	AccessKey    string `help:"Access key" default:"$JDCLOUD_ACCESS_KEY"`
-	AccessSecret string `help:"Secret" default:"$JDCLOUD_ACCESS_SECRET"`
+	AccessSecret string `help:"Secret" default:"$JDCLOUD_SECRET"`
 	RegionId     string `help:"RegionId" default:"$JDCLOUD_REGION"`
 	SUBCOMMAND   string `help:"jdcloudcli subcommand" subcommand:"true"`
 }
@@ -69,7 +69,7 @@ func showErrorAndExit(e error) {
 	os.Exit(1)
 }
 
-func newClient(options *Options) (*jdcloud.SRegion, error) {
+func newClient(options *Options) (*jdcloud.SJDCloudClient, error) {
 	if len(options.AccessKey) == 0 {
 		return nil, fmt.Errorf("Missing access key")
 	}
@@ -91,15 +91,16 @@ func newClient(options *Options) (*jdcloud.SRegion, error) {
 		return cfgProxyFunc(req.URL)
 	}
 
-	cfcg := &cloudprovider.ProviderConfig{
+	cfcg := cloudprovider.ProviderConfig{
 		ProxyFunc: proxyFunc,
 	}
 
-	region := jdcloud.NewRegion(regionId, options.AccessKey, options.AccessSecret, cfcg, options.Debug)
-	if region == nil {
-		return nil, fmt.Errorf("no such region %s", regionId)
-	}
-	return region, nil
+	return jdcloud.NewJDCloudClient(
+		jdcloud.NewJDCloudClientConfig(
+			options.AccessKey,
+			options.AccessSecret,
+		).CloudproviderConfig(cfcg).Debug(options.Debug),
+	)
 }
 
 func main() {
@@ -134,11 +135,15 @@ func main() {
 		fmt.Print(subparser.HelpString())
 		return
 	}
-	var region *jdcloud.SRegion
-	region, err = newClient(options)
+	client, err := newClient(options)
 	if err != nil {
 		showErrorAndExit(err)
 	}
+	region := client.GetRegion(options.RegionId)
+	if region == nil {
+		fmt.Printf("not found region: %s", options.RegionId)
+		return
+	}
 	err = subcmd.Invoke(region, suboptions)
 	if err != nil {
 		showErrorAndExit(err)

cmd/ldapcli/shell/search.go

@@ -40,6 +40,8 @@ func init() {
 		Search      []string `help:"search conditions, in format of field:value"`
 		Field       []string `help:"retrieve field info"`
 		Scope       string   `help:"query scope" choices:"one|sub" default:"sub"`
+		PageLimit   uint32   `help:"page size" default:"100"`
+		Limit       uint32   `help:"maximal output items"`
 	}
 	shellutils.R(&LdapSearchOptions{}, "search", "search ldap", func(cli *ldaputils.SLDAPClient, args *LdapSearchOptions) error {
 		search := make(map[string]string)
@@ -50,13 +52,14 @@ func init() {
 			}
 			search[s[:colonPos]] = s[(colonPos + 1):]
 		}
-		entries, err := cli.Search(args.Base, args.Objectclass, search, "", args.Field, queryScope(args.Scope))
+		total, err := cli.Search(args.Base, args.Objectclass, search, "", args.Field, queryScope(args.Scope), args.PageLimit, args.Limit, func(offset uint32, entry *ldap.Entry) error {
+			entry.PrettyPrint(2)
+			return nil
+		})
 		if err != nil {
 			return err
 		}
-		for _, entry := range entries {
-			entry.PrettyPrint(2)
-		}
+		fmt.Println("Total:", total)
 		return nil
 	})
 

go.mod

@@ -67,7 +67,7 @@ require (
 	github.com/serialx/hashring v0.0.0-20180504054112-49a4782e9908
 	github.com/sevlyar/go-daemon v0.1.5
 	github.com/shirou/gopsutil v3.21.11+incompatible
-	github.com/sirupsen/logrus v1.6.0
+	github.com/sirupsen/logrus v1.9.0
 	github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/stretchr/testify v1.7.2
@@ -86,12 +86,11 @@ require (
 	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
 	golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32
+	golang.org/x/sys v0.0.0-20220829200755-d48e67d00261
 	golang.org/x/text v0.3.7
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20191008142428-8d021180e987
-	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c
 	google.golang.org/grpc v1.38.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/fatih/set.v0 v0.2.1
@@ -192,7 +191,6 @@ require (
 	github.com/jstemmer/go-junit-report v0.9.1 // indirect
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/lestrrat-go/iter v0.0.0-20200422075355-fc1769541911 // indirect
 	github.com/lestrrat-go/pdebug v0.0.0-20200204225717-4d6bd78da58d // indirect
@@ -248,7 +246,9 @@ require (
 	golang.org/x/tools v0.1.2 // indirect
 	google.golang.org/api v0.30.0 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
+	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.44.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -261,5 +261,4 @@ require (
 replace (
 	github.com/go-logr/logr => github.com/go-logr/logr v0.4.0
 	github.com/jaypipes/ghw => github.com/zexi/ghw v0.9.1
-	google.golang.org/grpc => google.golang.org/grpc v1.29.0
 )

go.sum

@@ -1,5 +1,6 @@
 bazil.org/fuse v0.0.0-20180421153158-65cc252bf669 h1:FNCRpXiquG1aoyqcIWVFmpTSKVcx2bQD38uZZeGtdlw=
 bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -188,9 +189,11 @@ github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/o
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coredns/coredns v1.3.0 h1:Ubjr8ZrOpEj95/A9dqQqQAG2k5DWEFdg2ZB3tUXuULA=
 github.com/coredns/coredns v1.3.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=
 github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
@@ -229,7 +232,10 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg
 github.com/elgatito/upnp v0.0.0-20180711183757-2f244d205f9a h1:2Zw3pxDRTs4nX1WCLAEm27UN0hvjZSge7EaUUQexRZw=
 github.com/elgatito/upnp v0.0.0-20180711183757-2f244d205f9a/go.mod h1:afkYpY8JAIL4341N7Zj9xJ5yTovsg6BkWfBFlCzIoF4=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/farsightsec/golang-framestream v0.0.0-20181102145529-8a0cb8ba8710 h1:QdyRyGZWLEvJG5Kw3VcVJvhXJ5tZ1MkRgqpJOEZSySM=
@@ -371,6 +377,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -448,11 +455,11 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c h1:N7A4JCA2G+j5fuFxCsJqjFU/sZe0mj8H0sSoSwbaikw=
 github.com/koding/websocketproxy v0.0.0-20181220232114-7ed82d81a28c/go.mod h1:Nn5wlyECw3iJrzi0AhIWg+AJUb4PlRQVW4/3XHH1LZA=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -630,8 +637,9 @@ github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKl
 github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9 h1:lpEzuenPuO1XNTeikEmvqYFcU37GVLl8SRNblzyvGBE=
 github.com/skip2/go-qrcode v0.0.0-20190110000554-dc11ecdae0a9/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo=
 github.com/smartystreets/assertions v0.0.0-20180607162144-eb5b59917fa2/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
@@ -755,6 +763,7 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -779,6 +788,7 @@ golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -839,6 +849,7 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180724212812-e072cadbbdc8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -899,8 +910,10 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32 h1:Js08h5hqB5xyWR789+QqueR6sDE8mk+YvpETZ+F6X9Y=
-golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY=
+golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -921,6 +934,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -992,12 +1006,14 @@ google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0 h1:yfrXXP61wVuLb0vBcG6qaOoIoqYEzOQS8jum51jkv2w=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -1029,8 +1045,22 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/grpc v1.29.0 h1:2pJjwYOdkZ9HlN4sWRYBg9ttH5bCOlsueaM+b/oYjwo=
-google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0=
+google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -1050,8 +1080,9 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzyc
 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fatih/set.v0 v0.2.1 h1:Xvyyp7LXu34P0ROhCyfXkmQCAoOUKb1E2JS9I7SE5CY=
 gopkg.in/fatih/set.v0 v0.2.1/go.mod h1:5eLWEndGL4zGGemXWrKuts+wTJR0y+w+auqUJZbmyBg=

pkg/ansibleserver/models/ansibleplaybook_instance.go

@@ -215,7 +215,7 @@ var PlaybookWorker *workmanager.SWorkManager
 
 func taskFailed(ctx context.Context, reason string) {
 	if taskId := ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID); taskId != nil {
-		session := auth.GetAdminSessionWithInternal(ctx, "", "")
+		session := auth.GetAdminSessionWithInternal(ctx, "")
 		modules.TaskFailed(&compute.DevtoolTasks, session, taskId.(string), reason)
 	} else {
 		log.Warningf("Reqeuest task failed missing task id, with reason: %s", reason)
@@ -224,7 +224,7 @@ func taskFailed(ctx context.Context, reason string) {
 
 func taskCompleted(ctx context.Context, data jsonutils.JSONObject) {
 	if taskId := ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID); taskId != nil {
-		session := auth.GetAdminSessionWithInternal(ctx, "", "")
+		session := auth.GetAdminSessionWithInternal(ctx, "")
 		modules.TaskComplete(&compute.DevtoolTasks, session, taskId.(string), data)
 	} else {
 		log.Warningf("Reqeuest task failed missing task id, with data: %v", data)

pkg/ansibleserver/models/ansibleplaybooks_validator.go

@@ -102,7 +102,7 @@ func (v *ValidatorAnsiblePlaybook) Validate(data *jsonutils.JSONDict) error {
 }
 
 func (v *ValidatorAnsiblePlaybook) getHostAccessIp(name string) (string, error) {
-	sess := mcclient_auth.GetSession(context.Background(), v.userCred, "", "")
+	sess := mcclient_auth.GetSession(context.Background(), v.userCred, "")
 	hostJson, err := compute.Hosts.Get(sess, name, nil)
 	if err != nil {
 		return "", httperrors.NewBadRequestError("cannot find host %s", name)
@@ -118,7 +118,7 @@ func (v *ValidatorAnsiblePlaybook) getHostAccessIp(name string) (string, error)
 }
 
 func (v *ValidatorAnsiblePlaybook) getServerIp(name string) (string, error) {
-	sess := mcclient_auth.GetSession(context.Background(), v.userCred, "", "")
+	sess := mcclient_auth.GetSession(context.Background(), v.userCred, "")
 	serverJson, err := compute.Servers.Get(sess, name, nil)
 	if err != nil {
 		return "", httperrors.NewBadRequestError("find server %s: %v", name, err)

pkg/apigateway/handler/auth.go

@@ -78,6 +78,7 @@ func (h *AuthHandlers) AddMethods() {
 		NewHP(handleOIDCConfiguration, "oidc", ".well-known", "openid-configuration"),
 		NewHP(handleOIDCJWKeys, "oidc", "keys"),
 		NewHP(handleOIDCUserInfo, "oidc", "user"),
+		NewHP(handleOIDCRPInitLogout, "oidc", "logout"),
 	)
 	h.AddByMethod(POST, nil,
 		NewHP(h.initTotpSecrets, "initcredential"),
@@ -89,6 +90,7 @@ func (h *AuthHandlers) AddMethods() {
 		NewHP(h.handleSsoLogin, "ssologin"),
 		NewHP(h.handleIdpInitSsoLogin, "ssologin", "<idp_id>"),
 		NewHP(handleOIDCToken, "oidc", "token"),
+		NewHP(handleOIDCRPInitLogout, "oidc", "logout"),
 	)
 
 	// auth middleware handler
@@ -149,7 +151,7 @@ func (h *AuthHandlers) GetRegionsResponse(ctx context.Context, w http.ResponseWr
 		EncryptPasswd: true,
 	}
 
-	s := auth.GetAdminSession(ctx, regions[0], "")
+	s := auth.GetAdminSession(ctx, regions[0])
 	if options.Options.ReturnFullDomainList {
 		filters := jsonutils.NewDict()
 		if len(currentDomain) > 0 {
@@ -226,7 +228,7 @@ func (h *AuthHandlers) getUser(ctx context.Context, w http.ResponseWriter, req *
 
 func getStatsInfo(ctx context.Context, req *http.Request) (jsonutils.JSONObject, error) {
 	token := AppContextToken(ctx)
-	s := auth.GetSession(ctx, token, FetchRegion(req), "")
+	s := auth.GetSession(ctx, token, FetchRegion(req))
 	params, _ := jsonutils.ParseQueryString(req.URL.RawQuery)
 
 	if params == nil {
@@ -304,7 +306,7 @@ func doTenantLogin(ctx context.Context, req *http.Request, body jsonutils.JSONOb
 }
 
 func fetchUserInfoFromToken(ctx context.Context, req *http.Request, token mcclient.TokenCredential) (jsonutils.JSONObject, error) {
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	return fetchUserInfoById(s, token.GetUserId())
 }
 
@@ -353,9 +355,14 @@ func (h *AuthHandlers) doCredentialLogin(ctx context.Context, req *http.Request,
 		idpId, _ := body.GetString("idp_id")
 		redirectUri := getSsoCallbackUrl(ctx, req, idpId)
 		token, err = processSsoLoginData(body, cliIp, redirectUri)
-		if err != nil {
-			return nil, errors.Wrap(err, "processSsoLoginData")
-		}
+		// if err != nil {
+		//	return nil, errors.Wrap(err, "processSsoLoginData")
+		// }
+	} else if body.Contains("verify_code") { // verify by mobile
+		verifyCode, _ := body.GetString("verify_code")
+		uid, _ := body.GetString("uid")
+		contactType, _ := body.GetString("contact_type")
+		token, err = processVerifyLoginData(uid, contactType, verifyCode, cliIp)
 	} else {
 		return nil, httperrors.NewInputParameterError("missing credential")
 	}
@@ -381,7 +388,7 @@ func (h *AuthHandlers) doCredentialLogin(ctx context.Context, req *http.Request,
 	}
 	uname := token.GetUserName()
 	if len(tenant) > 0 {
-		s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+		s := auth.GetAdminSession(ctx, FetchRegion(req))
 		jsonProj, e := modules.Projects.GetById(s, tenant, nil)
 		if e != nil {
 			log.Errorf("fail to find preset project %s, reset to empty", tenant)
@@ -428,7 +435,7 @@ func (h *AuthHandlers) doCredentialLogin(ctx context.Context, req *http.Request,
 		}
 	}
 	if len(tenant) == 0 {
-		s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+		s := auth.GetAdminSession(ctx, FetchRegion(req))
 		projects, e := modules.UsersV3.GetProjects(s, token.GetUserId())
 		if e == nil && len(projects.Data) > 0 {
 			projectJson := projects.Data[0]
@@ -582,7 +589,7 @@ func (h *AuthHandlers) doLogin(ctx context.Context, w http.ResponseWriter, req *
 		if err != nil {
 			return err
 		}
-		s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+		s := auth.GetAdminSession(ctx, FetchRegion(req))
 		isTotpInit, err := isUserTotpCredInitialed(s, token.GetUserId())
 		if err != nil {
 			return err
@@ -622,7 +629,7 @@ func (h *AuthHandlers) doLogin(ctx context.Context, w http.ResponseWriter, req *
 	return nil
 }
 
-func (h *AuthHandlers) postLogoutHandler(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+func doLogout(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 	token, _, _ := fetchAuthInfo(ctx, req)
 	if token != nil {
 		// valid login, log the event
@@ -631,6 +638,10 @@ func (h *AuthHandlers) postLogoutHandler(ctx context.Context, w http.ResponseWri
 	}
 	clearAuthCookie(w)
 	appsrv.DisableClientCache(w)
+}
+
+func (h *AuthHandlers) postLogoutHandler(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+	doLogout(ctx, w, req)
 	appsrv.Send(w, "")
 }
 
@@ -838,7 +849,7 @@ func isHostAgentExists(s *mcclient.ClientSession) (bool, error) {
 
 func getUserInfo(ctx context.Context, req *http.Request) (*jsonutils.JSONDict, error) {
 	token := AppContextToken(ctx)
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	/*log.Infof("getUserInfo modules.UsersV3.Get")
 	usr, err := modules.UsersV3.Get(s, token.GetUserId(), nil)
 	if err != nil {
@@ -1111,7 +1122,7 @@ func (h *AuthHandlers) doCreatePolicies(ctx context.Context, w http.ResponseWrit
 		httperrors.InvalidInputError(ctx, w, "request body is empty")
 		return
 	}
-	s := auth.GetSession(ctx, t, FetchRegion(req), "")
+	s := auth.GetSession(ctx, t, FetchRegion(req))
 	result, err := policytool.PolicyCreate(s, body)
 	if err != nil {
 		httperrors.GeneralServerError(ctx, w, err)
@@ -1131,7 +1142,7 @@ func (h *AuthHandlers) doPatchPolicy(ctx context.Context, w http.ResponseWriter,
 		httperrors.InvalidInputError(ctx, w, "request body is empty")
 		return
 	}
-	s := auth.GetSession(ctx, t, FetchRegion(req), "")
+	s := auth.GetSession(ctx, t, FetchRegion(req))
 	result, err := policytool.PolicyPatch(s, params["<policy_id>"], body)
 	if err != nil {
 		httperrors.GeneralServerError(ctx, w, err)
@@ -1147,7 +1158,7 @@ func (h *AuthHandlers) doDeletePolicies(ctx context.Context, w http.ResponseWrit
 	// 	return
 	// }
 	_, query, _ := appsrv.FetchEnv(ctx, w, req)
-	s := auth.GetSession(ctx, t, FetchRegion(req), "")
+	s := auth.GetSession(ctx, t, FetchRegion(req))
 
 	idlist, e := query.GetArray("id")
 	if e != nil || len(idlist) == 0 {
@@ -1239,7 +1250,7 @@ func (h *AuthHandlers) resetUserPassword(ctx context.Context, w http.ResponseWri
 		return
 	}
 
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	// 2.如果已开启MFA，验证 随机密码正确
 	if isMfaEnabled(user) {
 		err = authToken.VerifyTotpPasscode(s, t.GetUserId(), input.Passcode)

pkg/apigateway/handler/auth_totp.go

@@ -214,7 +214,7 @@ func initTotpSecrets(ctx context.Context, w http.ResponseWriter, req *http.Reque
 		return
 	}
 
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	code, err := doCreateUserTotpCred(s, t)
 	if err != nil {
 		httperrors.GeneralServerError(ctx, w, err)
@@ -237,7 +237,7 @@ func validatePasscodeHandler(ctx context.Context, w http.ResponseWriter, req *ht
 		return
 	}
 
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	_, _, body := appsrv.FetchEnv(ctx, w, req)
 	if body == nil {
 		httperrors.InvalidInputError(ctx, w, "request body is empty")
@@ -276,7 +276,7 @@ func resetTotpSecrets(ctx context.Context, w http.ResponseWriter, req *http.Requ
 		return
 	}
 
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	_, _, body := appsrv.FetchEnv(ctx, w, req)
 	if body == nil {
 		httperrors.InvalidInputError(ctx, w, "request body is empty")
@@ -314,7 +314,7 @@ func listTotpRecoveryQuestions(ctx context.Context, w http.ResponseWriter, req *
 		return
 	}
 
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	// 做缓存？
 	ss, err := modules.Credentials.GetRecoverySecrets(s, t.GetUserId())
 	if len(ss) == 0 {
@@ -340,7 +340,7 @@ func resetTotpRecoveryQuestions(ctx context.Context, w http.ResponseWriter, req
 		return
 	}
 
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	_, _, body := appsrv.FetchEnv(ctx, w, req)
 	if body == nil {
 		httperrors.InvalidInputError(ctx, w, "request body is empty")

pkg/apigateway/handler/backendproxy.go

@@ -68,8 +68,8 @@ func (h *SBackendServiceProxyHandler) fetchReverseEndpoint() *proxy.SEndpointFac
 			return "", httperrors.NewBadRequestError("no service")
 		}
 		endpointType := "internalURL"
-		session := auth.GetAdminSession(ctx, FetchRegion(r), "")
-		ep, err := session.GetServiceURL(serviceName, endpointType)
+		session := auth.GetAdminSession(ctx, FetchRegion(r))
+		ep, err := session.GetServiceURL(serviceName, endpointType, "")
 		if err != nil {
 			return "", httperrors.NewBadRequestError("invalid service %s: %s", serviceName, err)
 		}

pkg/apigateway/handler/csrfexempt.go

@@ -44,7 +44,7 @@ func (h *CSRFResourceHandler) Bind(app *appsrv.Application) {
 	h.SHandlers.Bind(app)
 }
 
-func getAdminSession(ctx context.Context, apiVer string, region string, w http.ResponseWriter) *mcclient.ClientSession {
+func getAdminSession(ctx context.Context, region string, w http.ResponseWriter) *mcclient.ClientSession {
 	adminToken := auth.AdminCredential()
 	if adminToken == nil {
 		httperrors.NotFoundError(ctx, w, "get admin credential is nil")
@@ -60,7 +60,7 @@ func getAdminSession(ctx context.Context, apiVer string, region string, w http.R
 	if !ret {
 		httperrors.NotFoundError(ctx, w, "illegal region %s, please contact admin", region)
 	}
-	s := auth.GetAdminSession(ctx, region, apiVer)
+	s := auth.GetAdminSession(ctx, region)
 	return s
 }
 
@@ -109,7 +109,7 @@ func fetchEnvCsrf0(ctx context.Context, w http.ResponseWriter, r *http.Request)
 		return nil, nil, nil, nil
 	}
 	log.Infof("csrf region from url: %s", region)
-	session := getAdminSession(ctx, params[APIVer], region, w)
+	session := getAdminSession(ctx, region, w)
 	log.Infof("csrf got session: %s", region)
 	if session == nil {
 		return nil, nil, nil, nil

pkg/apigateway/handler/idp.go

@@ -49,7 +49,7 @@ func getSsoBaseCallbackUrl() string {
 
 func getSsoCallbackUrl(ctx context.Context, req *http.Request, idpId string) string {
 	baseUrl := getSsoBaseCallbackUrl()
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	input := api.GetIdpSsoCallbackUriInput{
 		RedirectUri: baseUrl,
 	}
@@ -110,7 +110,7 @@ func (h *AuthHandlers) getIdpSsoRedirectUri(ctx context.Context, w http.Response
 	query.(*jsonutils.JSONDict).Set("idp_nonce", jsonutils.NewString(utils.GenRequestId(4)))
 	state := base64.URLEncoding.EncodeToString([]byte(query.String()))
 	redirectUri := getSsoCallbackUrl(ctx, req, idpId)
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	input := api.GetIdpSsoRedirectUriInput{
 		RedirectUri: redirectUri,
 		State:       state,
@@ -145,7 +145,7 @@ func findExtUserId(input string) string {
 func (h *AuthHandlers) handleIdpInitSsoLogin(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 	params := appctx.AppContextParams(ctx)
 	idpId := params["<idp_id>"]
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	resp, err := modules.IdentityProviders.Get(s, idpId, nil)
 	if err != nil {
 		httperrors.GeneralServerError(ctx, w, err)
@@ -383,7 +383,7 @@ func linkWithExistingUser(ctx context.Context, req *http.Request, idpId, idpLink
 			IdpId:       idpId,
 			IdpEntityId: extUserId,
 		}
-		s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+		s := auth.GetAdminSession(ctx, FetchRegion(req))
 		_, err = modules.UsersV3.PerformAction(s, t.GetUserId(), "link-idp", jsonutils.Marshal(linkInput))
 		if err != nil {
 			return errors.Wrap(err, "link-idp")
@@ -413,7 +413,7 @@ func handleUnlinkIdp(ctx context.Context, w http.ResponseWriter, req *http.Reque
 		return
 	}
 
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	input := api.UserUnlinkIdpInput{
 		IdpId:       idpId,
 		IdpEntityId: idpEntityId,
@@ -438,7 +438,7 @@ func fetchIdpBasicConfig(ctx context.Context, w http.ResponseWriter, req *http.R
 }
 
 func getIdpBasicConfig(ctx context.Context, req *http.Request, idpId string) (jsonutils.JSONObject, error) {
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	baseUrl := getSsoBaseCallbackUrl()
 	input := api.GetIdpSsoCallbackUriInput{
 		RedirectUri: baseUrl,
@@ -468,7 +468,7 @@ func getIdpBasicConfig(ctx context.Context, req *http.Request, idpId string) (js
 }
 
 func fetchIdpSAMLMetadata(ctx context.Context, w http.ResponseWriter, req *http.Request) {
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	params := appctx.AppContextParams(ctx)
 	idpId := params["<idp_id>"]
 	query := jsonutils.NewDict()

pkg/apigateway/handler/imageutils.go

@@ -130,7 +130,7 @@ func imageUploadHandler(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	}
 
 	token := AppContextToken(ctx)
-	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+	s := auth.GetSession(ctx, token, FetchRegion(r))
 
 	res, e := modules.Images.Upload(s, params, f, imageSize)
 	if e != nil {
@@ -213,7 +213,7 @@ func imageDownload(ctx context.Context, w http.ResponseWriter, s *mcclient.Clien
 func imageDownloadHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
 	params, query, _ := appsrv.FetchEnv(ctx, w, r)
 	token := AppContextToken(ctx)
-	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+	s := auth.GetSession(ctx, token, FetchRegion(r))
 	// input params
 	imageId, ok := params["<image_id>"]
 	if !ok || len(imageId) == 0 {
@@ -273,7 +273,7 @@ func imageDownloadByUrlHandler(ctx context.Context, w http.ResponseWriter, r *ht
 		return
 	}
 	format, _ := d.GetString("format")
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	imageDownload(ctx, w, s, id, format)
 }
 

pkg/apigateway/handler/k8s.go

@@ -76,7 +76,7 @@ func (h *K8sResourceHandler) fetchEnv(ctx context.Context, req *http.Request) (*
 	namespace, _ := params.GetString("namespace")
 	cluster, _ := params.GetString("cluster")
 	token := AppContextToken(ctx)
-	s := auth.GetSession(ctx, token, FetchRegion(req), "")
+	s := auth.GetSession(ctx, token, FetchRegion(req))
 	return &k8sResourceEnv{
 		session:   s,
 		cluster:   cluster,

pkg/apigateway/handler/misc.go

@@ -70,9 +70,9 @@ var (
 	BatchHostPXERegisterTemplate = []string{HOST_NAME, HOST_IPMI_ADDR, HOST_IPMI_USERNAME, HOST_IPMI_PASSWORD, HOST_MNG_MAC_ADDR_OPTIONAL, HOST_MNG_IP_ADDR_OPTIONAL}
 )
 
-func FetchSession(ctx context.Context, r *http.Request, apiVersion string) *mcclient.ClientSession {
+func FetchSession(ctx context.Context, r *http.Request) *mcclient.ClientSession {
 	token := AppContextToken(ctx)
-	session := auth.GetSession(ctx, token, FetchRegion(r), apiVersion)
+	session := auth.GetSession(ctx, token, FetchRegion(r))
 	return session
 }
 
@@ -279,7 +279,7 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 	}
 
 	params := jsonutils.NewDict()
-	s := FetchSession(ctx, req, "")
+	s := FetchSession(ctx, req)
 	params.Set("hosts", jsonutils.NewString(hosts.String()))
 
 	// extra params
@@ -301,8 +301,8 @@ func (mh *MiscHandler) DoBatchHostRegister(ctx context.Context, w http.ResponseW
 }
 
 func (mh *MiscHandler) DoBatchUserRegister(ctx context.Context, w http.ResponseWriter, req *http.Request) {
-	adminS := auth.GetAdminSession(ctx, FetchRegion(req), "")
-	s := FetchSession(ctx, req, "")
+	adminS := auth.GetAdminSession(ctx, FetchRegion(req))
+	s := FetchSession(ctx, req)
 	files := req.MultipartForm.File
 
 	userfiles, ok := files["users"]
@@ -543,7 +543,7 @@ func (mh *MiscHandler) postS3UploadHandler(ctx context.Context, w http.ResponseW
 	acl, _ := p["acl"]
 
 	token := AppContextToken(ctx)
-	s := auth.GetSession(ctx, token, FetchRegion(r), "")
+	s := auth.GetSession(ctx, token, FetchRegion(r))
 
 	meta := http.Header{}
 	meta.Set("Content-Type", "application/octet-stream")

pkg/apigateway/handler/oidc.go

@@ -19,6 +19,7 @@ import (
 	"encoding/base64"
 	"encoding/binary"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
@@ -106,7 +107,7 @@ func handleOIDCAuth(ctx context.Context, w http.ResponseWriter, req *http.Reques
 
 func fetchOIDCCredential(ctx context.Context, req *http.Request, clientId string) (modules.SOpenIDConnectCredential, error) {
 	var oidcSecret modules.SOpenIDConnectCredential
-	s := auth.GetAdminSession(ctx, FetchRegion(req), "")
+	s := auth.GetAdminSession(ctx, FetchRegion(req))
 	secret, err := modules.Credentials.GetById(s, clientId, nil)
 	if err != nil {
 		return oidcSecret, errors.Wrap(err, "Request Credential")
@@ -342,12 +343,14 @@ func handleOIDCConfiguration(ctx context.Context, w http.ResponseWriter, req *ht
 	authUrl := httputils.JoinPath(options.Options.ApiServer, "api/v1/auth/oidc/auth")
 	tokenUrl := httputils.JoinPath(options.Options.ApiServer, "api/v1/auth/oidc/token")
 	userinfoUrl := httputils.JoinPath(options.Options.ApiServer, "api/v1/auth/oidc/user")
+	logoutUrl := httputils.JoinPath(options.Options.ApiServer, "api/v1/auth/oidc/logout")
 	jwksUrl := httputils.JoinPath(options.Options.ApiServer, "api/v1/auth/oidc/keys")
 	conf := oidcutils.SOIDCConfiguration{
 		Issuer:                httputils.JoinPath(options.Options.ApiServer, "api/v1/auth/oidc"),
 		AuthorizationEndpoint: authUrl,
 		TokenEndpoint:         tokenUrl,
 		UserinfoEndpoint:      userinfoUrl,
+		EndSessionEndpoint:    logoutUrl,
 		JwksUri:               jwksUrl,
 		ResponseTypesSupported: []string{
 			oidcutils.OIDC_RESPONSE_TYPE_CODE,
@@ -402,7 +405,7 @@ func handleOIDCUserInfo(ctx context.Context, w http.ResponseWriter, req *http.Re
 		return
 	}
 
-	s := auth.GetAdminSession(ctx, token.Info.Region, "")
+	s := auth.GetAdminSession(ctx, token.Info.Region)
 	data, err := getUserInfo2(s, token.Info.UserId, token.Info.ProjectId, token.Info.Ip.String())
 	if err != nil {
 		httperrors.NotFoundError(ctx, w, "%v", err)
@@ -410,3 +413,93 @@ func handleOIDCUserInfo(ctx context.Context, w http.ResponseWriter, req *http.Re
 	}
 	appsrv.SendJSON(w, data)
 }
+
+type SOIDCRPInitLogoutRequest struct {
+	// RECOMMENDED. ID Token previously issued by the OP to the RP passed to the Logout Endpoint
+	// as a hint about the End-User's current authenticated session with the Client. This is used
+	// as an indication of the identity of the End-User that the RP is requesting be logged out by the OP.
+	IdTokenHint string `json:"id_token_hint"`
+	// OPTIONAL. Hint to the Authorization Server about the End-User that is logging out. The value
+	// and meaning of this parameter is left up to the OP's discretion. For instance, the value might
+	// contain an email address, phone number, username, or session identifier pertaining to the RP's
+	// session with the OP for the End-User. (This parameter is intended to be analogous to the
+	// login_hint parameter defined in Section 3.1.2.1 of OpenID Connect Core 1.0 [OpenID.Core] that
+	// is used in Authentication Requests; whereas, logout_hint is used in RP-Initiated Logout Requests.)
+	LogoutHint string `json:"logout_hint"`
+	// OPTIONAL. OAuth 2.0 Client Identifier valid at the Authorization Server. When both client_id and
+	// id_token_hint are present, the OP MUST verify that the Client Identifier matches the one used when
+	// issuing the ID Token. The most common use case for this parameter is to specify the Client Identifier
+	// when post_logout_redirect_uri is used but id_token_hint is not. Another use is for symmetrically
+	// encrypted ID Tokens used as id_token_hint values that require the Client Identifier to be specified
+	// by other means, so that the ID Tokens can be decrypted by the OP.
+	ClientId string `json:"client_id"`
+	// OPTIONAL. URI to which the RP is requesting that the End-User's User Agent be redirected after a
+	// logout has been performed. This URI SHOULD use the https scheme and MAY contain port, path, and
+	// query parameter components; however, it MAY use the http scheme, provided that the Client Type is
+	// confidential, as defined in Section 2.1 of OAuth 2.0 [RFC6749], and provided the OP allows the use
+	// of http RP URIs. The URI MAY use an alternate scheme, such as one that is intended to identify a
+	// callback into a native application. The value MUST have been previously registered with the OP,
+	// either using the post_logout_redirect_uris Registration parameter or via another mechanism. An
+	// id_token_hint is also RECOMMENDED when this parameter is included.
+	PostLogoutRedirectUri string `json:"post_logout_redirect_uri"`
+	// OPTIONAL. Opaque value used by the RP to maintain state between the logout request and the callback
+	// to the endpoint specified by the post_logout_redirect_uri parameter. If included in the logout request,
+	// the OP passes this value back to the RP using the state parameter when redirecting the User Agent back to the RP.
+	State string `json:"state"`
+	// OPTIONAL. End-User's preferred languages and scripts for the user interface, represented as a
+	// space-separated list of BCP47 [RFC5646] language tag values, ordered by preference. For instance,
+	// the value "fr-CA fr en" represents a preference for French as spoken in Canada, then French (without
+	// a region designation), followed by English (without a region designation). An error SHOULD NOT result
+	// if some or all of the requested locales are not supported by the OpenID Provider.
+	UiLocales string `json:"ui_locales"`
+}
+
+func handleOIDCRPInitLogout(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+	params, err := fetchOIDCRPInitLogoutParam(req)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+	doLogout(ctx, w, req)
+	var redirUrl string
+	if len(params.PostLogoutRedirectUri) > 0 {
+		redirUrl = params.PostLogoutRedirectUri
+		if len(params.State) > 0 {
+			redirUrl = addQuery(redirUrl, jsonutils.Marshal(map[string]string{"state": params.State}))
+		}
+	} else {
+		redirUrl = getSsoAuthCallbackUrl()
+	}
+	appsrv.SendRedirect(w, redirUrl)
+}
+
+func fetchOIDCRPInitLogoutParam(req *http.Request) (*SOIDCRPInitLogoutRequest, error) {
+	var qs string
+	if req.Method == "GET" {
+		qs = req.URL.RawQuery
+	} else if req.Method == "POST" {
+		b, err := req.GetBody()
+		if err != nil {
+			return nil, errors.Wrap(err, "GetBody")
+		}
+		defer b.Close()
+		qsBytes, err := ioutil.ReadAll(b)
+		if err != nil {
+			return nil, errors.Wrap(err, "ioutil.ReadAll")
+		}
+		qs = string(qsBytes)
+	}
+	params := SOIDCRPInitLogoutRequest{}
+	if len(qs) == 0 {
+		return &params, nil
+	}
+	qsJson, err := jsonutils.ParseQueryString(qs)
+	if err != nil {
+		return nil, errors.Wrap(err, "jsonutils.ParseQueryString")
+	}
+	err = qsJson.Unmarshal(&params)
+	if err != nil {
+		return nil, errors.Wrap(err, "qsJson.Unmarshal")
+	}
+	return &params, nil
+}

pkg/apigateway/handler/proxy.go

@@ -68,8 +68,8 @@ func getEndpointSchemeHost(endpoint string) (string, error) {
 func fetchReverseEndpoint(serviceName string) *proxy.SEndpointFactory {
 	f := func(ctx context.Context, r *http.Request) (string, error) {
 		endpointType := "internalURL"
-		session := auth.GetAdminSession(ctx, FetchRegion(r), "")
-		ep, err := session.GetServiceURL(serviceName, endpointType)
+		session := auth.GetAdminSession(ctx, FetchRegion(r))
+		ep, err := session.GetServiceURL(serviceName, endpointType, "")
 		if err != nil {
 			return "", err
 		}

pkg/apigateway/handler/request.go

@@ -49,7 +49,7 @@ type Request struct {
 func newRequest(ctx context.Context, w http.ResponseWriter, r *http.Request) *Request {
 	params := appctx.AppContextParams(ctx)
 	token := AppContextToken(ctx)
-	session := auth.GetSession(ctx, token, FetchRegion(r), params[APIVer])
+	session := auth.GetSession(ctx, token, FetchRegion(r))
 	query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
 	req := &Request{
 		ctx:     ctx,

pkg/apigateway/handler/rpc.go

@@ -83,8 +83,8 @@ func RpcHandler(ctx context.Context, w http.ResponseWriter, req *http.Request) {
 		return
 	}
 	token := AppContextToken(ctx)
-	pathParams := appctx.AppContextParams(ctx)
-	s := auth.GetSession(ctx, token, FetchRegion(req), pathParams["<apiver>"])
+	// pathParams := appctx.AppContextParams(ctx)
+	s := auth.GetSession(ctx, token, FetchRegion(req))
 	funcname := verb + utils.Kebab2Camel(callName, "-")
 	mod, e := modulebase.GetModule(s, resType)
 	if e != nil || mod == nil {

pkg/apigateway/handler/syslog_webservices.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package handler
 
 import (
@@ -156,7 +170,7 @@ func fetchSyslogMessage(r *http.Request) jsonutils.JSONObject {
 		params.Add(jsonutils.NewString(moduleType), "service")
 	}
 
-	sess := auth.GetAdminSession(nil, "", "")
+	sess := auth.GetAdminSession(nil, "")
 	logs, err := modules.Actions.List(sess, params)
 	if err != nil {
 		ret.Code = 2

pkg/apigateway/handler/verify.go

@@ -0,0 +1,24 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+func processVerifyLoginData(uid, contactType, verifyCode string, cliIp string) (mcclient.TokenCredential, error) {
+	return auth.Client().AuthenticateVerify(uid, contactType, verifyCode, "", "", "", cliIp)
+}

pkg/apigateway/report/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package report // import "yunion.io/x/onecloud/pkg/apigateway/report"

pkg/apigateway/report/report.go

@@ -69,7 +69,7 @@ func Report(ctx context.Context, userCred mcclient.TokenCredential, isStart bool
 	getFunc := func() (*sReport, error) {
 		ret := &sReport{}
 		ret.Version = version.GetShortString()
-		s := auth.GetAdminSession(ctx, options.Options.Region, "")
+		s := auth.GetAdminSession(ctx, options.Options.Region)
 		system := jsonutils.Marshal(map[string]string{"scope": "system"})
 		user, err := identity.UsersV3.Get(s, idapi.SystemAdminUser, nil)
 		if err != nil {

pkg/apigateway/service/service.go

@@ -31,7 +31,6 @@ import (
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
-	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
 func StartService() {
@@ -46,10 +45,6 @@ func StartService() {
 
 	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
 
-	if opts.DisableModuleApiVersion {
-		mcclient.DisableApiVersionByModule()
-	}
-
 	if err := clientman.InitClient(); err != nil {
 		log.Fatalf("Init client token manager: %v", err)
 	}

pkg/apihelper/apihelper.go

@@ -128,7 +128,6 @@ func (h *APIHelper) adminClientSession(ctx context.Context) *mcclient.ClientSess
 	}
 
 	region := h.opts.CommonOptions.Region
-	apiVersion := "v2"
-	h.mcclientSession = auth.GetAdminSession(ctx, region, apiVersion)
+	h.mcclientSession = auth.GetAdminSession(ctx, region)
 	return h.mcclientSession
 }

pkg/apihelper/reflect.go

@@ -104,6 +104,7 @@ func GetModels(opts *GetModelsOptions) error {
 	listOptions := options.BaseListOptions{
 		System:       options.Bool(true),
 		Admin:        options.Bool(true),
+		Scope:        "system",
 		Details:      options.Bool(opts.IncludeDetails),
 		ShowEmulated: options.Bool(opts.IncludeEmulated),
 		Filter: []string{
@@ -118,8 +119,8 @@ func GetModels(opts *GetModelsOptions) error {
 		listOptions.Filter = append(listOptions.Filter,
 			"manager_id.isnullorempty()",  // len(manager_id) > 0 is for pubcloud objects
 			"external_id.isnullorempty()", // len(external_id) > 0 is for pubcloud objects
-			"cloud_env=onpremise",
 		)
+		listOptions.CloudEnv = "onpremise"
 	}
 	if inter, ok := opts.ModelSet.(IModelSetFilter); ok {
 		filter := inter.ModelFilter()

pkg/apis/apigateway/regions.go

@@ -32,8 +32,6 @@ type SRegionsReponse struct {
 	Regions []string `json:"regions,allowempty"`
 	Domains []string `json:"domains,allowempty"`
 
-	Captcha bool `json:"captcha,allowempty"`
-
 	Idps              []SIdp `json:"idps,allowempty"`
 	ReturnFullDomains bool   `json:"return_full_domains"`
 

pkg/apis/compute/api.go

@@ -440,8 +440,9 @@ type ServerCreateInput struct {
 	// 创建后自动启动
 	// 部分云创建后会自动启动例如: 腾讯云, AWS, OpenStack, ZStack, Ucloud, Huawei, Azure, 天翼云
 	// default: false
-	AutoStart     bool            `json:"auto_start"`
-	DeployConfigs []*DeployConfig `json:"deploy_configs"`
+	AutoStart      bool            `json:"auto_start"`
+	DeployConfigs  []*DeployConfig `json:"deploy_configs"`
+	DeployTelegraf bool            `json:"deploy_telegraf"`
 
 	// 包年包月时长
 	//

pkg/apis/compute/bucket.go

@@ -16,6 +16,7 @@ package compute
 
 import (
 	"net/http"
+	"strings"
 
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/regutils"
@@ -63,6 +64,33 @@ type BucketDetails struct {
 	AccessUrls []cloudprovider.SBucketAccessUrl `json:"access_urls"`
 }
 
+func (self BucketDetails) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"brand":          self.Brand,
+		"cloudregion":    self.Cloudregion,
+		"cloudregion_id": self.CloudregionId,
+		"domain_id":      self.DomainId,
+		"oss_id":         self.Id,
+		"oss_name":       self.Name,
+		"project_domain": self.ProjectDomain,
+		"region_ext_id":  self.RegionExtId,
+		"status":         self.Status,
+		"tenant":         self.Project,
+		"tenant_id":      self.ProjectId,
+	}
+	for k, v := range self.Metadata {
+		if strings.HasPrefix(k, apis.USER_TAG_PREFIX) {
+			ret[k] = v
+		}
+	}
+	return ret
+}
+
+func (self BucketDetails) GetMetricPairs() map[string]string {
+	ret := map[string]string{}
+	return ret
+}
+
 type BucketObjectsActionInput struct {
 	Key []string
 }

pkg/apis/compute/cloudaccount.go

@@ -15,7 +15,9 @@
 package compute
 
 import (
+	"fmt"
 	"reflect"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/gotypes"
@@ -317,6 +319,29 @@ type CloudaccountDetail struct {
 	ProjectMappingResourceInfo
 }
 
+func (self CloudaccountDetail) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"cloudaccount_id":   self.Id,
+		"cloudaccount_name": self.Name,
+		"brand":             self.Brand,
+		"domain_id":         self.DomainId,
+		"project_domain":    self.ProjectDomain,
+	}
+	for k, v := range self.Metadata {
+		if strings.HasPrefix(k, apis.USER_TAG_PREFIX) {
+			ret[k] = v
+		}
+	}
+	return ret
+}
+
+func (self CloudaccountDetail) GetMetricPairs() map[string]string {
+	ret := map[string]string{
+		"balance": fmt.Sprintf("%.2f", self.Balance),
+	}
+	return ret
+}
+
 type CloudaccountUpdateInput struct {
 	apis.EnabledStatusInfrasResourceBaseUpdateInput
 

pkg/apis/compute/dbinstance.go

@@ -15,6 +15,7 @@
 package compute
 
 import (
+	"strings"
 	"time"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -317,6 +318,42 @@ type DBInstanceDetails struct {
 	Zone2Name string `json:"zone2_name"`
 	// Zone3名称
 	Zone3Name string `json:"zone3_name"`
+
+	Databases []apis.IdNameDetails `json:"databases"`
+}
+
+func (self DBInstanceDetails) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"rds_id":         self.Id,
+		"rds_name":       self.Name,
+		"zone":           self.Zone1Name,
+		"zone_id":        self.Zone1,
+		"status":         self.Status,
+		"engine":         self.Engine,
+		"server_type":    strings.ToLower(self.Engine),
+		"cloudregion":    self.Cloudregion,
+		"cloudregion_id": self.CloudregionId,
+		"region_ext_id":  self.RegionExtId,
+		"tenant":         self.Project,
+		"tenant_id":      self.ProjectId,
+		"brand":          self.Brand,
+		"domain_id":      self.DomainId,
+		"project_domain": self.ProjectDomain,
+	}
+	if len(self.IpAddrs) > 0 {
+		ret["rds_ip"] = strings.ReplaceAll(self.IpAddrs, ",", "|")
+	}
+	for k, v := range self.Metadata {
+		if strings.HasPrefix(k, apis.USER_TAG_PREFIX) {
+			ret[k] = v
+		}
+	}
+	return ret
+}
+
+func (self DBInstanceDetails) GetMetricPairs() map[string]string {
+	ret := map[string]string{}
+	return ret
 }
 
 type DBInstanceResourceInfoBase struct {

pkg/apis/compute/disk.go

@@ -143,6 +143,9 @@ type DiskListInput struct {
 
 	FsFormat string `json:"fs_format"`
 
+	OrderByServer string `json:"order_by_server" choices:"asc|desc"`
+
+	OrderByGuestCount string `json:"order_by_guest_count" choices:"asc|desc"`
 	// 镜像
 	ImageId string `json:"image_id"`
 	// swagger:ignore

pkg/apis/compute/disk_const.go

@@ -65,3 +65,11 @@ const (
 )
 
 const DISK_META_EXISTING_PATH = "disk_existing_path"
+
+const (
+	DISK_DRIVER_VIRTIO = "virtio"
+	DISK_DRIVER_SCSI   = "scsi"
+	DISK_DRIVER_PVSCSI = "pvscsi"
+	DISK_DRIVER_IDE    = "ide"
+	DISK_DRIVER_SATA   = "sata"
+)

pkg/apis/compute/elasticcache.go

@@ -15,6 +15,7 @@
 package compute
 
 import (
+	"strings"
 	"time"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -37,6 +38,37 @@ type ElasticcacheDetails struct {
 	SlaveZoneInfos []apis.StandaloneShortDesc `json:"slave_zone_infos"`
 }
 
+func (self ElasticcacheDetails) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"redis_id":       self.Id,
+		"redis_ip":       self.PrivateIpAddr,
+		"redis_name":     self.Name,
+		"zone":           self.Zone,
+		"zone_id":        self.ZoneId,
+		"zone_ext_id":    self.ZoneExtId,
+		"status":         self.Status,
+		"cloudregion":    self.Cloudregion,
+		"cloudregion_id": self.CloudregionId,
+		"region_ext_id":  self.RegionExtId,
+		"tenant":         self.Project,
+		"tenant_id":      self.ProjectId,
+		"brand":          self.Brand,
+		"domain_id":      self.DomainId,
+		"project_domain": self.ProjectDomain,
+	}
+	for k, v := range self.Metadata {
+		if strings.HasPrefix(k, apis.USER_TAG_PREFIX) {
+			ret[k] = v
+		}
+	}
+	return ret
+}
+
+func (self ElasticcacheDetails) GetMetricPairs() map[string]string {
+	ret := map[string]string{}
+	return ret
+}
+
 type ElasticcacheResourceInfo struct {
 	// 弹性缓存实例名称
 	Elasticcache string `json:"elasticcache"`

pkg/apis/compute/guest_const.go

@@ -154,6 +154,9 @@ const (
 
 	VM_SYNC_ISOLATED_DEVICE_FAILED = "sync_isolated_device_failed"
 
+	VM_QGA_SET_PASSWORD      = "qga_set_password"
+	VM_QGA_COMMAND_EXECUTING = "qga_command_executing"
+
 	SHUTDOWN_STOP      = "stop"
 	SHUTDOWN_TERMINATE = "terminate"
 
@@ -186,6 +189,13 @@ const (
 	HYPERVISOR_DEFAULT = HYPERVISOR_KVM
 )
 
+const (
+	QGA_STATUS_UNKNOWN        = "unknown"
+	QGA_STATUS_EXCUTING       = "executing"
+	QGA_STATUS_EXECUTE_FAILED = "execute_failed"
+	QGA_STATUS_AVAILABLE      = "available"
+)
+
 const (
 	CPU_MODE_QEMU = "qemu"
 	CPU_MODE_HOST = "host"

pkg/apis/compute/guests.go

@@ -16,6 +16,7 @@ package compute
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -74,6 +75,7 @@ type ServerListInput struct {
 	// enum: asc,desc
 	OrderByDisk string `json:"order_by_disk"`
 
+	OrderByIp string `json:"order_by_ip"`
 	// 根据ip查找机器
 	IpAddr string `json:"ip_addr"`
 
@@ -244,6 +246,49 @@ type ServerDetails struct {
 	ScalingGroupId string `json:"scaling_group_id"`
 }
 
+func (self ServerDetails) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"res_type":            "guest",
+		"is_vm":               "true",
+		"paltform":            self.Hypervisor,
+		"host":                self.Host,
+		"host_id":             self.HostId,
+		"vm_id":               self.Id,
+		"vm_name":             self.Name,
+		"zone":                self.Zone,
+		"zone_id":             self.ZoneId,
+		"zone_ext_id":         self.ZoneExtId,
+		"os_type":             self.OsType,
+		"status":              self.Status,
+		"cloudregion":         self.Cloudregion,
+		"cloudregion_id":      self.CloudregionId,
+		"region_ext_id":       self.RegionExtId,
+		"tenant":              self.Project,
+		"tenant_id":           self.ProjectId,
+		"brand":               self.Brand,
+		"vm_scaling_group_id": self.ScalingGroupId,
+		"domain_id":           self.DomainId,
+		"project_domain":      self.ProjectDomain,
+		"account":             self.Account,
+		"account_id":          self.AccountId,
+	}
+	for k, v := range self.Metadata {
+		if strings.HasPrefix(k, apis.USER_TAG_PREFIX) {
+			ret[k] = v
+		}
+	}
+	return ret
+}
+
+func (self ServerDetails) GetMetricPairs() map[string]string {
+	ret := map[string]string{
+		"vcpu_count": fmt.Sprintf("%d", self.VcpuCount),
+		"vmem_size":  fmt.Sprintf("%d", self.VmemSize),
+		"disk":       fmt.Sprintf("%d", self.DiskSizeMb),
+	}
+	return ret
+}
+
 // GuestDiskInfo describe the information of disk on the guest.
 type GuestDiskInfo struct {
 	Id          string `json:"id"`
@@ -617,6 +662,8 @@ type ServerDeployInput struct {
 
 	// swagger: ignore
 	DeployConfigs []*DeployConfig `json:"deploy_configs"`
+	// swagger: ignore
+	DeployTelegraf bool `json:"deploy_telegraf"`
 }
 
 type ServerUserDataInput struct {
@@ -844,3 +891,21 @@ type ServerQemuInfo struct {
 	Version string `json:"version"`
 	Cmdline string `json:"cmdline"`
 }
+
+type ServerQgaSetPasswordInput struct {
+	Username string
+	Password string
+}
+
+type ServerQgaCommandInput struct {
+	Command string
+}
+
+type ServerSetPasswordInput struct {
+	Username string
+	Password string
+
+	// deploy params
+	ResetPassword bool
+	AutoStart     bool
+}

pkg/apis/compute/host.go

@@ -15,6 +15,8 @@
 package compute
 
 import (
+	"strings"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -192,7 +194,7 @@ type HostDetails struct {
 	// CPU超售比
 	CpuCommitBound float32 `json:"cpu_commit_bound"`
 	// 内存超售比
-	MemCommitBound float32 `json:"mem_commint_bound"`
+	MemCommitBound float32 `json:"mem_commit_bound"`
 	// 存储大小
 	Storage int64 `json:"storage"`
 	// 已使用存储大小
@@ -228,6 +230,38 @@ type HostDetails struct {
 	SysError string `json:"sys_error"`
 }
 
+func (self HostDetails) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"host_id":        self.Id,
+		"host_ip":        self.AccessIp,
+		"host":           self.Name,
+		"zone":           self.Zone,
+		"zone_id":        self.ZoneId,
+		"zone_ext_id":    self.ZoneExtId,
+		"status":         self.Status,
+		"cloudregion":    self.Cloudregion,
+		"cloudregion_id": self.CloudregionId,
+		"region_ext_id":  self.RegionExtId,
+		"brand":          self.Brand,
+		"domain_id":      self.DomainId,
+		"project_domain": self.ProjectDomain,
+		"account":        self.Account,
+		"res_type":       "host",
+		"account_id":     self.AccountId,
+	}
+	for k, v := range self.Metadata {
+		if strings.HasPrefix(k, apis.USER_TAG_PREFIX) {
+			ret[k] = v
+		}
+	}
+	return ret
+}
+
+func (self HostDetails) GetMetricPairs() map[string]string {
+	ret := map[string]string{}
+	return ret
+}
+
 type HostResourceInfo struct {
 	// 归属云订阅ID
 	ManagerId string `json:"manager_id"`

pkg/apis/compute/host_tap.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package compute
 
 const (

pkg/apis/compute/kube_cluster.go

@@ -45,10 +45,34 @@ type KubeClusterCreateInput struct {
 
 type KubeClusterDetails struct {
 	apis.EnabledStatusInfrasResourceBaseDetails
+
+	SKubeCluster
 	ManagedResourceInfo
 	CloudregionResourceInfo
 }
 
+func (self KubeClusterDetails) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"res_type":       "kube_cluster",
+		"cluster_id":     self.ExternalClusterId,
+		"cluster_name":   self.Name,
+		"status":         self.Status,
+		"cloudregion":    self.Cloudregion,
+		"cloudregion_id": self.CloudregionId,
+		"region_ext_id":  self.RegionExtId,
+		"domain_id":      self.DomainId,
+		"project_domain": self.ProjectDomain,
+		"account":        self.Account,
+		"account_id":     self.AccountId,
+	}
+	return ret
+}
+
+func (self KubeClusterDetails) GetMetricPairs() map[string]string {
+	ret := map[string]string{}
+	return ret
+}
+
 type KubeClusterUpdateInput struct {
 	apis.EnabledStatusInfrasResourceBaseUpdateInput
 }

pkg/apis/compute/networkaddress.go

@@ -53,6 +53,8 @@ type NetworkAddressListInput struct {
 	NetworkFilterListInput
 
 	GuestId []string
+
+	ManagedResourceListInput
 }
 
 type NetworkAddressDetails struct {

pkg/apis/compute/sku_const.go

@@ -83,6 +83,9 @@ type ServerSkuListInput struct {
 
 	// 预付费状态
 	PrepaidStatus string `json:"prepaid_status"`
+
+	// 按套餐名称去重
+	Distinct bool `json:"distinct"`
 }
 
 type ElasticcacheSkuListInput struct {

pkg/apis/compute/storage.go

@@ -15,6 +15,9 @@
 package compute
 
 import (
+	"strconv"
+	"strings"
+
 	"yunion.io/x/jsonutils"
 
 	"yunion.io/x/onecloud/pkg/apis"
@@ -147,6 +150,34 @@ type StorageDetails struct {
 	CommitBound float32 `json:"commit_bound"`
 }
 
+func (self StorageDetails) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"storage_id":     self.Id,
+		"storage_name":   self.Name,
+		"brand":          self.Brand,
+		"domain_id":      self.DomainId,
+		"project_domain": self.ProjectDomain,
+	}
+	for k, v := range self.Metadata {
+		if strings.HasPrefix(k, apis.USER_TAG_PREFIX) {
+			ret[k] = v
+		}
+	}
+	return ret
+}
+
+func (self StorageDetails) GetMetricPairs() map[string]string {
+	usageActive := "0"
+	if self.Capacity > 0 {
+		usageActive = strconv.FormatFloat(float64(self.ActualCapacityUsed/self.Capacity*100.0), 'f', -1, 64)
+	}
+	ret := map[string]string{
+		"free":         strconv.FormatFloat(float64(self.Capacity-self.ActualCapacityUsed), 'f', 2, 64),
+		"usage_active": usageActive,
+	}
+	return ret
+}
+
 type StorageResourceInfo struct {
 	// 归属云订阅ID
 	ManagerId string `json:"manager_id"`

pkg/apis/const.go

@@ -33,6 +33,7 @@ const (
 	SERVICE_TYPE_MONITOR           = "monitor"
 	SERVICE_TYPE_LOG               = "log"
 	SERVICE_TYPE_REGION            = "compute"
+	SERVICE_TYPE_CLOUDMON          = "cloudmon"
 
 	SERVICE_TYPE_ETCD     = "etcd"
 	SERVICE_TYPE_INFLUXDB = "influxdb"

pkg/apis/host/types.go

@@ -31,3 +31,9 @@ type HostTopology struct {
 type HostCPUInfo struct {
 	*cpu.Info
 }
+
+type GuestSetPasswordRequest struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+	Crypted  bool   `json:"crypted"`
+}

pkg/apis/identity/consts.go

@@ -37,6 +37,7 @@ const (
 	AUTH_METHOD_SAML     = "saml"
 	AUTH_METHOD_OIDC     = "oidc"
 	AUTH_METHOD_OAuth2   = "oauth2"
+	AUTH_METHOD_VERIFY   = "verify"
 
 	// AUTH_METHOD_ID_PASSWORD = 1
 	// AUTH_METHOD_ID_TOKEN    = 2

pkg/apis/list.go

@@ -377,3 +377,8 @@ type OpsLogListInput struct {
 
 	Until time.Time `json:"until"`
 }
+
+type IdNameDetails struct {
+	Id   string `json:"id"`
+	Name string `json:"name"`
+}

pkg/apis/monitor/alert.go

@@ -159,6 +159,8 @@ type AlertListInput struct {
 }
 
 type AlertDetails struct {
+	SAlert
+
 	apis.StatusStandaloneResourceDetails
 	apis.ScopedResourceBaseInfo
 }

pkg/apis/monitor/alertrecord.go

@@ -42,6 +42,8 @@ type AlertRecordListInput struct {
 }
 
 type AlertRecordDetails struct {
+	SAlertRecord
+
 	apis.StatusStandaloneResourceDetails
 	apis.ScopedResourceBaseInfo
 
@@ -50,6 +52,19 @@ type AlertRecordDetails struct {
 	TriggerTime time.Time
 }
 
+func (self AlertRecordDetails) GetMetricTags() map[string]string {
+	ret := map[string]string{
+		"alert_id":       self.AlertId,
+		"alert_name":     self.AlertName,
+		"domain_id":      self.DomainId,
+		"project_domain": self.ProjectDomain,
+		"res_type":       self.ResType,
+		"tenant":         self.Tenant,
+		"tenant_id":      self.TenantId,
+	}
+	return ret
+}
+
 type AlertRecordCreateInput struct {
 	apis.StandaloneResourceCreateInput
 

pkg/apis/notify/notification.go

@@ -63,6 +63,7 @@ type ReceiveDetail struct {
 	ReceiverId   string    `json:"receiver_id"`
 	ReceiverName string    `json:"receiver_name"`
 	Contact      string    `json:"contact"`
+	ReceiverType string    `json:"receiver_type"`
 	SendAt       time.Time `json:"sendAt"`
 	SendBy       string    `json:"send_by"`
 	Status       string    `json:"status"`

pkg/apis/notify/receiver.go

@@ -43,6 +43,9 @@ type ReceiverCreateInput struct {
 	// description: enabled contact types for user
 	// example: {"email", "mobile", "feishu", "dingtalk", "workwx"}
 	EnabledContactTypes []string `json:"enabled_contact_types"`
+
+	// force verified if admin create the records
+	ForceVerified bool `json:"force_verified"`
 }
 
 type SInternationalMobile struct {
@@ -129,6 +132,8 @@ type ReceiverUpdateInput struct {
 	// description: enabled contacts for user
 	// example: {"email", "mobile", "feishu", "dingtalk", "workwx"}
 	EnabledContactTypes []string `json:"enabled_contact_types"`
+
+	ForceVerified bool `json:"force_verified"`
 }
 
 type ReceiverTriggerVerifyInput struct {

pkg/appsrv/appsrv.go

@@ -67,6 +67,8 @@ type Application struct {
 	idleConnsClosed chan struct{}
 	httpServer      *http.Server
 	slaveHttpServer *http.Server
+
+	isTLS bool
 }
 
 const (
@@ -315,6 +317,9 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 	w.Header().Set("Server", "Yunion AppServer/Go/2018.4")
 	w.Header().Set("X-Frame-Options", "SAMEORIGIN")
 	w.Header().Set("X-XSS-Protection", "1; mode=block")
+	if app.isTLS {
+		w.Header().Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains")
+	}
 	isCors := app.handleCORS(w, r)
 	handler := app.getRoot(r.Method).Match(segs, params)
 	if handler != nil {
@@ -500,6 +505,7 @@ func (app *Application) ListenAndServeWithoutCleanup(addr, certFile, keyFile str
 }
 
 func (app *Application) ListenAndServeTLSWithCleanup2(addr string, certFile, keyFile string, onStop func(), isMaster bool) {
+	app.isTLS = true
 	httpSrv := app.initServer(addr)
 	if isMaster {
 		app.addDefaultHandlers()

pkg/appsrv/send.go

@@ -156,7 +156,7 @@ func SendStream(w http.ResponseWriter, isPartial bool, hdr http.Header, stream i
 
 func SendRedirect(w http.ResponseWriter, redirectUrl string) {
 	w.Header().Set("Location", redirectUrl)
-	w.WriteHeader(301)
+	w.WriteHeader(302)
 	w.Write([]byte{})
 }
 

pkg/baremetal/agent.go

@@ -81,7 +81,7 @@ func (agent *SBaremetalAgent) GetZoneId() string {
 }
 
 func (agent *SBaremetalAgent) GetAdminSession() *mcclient.ClientSession {
-	return auth.GetAdminSession(context.TODO(), o.Options.Region, "v2")
+	return auth.GetAdminSession(context.TODO(), o.Options.Region)
 }
 
 func (agent *SBaremetalAgent) GetListenIP() (net.IP, error) {

pkg/baremetal/cronjobs.go

@@ -138,7 +138,7 @@ func eventToJson(e redfish.SEvent) *jsonutils.JSONDict {
 
 func fetchLogs(baremetal *SBaremetalInstance, ctx context.Context, logType string) error {
 	params := jsonutils.NewDict()
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	params.Add(jsonutils.NewInt(1), "limit")
 	params.Add(jsonutils.NewString(baremetal.GetId()), "host_id")
 	params.Add(jsonutils.NewString(logType), "type")
@@ -216,8 +216,8 @@ func (job *SSendMetricsJob) Name() string {
 }
 
 func (job *SSendMetricsJob) Do(ctx context.Context, now time.Time) error {
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "")
-	urls, err := s.GetServiceURLs("influxdb", o.Options.SessionEndpointType)
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
+	urls, err := s.GetServiceURLs("influxdb", o.Options.SessionEndpointType, "")
 	if err != nil {
 		return errors.Wrap(err, "s.GetServiceURLs")
 	}

pkg/baremetal/manager.go

@@ -2847,7 +2847,9 @@ func (s *SBaremetalServer) DoDeploy(tool *disktool.SSHPartitionTool, term *ssh.C
 		password = seclib.RandomPassword(12)
 	}
 	deployInfo := deployapi.NewDeployInfo(publicKey, deployapi.JsonDeploysToStructs(deploys),
-		password, isInit, true, o.Options.LinuxDefaultRootUser, o.Options.WindowsDefaultAdminUser, false, "")
+		password, isInit, true, o.Options.LinuxDefaultRootUser, o.Options.WindowsDefaultAdminUser, false, "",
+		false, "",
+	)
 	return s.deployFs(tool, term, deployInfo)
 }
 

pkg/baremetal/tasks/bm_register.go

@@ -78,7 +78,7 @@ func NewBaremetalRegisterTask(
 }
 
 func (s *sBaremetalRegisterTask) getSession() *mcclient.ClientSession {
-	return auth.GetSession(context.Background(), s.userCred, o.Options.Region, "v2")
+	return auth.GetSession(context.Background(), s.userCred, o.Options.Region)
 }
 
 func (s *sBaremetalRegisterTask) getAccessDevMacAddr(ip string) (string, error) {

pkg/baremetal/utils/grub/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package grub // import "yunion.io/x/onecloud/pkg/baremetal/utils/grub"

pkg/cloudcommon/app/auth.go

@@ -100,7 +100,7 @@ func InitBaseAuth(options *common_options.BaseOptions) {
 }
 
 func FetchEtcdServiceInfo() (*identity.EndpointDetails, error) {
-	s := auth.GetAdminSession(context.Background(), "", "")
+	s := auth.GetAdminSession(context.Background(), "")
 	return s.GetCommonEtcdEndpoint()
 }
 

pkg/cloudcommon/cronman/cronman.go

@@ -38,6 +38,7 @@ var (
 )
 
 type TCronJobFunction func(ctx context.Context, userCred mcclient.TokenCredential, isStart bool)
+type TCronJobFunctionWithStartTime func(ctx context.Context, userCred mcclient.TokenCredential, start time.Time, isStart bool)
 
 var manager *SCronJobManager
 
@@ -72,11 +73,13 @@ func (t *TimerHour) Next(now time.Time) time.Time {
 }
 
 type SCronJob struct {
-	Name     string
-	job      TCronJobFunction
-	Timer    ICronTimer
-	Next     time.Time
-	StartRun bool
+	Name             string
+	job              TCronJobFunction
+	jobWithStartTime TCronJobFunctionWithStartTime
+	Timer            ICronTimer
+	Next             time.Time
+	StartRun         bool
+	times            []time.Time
 }
 
 type CronJobTimerHeap []*SCronJob
@@ -161,6 +164,38 @@ func (self *SCronJobManager) AddJobAtIntervals(name string, interval time.Durati
 	return self.AddJobAtIntervalsWithStartRun(name, interval, jobFunc, false)
 }
 
+func (self *SCronJobManager) AddJobAtIntervalsWithStarTime(name string, interval time.Duration, jobFunc TCronJobFunctionWithStartTime) error {
+	return self.AddJobAtIntervalsWithStarTimeStartRun(name, interval, jobFunc, false)
+}
+
+func (self *SCronJobManager) AddJobAtIntervalsWithStarTimeStartRun(name string, interval time.Duration, jobFunc TCronJobFunctionWithStartTime, startRun bool) error {
+	if interval <= 0 {
+		return errors.Error("AddJobAtIntervals: interval must > 0")
+	}
+	self.dataLock.Lock()
+	defer self.dataLock.Unlock()
+
+	if !self.IsNameUnique(name) {
+		return ErrCronJobNameConflict
+	}
+
+	t := Timer1{
+		dur: interval,
+	}
+	job := SCronJob{
+		Name:             name,
+		jobWithStartTime: jobFunc,
+		Timer:            &t,
+		StartRun:         startRun,
+	}
+	if !self.running {
+		self.jobs = append(self.jobs, &job)
+	} else {
+		self.addJob(&job)
+	}
+	return nil
+}
+
 func (self *SCronJobManager) AddJobAtIntervalsWithStartRun(name string, interval time.Duration, jobFunc TCronJobFunction, startRun bool) error {
 	if interval <= 0 {
 		return errors.Error("AddJobAtIntervals: interval must > 0")
@@ -268,7 +303,7 @@ func (self *SCronJobManager) addJob(newJob *SCronJob) {
 	now := time.Now()
 	newJob.Next = newJob.Timer.Next(now)
 	if newJob.StartRun {
-		newJob.runJob(true)
+		newJob.runJob(true, now)
 	}
 	heap.Push(&self.jobs, newJob)
 	go func() { self.add <- struct{}{} }()
@@ -335,7 +370,7 @@ func (self *SCronJobManager) init() {
 	for i := 0; i < len(self.jobs); i += 1 {
 		if self.jobs[i].StartRun {
 			self.jobs[i].StartRun = false
-			self.jobs[i].runJob(true)
+			self.jobs[i].runJob(true, now)
 		}
 	}
 }
@@ -369,7 +404,7 @@ func (self *SCronJobManager) runJobs(now time.Time) {
 	defer self.dataLock.Unlock()
 	for i := 0; i < len(self.jobs); i++ {
 		if !(self.jobs[i].Next.After(now) || self.jobs[i].Next.IsZero()) {
-			self.jobs[i].runJob(false)
+			self.jobs[i].runJob(false, now)
 			self.jobs[i].Next = self.jobs[i].Timer.Next(now)
 			heap.Fix(&self.jobs, i)
 		}
@@ -377,19 +412,25 @@ func (self *SCronJobManager) runJobs(now time.Time) {
 }
 
 func (job *SCronJob) Run() {
-	job.runJobInWorker(job.StartRun)
+	startTime := time.Now()
+	if len(job.times) > 0 {
+		startTime = job.times[0]
+		job.times = job.times[1:]
+	}
+	job.runJobInWorker(job.StartRun, startTime)
 }
 
 func (job *SCronJob) Dump() string {
 	return ""
 }
 
-func (job *SCronJob) runJob(isStart bool) {
+func (job *SCronJob) runJob(isStart bool, now time.Time) {
 	job.StartRun = isStart
+	job.times = append(job.times, now)
 	manager.workers.Run(job, nil, nil)
 }
 
-func (job *SCronJob) runJobInWorker(isStart bool) {
+func (job *SCronJob) runJobInWorker(isStart bool, startTime time.Time) {
 	defer func() {
 		if r := recover(); r != nil {
 			log.Errorf("CronJob task %s run error: %s", job.Name, r)
@@ -397,10 +438,14 @@ func (job *SCronJob) runJobInWorker(isStart bool) {
 		}
 	}()
 
-	log.Debugf("Cron job: %s started", job.Name)
+	log.Debugf("Cron job: %s started, startTime: %s", job.Name, startTime)
 	ctx := context.Background()
 	ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_APPNAME, "Cron-Service")
 	ctx = context.WithValue(ctx, appctx.APP_CONTEXT_KEY_TASKNAME, fmt.Sprintf("%s-%d", job.Name, time.Now().Unix()))
 	userCred := DefaultAdminSessionGenerator()
-	job.job(ctx, userCred, isStart)
+	if job.job != nil {
+		job.job(ctx, userCred, isStart)
+	} else if job.jobWithStartTime != nil {
+		job.jobWithStartTime(ctx, userCred, startTime, isStart)
+	}
 }

pkg/cloudcommon/db/encrypted.go

@@ -52,7 +52,7 @@ func (res *SEncryptedResource) GetEncryptInfo(
 	userCred mcclient.TokenCredential,
 ) (apis.SEncryptInfo, error) {
 	ret := apis.SEncryptInfo{}
-	session := auth.GetSession(ctx, userCred, consts.GetRegion(), "")
+	session := auth.GetSession(ctx, userCred, consts.GetRegion())
 	secKey, err := identity_modules.Credentials.GetEncryptKey(session, res.EncryptKeyId)
 	if err != nil {
 		return ret, errors.Wrap(err, "GetEncryptKey")
@@ -82,7 +82,7 @@ func (manager *SEncryptedResourceManager) ValidateCreateData(
 	input apis.EncryptedResourceCreateInput,
 ) (apis.EncryptedResourceCreateInput, error) {
 	if input.EncryptKeyId != nil && len(*input.EncryptKeyId) > 0 {
-		session := auth.GetSession(ctx, userCred, consts.GetRegion(), "v1")
+		session := auth.GetSession(ctx, userCred, consts.GetRegion())
 		keyObj, err := identity_modules.Credentials.Get(session, *input.EncryptKeyId, nil)
 		if err != nil {
 			return input, errors.Wrap(err, "Credentials get key")
@@ -103,7 +103,7 @@ func (manager *SEncryptedResourceManager) ValidateCreateData(
 func (res *SEncryptedResource) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, data jsonutils.JSONObject, nameHint string) error {
 	if len(res.EncryptKeyId) == 0 && jsonutils.QueryBoolean(data, "encrypt_key_new", false) && !jsonutils.QueryBoolean(data, "dry_run", false) {
 		// create new encrypt key
-		session := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+		session := auth.GetAdminSession(ctx, consts.GetRegion())
 		now := time.Now()
 		keyName := "key-" + nameHint + "-" + timeutils.ShortDate(now)
 		algName, _ := data.GetString("encrypt_key_alg")
@@ -127,7 +127,7 @@ func (manager *SEncryptedResourceManager) FetchCustomizeColumns(
 ) []apis.EncryptedResourceDetails {
 	rets := make([]apis.EncryptedResourceDetails, len(objs))
 
-	session := auth.GetSession(ctx, userCred, consts.GetRegion(), "")
+	session := auth.GetSession(ctx, userCred, consts.GetRegion())
 	encKeyMap := make(map[string]identity_modules.SEncryptKeySecret)
 	for i := range objs {
 		var base *SEncryptedResource

pkg/cloudcommon/db/opslog_const.go

@@ -150,6 +150,9 @@ const (
 	ACT_VM_DEPLOY      = "deploy"
 	ACT_VM_DEPLOY_FAIL = "deploy_fail"
 
+	ACT_SET_USER_PASSWORD      = "set_user_password"
+	ACT_SET_USER_PASSWORD_FAIL = "set_user_password_fail"
+
 	ACT_VM_IO_THROTTLE      = "io_throttle"
 	ACT_VM_IO_THROTTLE_FAIL = "io_throttle_fail"
 

pkg/cloudcommon/db/project.go

@@ -178,6 +178,10 @@ func (manager *SProjectizedResourceBaseManager) FetchCustomizeColumns(
 			}
 		}
 	}
+	domainRows := manager.SDomainizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	for i := range ret {
+		ret[i].DomainizedResourceInfo = domainRows[i]
+	}
 	return ret
 }
 

pkg/cloudcommon/db/rolecache.go

@@ -123,7 +123,7 @@ func (manager *SRoleCacheManager) FetchRoleFromKeystone(ctx context.Context, idS
 	query.Set("scope", jsonutils.NewString("system"))
 	query.Set("system", jsonutils.JSONTrue)
 
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	role, err := modules.RolesV3.GetById(s, idStr, query)
 	if err != nil {
 		if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 {
@@ -263,7 +263,7 @@ func (manager *SRoleCacheManager) StartWatchRoleInKeystone() error {
 		return nil
 	}
 	ctx := context.Background()
-	s := auth.GetAdminSession(ctx, "", "")
+	s := auth.GetAdminSession(ctx, "")
 	watchMan, err := informer.NewWatchManagerBySession(s)
 	if err != nil {
 		return err

pkg/cloudcommon/db/tenantcache.go

@@ -231,7 +231,7 @@ func (manager *STenantCacheManager) fetchTenantFromKeystone(ctx context.Context,
 	query := jsonutils.NewDict()
 	query.Set("scope", jsonutils.NewString("system"))
 
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	tenant, err := modules.Projects.GetById(s, idStr, query)
 	if err != nil {
 		if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 {
@@ -286,7 +286,7 @@ func (manager *STenantCacheManager) fetchDomainFromKeystone(ctx context.Context,
 		log.Debugf("fetch empty domain!!!!\n%s", debug.Stack())
 		return nil, fmt.Errorf("Empty idStr")
 	}
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	tenant, err := modules.Domains.GetById(s, idStr, nil)
 	if err != nil {
 		if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 {
@@ -421,7 +421,7 @@ func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(ctx context.C
 		return fmt.Errorf("Empty domainId")
 	}
 
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.NewString(domainId), "domain_id")
 	params.Add(jsonutils.JSONTrue, "details")

pkg/cloudcommon/db/tenantcachesync.go

@@ -69,7 +69,7 @@ func (w *tenantCacheSyncWorker) Dump() string {
 }
 
 func syncDomains(ctx context.Context) error {
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	query := jsonutils.NewDict()
 	query.Add(jsonutils.NewInt(1024), "limit")
 	query.Add(jsonutils.NewString(string(rbacutils.ScopeSystem)), "scope")
@@ -98,7 +98,7 @@ func syncDomains(ctx context.Context) error {
 }
 
 func syncProjects(ctx context.Context) error {
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	query := jsonutils.NewDict()
 	query.Add(jsonutils.NewInt(1024), "limit")
 	query.Add(jsonutils.NewString(string(rbacutils.ScopeSystem)), "scope")

pkg/cloudcommon/db/usercache.go

@@ -125,7 +125,7 @@ func (manager *SUserCacheManager) FetchUserFromKeystone(ctx context.Context, idS
 	query.Set("scope", jsonutils.NewString("system"))
 	query.Set("system", jsonutils.JSONTrue)
 
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	user, err := modules.UsersV3.GetById(s, idStr, query)
 	if err != nil {
 		if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 {

pkg/cloudcommon/db/virtualresource.go

@@ -61,7 +61,7 @@ type SVirtualResourceBase struct {
 	IsSystem bool `nullable:"true" default:"false" list:"admin" create:"optional" json:"is_system"`
 
 	// 资源放入回收站时间
-	PendingDeletedAt time.Time `json:"pending_deleted_at" list:"user"`
+	PendingDeletedAt time.Time `json:"pending_deleted_at" list:"user" update:"admin"`
 	// 资源是否处于回收站中
 	PendingDeleted bool `nullable:"false" default:"false" index:"true" get:"user" list:"user" json:"pending_deleted"`
 	// 资源是否被冻结

pkg/cloudcommon/notifyclient/notify.go

@@ -237,7 +237,7 @@ func (t *eventTask) Dump() string {
 }
 
 func (t *eventTask) Run() {
-	s, err := AdminSessionGenerator(context.Background(), "", "")
+	s, err := AdminSessionGenerator(context.Background(), "")
 	if err != nil {
 		log.Errorf("unable to get admin session: %v", err)
 		return
@@ -361,7 +361,7 @@ func IntelliNotify(ctx context.Context, recipientId []string, isGroup bool, chan
 }
 
 func FetchNotifyAdminRecipients(ctx context.Context, region string, users []string, groups []string) {
-	s, err := AdminSessionGenerator(ctx, region, "v1")
+	s, err := AdminSessionGenerator(ctx, region)
 	if err != nil {
 		log.Errorf("unable to get admin session: %v", err)
 	}

pkg/cloudcommon/notifyclient/notify_internal.go

@@ -72,7 +72,7 @@ func systemNotify(ctx context.Context, priority npk.TNotifyPriority, event strin
 }
 
 func notifyAll(ctx context.Context, recipientId []string, isGroup bool, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) error {
-	s, err := AdminSessionGenerator(ctx, consts.GetRegion(), "")
+	s, err := AdminSessionGenerator(ctx, consts.GetRegion())
 	if err != nil {
 		return err
 	}
@@ -184,7 +184,7 @@ func lang(ctx context.Context, contactType npk.TNotifyChannel, reIds []string, c
 
 func genMsgViaLang(ctx context.Context, p sNotifyParams) ([]npk.SNotifyMessage, error) {
 	reIds := make([]string, 0)
-	s, err := AdminSessionGenerator(context.Background(), consts.GetRegion(), "")
+	s, err := AdminSessionGenerator(context.Background(), consts.GetRegion())
 	if err != nil {
 		return nil, err
 	}
@@ -359,7 +359,7 @@ func (t *notifyTask) Dump() string {
 }
 
 func (t *notifyTask) Run() {
-	s, err := AdminSessionGenerator(t.ctx, consts.GetRegion(), "")
+	s, err := AdminSessionGenerator(t.ctx, consts.GetRegion())
 	if err != nil {
 		log.Errorf("fail to get session: %v", err)
 	}

pkg/cloudcommon/notifyclient/template.go

@@ -46,15 +46,15 @@ var (
 	checkTemplates        bool
 )
 
-type SAdminSessionGenerator func(ctx context.Context, region string, apiVersion string) (*mcclient.ClientSession, error)
+type SAdminSessionGenerator func(ctx context.Context, region string) (*mcclient.ClientSession, error)
 type SUserLangFetcher func(uids []string) (map[string]string, error)
 
-func getAdminSesion(ctx context.Context, region string, apiVersion string) (*mcclient.ClientSession, error) {
-	return auth.GetAdminSession(ctx, region, apiVersion), nil
+func getAdminSesion(ctx context.Context, region string) (*mcclient.ClientSession, error) {
+	return auth.GetAdminSession(ctx, region), nil
 }
 
 func getUserLang(uids []string) (map[string]string, error) {
-	s, err := AdminSessionGenerator(context.Background(), consts.GetRegion(), "")
+	s, err := AdminSessionGenerator(context.Background(), consts.GetRegion())
 	if err != nil {
 		return nil, err
 	}
@@ -79,7 +79,7 @@ func getUserLang(uids []string) (map[string]string, error) {
 }
 
 func getRobotLang(robots []string) (map[string]string, error) {
-	s, err := AdminSessionGenerator(context.Background(), consts.GetRegion(), "")
+	s, err := AdminSessionGenerator(context.Background(), consts.GetRegion())
 	if err != nil {
 		return nil, err
 	}

pkg/cloudcommon/options/mergeconf.go

@@ -79,7 +79,7 @@ func (s *mcclientServiceConfigSession) Merge(opts interface{}, serviceType strin
 	s.config = jsonutils.Marshal(opts).(*jsonutils.JSONDict)
 	region, _ := s.config.GetString("region")
 	// epType, _ := s.config.GetString("session_endpoint_type")
-	s.session = auth.GetAdminSession(context.Background(), region, "")
+	s.session = auth.GetAdminSession(context.Background(), region)
 	s.serviceId, _ = getServiceIdByType(s.session, serviceType, serviceVersion)
 	if len(s.serviceId) > 0 {
 		serviceConf, err := getServiceConfig(s.session, s.serviceId)

pkg/cloudcommon/policy/policy.go

@@ -239,14 +239,14 @@ type policyTask struct {
 }
 
 func (t *policyTask) Run() {
-	val := t.manager.policyCache.Get(t.key)
+	val := t.manager.policyCache.AtomicGet(t.key)
 	result := fetchResult{}
 	if gotypes.IsNil(val) {
 		pg, err := DefaultPolicyFetcher(context.Background(), t.userCred)
 		if err != nil {
 			result.err = errors.Wrap(err, "DefaultPolicyFetcher")
 		} else {
-			t.manager.policyCache.Set(t.key, pg)
+			t.manager.policyCache.AtomicSet(t.key, pg)
 			result.output = pg
 		}
 	} else {
@@ -297,7 +297,7 @@ func (manager *SPolicyManager) allow(scope rbacutils.TRbacScope, userCred mcclie
 		policySet = rbacutils.TPolicySet{}
 	}
 	result := manager.allowWithoutCache(policySet, scope, userCred, service, resource, action, extra...)
-	manager.permissionCache.Set(key, result)
+	manager.permissionCache.AtomicSet(key, result)
 	return result
 }
 
@@ -400,7 +400,7 @@ func explainPolicy(userCred mcclient.TokenCredential, policyReq jsonutils.JSONOb
 }
 
 func fetchPolicyDataByIdOrName(ctx context.Context, id string) (*sPolicyData, error) {
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	data, err := identity.Policies.Get(s, id, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "modules.Policies.Get")

pkg/cloudcommon/syncman/watcher/watcher.go

@@ -81,7 +81,7 @@ func (manager *SInformerSyncManager) StartWatching(resMan informer.IResourceMana
 func (manager *SInformerSyncManager) startWatcher() error {
 	log.Infof("[%s] Start resource informer watcher for %s", manager.Name(), manager.resourceManager.GetKeyword())
 	ctx := context.Background()
-	s := auth.GetAdminSession(ctx, consts.GetRegion(), "v1")
+	s := auth.GetAdminSession(ctx, consts.GetRegion())
 	informer.NewWatchManagerBySessionBg(s, func(watchMan *informer.SWatchManager) error {
 		if err := watchMan.For(manager.resourceManager).AddEventHandler(ctx, manager); err != nil {
 			return errors.Wrapf(err, "watch resource %s", manager.resourceManager.GetKeyword())

pkg/cloudevent/models/cloudproviders.go

@@ -329,7 +329,7 @@ type SCloudproviderDelegate struct {
 }
 
 func (self *SCloudprovider) GetDelegate() (*SCloudproviderDelegate, error) {
-	s := auth.GetAdminSession(context.Background(), options.Options.Region, "")
+	s := auth.GetAdminSession(context.Background(), options.Options.Region)
 	result, err := modules.Cloudproviders.Get(s, self.Id, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "modules.Cloudproviders.Get")

pkg/cloudid/models/cloudaccount.go

@@ -88,7 +88,7 @@ func (manager *SCloudaccountManager) GetResourceCount() ([]db.SScopeResourceCoun
 }
 
 func (manager *SCloudaccountManager) GetICloudaccounts() ([]SCloudaccount, error) {
-	s := auth.GetAdminSession(context.Background(), options.Options.Region, "")
+	s := auth.GetAdminSession(context.Background(), options.Options.Region)
 
 	data := []jsonutils.JSONObject{}
 	offset := int64(0)
@@ -449,7 +449,7 @@ func (manager *SCloudaccountManager) FetchAccount(ctx context.Context, id string
 	account, err := manager.FetchById(id)
 	if err != nil {
 		if err == sql.ErrNoRows {
-			session := auth.GetAdminSession(context.Background(), options.Options.Region, "")
+			session := auth.GetAdminSession(context.Background(), options.Options.Region)
 			result, err := modules.Cloudaccounts.Get(session, id, nil)
 			if err != nil {
 				return nil, errors.Wrap(err, "Cloudaccounts.Get")
@@ -493,7 +493,7 @@ type SCloudDelegate struct {
 }
 
 func (self *SCloudaccount) getCloudDelegate(ctx context.Context) (*SCloudDelegate, error) {
-	s := auth.GetAdminSession(ctx, options.Options.Region, "")
+	s := auth.GetAdminSession(ctx, options.Options.Region)
 	result, err := modules.Cloudaccounts.Get(s, self.Id, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "Cloudaccounts.Get")
@@ -515,7 +515,7 @@ func (self *SCloudaccount) GetProvider() (cloudprovider.ICloudProvider, error) {
 }
 
 func (self *SCloudaccount) GetCloudDelegaes(ctx context.Context) ([]SCloudDelegate, error) {
-	s := auth.GetAdminSession(ctx, options.Options.Region, "")
+	s := auth.GetAdminSession(ctx, options.Options.Region)
 	params := map[string]string{"cloudaccount": self.Id}
 	result, err := modules.Cloudproviders.List(s, jsonutils.Marshal(params))
 	if err != nil {
@@ -860,7 +860,7 @@ func (self *SCloudaccount) SyncSystemCloudpoliciesFromCloud(ctx context.Context,
 		return nil
 	}
 
-	s := auth.GetAdminSession(context.Background(), options.Options.Region, "")
+	s := auth.GetAdminSession(context.Background(), options.Options.Region)
 	transport := httputils.GetTransport(true)
 	transport.Proxy = options.Options.HttpTransportProxyFunc()
 	client := &http.Client{Transport: transport}
@@ -982,7 +982,7 @@ func (self *SCloudaccount) GetCloudproviders() ([]SCloudprovider, error) {
 }
 
 func (self *SCloudaccount) GetICloudprovider() ([]SCloudprovider, error) {
-	s := auth.GetAdminSession(context.Background(), options.Options.Region, "")
+	s := auth.GetAdminSession(context.Background(), options.Options.Region)
 	data := []jsonutils.JSONObject{}
 	offset := int64(0)
 	params := map[string]interface{}{
@@ -1895,7 +1895,7 @@ func (self *SCloudaccount) GetUserCloudgroups(userId string) ([]string, error) {
 		if len(cache.ExternalId) > 0 {
 			ret = append(ret, cache.Name)
 		} else {
-			s := auth.GetAdminSession(context.TODO(), options.Options.Region, "")
+			s := auth.GetAdminSession(context.TODO(), options.Options.Region)
 			_, err = cache.GetOrCreateICloudgroup(context.TODO(), s.GetToken())
 			if err != nil {
 				return []string{}, errors.Wrapf(err, "GetOrCreateICloudgroup")

pkg/cloudid/models/cloudprovider.go

@@ -83,7 +83,7 @@ func (manager *SCloudproviderManager) FetchProvider(ctx context.Context, id stri
 	provider, err := manager.FetchById(id)
 	if err != nil {
 		if err == sql.ErrNoRows {
-			session := auth.GetAdminSession(context.Background(), options.Options.Region, "")
+			session := auth.GetAdminSession(context.Background(), options.Options.Region)
 			result, err := modules.Cloudproviders.Get(session, id, nil)
 			if err != nil {
 				return nil, errors.Wrap(err, "Cloudproviders.Get")
@@ -105,7 +105,7 @@ func (manager *SCloudproviderManager) FetchProvider(ctx context.Context, id stri
 }
 
 func (self *SCloudprovider) getCloudDelegate(ctx context.Context) (*SCloudDelegate, error) {
-	s := auth.GetAdminSession(ctx, options.Options.Region, "")
+	s := auth.GetAdminSession(ctx, options.Options.Region)
 	result, err := modules.Cloudproviders.Get(s, self.Id, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "Cloudproviders.Get")

pkg/cloudid/models/samluser.go

@@ -205,7 +205,7 @@ func (self *SSamluser) SyncAzureGroup() error {
 		return errors.Wrapf(err, "group cache Register")
 	}
 	if len(cache.ExternalId) == 0 {
-		s := auth.GetAdminSession(context.TODO(), options.Options.Region, "")
+		s := auth.GetAdminSession(context.TODO(), options.Options.Region)
 		_, err = cache.GetOrCreateICloudgroup(context.TODO(), s.GetToken())
 		if err != nil {
 			return errors.Wrapf(err, "GetOrCreateICloudgroup")

pkg/cloudid/saml/providers/aliyun/driver.go

@@ -70,7 +70,10 @@ func (d *SAliyunSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCr
 			Values: []string{v},
 		})
 	}
-	data.RelayState = "https://homenew.console.aliyun.com/"
+	if len(redirectUrl) == 0 {
+		redirectUrl = "https://homenew.console.aliyun.com/"
+	}
+	data.RelayState = redirectUrl
 	return data, nil
 }
 

pkg/cloudid/saml/providers/aws/driver.go

@@ -87,7 +87,10 @@ func (d *SAWSSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCred
 			Values:       []string{v.value},
 		})
 	}
-	data.RelayState = "https://console.aws.amazon.com/"
+	if len(redirectUrl) == 0 {
+		redirectUrl = "https://console.aws.amazon.com/"
+	}
+	data.RelayState = redirectUrl
 
 	return data, nil
 }

pkg/cloudid/saml/providers/awscn/driver.go

@@ -86,7 +86,10 @@ func (d *SAWSCNSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCre
 			Values:       []string{v.value},
 		})
 	}
-	data.RelayState = "https://console.amazonaws.cn/"
+	if len(redirectUrl) == 0 {
+		redirectUrl = "https://console.amazonaws.cn/"
+	}
+	data.RelayState = redirectUrl
 
 	return data, nil
 }

pkg/cloudid/saml/providers/qcloud/driver.go

@@ -84,7 +84,10 @@ func (d *SQcloudSAMLDriver) GetIdpInitiatedLoginData(ctx context.Context, userCr
 			Values:       []string{v.value},
 		})
 	}
-	data.RelayState = "https://console.cloud.tencent.com/"
+	if len(redirectUrl) == 0 {
+		redirectUrl = "https://console.cloud.tencent.com/"
+	}
+	data.RelayState = redirectUrl
 
 	return data, nil
 }

pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryreport.go

@@ -1,142 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alertrecordhistorymon
-
-import (
-	"context"
-	"fmt"
-	"strconv"
-	"time"
-
-	"golang.org/x/sync/errgroup"
-
-	"yunion.io/x/jsonutils"
-	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/util/timeutils"
-
-	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
-	"yunion.io/x/onecloud/pkg/cloudmon/options"
-	"yunion.io/x/onecloud/pkg/mcclient"
-	modules "yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
-)
-
-func init() {
-	factory := SAlertRecordHistoryFactory{}
-	common.RegisterFactory(&factory)
-}
-
-type SAlertRecordHistoryFactory struct {
-}
-
-func (self *SAlertRecordHistoryFactory) MyRoutineInteval(monOptions options.CloudMonOptions) time.Duration {
-	return time.Duration(monOptions.AlertRecordHistoryInterval)
-}
-
-func (self *SAlertRecordHistoryFactory) MyRoutineFunc() common.RoutineFunc {
-	return common.MakePullMetricRoutineAtZeroPoint
-}
-
-func (self *SAlertRecordHistoryFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
-	args *options.ReportOptions,
-	operatorType string) common.ICloudReport {
-	return &SAlertRecordHistoryReport{
-		common.CloudReportBase{
-			SProvider: nil,
-			Session:   session,
-			Args:      args,
-			Operator:  string(common.ALERT_RECORD),
-		},
-	}
-}
-
-func (S *SAlertRecordHistoryFactory) GetId() string {
-	return string(common.ALERT_RECORD)
-}
-
-type SAlertRecordHistoryReport struct {
-	common.CloudReportBase
-}
-
-func (self *SAlertRecordHistoryReport) Report() error {
-	alerts, err := self.getMonitorCommonAlert()
-	if err != nil {
-		return err
-	}
-	errs := make([]error, 0)
-	recordGroup, _ := errgroup.WithContext(context.Background())
-	count := 0
-	for i, _ := range alerts {
-		tmp := alerts[i]
-		count++
-		recordGroup.Go(func() error {
-			alert_id, _ := tmp.GetString("id")
-			alertRecords, err := self.getAlertRecordsByAlertId(alert_id)
-			if err != nil {
-				return errors.Wrapf(err, "getAlertRecordsByAlertId:%s error", alert_id)
-			}
-			return self.collectMetric(alertRecords)
-		})
-		if count == 4 {
-			err := recordGroup.Wait()
-			if err != nil {
-				errs = append(errs, errors.Wrap(err, "alertRecordHistoryReport collectMetric error"))
-			}
-			count = 0
-		}
-	}
-	err = recordGroup.Wait()
-	if err != nil {
-		errs = append(errs, errors.Wrap(err, "alertRecordHistoryReport collectMetric error"))
-	}
-	return errors.NewAggregate(errs)
-}
-
-func (self *SAlertRecordHistoryReport) getMonitorCommonAlert() ([]jsonutils.JSONObject, error) {
-	alerts := make([]jsonutils.JSONObject, 0)
-	query := jsonutils.NewDict()
-	query.Add(jsonutils.NewString("0"), common.KEY_LIMIT)
-	//query.Add(jsonutils.NewBool(true), common.DETAILS)
-	query.Add(jsonutils.NewString("system"), "scope")
-
-	alerts, err := self.ListAllResource(modules.CommonAlertManager, query)
-	if err != nil {
-		return nil, errors.Wrap(err, "getMonitorCommonAlert error")
-	}
-	return alerts, nil
-}
-
-func (self *SAlertRecordHistoryReport) getAlertRecordsByAlertId(id string) ([]jsonutils.JSONObject, error) {
-	now := time.Now().UTC()
-	period64, err := strconv.ParseInt(self.Args.Interval, 10, 8)
-	if err != nil {
-		return nil, err
-	}
-	query := jsonutils.NewDict()
-	query.Add(jsonutils.NewString("0"), common.KEY_LIMIT)
-	query.Add(jsonutils.NewBool(true), common.DETAILS)
-	query.Add(jsonutils.NewString("system"), "scope")
-	query.Add(jsonutils.NewString(id), "alert_id")
-	query.Add(jsonutils.NewString("alerting"), "state")
-	query.Add(jsonutils.NewString(fmt.Sprintf(`created_at.between("%s","%s")`,
-		now.Add(-time.Hour*24*time.Duration(period64)).Format(timeutils.MysqlTimeFormat),
-		now.Format(timeutils.MysqlTimeFormat))),
-		"filter")
-
-	alertRecords, err := self.ListAllResource(modules.AlertRecordManager, query)
-	if err != nil {
-		return nil, err
-	}
-	return alertRecords, nil
-}

pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryservice.go

@@ -1,54 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alertrecordhistorymon
-
-import (
-	"time"
-
-	"yunion.io/x/jsonutils"
-
-	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
-	"yunion.io/x/onecloud/pkg/util/influxdb"
-)
-
-func (self *SAlertRecordHistoryReport) collectMetric(alertRecords []jsonutils.JSONObject) error {
-	dataList := make([]influxdb.SMetricData, 0)
-	matchRecord := self.getMaxEvalMatchOfAlertRecord(alertRecords)
-	if matchRecord == nil {
-		return nil
-	}
-
-	metric, err := self.NewMetricFromJson(matchRecord)
-	if err != nil {
-		return err
-	}
-	metric.Timestamp = time.Now()
-	metric.Name = ALERT_RECORD_HISTORY_MEASUREMENT
-	dataList = append(dataList, metric)
-	return common.SendMetrics(self.Session, dataList, self.Args.Debug, ALERT_RECORD_HISTORY_DATABASE)
-}
-
-func (self *SAlertRecordHistoryReport) getMaxEvalMatchOfAlertRecord(alertRecords []jsonutils.JSONObject) jsonutils.JSONObject {
-	var matchRecord jsonutils.JSONObject
-	maxCount := int64(0)
-	for _, record := range alertRecords {
-		resNum, _ := record.Int("res_num")
-		if resNum > maxCount {
-			maxCount = resNum
-			matchRecord = record
-		}
-	}
-	return matchRecord
-}

pkg/cloudmon/collectors/alertrecordhistorymon/doc.go

@@ -1,15 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alertrecordhistorymon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alertrecordhistorymon"

pkg/cloudmon/collectors/alimon/aliUtils.go

@@ -1,96 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alimon
-
-import "yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
-
-const (
-	SERVER_METRIC_NAMESPACE = "acs_ecs_dashboard"
-
-	REDIS_METRIC_NAMESPACE = "acs_kvstore"
-	RDS_METRIC_NAMESPACE   = "acs_rds_dashboard"
-	OSS_METRIC_NAMESPACE   = "acs_oss"
-	ELB_METRIC_NAMESPACE   = "acs_slb_dashboard"
-
-	K8S_METRIC_NAMESPACE = "acs_k8s"
-)
-
-//multiCloud查询指标列表组装
-var aliMetricSpecs = map[string][]string{
-	"CPUUtilization":  {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_CPU_USAGE},
-	"InternetInRate":  {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_NET_BPS_RX_INTERNET},
-	"IntranetInRate":  {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_NET_BPS_RX_INTRANET},
-	"InternetOutRate": {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_NET_BPS_TX_INTERNET},
-	"IntranetOutRate": {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_NET_BPS_TX_INTRANET},
-	"DiskReadBPS":     {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_DISK_READ_BPS},
-	"DiskWriteBPS":    {common.DEFAULT_STATISTICS, common.UNIT_BYTEPS, common.INFLUXDB_FIELD_DISK_WRITE_BPS},
-	"DiskReadIOPS":    {common.DEFAULT_STATISTICS, common.UNIT_CPS, common.INFLUXDB_FIELD_DISK_READ_IOPS},
-	"DiskWriteIOPS":   {common.DEFAULT_STATISTICS, common.UNIT_CPS, common.INFLUXDB_FIELD_DISK_WRITE_IOPS},
-}
-var aliRdsMetricSpecs = map[string][]string{
-	"CpuUsage":                {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_RDS_CPU_USAGE},
-	"MemoryUsage":             {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_RDS_MEM_USAGE},
-	"MySQL_NetworkInNew":      {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL},
-	"MySQL_NetworkOutNew":     {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL},
-	"DiskUsage":               {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_RDS_DISK_USAGE},
-	"SQLServer_NetworkInNew":  {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER},
-	"SQLServer_NetworkOutNew": {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER},
-	"ConnectionUsage":         {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_RDS_CONN_USAGE},
-}
-var aliRedisMetricSpecs = map[string][]string{
-	"CpuUsage":       {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_REDIS_CPU_USAGE},
-	"MemoryUsage":    {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_REDIS_MEM_USAGE},
-	"IntranetIn":     {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_REDIS_NET_BPS_RX},
-	"IntranetOut":    {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_REDIS_NET_BPS_TX},
-	"UsedConnection": {common.DEFAULT_STATISTICS, common.UNIT_COUNT, common.INFLUXDB_FIFLD_REDIS_CONN_USAGE},
-	"UsedQPS":        {common.DEFAULT_STATISTICS, common.UNIT_COUNT, common.INFLUXDB_FIFLD_REDIS_OPT_SES},
-	"Keys":           {common.DEFAULT_STATISTICS, common.UNIT_COUNT, common.INFLUXDB_FIFLD_REDIS_CACHE_KEYS},
-	"ExpiredKeys":    {common.DEFAULT_STATISTICS, common.UNIT_COUNT, common.INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS},
-	"UsedMemory":     {common.DEFAULT_STATISTICS, common.UNIT_MEM, common.INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE},
-}
-var aliOSSMetricSpecs = map[string][]string{
-	"InternetSend":         {common.DEFAULT_STATISTICS, common.UNIT_MEM, common.INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET},
-	"InternetRecv":         {common.DEFAULT_STATISTICS, common.UNIT_MEM, common.INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET},
-	"IntranetSend":         {common.DEFAULT_STATISTICS, common.UNIT_MEM, common.INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET},
-	"IntranetRecv":         {common.DEFAULT_STATISTICS, common.UNIT_MEM, common.INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET},
-	"GetObjectE2eLatency":  {common.DEFAULT_STATISTICS, common.UNIT_MSEC, common.INFLUXDB_FIELD_OSS_LATECY_GET},
-	"PostObjectE2eLatency": {common.DEFAULT_STATISTICS, common.UNIT_MSEC, common.INFLUXDB_FIELD_OSS_LATECY_POST},
-	"GetObjectCount":       {common.DEFAULT_STATISTICS, common.UNIT_COUNT, common.INFLUXDB_FIELD_OSS_REQ_COUNT_GET},
-	"PostObjectCount":      {common.DEFAULT_STATISTICS, common.UNIT_COUNT, common.INFLUXDB_FIELD_OSS_REQ_COUNT_POST},
-	"ServerErrorCount":     {common.DEFAULT_STATISTICS, common.UNIT_COUNT, common.INFLUXDB_FIELD_OSS_REQ_COUNT_5XX},
-}
-var aliElbMetricSpecs = map[string][]string{
-	"InstanceTrafficRX":     {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_ELB_NET_BPS_RX},
-	"InstanceTrafficTX":     {common.DEFAULT_STATISTICS, common.UNIT_BPS, common.INFLUXDB_FIELD_ELB_NET_BPS_TX},
-	"InstanceStatusCode2xx": {common.DEFAULT_STATISTICS, common.UNIT_COUNT_SEC, common.INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX},
-	"InstanceStatusCode3xx": {common.DEFAULT_STATISTICS, common.UNIT_COUNT_SEC, common.INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX},
-	"InstanceStatusCode4xx": {common.DEFAULT_STATISTICS, common.UNIT_COUNT_SEC, common.INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX},
-	"InstanceStatusCode5xx": {common.DEFAULT_STATISTICS, common.UNIT_COUNT_SEC, common.INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX},
-}
-
-var aliK8SClusterMetricSpecs = map[string][]string{
-	"cluster.cpu.utilization":    {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_K8S_CLUSTER_CPU_USAGE},
-	"cluster.memory.utilization": {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_K8S_CLUSTER_MEM_USAGE},
-}
-
-var aliK8SPodMetricSpecs = map[string][]string{
-	"pod.cpu.utilization":    {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_K8S_POD_CPU_USAGE},
-	"pod.memory.utilization": {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_K8S_POD_MEM_USAGE},
-}
-
-var aliK8SNodeMetricSpecs = map[string][]string{
-	"node.cpu.utilization":    {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_K8S_NODE_CPU_USAGE},
-	"node.memory.utilization": {common.DEFAULT_STATISTICS, common.UNIT_PERCENT, common.INFLUXDB_FIELD_K8S_NODE_MEM_USAGE},
-}

pkg/cloudmon/collectors/alimon/alicloudreport.go

@@ -1,143 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alimon
-
-import (
-	"yunion.io/x/jsonutils"
-
-	"yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
-	"yunion.io/x/onecloud/pkg/cloudmon/options"
-	"yunion.io/x/onecloud/pkg/mcclient"
-)
-
-func init() {
-	factory := SAliCloudReportFactory{}
-	common.RegisterFactory(&factory)
-	cluster := &AliK8sClusterHelper{
-		K8sClusterMetricBaseHelper: &common.K8sClusterMetricBaseHelper{
-			ModuleHelper: map[common.K8sClusterModuleType]common.IK8sClusterModuleHelper{},
-		},
-	}
-	cluster.RegisterModuleHelper(new(AliK8sClusterPodHelper))
-	cluster.RegisterModuleHelper(new(AliK8sClusterNodeHelper))
-	common.RegisterK8sClusterHelper(cluster)
-}
-
-type SAliCloudReportFactory struct {
-	common.CommonReportFactory
-}
-
-func (self *SAliCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
-	args *options.ReportOptions, operatorType string) common.ICloudReport {
-	return &SAliCloudReport{
-		common.CloudReportBase{
-			SProvider: provider,
-			Session:   session,
-			Args:      args,
-			Operator:  operatorType,
-		},
-	}
-}
-
-func (self *SAliCloudReportFactory) GetId() string {
-	return compute.CLOUD_PROVIDER_ALIYUN
-}
-
-type SAliCloudReport struct {
-	common.CloudReportBase
-}
-
-func (self *SAliCloudReport) Report() error {
-	var servers []jsonutils.JSONObject
-	var err error
-
-	servers, err = self.GetResourceByOperator()
-
-	providerInstance, err := self.InitProviderInstance()
-	if err != nil {
-		return err
-	}
-	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
-	if err != nil {
-		return err
-	}
-	for _, region := range regionList {
-		servers := regionServerMap[region.GetGlobalId()]
-		switch common.MonType(self.Operator) {
-		case common.REDIS:
-			err = self.collectRegionMetricOfRedis(region, servers)
-		case common.K8S:
-			err = self.collectRegionMetricOfResource(region, servers)
-			if err != nil {
-				return err
-			}
-			self.Impl = self
-			err = self.CollectRegionMetricOfK8sModules(region, servers)
-		default:
-			err = self.collectRegionMetricOfResource(region, servers)
-		}
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-type AliK8sClusterHelper struct {
-	*common.K8sClusterMetricBaseHelper
-}
-
-func (a AliK8sClusterHelper) HelperBrand() string {
-	return compute.CLOUD_PROVIDER_ALIYUN
-}
-
-type AliK8sClusterPodHelper struct {
-	common.K8sClusterModuleQueryHelper
-}
-
-func (q AliK8sClusterPodHelper) MyModuleType() common.K8sClusterModuleType {
-	return common.K8S_MODULE_POD
-}
-
-func (q AliK8sClusterPodHelper) MyResDimensionId() common.DimensionId {
-	return common.DimensionId{
-		LocalId: "name",
-		ExtId:   "pod",
-	}
-}
-
-func (q AliK8sClusterPodHelper) MyNamespaceAndMetrics() (string, map[string][]string) {
-	return K8S_METRIC_NAMESPACE, aliK8SPodMetricSpecs
-}
-
-type AliK8sClusterNodeHelper struct {
-	common.K8sClusterModuleQueryHelper
-}
-
-func (a AliK8sClusterNodeHelper) MyModuleType() common.K8sClusterModuleType {
-	return common.K8S_MODULE_NODE
-}
-
-func (a AliK8sClusterNodeHelper) MyResDimensionId() common.DimensionId {
-	return common.DimensionId{
-		LocalId: "name",
-		ExtId:   "node",
-	}
-}
-
-func (a AliK8sClusterNodeHelper) MyNamespaceAndMetrics() (string, map[string][]string) {
-	return K8S_METRIC_NAMESPACE, aliK8SNodeMetricSpecs
-}

pkg/cloudmon/collectors/alimon/aliservice.go

@@ -1,315 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package alimon
-
-import (
-	"strconv"
-	"strings"
-	"time"
-
-	"yunion.io/x/jsonutils"
-	"yunion.io/x/log"
-
-	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
-	"yunion.io/x/onecloud/pkg/cloudprovider"
-	"yunion.io/x/onecloud/pkg/multicloud/aliyun"
-	"yunion.io/x/onecloud/pkg/util/influxdb"
-)
-
-func (self *SAliCloudReport) collectRegionMetricOfResource(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
-	dataList := make([]influxdb.SMetricData, 0)
-	aliReg := region.(*aliyun.SRegion)
-	since, until, err := common.TimeRangeFromArgs(self.Args)
-	if err != nil {
-		return err
-	}
-	namespace, metrics := self.getMetricSpecs(servers)
-	dimensionId := self.getDimensionId()
-	dataCnt := 0
-	for metricName, influxDbSpecs := range metrics {
-		rtnArray, _, err := aliReg.DescribeMetricList(metricName, namespace, since, until, "", nil)
-		if err != nil {
-			log.Errorln(err)
-			continue
-		}
-		dataCnt = len(dataList)
-		if len(rtnArray) > 0 {
-			for _, rtnMetric := range rtnArray {
-				for _, server := range servers {
-					external_id, _ := server.GetString(dimensionId.LocalId)
-					if instanceId, _ := rtnMetric.GetString(dimensionId.ExtId); instanceId == external_id {
-						if self.Operator == string(common.SERVER) {
-							metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
-							if err != nil {
-								return err
-							}
-							dataList = append(dataList, metric)
-						}
-						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs, metricName)
-						if err != nil {
-							return err
-						}
-						dataList = append(dataList, serverMetric)
-					}
-				}
-			}
-		}
-		log.Debugf("%s %s %s report %d metric", self.SProvider.Name, self.Operator, metricName, len(dataList)-dataCnt)
-	}
-	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
-}
-
-func (self *SAliCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion,
-	servers []jsonutils.JSONObject) error {
-	dataList := make([]influxdb.SMetricData, 0)
-	aliReg := region.(*aliyun.SRegion)
-	since, until, err := common.TimeRangeFromArgs(self.Args)
-	if err != nil {
-		return err
-	}
-	redisDeployType := make(map[string]string)
-	for _, server := range servers {
-		local_category, err := server.GetString("local_category")
-		if err != nil {
-			return err
-		}
-		switch local_category {
-		case "single":
-			redisDeployType["single"] = "Standard"
-		case "master":
-			redisDeployType["master"] = "Standard"
-		case "cluster":
-			redisDeployType["cluster"] = "Sharding"
-		case "rwsplit":
-			redisDeployType["rwsplit"] = "Splitrw"
-		}
-
-	}
-	dataCnt := 0
-	for metricName, influxDbSpecs := range aliRedisMetricSpecs {
-		for _, pre := range redisDeployType {
-			rtnArray, _, err := aliReg.DescribeMetricList(pre+metricName, "acs_kvstore", since, until, "", nil)
-			if err != nil {
-				log.Errorln(err)
-				continue
-			}
-			dataCnt = len(dataList)
-			if len(rtnArray) > 0 {
-				for _, rtnMetric := range rtnArray {
-					for _, server := range servers {
-						external_id, _ := server.GetString("external_id")
-						if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
-							serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs, "")
-							node_id, _ := rtnMetric.GetString("nodeId")
-							serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: node_id})
-							if err != nil {
-								return err
-							}
-							dataList = append(dataList, serverMetric)
-						}
-					}
-				}
-			}
-		}
-		log.Debugf("%s %s %s report %d metric", self.SProvider.Name, self.Operator, metricName, len(dataList)-dataCnt)
-	}
-	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
-}
-
-func (self *SAliCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject, influxDbSpecs []string, valueKey string) (influxdb.SMetricData, error) {
-	metric, err := self.NewMetricFromJson(server)
-	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
-	if err != nil {
-		return influxdb.SMetricData{}, err
-	}
-	timestamp, _ := rtnMetric.Get("timestamp")
-	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
-	fieldValue, err := self.getDataPointValue(valueKey, influxDbSpecs, rtnMetric)
-	if err != nil {
-		return influxdb.SMetricData{}, err
-	}
-	//根据条件拼装metric的tag和metirc信息
-	influxDbSpec := influxDbSpecs[2]
-	measurement := common.SubstringBefore(influxDbSpec, ".")
-	var pairsKey string
-	if strings.Contains(influxDbSpec, ",") {
-		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
-	} else {
-		pairsKey = common.SubstringAfter(influxDbSpec, ".")
-	}
-	if influxDbSpecs[1] == common.UNIT_BYTEPS && strings.Contains(pairsKey, common.UNIT_BPS) {
-		fieldValue = fieldValue * 8
-	}
-	tag := common.SubstringAfter(influxDbSpec, ",")
-	if tag != "" && strings.Contains(influxDbSpec, "=") {
-		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
-			Key:   common.SubstringBefore(tag, "="),
-			Value: common.SubstringAfter(tag, "="),
-		})
-	}
-	cpu_cout, err := server.Get("vcpu_count")
-	if err == nil {
-		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
-			Key:   "cpu_count",
-			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
-		})
-	}
-	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
-		Key:   pairsKey,
-		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
-	})
-	metric.Name = measurement
-	return metric, nil
-}
-
-func (self *SAliCloudReport) getDataPointValue(valueKey string, influxDbSpecs []string, rtnMetric jsonutils.JSONObject) (float64, error) {
-	key := common.UNIT_AVERAGE
-	switch common.MonType(self.Operator) {
-	case common.OSS:
-		key = valueKey
-	case common.K8S:
-		key = "Value"
-	}
-	fieldValue, err := rtnMetric.Float(key)
-	if err != nil {
-		return 0, err
-	}
-	return fieldValue, nil
-}
-
-func (self *SAliCloudReport) getMetricSpecs(res []jsonutils.JSONObject) (string, map[string][]string) {
-	switch common.MonType(self.Operator) {
-	case common.SERVER:
-		return SERVER_METRIC_NAMESPACE, aliMetricSpecs
-	case common.REDIS:
-		return REDIS_METRIC_NAMESPACE, aliRedisMetricSpecs
-	case common.RDS:
-		return RDS_METRIC_NAMESPACE, aliRdsMetricSpecs
-	case common.OSS:
-		return OSS_METRIC_NAMESPACE, aliOSSMetricSpecs
-	case common.ELB:
-		return ELB_METRIC_NAMESPACE, aliElbMetricSpecs
-	case common.K8S:
-		return K8S_METRIC_NAMESPACE, aliK8SClusterMetricSpecs
-	default:
-		return SERVER_METRIC_NAMESPACE, aliMetricSpecs
-	}
-}
-
-func (self *SAliCloudReport) getDimensionId() *common.DimensionId {
-	switch common.MonType(self.Operator) {
-	case common.SERVER:
-		return &common.DimensionId{
-			LocalId: "external_id",
-			ExtId:   "instanceId",
-		}
-	case common.REDIS:
-		return &common.DimensionId{
-			LocalId: "external_id",
-			ExtId:   "instanceId",
-		}
-	case common.RDS:
-		return &common.DimensionId{
-			LocalId: "external_id",
-			ExtId:   "instanceId",
-		}
-	case common.OSS:
-		return &common.DimensionId{
-			LocalId: "name",
-			ExtId:   "BucketName",
-		}
-	case common.ELB:
-		return &common.DimensionId{
-			LocalId: "external_id",
-			ExtId:   "InstanceId",
-		}
-	case common.K8S:
-		return &common.DimensionId{
-			LocalId: "external_cloud_cluster_id",
-			ExtId:   "cluster",
-		}
-	}
-	return nil
-}
-
-func (self *SAliCloudReport) CollectK8sModuleMetric(region cloudprovider.ICloudRegion, cluster jsonutils.JSONObject,
-	helper common.IK8sClusterModuleHelper) error {
-	aliReg := region.(*aliyun.SRegion)
-	since, until, err := common.TimeRangeFromArgs(self.Args)
-	id, _ := cluster.GetString("id")
-	resources, err := common.ListK8sClusterModuleResources(helper.MyModuleType(), id, self.Session, nil)
-	if err != nil {
-		log.Errorf("ListK8sClusterModuleResources err: %v", err)
-		return err
-	}
-	namespace, metricSpecs := helper.MyNamespaceAndMetrics()
-	metricNames := make([]string, 0)
-	for metricName := range metricSpecs {
-		metricNames = append(metricNames, metricName)
-	}
-	dimensionId := helper.MyResDimensionId()
-	dataList := make([]influxdb.SMetricData, 0)
-	dataCnt := 0
-	for metricName, influxDbSpecs := range metricSpecs {
-		rtnArray, _, err := aliReg.DescribeMetricList(metricName, namespace, since, until, "", nil)
-		if err != nil {
-			log.Errorln(err)
-			continue
-		}
-		dataCnt = len(dataList)
-		if len(rtnArray) > 0 {
-			for _, rtnMetric := range rtnArray {
-				for _, resource := range resources {
-					external_id, _ := resource.GetString(dimensionId.LocalId)
-					if instanceId, _ := rtnMetric.GetString(dimensionId.ExtId); instanceId == external_id {
-						serverMetric, err := self.collectMetricFromThisServer(resource, rtnMetric, influxDbSpecs, metricName)
-						if err != nil {
-							return err
-						}
-						dataList = append(dataList, serverMetric)
-					}
-				}
-			}
-		}
-		log.Debugf("%s %s %s report %d metric", self.SProvider.Name, self.Operator, metricName, len(dataList)-dataCnt)
-	}
-	err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
-	if err != nil {
-		log.Errorf("K8s metricName: %s SendMetrics err: %#v", "", err)
-	}
-	return nil
-}
-
-func (self *SAliCloudReport) getK8sModuleDimensions(helper common.IK8sClusterModuleHelper, module jsonutils.JSONObject,
-	clusterId string) []aliyun.SResourceLabel {
-	dimensions := make([]aliyun.SResourceLabel, 0)
-	dimensionId := helper.MyResDimensionId()
-
-	localIds := strings.Split(dimensionId.LocalId, ",")
-	extIds := strings.Split(dimensionId.ExtId, ",")
-	if len(localIds) != len(extIds) {
-		return dimensions
-	}
-	for index, localId := range localIds {
-		localIdKey := strings.Split(localId, ".")
-		val, _ := module.GetString(localIdKey...)
-		dimensions = append(dimensions, aliyun.SResourceLabel{
-			Name:  extIds[index],
-			Value: val,
-		})
-
-	}
-	return dimensions
-}

pkg/cloudmon/collectors/apsaramon/apsaracloudreport.go

@@ -1,86 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package apsaramon
-
-import (
-	"yunion.io/x/jsonutils"
-
-	"yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
-	"yunion.io/x/onecloud/pkg/cloudmon/options"
-	"yunion.io/x/onecloud/pkg/mcclient"
-)
-
-func init() {
-	factory := SApsaraCloudReportFactory{}
-	common.RegisterFactory(&factory)
-}
-
-type SApsaraCloudReportFactory struct {
-	common.CommonReportFactory
-}
-
-func (self *SApsaraCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
-	args *options.ReportOptions, operatorType string) common.ICloudReport {
-	return &SApsaraCloudReport{
-		common.CloudReportBase{
-			SProvider: provider,
-			Session:   session,
-			Args:      args,
-			Operator:  operatorType,
-		},
-	}
-}
-
-func (self *SApsaraCloudReportFactory) GetId() string {
-	return compute.CLOUD_PROVIDER_APSARA
-}
-
-type SApsaraCloudReport struct {
-	common.CloudReportBase
-}
-
-func (self *SApsaraCloudReport) Report() error {
-	var servers []jsonutils.JSONObject
-	var err error
-	servers, err = self.GetResourceByOperator()
-	providerInstance, err := self.InitProviderInstance()
-	if err != nil {
-		return err
-	}
-	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
-	if err != nil {
-		return err
-	}
-	for _, region := range regionList {
-		servers := regionServerMap[region.GetGlobalId()]
-		switch self.Operator {
-		case "server":
-			err = self.collectRegionMetricOfHost(region, servers)
-		case "redis":
-			err = self.collectRegionMetricOfRedis(region, servers)
-		case "rds":
-			err = self.collectRegionMetricOfRds(region, servers)
-		case "oss":
-			err = self.collectRegionMetricOfOss(region, servers)
-		case "elb":
-			err = self.collectRegionMetricOfElb(region, servers)
-		}
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}

pkg/cloudmon/collectors/apsaramon/apsaraservice.go

@@ -1,373 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package apsaramon
-
-import (
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"yunion.io/x/jsonutils"
-	"yunion.io/x/log"
-	"yunion.io/x/pkg/errors"
-
-	api "yunion.io/x/onecloud/pkg/apis/compute"
-	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
-	"yunion.io/x/onecloud/pkg/cloudprovider"
-	iden_modules "yunion.io/x/onecloud/pkg/mcclient/modules/identity"
-	"yunion.io/x/onecloud/pkg/multicloud/apsara"
-	"yunion.io/x/onecloud/pkg/util/influxdb"
-)
-
-type SApsaraMetric struct {
-	Timestamp  int64
-	UserId     string
-	InstanceId string
-	Maximum    float64
-	Minimum    float64
-	Average    float64
-	NodeId     string
-}
-
-func (self *SApsaraCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
-	dataList := make([]influxdb.SMetricData, 0)
-	aliReg := region.(*apsara.SRegion)
-	since, until, err := common.TimeRangeFromArgs(self.Args)
-	if err != nil {
-		return errors.Wrapf(err, "common.TimeRangeFromArgs")
-	}
-
-	instances := []api.ServerDetails{}
-	instanceMaps := map[string]api.ServerDetails{}
-	jsonutils.Update(&instances, servers)
-	for i := range instances {
-		if len(instances[i].ExternalId) == 0 {
-			continue
-		}
-		instanceMaps[instances[i].ExternalId] = instances[i]
-		metric, err := common.FillVMCapacity(jsonutils.Marshal(instances[i]).(*jsonutils.JSONDict))
-		if err != nil {
-			return errors.Wrapf(err, "common.FillVMCapacity")
-		}
-		dataList = append(dataList, metric)
-	}
-
-	err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
-	if err != nil {
-		log.Errorf("send server base metric error: %v", err)
-	}
-
-	var wg sync.WaitGroup
-	wg.Add(len(apsaraMetricSpecs))
-	for _metricName, _influxDbSpecs := range apsaraMetricSpecs {
-		go func(metricName string, influxDbSpecs []string) {
-			defer wg.Done()
-			dataList = []influxdb.SMetricData{}
-			metricArray, err := aliReg.FetchMetricData(metricName, "acs_ecs_dashboard", since, until)
-			if err != nil {
-				log.Errorln(err)
-				return
-			}
-
-			metrics := []SApsaraMetric{}
-			jsonutils.Update(&metrics, metricArray)
-
-			for _, rtnMetric := range metrics {
-				server, ok := instanceMaps[rtnMetric.InstanceId]
-				if ok {
-					serverMetric, err := self.collectMetricFromThisServer(jsonutils.Marshal(server), rtnMetric, influxDbSpecs)
-					if err != nil {
-						log.Errorf("collect %s error: %v", metricName, err)
-						continue
-					}
-
-					project, err := self.GetResourceById(server.ProjectId, &iden_modules.Projects)
-					if err != nil {
-						log.Errorf("server: %s getProject: %s err: %v", server.Name, server.ProjectId, err)
-						continue
-					}
-					metaMap, _ := project.GetMap("metadata")
-					if len(metaMap) > 0 {
-						for key, valObj := range metaMap {
-							if strings.Contains(key, "user:") {
-								val, _ := valObj.GetString()
-								serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{
-									Key:   key,
-									Value: val,
-								})
-							}
-						}
-					}
-					dataList = append(dataList, serverMetric)
-				}
-			}
-			log.Infof("SendMetrics %s length: %d", metricName, len(dataList))
-			err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
-			if err != nil {
-				log.Errorf("SendMetrics %s length: %d error: %v", metricName, len(dataList), err)
-			}
-
-		}(_metricName, _influxDbSpecs)
-	}
-	wg.Wait()
-	return nil
-}
-
-func (self *SApsaraCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion,
-	servers []jsonutils.JSONObject) error {
-	dataList := make([]influxdb.SMetricData, 0)
-	aliReg := region.(*apsara.SRegion)
-	since, until, err := common.TimeRangeFromArgs(self.Args)
-	if err != nil {
-		return err
-	}
-	redisDeployType := make(map[string]string)
-	for _, server := range servers {
-		local_category, err := server.GetString("local_category")
-		if err != nil {
-			return err
-		}
-		switch local_category {
-		case "single":
-			redisDeployType["single"] = "Standard"
-		case "master":
-			redisDeployType["master"] = "Standard"
-		case "cluster":
-			redisDeployType["cluster"] = "Sharding"
-		case "rwsplit":
-			redisDeployType["rwsplit"] = "Splitrw"
-		}
-
-	}
-	for metricName, influxDbSpecs := range aliRedisMetricSpecs {
-		for _, pre := range redisDeployType {
-			rtnArray, err := aliReg.FetchMetricData(pre+metricName, "acs_kvstore", since, until)
-			if err != nil {
-				log.Errorln(err)
-				continue
-			}
-
-			metrics := []SApsaraMetric{}
-			jsonutils.Update(&metrics, rtnArray)
-
-			for _, rtnMetric := range metrics {
-				for _, server := range servers {
-					external_id, _ := server.GetString("external_id")
-					if rtnMetric.InstanceId == external_id {
-						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
-						serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: rtnMetric.NodeId})
-						if err != nil {
-							return err
-						}
-						dataList = append(dataList, serverMetric)
-					}
-				}
-			}
-		}
-	}
-	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
-}
-
-func (self *SApsaraCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
-	dataList := make([]influxdb.SMetricData, 0)
-	aliReg := region.(*apsara.SRegion)
-	since, until, err := common.TimeRangeFromArgs(self.Args)
-	if err != nil {
-		return err
-	}
-	for metricName, influxDbSpecs := range aliRdsMetricSpecs {
-		rtnArray, err := aliReg.FetchMetricData(metricName, "acs_rds_dashboard", since, until)
-		if err != nil {
-			log.Errorln(err)
-			continue
-		}
-		metrics := []SApsaraMetric{}
-		jsonutils.Update(&metrics, rtnArray)
-
-		for _, rtnMetric := range metrics {
-			for _, server := range servers {
-				external_id, _ := server.GetString("external_id")
-				if rtnMetric.InstanceId == external_id {
-					metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
-					if err != nil {
-						return err
-					}
-					dataList = append(dataList, metric)
-					serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
-					if err != nil {
-						return err
-					}
-					dataList = append(dataList, serverMetric)
-				}
-			}
-		}
-	}
-	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
-}
-
-func (self *SApsaraCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
-	dataList := make([]influxdb.SMetricData, 0)
-	aliReg := region.(*apsara.SRegion)
-	since, until, err := common.TimeRangeFromArgs(self.Args)
-	if err != nil {
-		return err
-	}
-	for metricName, influxDbSpecs := range aliOSSMetricSpecs {
-		rtnArray, err := aliReg.FetchMetricData(metricName, "acs_oss", since, until)
-		if err != nil {
-			log.Errorln(err)
-			continue
-		}
-		if len(rtnArray) > 0 {
-			for _, rtnMetric := range rtnArray {
-				for _, server := range servers {
-					name, _ := server.GetString("name")
-					if bucketName, _ := rtnMetric.GetString("BucketName"); bucketName == name {
-						serverMetric, err := self.collectOssMetricFromThisServer(server, rtnMetric, metricName,
-							influxDbSpecs)
-						if err != nil {
-							return err
-						}
-						dataList = append(dataList, serverMetric)
-					}
-				}
-			}
-		}
-	}
-	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
-}
-
-func (self *SApsaraCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
-	dataList := make([]influxdb.SMetricData, 0)
-	aliReg := region.(*apsara.SRegion)
-	since, until, err := common.TimeRangeFromArgs(self.Args)
-	if err != nil {
-		return err
-	}
-	for metricName, influxDbSpecs := range aliElbMetricSpecs {
-		rtnArray, err := aliReg.FetchMetricData(metricName, "acs_slb_dashboard", since, until)
-		if err != nil {
-			log.Errorln(err)
-			continue
-		}
-
-		metrics := []SApsaraMetric{}
-		jsonutils.Update(&metrics, rtnArray)
-
-		for _, rtnMetric := range metrics {
-			for _, server := range servers {
-				external_id, _ := server.GetString("external_id")
-				if rtnMetric.InstanceId == external_id {
-					serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
-					if err != nil {
-						return err
-					}
-					dataList = append(dataList, serverMetric)
-				}
-			}
-		}
-		log.Infof("SendMetrics %s length: %d", metricName, len(dataList))
-	}
-	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
-}
-
-func (self *SApsaraCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric SApsaraMetric, influxDbSpecs []string) (influxdb.SMetricData, error) {
-	metric, err := self.NewMetricFromJson(server)
-	if err != nil {
-		return influxdb.SMetricData{}, err
-	}
-	metric.Timestamp = time.Unix(rtnMetric.Timestamp/1000, 0)
-	fieldValue := rtnMetric.Average
-	//根据条件拼装metric的tag和metirc信息
-	influxDbSpec := influxDbSpecs[2]
-	measurement := common.SubstringBefore(influxDbSpec, ".")
-	var pairsKey string
-	if strings.Contains(influxDbSpec, ",") {
-		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
-	} else {
-		pairsKey = common.SubstringAfter(influxDbSpec, ".")
-	}
-	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
-		fieldValue = fieldValue * 8
-	}
-	tag := common.SubstringAfter(influxDbSpec, ",")
-	if tag != "" && strings.Contains(influxDbSpec, "=") {
-		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
-			Key:   common.SubstringBefore(tag, "="),
-			Value: common.SubstringAfter(tag, "="),
-		})
-	}
-	cpu_cout, err := server.Get("vcpu_count")
-	if err == nil {
-		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
-			Key:   "cpu_count",
-			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
-		})
-	}
-	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
-		Key:   pairsKey,
-		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
-	})
-	metric.Name = measurement
-	return metric, nil
-}
-
-func (self *SApsaraCloudReport) collectOssMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
-	metricName string, influxDbSpecs []string) (influxdb.SMetricData, error) {
-	metric, err := self.NewMetricFromJson(server)
-	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
-	if err != nil {
-		return influxdb.SMetricData{}, err
-	}
-	timestamp, _ := rtnMetric.Get("timestamp")
-	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
-	fieldValue, err := rtnMetric.Float(metricName)
-	if err != nil {
-		return influxdb.SMetricData{}, err
-	}
-	//根据条件拼装metric的tag和metirc信息
-	influxDbSpec := influxDbSpecs[2]
-	measurement := common.SubstringBefore(influxDbSpec, ".")
-	var pairsKey string
-	if strings.Contains(influxDbSpec, ",") {
-		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
-	} else {
-		pairsKey = common.SubstringAfter(influxDbSpec, ".")
-	}
-	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
-		fieldValue = fieldValue * 8
-	}
-	tag := common.SubstringAfter(influxDbSpec, ",")
-	if tag != "" && strings.Contains(influxDbSpec, "=") {
-		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
-			Key:   common.SubstringBefore(tag, "="),
-			Value: common.SubstringAfter(tag, "="),
-		})
-	}
-	cpu_cout, err := server.Get("vcpu_count")
-	if err == nil {
-		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
-			Key:   "cpu_count",
-			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
-		})
-	}
-	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
-		Key:   pairsKey,
-		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
-	})
-	metric.Name = measurement
-	return metric, nil
-}

pkg/cloudmon/collectors/apsaramon/apsarautils.go

@@ -1,190 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package apsaramon
-
-import "yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
-
-const (
-	PERIOD             = 60
-	UNIT_AVERAGE       = "Average"
-	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
-	UNIT_PERCENT       = "Percent"
-	UNIT_BPS           = "bps"
-	UNIT_MBPS          = "Mbps"
-	UNIT_BYTEPS        = "Bps"
-	UNIT_CPS           = "cps"
-	UNIT_COUNT         = "count"
-	UNIT_MEM           = "byte"
-	UNIT_MSEC          = "ms"
-	UNIT_COUNT_SEC     = "count/s"
-
-	//ESC监控指标
-	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
-	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
-	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
-	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
-	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
-	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
-	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
-	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
-	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
-	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
-	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
-	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
-	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
-	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
-	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
-	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
-
-	//RDS监控指标
-	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
-	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
-	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
-	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
-	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
-	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_sent"
-	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
-	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
-	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
-	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
-	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
-	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
-	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
-
-	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
-	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
-	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
-	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
-
-	//REDIS监控指标
-	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
-	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
-	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
-	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
-	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
-	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
-	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
-	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
-	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
-
-	//对象存储OSS监控指标
-	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
-	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
-	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
-	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
-	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
-	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
-	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
-	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
-	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
-	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
-	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
-	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
-	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
-	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
-
-	//负载均衡监控指标
-	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
-	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
-	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
-	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
-	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
-	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
-	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
-	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
-	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
-	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
-	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
-	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
-	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
-	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
-
-	KEY_VMS   = "vms"
-	KEY_CPUS  = "cpus"
-	KEY_MEMS  = "mems"
-	KEY_DISKS = "disks"
-
-	KEY_LIMIT  = "limit"
-	KEY_ADMIN  = "admin"
-	KEY_USABLE = "usable"
-)
-
-//multiCloud查询指标列表组装
-var aliMetricSpecs = map[string][]string{
-	"CPUUtilization":  {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
-	"InternetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET},
-	"IntranetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET},
-	"InternetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET},
-	"IntranetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET},
-	"DiskReadBPS":     {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},
-	"DiskWriteBPS":    {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
-	"DiskReadIOPS":    {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
-	"DiskWriteIOPS":   {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
-}
-var apsaraMetricSpecs = map[string][]string{
-	"vm.MemoryUtilization":  {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE},
-	"vm.CPUUtilization":     {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
-	"diskusage_utilization": {DEFAULT_STATISTICS, UNIT_PERCENT, common.INFLUXDB_FIELD_DISK_USAGE},
-	"InternetInRate":        {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET},
-	"IntranetInRate":        {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET},
-	"InternetOutRate":       {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET},
-	"IntranetOutRate":       {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET},
-	"DiskReadBPS":           {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},
-	"DiskWriteBPS":          {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
-	"DiskReadIOPS":          {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
-	"DiskWriteIOPS":         {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
-}
-var aliRdsMetricSpecs = map[string][]string{
-	"CpuUsage":                {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
-	"MemoryUsage":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
-	"MySQL_NetworkInNew":      {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL},
-	"MySQL_NetworkOutNew":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL},
-	"DiskUsage":               {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
-	"SQLServer_NetworkInNew":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER},
-	"SQLServer_NetworkOutNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER},
-	"ConnectionUsage":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_USAGE},
-}
-var aliRedisMetricSpecs = map[string][]string{
-	"CpuUsage":       {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE},
-	"MemoryUsage":    {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE},
-	"IntranetIn":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX},
-	"IntranetOut":    {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX},
-	"UsedConnection": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE},
-	"UsedQPS":        {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES},
-	"Keys":           {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS},
-	"ExpiredKeys":    {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS},
-	"UsedMemory":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE},
-}
-var aliOSSMetricSpecs = map[string][]string{
-	"InternetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET},
-	"InternetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET},
-	"IntranetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET},
-	"IntranetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET},
-	"GetObjectE2eLatency":  {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET},
-	"PostObjectE2eLatency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_POST},
-	"GetObjectCount":       {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET},
-	"PostObjectCount":      {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_POST},
-	"ServerErrorCount":     {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX},
-}
-var aliElbMetricSpecs = map[string][]string{
-	"InstanceTrafficRX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_RX},
-	"TrafficRXNew":          {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_RX},
-	"InstanceTrafficTX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_TX},
-	"TrafficTXNew":          {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_TX},
-	"InstanceStatusCode2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX},
-	"InstanceStatusCode3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX},
-	"InstanceStatusCode4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX},
-	"InstanceStatusCode5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX},
-}

pkg/cloudmon/collectors/apsaramon/doc.go

@@ -1,15 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package apsaramon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon"