feature: metric data collector for aliyun&huawei

This commit is contained in:
Qiu Jian
2019-11-27 03:14:14 +08:00
parent 937de0a3aa
commit afbafb1e29
17 changed files with 643 additions and 2 deletions

View File

@@ -1 +1,15 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cloudevent // import "yunion.io/x/onecloud/pkg/apis/cloudevent"

View File

@@ -1,3 +1,17 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package compute
import (

View File

@@ -1 +1,15 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package models // import "yunion.io/x/onecloud/pkg/cloudevent/models"

View File

@@ -1 +1,15 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package options // import "yunion.io/x/onecloud/pkg/cloudevent/options"

View File

@@ -1 +1,15 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package service // import "yunion.io/x/onecloud/pkg/cloudevent/service"

View File

@@ -1 +1,15 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tasks // import "yunion.io/x/onecloud/pkg/cloudevent/tasks"

View File

@@ -1,3 +1,17 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package guestman
import (

View File

@@ -76,6 +76,8 @@ const (
ErrInvalidProvider = errors.Error("InvalidProvider")
ErrNoBalancePermission = errors.Error("NoBalancePermission")
ErrTooLarge = errors.Error("TooLargeEntity")
)
var (
@@ -141,6 +143,8 @@ var (
ErrTimeout: 504,
ErrProtectedResource: 403,
ErrNoProject: 403,
ErrTooLarge: 413,
}
)

View File

@@ -199,3 +199,7 @@ func NewClientError(msg string, params ...interface{}) *httputils.JSONClientErro
func NewUnclassifiedError(msg string, params ...interface{}) *httputils.JSONClientError {
return httputils.NewJsonClientError(httpErrorCode[errors.ErrUnclassified], string(errors.ErrUnclassified), msg, params)
}
func NewTooLargeEntityError(msg string, params ...interface{}) *httputils.JSONClientError {
return httputils.NewJsonClientError(httpErrorCode[ErrTooLarge], string(ErrTooLarge), msg, params)
}

View File

@@ -0,0 +1,189 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package aliyun
import (
"strconv"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
)
const (
ALIYUN_API_VERSION_METRICS = "2019-01-01"
)
func (r *SRegion) metricsRequest(action string, params map[string]string) (jsonutils.JSONObject, error) {
client, err := r.getSdkClient()
if err != nil {
return nil, errors.Wrap(err, "r.getSdkClient")
}
return jsonRequest(client, "metrics.aliyuncs.com", ALIYUN_API_VERSION_METRICS, action, params, r.client.Debug)
}
type SResourceLabel struct {
Name string `json:"name"`
Value string `json:"value"`
}
type SResource struct {
Description string `json:"Description"`
Labels string `json:"Labels"`
Namespace string `json:"Namespace"`
}
func (r *SRegion) DescribeProjectMeta(limit, offset int) (int, []SResource, error) {
params := make(map[string]string)
if limit <= 0 {
limit = 30
}
params["PageSize"] = strconv.FormatInt(int64(limit), 10)
if offset > 0 {
pageNum := (offset / limit) + 1
params["PageNumber"] = strconv.FormatInt(int64(pageNum), 10)
}
body, err := r.metricsRequest("DescribeProjectMeta", params)
if err != nil {
return 0, nil, errors.Wrap(err, "r.metricsRequest DescribeProjectMeta")
}
total, _ := body.Int("Total")
res := make([]SResource, 0)
err = body.Unmarshal(&res, "Resources", "Resource")
if err != nil {
return 0, nil, errors.Wrap(err, "body.Unmarshal Resources Resource")
}
return int(total), res, nil
}
func (r *SRegion) FetchNamespaces() ([]SResource, error) {
resources := make([]SResource, 0)
total := -1
for total < 0 || len(resources) < total {
ntotal, res, err := r.DescribeProjectMeta(1000, len(resources))
if err != nil {
return nil, errors.Wrap(err, "r.DescribeProjectMeta")
}
if len(res) == 0 {
break
}
resources = append(resources, res...)
total = ntotal
}
return resources, nil
}
type SMetricMeta struct {
Description string `json:"Description"`
MetricName string `json:"MetricName"`
Statistics string `json:"Statistics"`
Labels string `json:"Labels"`
Dimensions string `json:"Dimensions"`
Namespace string `json:"Namespace"`
Periods string `json:"Periods"`
Unit string `json:"Unit"`
}
func (r *SRegion) DescribeMetricMetaList(ns string, limit, offset int) (int, []SMetricMeta, error) {
params := make(map[string]string)
if limit <= 0 {
limit = 30
}
params["Namespace"] = ns
params["PageSize"] = strconv.FormatInt(int64(limit), 10)
if offset > 0 {
pageNum := (offset / limit) + 1
params["PageNumber"] = strconv.FormatInt(int64(pageNum), 10)
}
body, err := r.metricsRequest("DescribeMetricMetaList", params)
if err != nil {
return 0, nil, errors.Wrap(err, "r.metricsRequest DescribeMetricMetaList")
}
total, _ := body.Int("TotalCount")
res := make([]SMetricMeta, 0)
err = body.Unmarshal(&res, "Resources", "Resource")
if err != nil {
return 0, nil, errors.Wrap(err, "body.Unmarshal Resources Resource")
}
return int(total), res, nil
}
func (r *SRegion) FetchMetrics(ns string) ([]SMetricMeta, error) {
metrics := make([]SMetricMeta, 0)
total := -1
for total < 0 || len(metrics) < total {
ntotal, res, err := r.DescribeMetricMetaList(ns, 1000, len(metrics))
if err != nil {
return nil, errors.Wrap(err, "r.DescribeMetricMetaList")
}
if len(res) == 0 {
break
}
metrics = append(metrics, res...)
total = ntotal
}
return metrics, nil
}
func (r *SRegion) DescribeMetricList(name string, ns string, since time.Time, until time.Time, nextToken string) ([]jsonutils.JSONObject, string, error) {
params := make(map[string]string)
params["MetricName"] = name
params["Namespace"] = ns
params["Length"] = "2000"
if len(nextToken) > 0 {
params["NextToken"] = nextToken
}
if !since.IsZero() {
params["StartTime"] = strconv.FormatInt(since.UnixNano()/1000, 10)
}
if !until.IsZero() {
params["EndTime"] = strconv.FormatInt(until.UnixNano()/1000, 10)
}
body, err := r.metricsRequest("DescribeMetricList", params)
if err != nil {
return nil, "", errors.Wrap(err, "region.MetricRequest")
}
nToken, _ := body.GetString("NextToken")
dataStr, _ := body.GetString("Datapoints")
if len(dataStr) == 0 {
return nil, "", nil
}
dataJson, err := jsonutils.ParseString(dataStr)
if err != nil {
return nil, "", errors.Wrap(err, "jsonutils.ParseString")
}
dataArray, err := dataJson.GetArray()
if err != nil {
return nil, "", errors.Wrap(err, "dataJson.GetArray")
}
return dataArray, nToken, nil
}
func (r *SRegion) FetchMetricData(name string, ns string, since time.Time, until time.Time) ([]jsonutils.JSONObject, error) {
data := make([]jsonutils.JSONObject, 0)
nextToken := ""
for {
datArray, next, err := r.DescribeMetricList(name, ns, since, until, nextToken)
if err != nil {
return nil, errors.Wrap(err, "r.DescribeMetricList")
}
data = append(data, datArray...)
if len(next) == 0 {
break
}
nextToken = next
}
return data, nil
}

View File

@@ -0,0 +1,85 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shell
import (
"time"
"yunion.io/x/pkg/util/timeutils"
"yunion.io/x/onecloud/pkg/mcclient/modulebase"
"yunion.io/x/onecloud/pkg/multicloud/aliyun"
"yunion.io/x/onecloud/pkg/util/printutils"
"yunion.io/x/onecloud/pkg/util/shellutils"
)
func init() {
type NamespaceListOptions struct {
}
shellutils.R(&NamespaceListOptions{}, "namespace-list", "List monbitor metric namespaces", func(cli *aliyun.SRegion, args *NamespaceListOptions) error {
nslist, err := cli.FetchNamespaces()
if err != nil {
return err
}
printList(nslist, 0, 0, 0, nil)
return nil
})
type MetricListOptions struct {
NAMESPACE string `help:"namespace"`
}
shellutils.R(&MetricListOptions{}, "metrics-list", "List metrics in a namespace", func(cli *aliyun.SRegion, args *MetricListOptions) error {
metrics, err := cli.FetchMetrics(args.NAMESPACE)
if err != nil {
return err
}
printList(metrics, 0, 0, 0, nil)
return nil
})
type DescribeMetricListOptions struct {
METRIC string `help:"metric name"`
NAMESPACE string `help:"name space"`
Since string `help:"since, 2019-11-29T11:22:00Z"`
Until string `help:"since, 2019-11-30T11:22:00Z"`
}
shellutils.R(&DescribeMetricListOptions{}, "metric-data-list", "DescribeMetricList", func(cli *aliyun.SRegion, args *DescribeMetricListOptions) error {
var since time.Time
var err error
if len(args.Since) > 0 {
since, err = timeutils.ParseTimeStr(args.Since)
if err != nil {
return err
}
}
var until time.Time
if len(args.Until) > 0 {
until, err = timeutils.ParseTimeStr(args.Until)
if err != nil {
return err
}
}
data, err := cli.FetchMetricData(args.METRIC, args.NAMESPACE, since, until)
if err != nil {
return err
}
result := &modulebase.ListResult{
Data: data,
Total: len(data),
}
printutils.PrintJSONList(result, nil)
return nil
})
}

View File

@@ -78,6 +78,7 @@ type Client struct {
DBInstanceFlavor *modules.SDBInstanceFlavorManager
DBInstanceJob *modules.SDBInstanceJobManager
Traces *modules.STraceManager
CloudEye *modules.SCloudEyeManager
}
func (self *Client) Init() error {
@@ -160,6 +161,7 @@ func (self *Client) initManagers() {
self.DBInstanceFlavor = modules.NewDBInstanceFlavorManager(self.regionId, self.projectId, self.signer, self.debug)
self.DBInstanceJob = modules.NewDBInstanceJobManager(self.regionId, self.projectId, self.signer, self.debug)
self.Traces = modules.NewTraceManager(self.regionId, self.projectId, self.signer, self.debug)
self.CloudEye = modules.NewCloudEyeManager(self.regionId, self.projectId, self.signer, self.debug)
}
self.init = true

View File

@@ -34,7 +34,7 @@ const (
ServiceNameECS ServiceNameType = "ecs" // 弹性云服务
ServiceNameCCE ServiceNameType = "cce" // 云容器服务
ServiceNameAS ServiceNameType = "as" // 弹性伸缩服务
ServiceNameIAM ServiceNameType = "iam" // 统一身份认证服务
ServiceNameIAM ServiceNameType = "iam" // 统一身份认证服务
ServiceNameIMS ServiceNameType = "ims" // 镜像服务
ServiceNameCSBS ServiceNameType = "csbs" // 云服务器备份服务
ServiceNameCCI ServiceNameType = "cci" // 云容器实例 CCI
@@ -48,7 +48,8 @@ const (
ServiceNameNAT ServiceNameType = "nat" // Nat网关 NAT
ServiceNameDCS ServiceNameType = "dcs" // 分布式缓存服务
ServiceNameRDS ServiceNameType = "rds" // 关系型数据库 RDS
ServiceNameCTS ServiceNameType = "cts" //云审计服务
ServiceNameCTS ServiceNameType = "cts" // 云审计服务
ServiceNameCES ServiceNameType = "ces" // 监控服务 CloudEye
)
type SManagerContext struct {

View File

@@ -0,0 +1,153 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package modules
import (
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/multicloud/huawei/client/auth"
"yunion.io/x/onecloud/pkg/multicloud/huawei/client/requests"
)
type SCloudEyeManager struct {
SResourceManager
}
func NewCloudEyeManager(regionId string, projectId string, signer auth.Signer, debug bool) *SCloudEyeManager {
return &SCloudEyeManager{SResourceManager: SResourceManager{
SBaseManager: NewBaseManager(signer, debug),
ServiceName: ServiceNameCES,
Region: regionId,
ProjectId: projectId,
version: "V1.0",
Keyword: "",
KeywordPlural: "metrics",
ResourceKeyword: "metrics",
}}
}
type SMetricDimension struct {
Name string `json:"name"`
Value string `json:"value"`
}
type SDatapoint struct {
Timestamp int64 `json:"timestamp"`
Max float64 `json:"max,omitzero"`
Min float64 `json:"min,omitzero"`
Average float64 `json:"average,omitzero"`
Sum float64 `json:"sum,omitzero"`
Variance float64 `json:"variance,omitzero"`
}
type SMetricData struct {
SMetricMeta
Datapoints []SDatapoint
}
type SMetricMeta struct {
SMetric
Unit string `json:"unit"`
}
type SMetric struct {
MetricName string `json:"metric_name"`
Namespace string `json:"namespace"`
Dimensions []SMetricDimension `json:"dimensions"`
}
func (ces *SCloudEyeManager) ListMetrics() ([]SMetricMeta, error) {
metrics := make([]SMetricMeta, 0)
next := ""
for {
marker, data, err := ces.listMetricsInternal(next)
if err != nil {
return nil, errors.Wrap(err, "ces.listMetricsInternal")
}
if len(data) == 0 {
break
}
metrics = append(metrics, data...)
next = marker
}
return metrics, nil
}
func (ces *SCloudEyeManager) listMetricsInternal(start string) (string, []SMetricMeta, error) {
request := requests.NewResourceRequest("GET", string(ces.ServiceName), ces.version, ces.Region, ces.ProjectId, ces.ResourceKeyword)
request.AddQueryParam("limit", "1000")
if len(start) > 0 {
request.AddQueryParam("start", start)
}
_, resp, err := ces.jsonRequest(request)
if err != nil {
return "", nil, errors.Wrap(err, "ces.jsonRequest")
}
marker, _ := resp.GetString("meta_data", "marker")
metrics := make([]SMetricMeta, 0)
err = resp.Unmarshal(&metrics, "metrics")
if err != nil {
return "", nil, errors.Wrap(err, "resp.Unmarshal metrics")
}
return marker, metrics, nil
}
type SBatchQueryMetricDataInput struct {
Metrics []SMetric `json:"metrics"`
From int64 `json:"from"`
To int64 `json:"to"`
Period string `json:"period"`
Filter string `json:"filter"`
}
func (ces *SCloudEyeManager) GetMetricsData(metrics []SMetricMeta, since time.Time, until time.Time) ([]SMetricData, error) {
if len(metrics) > 10 {
return nil, errors.Wrap(httperrors.ErrTooLarge, "request more than 10 metrics")
}
metricReq := make([]SMetric, len(metrics))
for i := range metrics {
metricReq[i] = metrics[i].SMetric
}
request := requests.NewResourceRequest("POST", string(ces.ServiceName), ces.version, ces.Region, ces.ProjectId, "batch-query-metric-data")
input := SBatchQueryMetricDataInput{
Metrics: metricReq,
From: since.UnixNano() / 1000,
To: until.UnixNano() / 1000,
Period: "1",
Filter: "average",
}
body := jsonutils.Marshal(&input).String()
request.SetContent([]byte(body))
_, resp, err := ces.jsonRequest(request)
if err != nil {
return nil, errors.Wrap(err, "ces.jsonRequest")
}
log.Debugf("%s", resp)
result := make([]SMetricData, 0)
err = resp.Unmarshal(&result, "metrics")
if err != nil {
return nil, errors.Wrap(err, "resp.Unmarshal")
}
return result, nil
}

View File

@@ -0,0 +1,29 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package huawei
import (
"time"
"yunion.io/x/onecloud/pkg/multicloud/huawei/client/modules"
)
func (r *SRegion) GetMetrics() ([]modules.SMetricMeta, error) {
return r.ecsClient.CloudEye.ListMetrics()
}
func (r *SRegion) GetMetricsData(metrics []modules.SMetricMeta, since time.Time, until time.Time) ([]modules.SMetricData, error) {
return r.ecsClient.CloudEye.GetMetricsData(metrics, since, until)
}

View File

@@ -0,0 +1,62 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package shell
import (
"yunion.io/x/pkg/util/timeutils"
"yunion.io/x/onecloud/pkg/multicloud/huawei"
"yunion.io/x/onecloud/pkg/util/shellutils"
)
func init() {
type MetricListOptions struct {
}
shellutils.R(&MetricListOptions{}, "metrics-list", "List metrics", func(cli *huawei.SRegion, args *MetricListOptions) error {
metrics, err := cli.GetMetrics()
if err != nil {
return err
}
printList(metrics, 0, 0, 0, nil)
return nil
})
type MetricDataOptions struct {
START int `help:"Start metrics"`
Count int `help:"Metric count" default:"1"`
SINCE string `help:"since"`
UNTIL string `help:"until"`
}
shellutils.R(&MetricDataOptions{}, "metrics-data-list", "List metrics", func(cli *huawei.SRegion, args *MetricDataOptions) error {
metrics, err := cli.GetMetrics()
if err != nil {
return err
}
since, err := timeutils.ParseTimeStr(args.SINCE)
if err != nil {
return err
}
until, err := timeutils.ParseTimeStr(args.UNTIL)
if err != nil {
return err
}
data, err := cli.GetMetricsData(metrics[args.START:args.START+args.Count], since, until)
if err != nil {
return err
}
printList(data, 0, 0, 0, nil)
return nil
})
}

View File

@@ -1,3 +1,17 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package httputils
import (