fanhuadesenlinnn/cloudpods
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git synced 2026-09-20 08:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

s3gateway round 2

Browse Source
This commit is contained in:
Qiu Jian
2019-07-23 01:30:09 +08:00
parent 870813f04b
commit 93b8bac5bb
103 changed files with 5509 additions and 1632 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
cmd/climc/shell/buckets.go
View File

@@ -3,6 +3,8 @@ package shell
import (
"yunion.io/x/jsonutils"
"io"
"os"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/modules"
"yunion.io/x/onecloud/pkg/mcclient/options"
@@ -84,4 +86,88 @@ func init() {
printObject(result)
return nil
})
type BucketListObjectsOptions struct {
ID string `help:"ID or name of bucket" json:"-"`
Prefix string `help:"List objects with prefix"`
Recursive bool `help:"List objects recursively"`
}
R(&BucketListObjectsOptions{}, "bucket-object-list", "List objects in a bucket", func(s *mcclient.ClientSession, args *BucketListObjectsOptions) error {
params, err := options.StructToParams(args)
if err != nil {
return err
}
result, err := modules.Buckets.GetSpecific(s, args.ID, "objects", params)
if err != nil {
return err
}
arrays, _ := result.(*jsonutils.JSONArray).GetArray()
listResult := modules.ListResult{Data: arrays}
printList(&listResult, []string{})
return nil
})
type BucketDeleteObjectsOptions struct {
ID string `help:"ID or name of bucket" json:"-"`
KEYS []string `help:"List of objects to delete"`
}
R(&BucketDeleteObjectsOptions{}, "bucket-object-delete", "Delete objects in a bucket", func(s *mcclient.ClientSession, args *BucketDeleteObjectsOptions) error {
params, err := options.StructToParams(args)
if err != nil {
return err
}
result, err := modules.Buckets.PerformAction(s, args.ID, "delete", params)
if err != nil {
return err
}
printObject(result)
return nil
})
type BucketUploadObjectsOptions struct {
ID string `help:"ID or name of bucket" json:"-"`
KEY string `help:"Key of object to upload"`
Path string `help:"Path to file to upload"`
ContentType string `help:"Content type"`
StorageClass string `help:"storage CLass"`
}
R(&BucketUploadObjectsOptions{}, "bucket-object-upload", "Upload an object into a bucket", func(s *mcclient.ClientSession, args *BucketUploadObjectsOptions) error {
var body io.Reader
if len(args.Path) > 0 {
file, err := os.Open(args.Path)
if err != nil {
return err
}
defer file.Close()
body = file
} else {
body = os.Stdin
}
err := modules.Buckets.Upload(s, args.ID, args.KEY, body, args.ContentType, args.StorageClass)
if err != nil {
return err
}
return nil
})
type BucketPresignObjectsOptions struct {
ID string `help:"ID or name of bucket" json:"-"`
KEY string `help:"Key of object to upload"`
Method string `help:"Request method" choices:"GET|PUT|DELETE"`
ExpireSeconds int `help:"expire in seconds" default:"60"`
}
R(&BucketPresignObjectsOptions{}, "bucket-object-tempurl", "Get temporal URL for an object in a bucket", func(s *mcclient.ClientSession, args *BucketPresignObjectsOptions) error {
params, err := options.StructToParams(args)
if err != nil {
return err
}
result, err := modules.Buckets.PerformAction(s, args.ID, "temp-url", params)
if err != nil {
return err
}
printObject(result)
return nil
})
}

9
go.mod
View File

@@ -64,6 +64,7 @@ require (
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
github.com/golang/protobuf v1.3.1
github.com/google/btree v1.0.0 // indirect
github.com/google/go-querystring v1.0.0 // indirect
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect
github.com/google/gopacket v1.1.17
github.com/google/uuid v1.1.0 // indirect
@@ -102,8 +103,8 @@ require (
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/moul/http2curl v1.0.0
github.com/mozillazg/go-httpheader v0.2.1 // indirect
github.com/mozillazg/go-pinyin v0.15.0
github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect
github.com/opentracing/opentracing-go v1.0.2 // indirect
github.com/openzipkin/zipkin-go-opentracing v0.3.4 // indirect
@@ -123,6 +124,7 @@ require (
github.com/spf13/pflag v1.0.3 // indirect
github.com/stretchr/testify v1.3.0
github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f
github.com/tinylib/msgp v1.1.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
github.com/tredoe/osutil v0.0.0-20161130133508-7d3ee1afa71c
@@ -144,7 +146,10 @@ require (
google.golang.org/genproto v0.0.0-20181218023534-67d6565462c5 // indirect
google.golang.org/grpc v1.19.0
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
gopkg.in/go-playground/validator.v8 v8.18.2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.44.0 // indirect
gopkg.in/ldap.v3 v3.0.3
gopkg.in/yaml.v2 v2.2.2
k8s.io/api v0.0.0-20181004124137-fd83cbc87e76
@@ -157,5 +162,5 @@ require (
yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30
yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda
yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe
yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d
)

18
go.sum
View File

@@ -179,6 +179,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck=
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
@@ -288,12 +290,12 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs=
github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
github.com/mozillazg/go-pinyin v0.15.0 h1:sSwlnsogK/WMzcf0HnjgxyAI4GU6LFqwXnhr77q1Z80=
github.com/mozillazg/go-pinyin v0.15.0/go.mod h1:bO+dztNW6O2lSJdYLha7LO3bujXzjjU3UvKb2IGANfg=
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY=
github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80 h1:1tGm26e9ktdEIa7LHD7xf7oMUCE0V2wP4URkUAvEttA=
github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80/go.mod h1:UZaoQ2hntRH4P8MwrKOxcVXvNgg1N4atxfa7NP+1wWg=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU=
@@ -362,6 +364,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M=
github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e h1:CtKVGXKh2bfmepZ/YogAjvL/CrSy9NGZET500K7arf4=
github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f h1:TzE7Cs9HhTyfot4WIoMnbD1rWfD4Jkwy2M3Zs66CaRE=
github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f/go.mod h1:/4BhymH1yO6ljUGQgcKsd7L3W+pdKRxoRiOuoZPLnGg=
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s=
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -453,8 +457,14 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ=
gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.44.0 h1:YRJzTUp0kSYWUVFF5XAbDFfyiqwsl0Vb9R8TVP5eRi0=
gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ldap.v3 v3.0.3 h1:YKRHW/2sIl05JsCtx/5ZuUueFuJyoj/6+DGXe3wp6ro=
gopkg.in/ldap.v3 v3.0.3/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
@@ -488,5 +498,5 @@ yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30 h1:6CkrwtX4xeYFqqpdWtQPAVqEnD
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda h1:i+/3Hh+kVmPZM1P+j3wfViEzb0XlttWyNvDSYA5DDDU=
yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe h1:Wr2EXv72ynwtW+x0VCqWwpsfUBWecaZWnLMgaP3UiTo=
yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d h1:00kGV39weRaYPldUUh5mllj4aHcGMOZZx4m3CotRESw=
yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

3
pkg/apis/compute/bucket.go
View File

@@ -9,4 +9,7 @@ const (
BUCKET_STATUS_DELETING = "deleting"
BUCKET_STATUS_DELETED = "deleted"
BUCKET_STATUS_DELETE_FAIL = "delete_fail"
BUCKET_UPLOAD_OBJECT_KEY_HEADER = "X-Yunion-Bucket-Upload-Key"
BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER = "X-Yunion-Bucket-Upload-Storageclass"
)

3
pkg/appsrv/appsrv.go
View File

@@ -250,7 +250,8 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
to = app.processTimeout
}
var (
ctx context.Context = app.context
ctx = app.context
cancel context.CancelFunc = nil
)
if to > 0 {

43
pkg/appsrv/handlerinfo.go
View File

@@ -26,27 +26,38 @@ type handlerRequestCounter struct {
duration float64
}
type TProcessTimeoutCallback func(*SHandlerInfo, *http.Request) time.Duration
type SHandlerInfo struct {
method string
path []string
name string
handler func(context.Context, http.ResponseWriter, *http.Request)
metadata map[string]interface{}
tags map[string]string
counter2XX handlerRequestCounter
counter4XX handlerRequestCounter
counter5XX handlerRequestCounter
processTimeout time.Duration
workerMan *SWorkerManager
skipLog bool
method string
path []string
name string
handler func(context.Context, http.ResponseWriter, *http.Request)
metadata map[string]interface{}
tags map[string]string
counter2XX handlerRequestCounter
counter4XX handlerRequestCounter
counter5XX handlerRequestCounter
workerMan *SWorkerManager
skipLog bool
processTimeout time.Duration
processTimeoutCallback TProcessTimeoutCallback
}
func (this *SHandlerInfo) FetchProcessTimeout(r *http.Request) time.Duration {
if r.Method == http.MethodGet && len(r.URL.Query().Get("export_keys")) > 0 {
return time.Hour * 2
} else {
return this.processTimeout
var tm time.Duration
if this.processTimeoutCallback != nil {
tm = this.processTimeoutCallback(this, r)
}
if tm < this.processTimeout {
tm = this.processTimeout
}
return tm
}
func (this *SHandlerInfo) SetProcessTimeoutCallback(callback TProcessTimeoutCallback) {
this.processTimeoutCallback = callback
}
func (this *SHandlerInfo) GetName(params map[string]string) string {

3
pkg/cloudcommon/db/interface.go
View File

@@ -21,6 +21,7 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/sqlchemy"
"time"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/cloudcommon/object"
@@ -90,6 +91,8 @@ type IModelManager interface {
InitializeData() error
CustomizeHandlerInfo(info *appsrv.SHandlerInfo)
SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration
FetchCreateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error)
FetchUpdateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error)
IsCustomizedGetDetailsBody() bool

17
pkg/cloudcommon/db/modelbase.go
View File

@@ -22,6 +22,8 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/sqlchemy"
"log"
"time"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/cloudcommon/object"
"yunion.io/x/onecloud/pkg/httperrors"
@@ -53,7 +55,11 @@ func NewModelBaseManager(model interface{}, tableName string, keyword string, ke
}
func (manager *SModelBaseManager) GetIModelManager() IModelManager {
return manager.GetVirtualObject().(IModelManager)
virt := manager.GetVirtualObject()
if virt == nil {
log.Fatalf("%s.GetIModelManager got nil!", manager.keywordPlural)
}
return virt.(IModelManager)
}
func (manager *SModelBaseManager) SetAlias(alias string, aliasPlural string) {
@@ -203,7 +209,14 @@ func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, query
}
func (manager *SModelBaseManager) CustomizeHandlerInfo(info *appsrv.SHandlerInfo) {
// do nothing
info.SetProcessTimeoutCallback(manager.GetIModelManager().SetHandlerProcessTimeout)
}
func (manager *SModelBaseManager) SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration {
if r.Method == http.MethodGet && len(r.URL.Query().Get("export_keys")) > 0 {
return time.Hour * 2
}
return -time.Second
}
func (manager *SModelBaseManager) FetchCreateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error) {

131
pkg/cloudprovider/objectstore.go
View File

@@ -1,8 +1,11 @@
package cloudprovider
import (
"context"
"io"
"time"
"strings"
"yunion.io/x/pkg/errors"
)
@@ -11,6 +14,22 @@ type SBucketAccessUrl struct {
Description string
}
type SBaseCloudObject struct {
Key string
SizeBytes int64
StorageClass string
ETag string
LastModified time.Time
ContentType string
}
type SListObjectResult struct {
Objects []ICloudObject
NextMarker string
CommonPrefixes []ICloudObject
IsTruncated bool
}
type ICloudBucket interface {
IVirtualResource
@@ -22,17 +41,125 @@ type ICloudBucket interface {
GetCreateAt() time.Time
GetStorageClass() string
GetAccessUrls() []SBucketAccessUrl
ListObjects(prefix string, marker string, delimiter string, maxCount int) (SListObjectResult, error)
GetIObjects(prefix string, isRecursive bool) ([]ICloudObject, error)
PutObject(ctx context.Context, key string, input io.ReadSeeker, contType string, storageClass string) error
DeleteObject(ctx context.Context, keys string) error
GetTempUrl(method string, key string, expire time.Duration) (string, error)
// ObjectExist(key string) (bool, error)
}
func GetIBucketByName(region ICloudRegion, name string) (ICloudBucket, error) {
type ICloudObject interface {
GetIBucket() ICloudBucket
GetKey() string
GetSizeBytes() int64
GetLastModified() time.Time
GetStorageClass() string
GetETag() string
GetContentType() string
}
func ICloudObject2BaseCloudObject(obj ICloudObject) SBaseCloudObject {
return SBaseCloudObject{
Key: obj.GetKey(),
SizeBytes: obj.GetSizeBytes(),
StorageClass: obj.GetStorageClass(),
ETag: obj.GetETag(),
LastModified: obj.GetLastModified(),
ContentType: obj.GetContentType(),
}
}
func (o *SBaseCloudObject) GetKey() string {
return o.Key
}
func (o *SBaseCloudObject) GetSizeBytes() int64 {
return o.SizeBytes
}
func (o *SBaseCloudObject) GetLastModified() time.Time {
return o.LastModified
}
func (o *SBaseCloudObject) GetStorageClass() string {
return o.StorageClass
}
func (o *SBaseCloudObject) GetETag() string {
return o.ETag
}
func (o *SBaseCloudObject) GetContentType() string {
return o.ContentType
}
func GetIBucketById(region ICloudRegion, name string) (ICloudBucket, error) {
buckets, err := region.GetIBuckets()
if err != nil {
return nil, errors.Wrap(err, "region.GetIBuckets")
}
for i := range buckets {
if buckets[i].GetName() == name {
if buckets[i].GetGlobalId() == name {
return buckets[i], nil
}
}
return nil, ErrNotFound
}
func GetIObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool) ([]ICloudObject, error) {
delimiter := "/"
if isRecursive {
delimiter = ""
}
ret := make([]ICloudObject, 0)
// Save marker for next request.
var marker string
for {
// Get list of objects a maximum of 1000 per request.
result, err := bucket.ListObjects(objectPrefix, marker, delimiter, 1000)
if err != nil {
return nil, errors.Wrap(err, "bucket.ListObjects")
}
// Send all objects
if len(result.Objects) > 0 {
ret = append(ret, result.Objects...)
marker = result.Objects[len(result.Objects)-1].GetKey()
}
// Send all common prefixes if any.
// NOTE: prefixes are only present if the request is delimited.
if len(result.CommonPrefixes) > 0 {
ret = append(ret, result.CommonPrefixes...)
}
// If next marker present, save it for next request.
if result.NextMarker != "" {
marker = result.NextMarker
}
// Listing ends result is not truncated, break the loop
if !result.IsTruncated {
break
}
}
return ret, nil
}
func Makedir(ctx context.Context, bucket ICloudBucket, key string) error {
segs := make([]string, 0)
for _, seg := range strings.Split(key, "/") {
if len(seg) > 0 {
segs = append(segs, seg)
}
}
path := strings.Join(segs, "/") + "/"
err := bucket.PutObject(ctx, path, strings.NewReader(""), "", "")
if err != nil {
return errors.Wrap(err, "PutObject")
}
return nil
}

2
pkg/cloudprovider/resources.go
View File

@@ -114,7 +114,7 @@ type ICloudRegion interface {
CreateIBucket(name string, storageClassStr string, acl string) error
DeleteIBucket(name string) error
IBucketExist(name string) (bool, error)
GetIBucketByName(name string) (ICloudBucket, error)
GetIBucketById(name string) (ICloudBucket, error)
GetProvider() string
}

224
pkg/compute/models/buckets.go
View File

@@ -3,7 +3,11 @@ package models
import (
"context"
"database/sql"
"regexp"
"net/http"
"strings"
"time"
"github.com/minio/minio-go/pkg/s3utils"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
@@ -11,7 +15,9 @@ import (
"yunion.io/x/pkg/util/compare"
"yunion.io/x/sqlchemy"
"io"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
@@ -19,6 +25,7 @@ import (
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/modules"
)
type SBucketManager struct {
@@ -52,6 +59,14 @@ type SBucket struct {
Acl string `width:"36" charset:"ascii" nullable:"false" list:"user"`
}
func (manager *SBucketManager) SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration {
if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/upload") && r.Header.Get(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER) != "" {
log.Debugf("upload object, set process timeout to 2 hour!!!")
return 2 * time.Hour
}
return manager.SVirtualResourceBaseManager.SetHandlerProcessTimeout(info, r)
}
func (manager *SBucketManager) fetchBuckets(provider *SCloudprovider, region *SCloudregion) ([]SBucket, error) {
q := manager.Query()
if provider != nil {
@@ -267,10 +282,16 @@ func (bucket *SBucket) GetIRegion() (cloudprovider.ICloudRegion, error) {
return provider.GetIRegionById(region.GetExternalId())
}
var BUCKET_NAME_REG = regexp.MustCompile(`^[a-zA-Z0-9-]+$`)
func (bucket *SBucket) GetIBucket() (cloudprovider.ICloudBucket, error) {
iregion, err := bucket.GetIRegion()
if err != nil {
return nil, errors.Wrap(err, "bucket.GetIRegion")
}
return iregion.GetIBucketById(bucket.ExternalId)
}
func isValidBucketName(name string) bool {
return BUCKET_NAME_REG.MatchString(name)
func isValidBucketName(name string) error {
return s3utils.CheckValidBucketNameStrict(name)
}
func (manager *SBucketManager) ValidateCreateData(
@@ -293,8 +314,9 @@ func (manager *SBucketManager) ValidateCreateData(
if len(nameStr) == 0 {
return nil, httperrors.NewInputParameterError("missing name")
}
if !isValidBucketName(nameStr) {
return nil, httperrors.NewInputParameterError("invalid name, only alphabets, digits and hyphen(-) allowed")
err := isValidBucketName(nameStr)
if err != nil {
return nil, httperrors.NewInputParameterError("invalid bucket name: %s", err)
}
return manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data)
}
@@ -323,8 +345,9 @@ func (bucket *SBucket) ValidateUpdateData(
) (*jsonutils.JSONDict, error) {
nameStr, _ := data.GetString("name")
if len(nameStr) > 0 {
if !isValidBucketName(nameStr) {
return nil, httperrors.NewInputParameterError("invalid name, only alphabets, digits and hyphen(-) allowed")
err := isValidBucketName(nameStr)
if err != nil {
return nil, httperrors.NewInputParameterError("invalid bucket name: %s", err)
}
}
return bucket.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
@@ -343,7 +366,7 @@ func (bucket *SBucket) RemoteCreate(ctx context.Context, userCred mcclient.Token
if err != nil {
return errors.Wrap(err, "db.SetExternalId")
}
extBucket, err := iregion.GetIBucketByName(bucket.Name)
extBucket, err := iregion.GetIBucketById(bucket.Name)
if err != nil {
return errors.Wrap(err, "iregion.GetIBucketByName")
}
@@ -401,3 +424,186 @@ func (manager *SBucketManager) ListItemFilter(ctx context.Context, q *sqlchemy.S
return q, nil
}
func (bucket *SBucket) AllowGetDetailsObjects(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
) bool {
return bucket.IsOwner(userCred)
}
func (bucket *SBucket) GetDetailsObjects(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
) (jsonutils.JSONObject, error) {
iBucket, err := bucket.GetIBucket()
if err != nil {
return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
}
prefix, _ := query.GetString("prefix")
isRecursive := jsonutils.QueryBoolean(query, "recursive", false)
objects, err := iBucket.GetIObjects(prefix, isRecursive)
if err != nil {
return nil, httperrors.NewInternalServerError("fail to get objects: %s", err)
}
retArray := jsonutils.NewArray()
for i := range objects {
retArray.Add(jsonutils.Marshal(cloudprovider.ICloudObject2BaseCloudObject(objects[i])))
}
return retArray, nil
}
func (bucket *SBucket) AllowPerformTempUrl(ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject,
) bool {
return bucket.IsOwner(userCred)
}
func (bucket *SBucket) PerformTempUrl(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject,
) (jsonutils.JSONObject, error) {
method, _ := data.GetString("method")
key, _ := data.GetString("key")
expire, _ := data.Int("expire_seconds")
if len(method) == 0 {
method = "GET"
}
if len(key) == 0 {
return nil, httperrors.NewInputParameterError("missing key")
}
if expire == 0 {
expire = 60 // default 60 seconds
}
iBucket, err := bucket.GetIBucket()
if err != nil {
return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
}
tmpUrl, err := iBucket.GetTempUrl(method, key, time.Duration(expire)*time.Second)
if err != nil {
return nil, httperrors.NewInternalServerError("fail to generate temp url: %s", err)
}
ret := jsonutils.NewDict()
ret.Add(jsonutils.NewString(tmpUrl), "url")
return ret, nil
}
func (bucket *SBucket) AllowPerformMakedir(ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject,
) bool {
return bucket.IsOwner(userCred)
}
func (bucket *SBucket) PerformMakedir(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject,
) (jsonutils.JSONObject, error) {
key, _ := data.GetString("key")
if key[len(key)-1] != '/' {
return nil, httperrors.NewInputParameterError("directory must ends with /")
}
err := s3utils.CheckValidObjectName(key)
if err != nil {
return nil, httperrors.NewInputParameterError("invalid key: %s", err)
}
iBucket, err := bucket.GetIBucket()
if err != nil {
return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
}
err = cloudprovider.Makedir(ctx, iBucket, key)
if err != nil {
return nil, httperrors.NewInternalServerError("fail to mkdir: %s", err)
}
return nil, nil
}
func (bucket *SBucket) AllowPerformDelete(ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject,
) bool {
return bucket.IsOwner(userCred)
}
func (bucket *SBucket) PerformDelete(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject,
) (jsonutils.JSONObject, error) {
keys, _ := data.Get("keys")
if keys == nil {
return nil, httperrors.NewInputParameterError("missing keys")
}
keyStrs := keys.(*jsonutils.JSONArray).GetStringArray()
if len(keyStrs) == 0 {
return nil, httperrors.NewInputParameterError("empty keys")
}
iBucket, err := bucket.GetIBucket()
if err != nil {
return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
}
ok := jsonutils.NewDict()
results := modules.BatchDo(keyStrs, func(key string) (jsonutils.JSONObject, error) {
err := iBucket.DeleteObject(ctx, key)
if err != nil {
return nil, err
} else {
return ok, nil
}
})
return modules.SubmitResults2JSON(results), nil
}
func (bucket *SBucket) AllowPerformUpload(ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject,
) bool {
return bucket.IsOwner(userCred)
}
func (bucket *SBucket) PerformUpload(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject,
) (jsonutils.JSONObject, error) {
appParams := appsrv.AppContextGetParams(ctx)
key := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER)
err := s3utils.CheckValidObjectName(key)
if err != nil {
return nil, httperrors.NewInputParameterError("invalid object key: %s", err)
}
iBucket, err := bucket.GetIBucket()
if err != nil {
return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
}
contType := appParams.Request.Header.Get("Content-Type")
storageClass := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER)
err = iBucket.PutObject(ctx, key, appParams.Request.Body.(io.ReadSeeker), contType, storageClass)
if err != nil {
return nil, httperrors.NewInternalServerError("put object error %s", err)
}
return nil, nil
}

11
pkg/compute/models/guest_actions.go
View File

@@ -82,12 +82,17 @@ func (self *SGuest) GetDetailsVnc(ctx context.Context, userCred mcclient.TokenCr
func (self *SGuest) AllowPerformMonitor(ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject) bool {
data jsonutils.JSONObject,
) bool {
return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "monitor")
}
func (self *SGuest) PerformMonitor(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject,
data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
func (self *SGuest) PerformMonitor(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
data jsonutils.JSONObject,
) (jsonutils.JSONObject, error) {
if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_BLOCK_STREAM}) {
cmd, err := data.GetString("command")
if err != nil {

1
pkg/compute/models/natdtable.go
View File

@@ -46,6 +46,7 @@ func init() {
"natdtables",
),
}
NatDTableManager.SetVirtualObject(NatDTableManager)
}
type SNatDEntry struct {

1
pkg/compute/models/natgateways.go
View File

@@ -46,6 +46,7 @@ func init() {
"natgateways",
),
}
NatGatewayManager.SetVirtualObject(NatGatewayManager)
}
type SNatGateway struct {

1
pkg/compute/models/natstable.go
View File

@@ -46,6 +46,7 @@ func init() {
"natstables",
),
}
NatSTableManager.SetVirtualObject(NatSTableManager)
}
type SNatSEntry struct {

1
pkg/compute/models/snapshotpolicy.go
View File

@@ -62,6 +62,7 @@ func init() {
"snapshotpolicies",
),
}
SnapshotPolicyManager.SetVirtualObject(SnapshotPolicyManager)
}
func (manager *SSnapshotPolicyManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {

2
pkg/image/models/images.go
View File

@@ -122,6 +122,8 @@ type SImage struct {
}
func (manager *SImageManager) CustomizeHandlerInfo(info *appsrv.SHandlerInfo) {
manager.SVirtualResourceBaseManager.CustomizeHandlerInfo(info)
switch info.GetName(nil) {
case "get_details", "create", "update":
info.SetProcessTimeout(time.Minute * 120).SetWorkerManager(imgStreamingWorkerMan)

50
pkg/mcclient/modules/mod_buckets.go
View File

@@ -14,17 +14,53 @@
package modules
import (
"fmt"
"io"
"net/http"
"github.com/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/util/httputils"
)
type SBucketManager struct {
ResourceManager
}
func (manager *SBucketManager) Upload(s *mcclient.ClientSession, bucketId string, key string, body io.Reader, contType string, storageClass string) error {
method := httputils.POST
path := fmt.Sprintf("/%s/%s/upload", manager.URLPath(), bucketId)
headers := http.Header{}
headers.Set(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER, key)
if len(contType) > 0 {
headers.Set("Content-Type", contType)
}
if len(storageClass) > 0 {
headers.Set(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER, storageClass)
}
_, err := manager.rawRequest(s, method, path, headers, body)
if err != nil {
return errors.Wrap(err, "rawRequest")
}
return nil
}
var (
Buckets ResourceManager
Buckets SBucketManager
)
func init() {
Buckets = NewComputeManager("bucket", "buckets",
[]string{"ID", "Name", "Storage_Class",
"Status", "location", "acl",
"region",
},
[]string{})
Buckets = SBucketManager{
NewComputeManager("bucket", "buckets",
[]string{"ID", "Name", "Storage_Class",
"Status", "location", "acl",
"region", "manager_id",
},
[]string{}),
}
registerCompute(&Buckets)
}

2
pkg/multicloud/no_storage_region.go
View File

@@ -22,7 +22,7 @@ func (cli *SNoObjectStorageRegion) IBucketExist(name string) (bool, error) {
return false, cloudprovider.ErrNotSupported
}
func (cli *SNoObjectStorageRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
func (cli *SNoObjectStorageRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
return nil, cloudprovider.ErrNotSupported
}

74
pkg/multicloud/objectstore/buckets.go
View File

@@ -1,10 +1,16 @@
package objectstore
import (
"context"
"fmt"
"io"
"path"
"time"
"github.com/minio/minio-go"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/cloudprovider"
)
@@ -58,3 +64,71 @@ func (bucket *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
},
}
}
func (bucket *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
isRecursive := true
if delimiter == "/" {
isRecursive = false
}
result := cloudprovider.SListObjectResult{}
var err error
result.Objects, err = bucket.GetIObjects(prefix, isRecursive)
return result, err
}
func (bucket *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
doneCh := make(chan struct{})
defer close(doneCh)
ret := make([]cloudprovider.ICloudObject, 0)
objectCh := bucket.client.client.ListObjects(bucket.Name, prefix, isRecursive, doneCh)
for object := range objectCh {
if object.Err != nil {
return nil, errors.Wrap(object.Err, "ListObjects")
}
obj := &SObject{
bucket: bucket,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
StorageClass: object.StorageClass,
Key: object.Key,
SizeBytes: object.Size,
ETag: object.ETag,
LastModified: object.LastModified,
ContentType: object.ContentType,
},
}
ret = append(ret, obj)
}
return ret, nil
}
func (bucket *SBucket) PutObject(ctx context.Context, key string, input io.ReadSeeker, contType string, storageClass string) error {
opts := minio.PutObjectOptions{}
if len(contType) > 0 {
opts.ContentType = contType
}
if len(storageClass) > 0 {
opts.StorageClass = storageClass
}
_, err := bucket.client.client.PutObjectWithContext(ctx, bucket.Name, key, input, -1, opts)
return err
}
func (bucket *SBucket) DeleteObject(ctx context.Context, key string) error {
err := bucket.client.client.RemoveObject(bucket.Name, key)
if err != nil {
return errors.Wrap(err, "RemoveObject")
}
return nil
}
func (bucket *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
if method != "GET" && method != "PUT" && method != "DELETE" {
return "", errors.Error("unsupported method")
}
url, err := bucket.client.client.Presign(method, bucket.Name, key, expire, nil)
if err != nil {
return "", errors.Wrap(err, "Presign")
}
return url.String(), nil
}

15
pkg/multicloud/objectstore/object.go Normal file
View File

@@ -0,0 +1,15 @@
package objectstore
import (
"yunion.io/x/onecloud/pkg/cloudprovider"
)
type SObject struct {
bucket *SBucket
cloudprovider.SBaseCloudObject
}
func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
return o.bucket
}

9
pkg/multicloud/objectstore/objectstore.go
View File

@@ -9,10 +9,12 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/util/secrules"
"time"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/object"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/multicloud"
"yunion.io/x/onecloud/pkg/util/httputils"
)
type SObjectStoreClient struct {
@@ -54,6 +56,9 @@ func NewObjectStoreClient(providerId string, providerName string, endpoint strin
return nil, errors.Wrap(err, "minio.New")
}
tr := httputils.GetTransport(true, time.Second*5)
cli.SetCustomTransport(tr)
client.client = cli
return &client, nil
@@ -319,6 +324,6 @@ func (cli *SObjectStoreClient) IBucketExist(name string) (bool, error) {
return exist, nil
}
func (cli *SObjectStoreClient) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketByName(cli, name)
func (cli *SObjectStoreClient) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketById(cli, name)
}

190
pkg/multicloud/objectstore/shell.go Normal file
View File

@@ -0,0 +1,190 @@
package objectstore
import (
"context"
"fmt"
"io"
"os"
"time"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/util/printutils"
"yunion.io/x/onecloud/pkg/util/shellutils"
)
func S3Shell() {
type BucketListOptions struct {
}
shellutils.R(&BucketListOptions{}, "bucket-list", "List all bucket", func(cli cloudprovider.ICloudRegion, args *BucketListOptions) error {
buckets, err := cli.GetIBuckets()
if err != nil {
return err
}
printutils.PrintGetterList(buckets, nil)
return nil
})
type BucketCreateOptions struct {
NAME string `help:"name of bucket to create"`
}
shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli cloudprovider.ICloudRegion, args *BucketCreateOptions) error {
err := cli.CreateIBucket(args.NAME, "", "")
if err != nil {
return err
}
return nil
})
type BucketDeleteOptions struct {
NAME string `help:"name of bucket to delete"`
}
shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete bucket", func(cli cloudprovider.ICloudRegion, args *BucketDeleteOptions) error {
err := cli.DeleteIBucket(args.NAME)
if err != nil {
return err
}
return nil
})
type BucketObjectsOptions struct {
BUCKET string `help:"name of bucket to list objects"`
Prefix string `help:"prefix"`
Marker string `help:"marker"`
Demiliter string `help:"delimiter"`
Max int `help:"Max count"`
}
shellutils.R(&BucketObjectsOptions{}, "bucket-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketObjectsOptions) error {
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}
result, err := bucket.ListObjects(args.Prefix, args.Marker, args.Demiliter, args.Max)
if err != nil {
return err
}
if result.IsTruncated {
fmt.Println("NextMarker: %s IsTruncated: %v", result.NextMarker, result.IsTruncated)
}
fmt.Println("Common prefixes:")
printutils.PrintGetterList(result.CommonPrefixes, []string{"key", "size_bytes"})
fmt.Println("Objects:")
printutils.PrintGetterList(result.Objects, []string{"key", "size_bytes"})
return nil
})
type BucketListObjectsOptions struct {
BUCKET string `help:"name of bucket to list objects"`
Prefix string `help:"prefix"`
}
shellutils.R(&BucketListObjectsOptions{}, "bucket-list-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error {
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}
objects, err := bucket.GetIObjects(args.Prefix, true)
if err != nil {
return err
}
printutils.PrintGetterList(objects, []string{"key", "size_bytes"})
return nil
})
shellutils.R(&BucketListObjectsOptions{}, "bucket-dir-object", "List objects in a bucket like directory", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error {
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}
objects, err := bucket.GetIObjects(args.Prefix, false)
if err != nil {
return err
}
printutils.PrintGetterList(objects, []string{"key", "size_bytes"})
return nil
})
type BucketMakrdirOptions struct {
BUCKET string `help:"name of bucket to put object"`
DIR string `help:"dir to make"`
}
shellutils.R(&BucketMakrdirOptions{}, "bucket-mkdir", "Mkdir in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketMakrdirOptions) error {
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}
err = cloudprovider.Makedir(context.Background(), bucket, args.DIR)
if err != nil {
return err
}
fmt.Printf("Mkdir success\n")
return nil
})
type BucketPutObjectOptions struct {
BUCKET string `help:"name of bucket to put object"`
KEY string `help:"key of object"`
Path string `help:"Path of file to upload"`
ContentType string `help:"content-type"`
StorageClass string `help:"storage class"`
}
shellutils.R(&BucketPutObjectOptions{}, "put-object", "Put object into a bucket", func(cli cloudprovider.ICloudRegion, args *BucketPutObjectOptions) error {
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}
var input io.ReadSeeker
if len(args.Path) > 0 {
file, err := os.Open(args.Path)
if err != nil {
return err
}
defer file.Close()
input = file
} else {
input = os.Stdout
}
err = bucket.PutObject(context.Background(), args.KEY, input, args.ContentType, args.StorageClass)
if err != nil {
return err
}
fmt.Printf("Upload success\n")
return nil
})
type BucketDeleteObjectOptions struct {
BUCKET string `help:"name of bucket to put object"`
KEY string `help:"key of object"`
}
shellutils.R(&BucketDeleteObjectOptions{}, "delete-object", "Delete object from a bucket", func(cli cloudprovider.ICloudRegion, args *BucketDeleteObjectOptions) error {
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}
err = bucket.DeleteObject(context.Background(), args.KEY)
if err != nil {
return err
}
fmt.Printf("Delete success\n")
return nil
})
type BucketTempUrlOption struct {
BUCKET string `help:"name of bucket to put object"`
METHOD string `help:"http method" choices:"GET|PUT|DELETE"`
KEY string `help:"key of object"`
Duration int `help:"duration in seconds" default:"60"`
}
shellutils.R(&BucketTempUrlOption{}, "temp-url", "generate temp url", func(cli cloudprovider.ICloudRegion, args *BucketTempUrlOption) error {
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}
urlStr, err := bucket.GetTempUrl(args.METHOD, args.KEY, time.Duration(args.Duration)*time.Second)
if err != nil {
return err
}
fmt.Println(urlStr)
return nil
})
}

60
pkg/multicloud/objectstore/shell/bucket.go
View File

@@ -1,63 +1,7 @@
package shell
import (
"fmt"
"yunion.io/x/onecloud/pkg/multicloud/objectstore"
"yunion.io/x/onecloud/pkg/util/shellutils"
)
import "yunion.io/x/onecloud/pkg/multicloud/objectstore"
func init() {
type BucketListOptions struct {
}
shellutils.R(&BucketListOptions{}, "bucket-list", "List all bucket", func(cli *objectstore.SObjectStoreClient, args *BucketListOptions) error {
buckets, err := cli.GetIBuckets()
if err != nil {
return err
}
printList(buckets, nil)
return nil
})
type BucketCreateOptions struct {
NAME string `help:"name of bucket to create"`
}
shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli *objectstore.SObjectStoreClient, args *BucketCreateOptions) error {
err := cli.CreateIBucket(args.NAME, "", "")
if err != nil {
return err
}
return nil
})
type BucketDeleteOptions struct {
NAME string `help:"name of bucket to delete"`
}
shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete bucket", func(cli *objectstore.SObjectStoreClient, args *BucketDeleteOptions) error {
err := cli.DeleteIBucket(args.NAME)
if err != nil {
return err
}
return nil
})
type BucketPolicyOptions struct {
NAME string `help:"name of bucket to get policy"`
}
shellutils.R(&BucketPolicyOptions{}, "bucket-policy", "Get bucket policy", func(cli *objectstore.SObjectStoreClient, args *BucketPolicyOptions) error {
policy, err := cli.GetIBucketPolicy(args.NAME)
if err != nil {
return err
}
fmt.Println(policy)
return nil
})
shellutils.R(&BucketPolicyOptions{}, "bucket-lifecycle", "Get bucket lifecycle", func(cli *objectstore.SObjectStoreClient, args *BucketPolicyOptions) error {
lifecycle, err := cli.GetIBucketLiftcycle(args.NAME)
if err != nil {
return err
}
fmt.Println(lifecycle)
return nil
})
objectstore.S3Shell()
}

124
pkg/util/aliyun/bucket.go
View File

@@ -1,9 +1,15 @@
package aliyun
import (
"context"
"fmt"
"io"
"time"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/cloudprovider"
)
@@ -61,3 +67,121 @@ func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
},
}
}
func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}
osscli, err := b.region.GetOssClient()
if err != nil {
return result, errors.Wrap(err, "GetOssClient")
}
bucket, err := osscli.Bucket(b.Name)
if err != nil {
return result, errors.Wrap(err, "Bucket")
}
opts := make([]oss.Option, 0)
if len(prefix) > 0 {
opts = append(opts, oss.Prefix(prefix))
}
if len(delimiter) > 0 {
opts = append(opts, oss.Delimiter(delimiter))
}
if len(marker) > 0 {
opts = append(opts, oss.Marker(marker))
}
if maxCount > 0 {
opts = append(opts, oss.MaxKeys(maxCount))
}
oResult, err := bucket.ListObjects(opts...)
if err != nil {
return result, errors.Wrap(err, "ListObjects")
}
result.Objects = make([]cloudprovider.ICloudObject, 0)
for _, object := range oResult.Objects {
obj := &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
StorageClass: object.StorageClass,
Key: object.Key,
SizeBytes: object.Size,
ETag: object.ETag,
LastModified: object.LastModified,
ContentType: object.Type,
},
}
result.Objects = append(result.Objects, obj)
}
if oResult.CommonPrefixes != nil {
result.CommonPrefixes = make([]cloudprovider.ICloudObject, len(oResult.CommonPrefixes))
for i, commPrefix := range oResult.CommonPrefixes {
result.CommonPrefixes[i] = &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{Key: commPrefix},
}
}
}
result.IsTruncated = oResult.IsTruncated
result.NextMarker = oResult.NextMarker
return result, nil
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SBucket) PutObject(ctx context.Context, key string, input io.ReadSeeker, contType string, storageClassStr string) error {
osscli, err := b.region.GetOssClient()
if err != nil {
return errors.Wrap(err, "GetOssClient")
}
bucket, err := osscli.Bucket(b.Name)
if err != nil {
return errors.Wrap(err, "Bucket")
}
opts := make([]oss.Option, 0)
if len(contType) > 0 {
opts = append(opts, oss.ContentType(contType))
}
if len(storageClassStr) > 0 {
storageClass, err := str2StorageClass(storageClassStr)
if err != nil {
return errors.Wrap(err, "str2StorageClass")
}
opts = append(opts, oss.ObjectStorageClass(storageClass))
}
return bucket.PutObject(key, input, opts...)
}
func (b *SBucket) DeleteObject(ctx context.Context, key string) error {
osscli, err := b.region.GetOssClient()
if err != nil {
return errors.Wrap(err, "GetOssClient")
}
bucket, err := osscli.Bucket(b.Name)
if err != nil {
return errors.Wrap(err, "Bucket")
}
err = bucket.DeleteObject(key)
if err != nil {
return errors.Wrap(err, "DeleteObject")
}
return nil
}
func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
if method != "GET" && method != "PUT" && method != "DELETE" {
return "", errors.Error("unsupported method")
}
osscli, err := b.region.GetOssClient()
if err != nil {
return "", errors.Wrap(err, "GetOssClient")
}
bucket, err := osscli.Bucket(b.Name)
if err != nil {
return "", errors.Wrap(err, "Bucket")
}
urlStr, err := bucket.SignURL(key, oss.HTTPMethod(method), int64(expire/time.Second))
if err != nil {
return "", errors.Wrap(err, "SignURL")
}
return urlStr, nil
}

15
pkg/util/aliyun/objects.go Normal file
View File

@@ -0,0 +1,15 @@
package aliyun
import (
"yunion.io/x/onecloud/pkg/cloudprovider"
)
type SObject struct {
bucket *SBucket
cloudprovider.SBaseCloudObject
}
func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
return o.bucket
}

60
pkg/util/aliyun/region.go
View File

@@ -953,6 +953,34 @@ func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
return ret, nil
}
func str2StorageClass(storageClassStr string) (oss.StorageClassType, error) {
storageClass := oss.StorageStandard
if strings.EqualFold(storageClassStr, string(oss.StorageStandard)) {
//
} else if strings.EqualFold(storageClassStr, string(oss.StorageIA)) {
storageClass = oss.StorageIA
} else if strings.EqualFold(storageClassStr, string(oss.StorageArchive)) {
storageClass = oss.StorageArchive
} else {
return storageClass, errors.Error("not supported storageClass")
}
return storageClass, nil
}
func str2Acl(aclStr string) (oss.ACLType, error) {
acl := oss.ACLPrivate
if strings.EqualFold(aclStr, string(oss.ACLPrivate)) {
// private, default
} else if strings.EqualFold(aclStr, string(oss.ACLPublicRead)) {
acl = oss.ACLPublicRead
} else if strings.EqualFold(aclStr, string(oss.ACLPublicReadWrite)) {
acl = oss.ACLPublicReadWrite
} else {
return acl, errors.Error("not supported acl")
}
return acl, nil
}
func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
osscli, err := region.GetOssClient()
if err != nil {
@@ -960,32 +988,18 @@ func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr
}
opts := make([]oss.Option, 0)
if len(storageClassStr) > 0 {
storageClass := oss.StorageStandard
if strings.EqualFold(storageClassStr, string(oss.StorageStandard)) {
//
} else if strings.EqualFold(storageClassStr, string(oss.StorageIA)) {
storageClass = oss.StorageIA
} else if strings.EqualFold(storageClassStr, string(oss.StorageArchive)) {
storageClass = oss.StorageArchive
} else {
return errors.Error("not supported storageClass")
storageClass, err := str2StorageClass(storageClassStr)
if err != nil {
return err
}
opt := oss.StorageClass(storageClass)
opts = append(opts, opt)
opts = append(opts, oss.StorageClass(storageClass))
}
if len(aclStr) > 0 {
acl := oss.ACLPrivate
if strings.EqualFold(aclStr, string(oss.ACLPrivate)) {
// private, default
} else if strings.EqualFold(aclStr, string(oss.ACLPublicRead)) {
acl = oss.ACLPublicRead
} else if strings.EqualFold(aclStr, string(oss.ACLPublicReadWrite)) {
acl = oss.ACLPublicReadWrite
} else {
return errors.Error("not supported acl")
acl, err := str2Acl(aclStr)
if err != nil {
return err
}
opt := oss.ACL(acl)
opts = append(opts, opt)
opts = append(opts, oss.ACL(acl))
}
err = osscli.CreateBucket(name, opts...)
if err != nil {
@@ -1031,7 +1045,7 @@ func (region *SRegion) IBucketExist(name string) (bool, error) {
return exist, nil
}
func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
osscli, err := region.GetOssClient()
if err != nil {
return nil, errors.Wrap(err, "region.GetOssClient")

7
pkg/util/aliyun/shell/bucket.go Normal file
View File

@@ -0,0 +1,7 @@
package shell
import "yunion.io/x/onecloud/pkg/multicloud/objectstore"
func init() {
objectstore.S3Shell()
}

133
pkg/util/aws/bucket.go
View File

@@ -1,9 +1,16 @@
package aws
import (
"context"
"fmt"
"io"
"time"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/s3"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/cloudprovider"
)
@@ -59,3 +66,129 @@ func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
},
}
}
func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}
s3cli, err := b.region.GetS3Client()
if err != nil {
return result, errors.Wrap(err, "GetS3Client")
}
input := &s3.ListObjectsInput{}
input.SetBucket(b.Name)
if len(prefix) > 0 {
input.SetPrefix(prefix)
}
if len(marker) > 0 {
input.SetMarker(marker)
}
if len(delimiter) > 0 {
input.SetDelimiter(delimiter)
}
if maxCount > 0 {
input.SetMaxKeys(int64(maxCount))
}
oResult, err := s3cli.ListObjects(input)
if err != nil {
return result, errors.Wrap(err, "ListObjects")
}
result.Objects = make([]cloudprovider.ICloudObject, 0)
for _, object := range oResult.Contents {
obj := &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
StorageClass: *object.StorageClass,
Key: *object.Key,
SizeBytes: *object.Size,
ETag: *object.ETag,
LastModified: *object.LastModified,
ContentType: "",
},
}
result.Objects = append(result.Objects, obj)
}
if oResult.CommonPrefixes != nil {
result.CommonPrefixes = make([]cloudprovider.ICloudObject, len(oResult.CommonPrefixes))
for i, commPrefix := range oResult.CommonPrefixes {
result.CommonPrefixes[i] = &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{Key: *commPrefix.Prefix},
}
}
}
if oResult.IsTruncated != nil {
result.IsTruncated = *oResult.IsTruncated
}
if oResult.NextMarker != nil {
result.NextMarker = *oResult.NextMarker
}
return result, nil
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SBucket) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error {
s3cli, err := b.region.GetS3Client()
if err != nil {
return errors.Wrap(err, "GetS3Client")
}
input := &s3.PutObjectInput{}
input.SetBucket(b.Name)
input.SetKey(key)
input.SetBody(reader)
if len(storageClassStr) > 0 {
input.SetStorageClass(storageClassStr)
}
if len(contType) > 0 {
input.SetContentType(contType)
}
_, err = s3cli.PutObjectWithContext(ctx, input)
if err != nil {
return errors.Wrap(err, "PutObjectWithContext")
}
return nil
}
func (b *SBucket) DeleteObject(ctx context.Context, key string) error {
s3cli, err := b.region.GetS3Client()
if err != nil {
return errors.Wrap(err, "GetS3Client")
}
input := &s3.DeleteObjectInput{}
input.SetBucket(b.Name)
input.SetKey(key)
_, err = s3cli.DeleteObjectWithContext(ctx, input)
if err != nil {
return errors.Wrap(err, "DeleteObject")
}
return nil
}
func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
s3cli, err := b.region.GetS3Client()
if err != nil {
return "", errors.Wrap(err, "GetS3Client")
}
var request *request.Request
switch method {
case "GET":
input := &s3.GetObjectInput{}
input.SetBucket(b.Name)
input.SetKey(key)
request, _ = s3cli.GetObjectRequest(input)
case "PUT":
input := &s3.PutObjectInput{}
input.SetBucket(b.Name)
input.SetKey(key)
request, _ = s3cli.PutObjectRequest(input)
case "DELETE":
input := &s3.DeleteObjectInput{}
input.SetBucket(b.Name)
input.SetKey(key)
request, _ = s3cli.DeleteObjectRequest(input)
default:
return "", errors.Error("unsupported method")
}
return request.Presign(expire)
}

4
pkg/util/aws/region.go
View File

@@ -610,8 +610,8 @@ func (region *SRegion) IBucketExist(name string) (bool, error) {
return true, nil
}
func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketByName(region, name)
func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketById(region, name)
}
func (region *SRegion) getBaseEndpoint() string {

13
pkg/util/aws/s3object.go Normal file
View File

@@ -0,0 +1,13 @@
package aws
import "yunion.io/x/onecloud/pkg/cloudprovider"
type SObject struct {
bucket *SBucket
cloudprovider.SBaseCloudObject
}
func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
return o.bucket
}

7
pkg/util/aws/shell/bucket.go Normal file
View File

@@ -0,0 +1,7 @@
package shell
import "yunion.io/x/onecloud/pkg/multicloud/objectstore"
func init() {
objectstore.S3Shell()
}

13
pkg/util/azure/blobobject.go Normal file
View File

@@ -0,0 +1,13 @@
package azure
import "yunion.io/x/onecloud/pkg/cloudprovider"
type SObject struct {
container *SContainer
cloudprovider.SBaseCloudObject
}
func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
return o.container.storageaccount
}

3
pkg/util/azure/classic_disk.go
View File

@@ -22,6 +22,7 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"github.com/Azure/azure-sdk-for-go/storage"
billing_api "yunion.io/x/onecloud/pkg/apis/billing"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -67,7 +68,7 @@ func (self *SRegion) GetStorageAccountDisksWithSnapshots(storageaccount SStorage
}
for _, container := range containers {
if container.Name == "vhds" {
files, err := container.ListFiles()
files, err := container.ListAllFiles(&storage.IncludeBlobDataset{Snapshots: true, Metadata: true})
if err != nil {
log.Errorf("List storage %s container %s files error: %v", storageaccount.Name, container.Name, err)
return nil, nil, err

4
pkg/util/azure/region.go
View File

@@ -651,6 +651,6 @@ func (region *SRegion) IBucketExist(name string) (bool, error) {
return region.checkStorageAccountNameExist(name)
}
func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketByName(region, name)
func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketById(region, name)
}

50
pkg/util/azure/shell/bucket.go
View File

@@ -1,55 +1,9 @@
package shell
import (
"yunion.io/x/onecloud/pkg/util/azure"
"yunion.io/x/onecloud/pkg/util/printutils"
"yunion.io/x/onecloud/pkg/util/shellutils"
"yunion.io/x/onecloud/pkg/multicloud/objectstore"
)
func init() {
type BucketListOptions struct {
}
shellutils.R(&BucketListOptions{}, "bucket-list", "List buckets", func(cli *azure.SRegion, args *BucketListOptions) error {
buckets, err := cli.GetIBuckets()
if err != nil {
return err
}
printutils.PrintGetterList(buckets, nil)
return nil
})
type BucketCreateOptions struct {
BUCKET string `help:"bucket name"`
STORAGECLASS string `help:"storage class"`
}
shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli *azure.SRegion, args *BucketCreateOptions) error {
err := cli.CreateIBucket(args.BUCKET, args.STORAGECLASS, "")
if err != nil {
return err
}
return nil
})
type BucketDeleteOptions struct {
BUCKET string `help:"bucket name"`
}
shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete a bucket", func(cli *azure.SRegion, args *BucketDeleteOptions) error {
err := cli.DeleteIBucket(args.BUCKET)
if err != nil {
return err
}
return nil
})
type BucketShowOptions struct {
BUCKET string `help:"bucket name"`
}
shellutils.R(&BucketShowOptions{}, "bucket-show", "Show a bucket", func(cli *azure.SRegion, args *BucketShowOptions) error {
bucket, err := cli.GetIBucketByName(args.BUCKET)
if err != nil {
return err
}
printObject(bucket)
return nil
})
objectstore.S3Shell()
}

2
pkg/util/azure/shell/storageaccount.go
View File

@@ -98,7 +98,7 @@ func init() {
if err != nil {
return err
}
blobs, err := container.ListFiles()
blobs, err := container.ListAllFiles(nil)
if err != nil {
return err
}

300
pkg/util/azure/storageaccount.go
View File

@@ -15,7 +15,9 @@
package azure
import (
"context"
"fmt"
"io"
"math/rand"
"path"
"strings"
@@ -27,9 +29,9 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/pkg/utils"
)
type SContainer struct {
@@ -75,7 +77,8 @@ type AccountProperties struct {
}
type SStorageAccount struct {
region *SRegion
region *SRegion
accountKey string
Sku SSku `json:"sku,omitempty"`
Kind string `json:"kind,omitempty"`
@@ -416,18 +419,52 @@ func (self *SStorageAccount) GetContainer(name string) (*SContainer, error) {
return nil, cloudprovider.ErrNotFound
}
func (self *SContainer) ListFiles() ([]storage.Blob, error) {
func (self *SContainer) ListAllFiles(include *storage.IncludeBlobDataset) ([]storage.Blob, error) {
blobs := make([]storage.Blob, 0)
var marker string
for {
result, err := self.ListFiles("", marker, "", 5000, include)
if err != nil {
return nil, errors.Wrap(err, "ListFiles")
}
if len(result.Blobs) > 0 {
blobs = append(blobs, result.Blobs...)
}
if len(result.NextMarker) == 0 {
break
} else {
marker = result.NextMarker
}
}
return blobs, nil
}
func (self *SContainer) ListFiles(prefix string, marker string, delimiter string, maxCount int, include *storage.IncludeBlobDataset) (storage.BlobListResponse, error) {
var result storage.BlobListResponse
storageaccount := self.storageaccount
client, err := storage.NewBasicClientOnSovereignCloud(storageaccount.Name, storageaccount.accountKey, storageaccount.region.client.env)
if err != nil {
return nil, err
return result, err
}
blobService := client.GetBlobService()
result, err := blobService.GetContainerReference(self.Name).ListBlobs(storage.ListBlobsParameters{Include: &storage.IncludeBlobDataset{Snapshots: true, Metadata: true}})
if err != nil {
return nil, err
params := storage.ListBlobsParameters{Include: include}
if len(prefix) > 0 {
params.Prefix = prefix
}
return result.Blobs, nil
if len(marker) > 0 {
params.Marker = marker
}
if len(delimiter) > 0 {
params.Delimiter = delimiter
}
if maxCount > 0 {
params.MaxResults = uint(maxCount)
}
result, err = blobService.GetContainerReference(self.Name).ListBlobs(params)
if err != nil {
return result, err
}
return result, nil
}
func (self *SContainer) getClient() (storage.Client, error) {
@@ -471,6 +508,51 @@ func (self *SContainer) CopySnapshot(snapshotId, fileName string) (*storage.Blob
return blobRef, blobRef.GetProperties(&storage.GetBlobPropertiesOptions{})
}
func (self *SContainer) UploadStream(key string, reader io.ReadSeeker, contType string) error {
storageaccount := self.storageaccount
client, err := storage.NewBasicClientOnSovereignCloud(storageaccount.Name, storageaccount.accountKey, storageaccount.region.client.env)
if err != nil {
return errors.Wrap(err, "NewBasicClientOnSovereignCloud")
}
blobService := client.GetBlobService()
containerRef := blobService.GetContainerReference(self.Name)
blobRef := containerRef.GetBlobReference(key)
blobRef.Properties.BlobType = storage.BlobTypeBlock
blobRef.Properties.ContentType = contType
return blobRef.CreateBlockBlobFromReader(reader, &storage.PutBlobOptions{})
}
func (self *SContainer) SignUrl(method string, key string, expire time.Duration) (string, error) {
storageaccount := self.storageaccount
client, err := storage.NewBasicClientOnSovereignCloud(storageaccount.Name, storageaccount.accountKey, storageaccount.region.client.env)
if err != nil {
return "", errors.Wrap(err, "NewBasicClientOnSovereignCloud")
}
blobService := client.GetBlobService()
containerRef := blobService.GetContainerReference(self.Name)
sas := storage.ContainerSASOptions{}
sas.Start = time.Now()
sas.Expiry = sas.Start.Add(expire)
sas.UseHTTPS = true
sas.Identifier = key
switch method {
case "GET":
sas.Read = true
case "PUT":
sas.Read = true
sas.Add = true
sas.Create = true
sas.Write = true
case "DELETE":
sas.Read = true
sas.Write = true
sas.Delete = true
default:
return "", errors.Error("unsupport method")
}
return containerRef.GetSASURI(sas)
}
func (self *SContainer) UploadFile(filePath string) (string, error) {
storageaccount := self.storageaccount
client, err := storage.NewBasicClientOnSovereignCloud(storageaccount.Name, storageaccount.accountKey, storageaccount.region.client.env)
@@ -525,11 +607,19 @@ func (self *SContainer) UploadFile(filePath string) (string, error) {
}
func (self *SStorageAccount) UploadFile(containerName string, filePath string) (string, error) {
container, err := self.getOrCreateContainer(containerName, true)
if err != nil {
return "", errors.Wrap(err, "getOrCreateContainer")
}
return container.UploadFile(filePath)
}
func (self *SStorageAccount) getOrCreateContainer(containerName string, create bool) (*SContainer, error) {
containers, err := self.GetContainers()
if err != nil {
return "", err
return nil, errors.Wrap(err, "GetContainers")
}
container := &SContainer{}
var container *SContainer
find := false
for i := 0; i < len(containers); i++ {
if containers[i].Name == containerName {
@@ -539,12 +629,23 @@ func (self *SStorageAccount) UploadFile(containerName string, filePath string) (
}
}
if !find {
if !create {
return nil, cloudprovider.ErrNotFound
}
container, err = self.CreateContainer(containerName)
if err != nil {
return "", err
return nil, errors.Wrap(err, "CreateContainer")
}
}
return container.UploadFile(filePath)
return container, nil
}
func (self *SStorageAccount) UploadStream(containerName string, key string, reader io.ReadSeeker, contType string) error {
container, err := self.getOrCreateContainer(containerName, true)
if err != nil {
return errors.Wrap(err, "getOrCreateContainer")
}
return container.UploadStream(key, reader, contType)
}
func (b *SStorageAccount) GetProjectId() string {
@@ -624,3 +725,178 @@ func (b *SStorageAccount) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
}
return primary
}
func (b *SStorageAccount) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SStorageAccount) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}
containers, err := b.GetContainers()
if err != nil {
return result, errors.Wrap(err, "GetContainers")
}
result.Objects = make([]cloudprovider.ICloudObject, 0)
result.CommonPrefixes = make([]cloudprovider.ICloudObject, 0)
for i := 0; i < len(containers); i += 1 {
container := containers[i]
matchLen := len(container.Name)
if matchLen > len(prefix) {
matchLen = len(prefix)
}
var subMarker string
if len(marker) < len(container.Name) {
if marker > container.Name {
continue
}
subMarker = ""
} else {
containerMarker := marker[:len(container.Name)]
if containerMarker > container.Name {
continue
}
subMarker = marker[len(container.Name)+1:]
}
if marker > container.Name {
continue
}
if maxCount <= 0 {
break
}
// container name matches prefix
if matchLen == 0 || container.Name[:matchLen] == prefix[:matchLen] {
if delimiter == "/" && (len(prefix) == 0 || prefix == container.Name+delimiter) {
// populate CommonPrefixes
o := &SObject{
container: &container,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
Key: container.Name + "/",
},
}
result.CommonPrefixes = append(result.CommonPrefixes, o)
maxCount -= 1
} else if len(prefix) <= len(container.Name)+1 {
// returns contain names only
o := &SObject{
container: &container,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
Key: container.Name + "/",
},
}
result.Objects = append(result.Objects, o)
maxCount -= 1
}
if delimiter == "" || len(prefix) >= len(container.Name)+1 {
subPrefix := ""
if len(prefix) >= len(container.Name) {
subPrefix = prefix[len(container.Name)+1:]
}
oResult, err := container.ListFiles(subPrefix, subMarker, delimiter, maxCount, nil)
if err != nil {
return result, errors.Wrap(err, "ListFiles")
}
for i := range oResult.Blobs {
blob := oResult.Blobs[i]
o := &SObject{
container: &container,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
Key: container.Name + "/" + blob.Name,
SizeBytes: blob.Properties.ContentLength,
StorageClass: "",
ETag: blob.Properties.Etag,
LastModified: time.Time(blob.Properties.LastModified),
ContentType: blob.Properties.ContentType,
},
}
result.Objects = append(result.Objects, o)
maxCount -= 1
if maxCount == 0 {
break
}
result.NextMarker = blob.Name
}
for i := range oResult.BlobPrefixes {
o := &SObject{
container: &container,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
Key: container.Name + "/" + oResult.BlobPrefixes[i],
},
}
result.CommonPrefixes = append(result.CommonPrefixes, o)
maxCount -= 1
}
if len(oResult.NextMarker) > 0 {
result.NextMarker = container.Name + "/" + oResult.NextMarker
result.IsTruncated = true
break
}
}
}
}
return result, nil
}
func splitKey(key string) (string, string, error) {
slashPos := strings.IndexByte(key, '/')
if slashPos <= 0 {
return "", "", errors.Error("cannot put object to root")
}
containerName := key[:slashPos]
key = key[slashPos+1:]
if len(key) == 0 {
return "", "", errors.Error("empty blob path")
}
return containerName, key, nil
}
func (b *SStorageAccount) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error {
containerName, blob, err := splitKey(key)
if err != nil {
return errors.Wrap(err, "splitKey")
}
err = b.UploadStream(containerName, blob, reader, contType)
if err != nil {
return errors.Wrap(err, "UploadStream")
}
return nil
}
func (b *SStorageAccount) DeleteObject(ctx context.Context, key string) error {
containerName, blob, err := splitKey(key)
if err != nil {
return errors.Wrap(err, "splitKey")
}
client, err := storage.NewBasicClientOnSovereignCloud(b.Name, b.accountKey, b.region.client.env)
if err != nil {
return errors.Wrap(err, "storage.NewBasicClientOnSovereignCloud")
}
blobService := client.GetBlobService()
containerRef := blobService.GetContainerReference(containerName)
if len(blob) > 0 {
// delete object
blobRef := containerRef.GetBlobReference(blob)
_, err = blobRef.DeleteIfExists(nil)
if err != nil {
return errors.Wrap(err, "blobRef.DeleteIfExists")
}
} else {
// delete container
_, err = containerRef.DeleteIfExists(nil)
if err != nil {
return errors.Wrap(err, "containerRef.DeleteIfExists")
}
}
return nil
}
func (b *SStorageAccount) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
containerName, blob, err := splitKey(key)
if err != nil {
return "", errors.Wrap(err, "splitKey")
}
container, err := b.getOrCreateContainer(containerName, false)
if err != nil {
return "", errors.Wrap(err, "getOrCreateContainer")
}
return container.SignUrl(method, blob, expire)
}

8
pkg/util/httputils/httputils.go
View File

@@ -127,8 +127,8 @@ func GetAddrPort(urlStr string) (string, int, error) {
}
}
func GetClient(insecure bool, timeout time.Duration) *http.Client {
tr := &http.Transport{
func GetTransport(insecure bool, timeout time.Duration) *http.Transport {
return &http.Transport{
DialContext: (&net.Dialer{
Timeout: 5 * time.Second,
}).DialContext,
@@ -137,6 +137,10 @@ func GetClient(insecure bool, timeout time.Duration) *http.Client {
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure},
}
}
func GetClient(insecure bool, timeout time.Duration) *http.Client {
tr := GetTransport(insecure, timeout)
return &http.Client{
Transport: tr,
Timeout: timeout,

128
pkg/util/huawei/bucket.go
View File

@@ -4,7 +4,12 @@ import (
"fmt"
"time"
"yunion.io/x/pkg/errors"
"context"
"io"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/util/huawei/obs"
)
type SBucket struct {
@@ -73,3 +78,126 @@ func (b *SBucket) GetSizeByte() int64 {
func (b *SBucket) GetObjectNumber() int {
return b.ObjectNumber
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}
obscli, err := b.region.getOBSClient()
if err != nil {
return result, errors.Wrap(err, "GetOBSClient")
}
input := &obs.ListObjectsInput{}
input.Bucket = b.Name
if len(prefix) > 0 {
input.Prefix = prefix
}
if len(marker) > 0 {
input.Marker = marker
}
if len(delimiter) > 0 {
input.Delimiter = delimiter
}
if maxCount > 0 {
input.MaxKeys = maxCount
}
oResult, err := obscli.ListObjects(input)
if err != nil {
return result, errors.Wrap(err, "ListObjects")
}
result.Objects = make([]cloudprovider.ICloudObject, 0)
for _, object := range oResult.Contents {
obj := &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
StorageClass: string(object.StorageClass),
Key: object.Key,
SizeBytes: object.Size,
ETag: object.ETag,
LastModified: object.LastModified,
ContentType: "",
},
}
result.Objects = append(result.Objects, obj)
}
if oResult.CommonPrefixes != nil {
result.CommonPrefixes = make([]cloudprovider.ICloudObject, 0)
for _, commonPrefix := range oResult.CommonPrefixes {
obj := &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
Key: commonPrefix,
},
}
result.CommonPrefixes = append(result.CommonPrefixes, obj)
}
}
result.IsTruncated = oResult.IsTruncated
result.NextMarker = oResult.NextMarker
return result, nil
}
func (b *SBucket) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error {
obscli, err := b.region.getOBSClient()
if err != nil {
return errors.Wrap(err, "GetOBSClient")
}
input := &obs.PutObjectInput{}
input.Bucket = b.Name
input.Key = key
input.Body = reader
if len(storageClassStr) > 0 {
input.StorageClass, err = str2StorageClass(storageClassStr)
if err != nil {
return err
}
}
if len(contType) > 0 {
input.ContentType = contType
}
_, err = obscli.PutObject(input)
if err != nil {
return errors.Wrap(err, "PutObject")
}
return nil
}
func (b *SBucket) DeleteObject(ctx context.Context, key string) error {
obscli, err := b.region.getOBSClient()
if err != nil {
return errors.Wrap(err, "GetOBSClient")
}
input := &obs.DeleteObjectInput{}
input.Bucket = b.Name
input.Key = key
_, err = obscli.DeleteObject(input)
if err != nil {
return errors.Wrap(err, "DeleteObject")
}
return nil
}
func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
obscli, err := b.region.getOBSClient()
if err != nil {
return "", errors.Wrap(err, "GetOBSClient")
}
input := obs.CreateSignedUrlInput{}
input.Bucket = b.Name
input.Key = key
input.Expires = int(expire / time.Second)
switch method {
case "GET":
input.Method = obs.HttpMethodGet
case "PUT":
input.Method = obs.HttpMethodPut
case "DELETE":
input.Method = obs.HttpMethodDelete
default:
return "", errors.Error("unsupported method")
}
output, err := obscli.CreateSignedUrl(&input)
return output.SignedUrl, nil
}

13
pkg/util/huawei/object.go Normal file
View File

@@ -0,0 +1,13 @@
package huawei
import "yunion.io/x/onecloud/pkg/cloudprovider"
type SObject struct {
bucket *SBucket
cloudprovider.SBaseCloudObject
}
func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
return o.bucket
}

25
pkg/util/huawei/region.go
View File

@@ -705,6 +705,18 @@ func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
return ret, nil
}
func str2StorageClass(storageClassStr string) (obs.StorageClassType, error) {
if strings.EqualFold(storageClassStr, string(obs.StorageClassStandard)) {
return obs.StorageClassStandard, nil
} else if strings.EqualFold(storageClassStr, string(obs.StorageClassWarm)) {
return obs.StorageClassWarm, nil
} else if strings.EqualFold(storageClassStr, string(obs.StorageClassCold)) {
return obs.StorageClassCold, nil
} else {
return obs.StorageClassStandard, errors.Error("unsupported storageClass")
}
}
func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
obsClient, err := region.getOBSClient()
if err != nil {
@@ -725,14 +737,9 @@ func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr
}
}
if len(storageClassStr) > 0 {
if strings.EqualFold(storageClassStr, string(obs.StorageClassStandard)) {
input.StorageClass = obs.StorageClassStandard
} else if strings.EqualFold(storageClassStr, string(obs.StorageClassWarm)) {
input.StorageClass = obs.StorageClassWarm
} else if strings.EqualFold(storageClassStr, string(obs.StorageClassCold)) {
input.StorageClass = obs.StorageClassCold
} else {
return errors.Error("unsupported storageClass")
input.StorageClass, err = str2StorageClass(storageClassStr)
if err != nil {
return err
}
}
_, err = obsClient.CreateBucket(input)
@@ -769,7 +776,7 @@ func (region *SRegion) IBucketExist(name string) (bool, error) {
return true, nil
}
func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
obsClient, err := region.getOBSClient()
if err != nil {
return nil, errors.Wrap(err, "region.getOBSClient")

9
pkg/util/huawei/shell/bucket.go Normal file
View File

@@ -0,0 +1,9 @@
package shell
import (
"yunion.io/x/onecloud/pkg/multicloud/objectstore"
)
func init() {
objectstore.S3Shell()
}

2
pkg/util/huawei/shell/obs.go
View File

@@ -30,7 +30,7 @@ func init() {
BUCKET string `help:"bucket name to show"`
}
shellutils.R(&ObsBucketShowOptions{}, "obs-show", "Show bucket detail", func(cli *huawei.SRegion, args *ObsBucketShowOptions) error {
bucket, err := cli.GetIBucketByName(args.BUCKET)
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}

2
pkg/util/openstack/region.go
View File

@@ -499,6 +499,6 @@ func (region *SRegion) IBucketExist(name string) (bool, error) {
return false, cloudprovider.ErrNotImplemented
}
func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
return nil, cloudprovider.ErrNotImplemented
}

122
pkg/util/qcloud/bucket.go
View File

@@ -1,9 +1,16 @@
package qcloud
import (
"context"
"fmt"
"io"
"time"
"github.com/tencentyun/cos-go-sdk-v5"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/timeutils"
"yunion.io/x/onecloud/pkg/cloudprovider"
)
@@ -49,10 +56,14 @@ func (b *SBucket) GetAcl() string {
return b.Acl
}
func (b *SBucket) getBucketUrl() string {
return fmt.Sprintf("https://%s.%s", b.FullName, b.region.getCosEndpoint())
}
func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
return []cloudprovider.SBucketAccessUrl{
{
Url: fmt.Sprintf("https://%s.%s", b.FullName, b.region.getCosEndpoint()),
Url: b.getBucketUrl(),
Description: "bucket domain",
},
{
@@ -61,3 +72,112 @@ func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
},
}
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}
coscli, err := b.region.GetCosClient(b)
if err != nil {
return result, errors.Wrap(err, "GetCosClient")
}
opts := &cos.BucketGetOptions{}
if len(prefix) > 0 {
opts.Prefix = prefix
}
if len(marker) > 0 {
opts.Marker = marker
}
if len(delimiter) > 0 {
opts.Delimiter = delimiter
}
if maxCount > 0 {
opts.MaxKeys = maxCount
}
oResult, _, err := coscli.Bucket.Get(context.Background(), opts)
if err != nil {
return result, errors.Wrap(err, "coscli.Bucket.Get")
}
result.Objects = make([]cloudprovider.ICloudObject, 0)
for _, object := range oResult.Contents {
lastModified, _ := timeutils.ParseTimeStr(object.LastModified)
obj := &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
StorageClass: string(object.StorageClass),
Key: object.Key,
SizeBytes: int64(object.Size),
ETag: object.ETag,
LastModified: lastModified,
ContentType: "",
},
}
result.Objects = append(result.Objects, obj)
}
if oResult.CommonPrefixes != nil {
result.CommonPrefixes = make([]cloudprovider.ICloudObject, 0)
for _, commPrefix := range oResult.CommonPrefixes {
obj := &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
Key: commPrefix,
},
}
result.CommonPrefixes = append(result.CommonPrefixes, obj)
}
}
result.IsTruncated = oResult.IsTruncated
result.NextMarker = oResult.NextMarker
return result, nil
}
func (b *SBucket) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error {
coscli, err := b.region.GetCosClient(b)
if err != nil {
return errors.Wrap(err, "GetCosClient")
}
opts := &cos.ObjectPutOptions{}
if len(contType) > 0 {
opts.ContentType = contType
}
if len(storageClassStr) > 0 {
opts.XCosStorageClass = storageClassStr
}
_, err = coscli.Object.Put(ctx, key, reader, opts)
if err != nil {
return errors.Wrap(err, "coscli.Object.Put")
}
return nil
}
func (b *SBucket) DeleteObject(ctx context.Context, key string) error {
coscli, err := b.region.GetCosClient(b)
if err != nil {
return errors.Wrap(err, "GetCosClient")
}
_, err = coscli.Object.Delete(ctx, key)
if err != nil {
return errors.Wrap(err, "coscli.Object.Delete")
}
return nil
}
func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
if method != "GET" && method != "PUT" && method != "DELETE" {
return "", errors.Error("unsupported method")
}
coscli, err := b.region.GetCosClient(b)
if err != nil {
return "", errors.Wrap(err, "GetCosClient")
}
url, err := coscli.Object.GetPresignedURL(context.Background(), method, key,
b.region.client.SecretKey,
b.region.client.SecretID,
expire, nil)
if err != nil {
return "", errors.Wrap(err, "coscli.Object.GetPresignedURL")
}
return url.String(), nil
}

13
pkg/util/qcloud/object.go Normal file
View File

@@ -0,0 +1,13 @@
package qcloud
import "yunion.io/x/onecloud/pkg/cloudprovider"
type SObject struct {
bucket *SBucket
cloudprovider.SBaseCloudObject
}
func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
return o.bucket
}

106
pkg/util/qcloud/qcloud.go
View File

@@ -20,9 +20,6 @@ import (
"strings"
"time"
"yunion.io/x/pkg/utils"
coslib "github.com/nelsonken/cos-go-sdk-v5/cos"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
sdkerrors "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors"
tchttp "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http"
@@ -30,12 +27,9 @@ import (
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/appctx"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/pkg/util/timeutils"
)
const (
@@ -587,103 +581,3 @@ func (client *SQcloudClient) GetIProjects() ([]cloudprovider.ICloudProject, erro
}
return iprojects, nil
}
func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
ctx := appctx.Background
cos, err := region.GetCosClient()
if err != nil {
return nil, errors.Wrap(err, "GetCosClient")
}
result, err := cos.GetBucketList(ctx)
if err != nil {
return nil, errors.Wrap(err, "GetBucketList")
}
ret := make([]cloudprovider.ICloudBucket, 0)
for i := range result.Buckets.Bucket {
bInfo := result.Buckets.Bucket[i]
// ignore buckets not belong to this region
if bInfo.Location != region.GetId() {
continue
}
createAt, _ := timeutils.ParseTimeStr(bInfo.CreateDate)
name := bInfo.Name
// name = name[:len(name)-len(result.Owner.ID)-1]
name = name[:strings.LastIndexByte(name, '-')]
b := SBucket{
region: region,
Name: name,
FullName: bInfo.Name,
Location: bInfo.Location,
CreateDate: createAt,
}
ret = append(ret, &b)
}
return ret, nil
}
func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
ctx := appctx.Background
cos, err := region.GetCosClient()
if err != nil {
return errors.Wrap(err, "GetCosClient")
}
acl := &coslib.AccessControl{}
if len(aclStr) > 0 {
if utils.IsInStringArray(aclStr, []string{
"private", "public-read", "public-read-write", "authenticated-read",
}) {
acl.ACL = aclStr
} else {
return errors.Error("invalid acl")
}
}
err = cos.CreateBucket(ctx, name, acl)
if err != nil {
return errors.Wrap(err, "oss.CreateBucket")
}
return nil
}
func cosHttpCode(err error) int {
if httpErr, ok := err.(coslib.HTTPError); ok {
return httpErr.Code
}
if httpErr, ok := err.(*coslib.HTTPError); ok {
return httpErr.Code
}
return -1
}
func (region *SRegion) DeleteIBucket(name string) error {
ctx := appctx.Background
cos, err := region.GetCosClient()
if err != nil {
return errors.Wrap(err, "GetCosClient")
}
err = cos.DeleteBucket(ctx, name)
if err != nil {
if cosHttpCode(err) == 404 {
return nil
}
return errors.Wrap(err, "DeleteBucket")
}
return nil
}
func (region *SRegion) IBucketExist(name string) (bool, error) {
ctx := appctx.Background
cos, err := region.GetCosClient()
if err != nil {
return false, errors.Wrap(err, "GetCosClient")
}
err = cos.BucketExists(ctx, name)
if err != nil {
return false, errors.Wrap(err, "BucketExists")
}
return true, nil
}
func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketByName(region, name)
}

152
pkg/util/qcloud/region.go
View File

@@ -15,15 +15,22 @@
package qcloud
import (
"context"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/nelsonken/cos-go-sdk-v5/cos"
"github.com/tencentyun/cos-go-sdk-v5"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/timeutils"
"yunion.io/x/pkg/utils"
"github.com/tencentyun/cos-go-sdk-v5/debug"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/multicloud"
@@ -32,8 +39,7 @@ import (
type SRegion struct {
multicloud.SRegion
client *SQcloudClient
cosClient *cos.Client
client *SQcloudClient
izones []cloudprovider.ICloudZone
ivpcs []cloudprovider.ICloudVpc
@@ -266,16 +272,30 @@ func (self *SRegion) CreateIVpc(name string, desc string, cidr string) (cloudpro
return self.GetIVpcById(vpcId)
}
func (self *SRegion) GetCosClient() (*cos.Client, error) {
if self.cosClient == nil {
self.cosClient = cos.New(&cos.Option{
AppID: self.client.AppID,
SecretID: self.client.SecretID,
SecretKey: self.client.SecretKey,
Region: self.Region,
})
func (self *SRegion) GetCosClient(bucket *SBucket) (*cos.Client, error) {
var baseUrl *cos.BaseURL
if bucket != nil {
u, _ := url.Parse(bucket.getBucketUrl())
baseUrl = &cos.BaseURL{
BucketURL: u,
}
}
return self.cosClient, nil
cosClient := cos.NewClient(
baseUrl,
&http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: self.client.SecretID,
SecretKey: self.client.SecretKey,
Transport: &debug.DebugRequestTransport{
RequestHeader: self.client.Debug,
RequestBody: self.client.Debug,
ResponseHeader: self.client.Debug,
ResponseBody: self.client.Debug,
},
},
},
)
return cosClient, nil
}
func (self *SRegion) GetClient() *SQcloudClient {
@@ -820,3 +840,111 @@ func (self *SRegion) QueryAccountBalance() (*SAccountBalance, error) {
func (self *SRegion) getCosEndpoint() string {
return fmt.Sprintf("cos.%s.myqcloud.com", self.GetId())
}
func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
coscli, err := region.GetCosClient(nil)
if err != nil {
return nil, errors.Wrap(err, "GetCosClient")
}
s, _, err := coscli.Service.Get(context.Background())
if err != nil {
return nil, errors.Wrap(err, "coscli.Service.Get")
}
ret := make([]cloudprovider.ICloudBucket, 0)
for i := range s.Buckets {
bInfo := s.Buckets[i]
// ignore buckets not belong to this region
if bInfo.Region != region.GetId() {
continue
}
createAt, _ := timeutils.ParseTimeStr(bInfo.CreationDate)
name := bInfo.Name
// name = name[:len(name)-len(result.Owner.ID)-1]
name = name[:strings.LastIndexByte(name, '-')]
b := SBucket{
region: region,
Name: name,
FullName: bInfo.Name,
Location: bInfo.Region,
CreateDate: createAt,
}
ret = append(ret, &b)
}
return ret, nil
}
func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
bucket := &SBucket{
region: region,
Name: name,
FullName: fmt.Sprintf("%s-%s", name, region.client.AppID),
}
coscli, err := region.GetCosClient(bucket)
if err != nil {
return errors.Wrap(err, "GetCosClient")
}
opts := &cos.BucketPutOptions{}
if len(aclStr) > 0 {
if utils.IsInStringArray(aclStr, []string{
"private", "public-read", "public-read-write", "authenticated-read",
}) {
opts.XCosACL = aclStr
} else {
return errors.Error("invalid acl")
}
}
_, err = coscli.Bucket.Put(context.Background(), opts)
if err != nil {
return errors.Wrap(err, "coscli.Bucket.Put")
}
return nil
}
func cosHttpCode(err error) int {
if httpErr, ok := err.(*cos.ErrorResponse); ok {
return httpErr.Response.StatusCode
}
return -1
}
func (region *SRegion) DeleteIBucket(name string) error {
bucket := &SBucket{
region: region,
Name: name,
FullName: fmt.Sprintf("%s-%s", name, region.client.AppID),
}
coscli, err := region.GetCosClient(bucket)
if err != nil {
return errors.Wrap(err, "GetCosClient")
}
_, err = coscli.Bucket.Delete(context.Background())
if err != nil {
if cosHttpCode(err) == 404 {
return nil
}
return errors.Wrap(err, "DeleteBucket")
}
return nil
}
func (region *SRegion) IBucketExist(name string) (bool, error) {
bucket := &SBucket{
region: region,
Name: name,
FullName: fmt.Sprintf("%s-%s", name, region.client.AppID),
}
coscli, err := region.GetCosClient(bucket)
if err != nil {
return false, errors.Wrap(err, "GetCosClient")
}
_, err = coscli.Bucket.Head(context.Background())
if err != nil {
return false, errors.Wrap(err, "BucketExists")
}
return true, nil
}
func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketById(region, name)
}

131
pkg/util/qcloud/shell/cos.go
View File

@@ -15,136 +15,9 @@
package shell
import (
"context"
"os"
coslib "github.com/nelsonken/cos-go-sdk-v5/cos"
"yunion.io/x/onecloud/pkg/util/printutils"
"yunion.io/x/onecloud/pkg/util/qcloud"
"yunion.io/x/onecloud/pkg/util/shellutils"
"yunion.io/x/onecloud/pkg/multicloud/objectstore"
)
func init() {
type CosListOptions struct {
}
shellutils.R(&CosListOptions{}, "cos-list", "List COS buckets", func(cli *qcloud.SRegion, args *CosListOptions) error {
buckets, err := cli.GetIBuckets()
if err != nil {
return err
}
printList(buckets, 0, 0, 0, nil)
return nil
})
shellutils.R(&CosListOptions{}, "bucket-list", "List COS buckets", func(cli *qcloud.SRegion, args *CosListOptions) error {
buckets, err := cli.GetIBuckets()
if err != nil {
return err
}
printutils.PrintGetterList(buckets, nil)
return nil
})
type CosCreateBucketOptions struct {
BUCKET string `help:"name of bucket to create"`
Acl string `help:"Acl"`
}
shellutils.R(&CosCreateBucketOptions{}, "cos-create-bucket", "Create a COS bucket", func(cli *qcloud.SRegion, args *CosCreateBucketOptions) error {
err := cli.CreateIBucket(args.BUCKET, "", args.Acl)
if err != nil {
return err
}
return nil
})
type CosDeleteBucketOptions struct {
BUCKET string `help:"name of bucket to delete"`
}
shellutils.R(&CosDeleteBucketOptions{}, "cos-delete-bucket", "Delete a COS bucket", func(cli *qcloud.SRegion, args *CosDeleteBucketOptions) error {
err := cli.DeleteIBucket(args.BUCKET)
if err != nil {
return err
}
return nil
})
type CosListBucketOptions struct {
BUCKET string `help:"bucket name"`
}
shellutils.R(&CosListBucketOptions{}, "cos-bucket-list", "List content of a OSS bucket", func(cli *qcloud.SRegion, args *CosListBucketOptions) error {
cos, err := cli.GetCosClient()
if err != nil {
return err
}
result, err := cos.ListBucketContents(context.Background(), args.BUCKET, &coslib.QueryCondition{})
if err != nil {
return err
}
printList(result.Contents, len(result.Contents), 0, len(result.Contents), nil)
return nil
})
shellutils.R(&CosListBucketOptions{}, "cos-bucket-create", "Create a OSS bucket", func(cli *qcloud.SRegion, args *CosListBucketOptions) error {
cos, err := cli.GetCosClient()
if err != nil {
return err
}
return cos.CreateBucket(context.Background(), args.BUCKET, &coslib.AccessControl{})
})
type CosUploadOptions struct {
BUCKET string `help:"bucket name"`
KEY string `help:"Object key"`
FILE string `help:"Local file path"`
Acl string `help:"Object ACL" choices:"private|public-read|public-read-write"`
}
shellutils.R(&CosUploadOptions{}, "cos-upload", "Upload a file to a Cos bucket", func(cli *qcloud.SRegion, args *CosUploadOptions) error {
cos, err := cli.GetCosClient()
if err != nil {
return err
}
return cos.Bucket(args.BUCKET).UploadObjectBySlice(context.Background(), args.KEY, args.FILE, 3, nil)
})
type CosDownloadOptions struct {
BUCKET string `help:"bucket name"`
NAME string `help:"File name"`
}
shellutils.R(&CosDownloadOptions{}, "cos-download", "Download a file", func(cli *qcloud.SRegion, args *CosDownloadOptions) error {
cos, err := cli.GetCosClient()
if err != nil {
return err
}
//file
return cos.Bucket(args.BUCKET).DownloadObject(context.Background(), args.NAME, os.Stdout)
//return cos.Bucket(args.BUCKET).UploadObjectBySlice(context.Background(), args.KEY, args.FILE, 3, nil)
})
type CosObjectAclOptions struct {
BUCKET string `help:"bucket name"`
KEY string `help:"object key"`
ACL string `help:"ACL" choices:"private|public-read|public-read-write"`
}
shellutils.R(&CosObjectAclOptions{}, "cos-set-acl", "Set acl for a object", func(cli *qcloud.SRegion, args *CosObjectAclOptions) error {
cos, err := cli.GetCosClient()
if err != nil {
return err
}
return cos.SetBucketACL(context.Background(), args.KEY, &coslib.AccessControl{ACL: args.ACL})
})
type CosDeleteOptions struct {
BUCKET string `help:"bucket name"`
KEY string `help:"Object key"`
}
shellutils.R(&CosDeleteOptions{}, "cos-delete", "Delete a file from a Cos bucket", func(cli *qcloud.SRegion, args *CosDeleteOptions) error {
cos, err := cli.GetCosClient()
if err != nil {
return err
}
return cos.Bucket(args.BUCKET).DeleteObject(context.Background(), args.KEY)
})
objectstore.S3Shell()
}

42
pkg/util/qcloud/storagecache.go
View File

@@ -18,15 +18,13 @@ import (
"context"
"fmt"
"io"
"os"
"strings"
"time"
coslib "github.com/nelsonken/cos-go-sdk-v5/cos"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"github.com/pkg/errors"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/compute/options"
@@ -165,33 +163,16 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To
// first upload image to oss
s := auth.GetAdminSession(ctx, options.Options.Region, "")
meta, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false)
_, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false)
if err != nil {
return "", err
}
tmpFile := fmt.Sprintf("%s/%s", options.Options.TempPath, image.ImageId)
defer os.Remove(tmpFile)
f, err := os.Create(tmpFile)
if err != nil {
return "", err
}
defer f.Close()
if _, err := io.Copy(f, reader); err != nil {
return "", err
}
log.Infof("meta data %s", meta)
cos, err := self.region.GetCosClient()
if err != nil {
log.Errorf("GetOssClient err %s", err)
return "", err
}
bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s", self.region.GetId()))
err = cos.BucketExists(context.Background(), bucketName)
if err != nil {
bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s-%s", self.region.GetId(), image.ImageId))
exists, _ := self.region.IBucketExist(bucketName)
if !exists {
log.Debugf("Bucket %s not exists, to create ...", bucketName)
err := cos.CreateBucket(context.Background(), bucketName, &coslib.AccessControl{ACL: "public-read"})
err := self.region.CreateIBucket(bucketName, "", "public-read")
if err != nil {
log.Errorf("Create bucket error %s", err)
return "", err
@@ -199,14 +180,19 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To
} else {
log.Debugf("Bucket %s exists", bucketName)
}
defer self.region.DeleteIBucket(bucketName)
log.Debugf("To upload image to bucket %s ...", bucketName)
err = cos.Bucket(bucketName).UploadObjectBySlice(context.Background(), image.ImageId, tmpFile, 3, map[string]string{})
bucket, err := self.region.GetIBucketById(bucketName)
if err != nil {
return "", errors.Wrap(err, "GetIBucketByName")
}
err = bucket.PutObject(context.Background(), image.ImageId, reader.(io.ReadSeeker), "", "")
if err != nil {
log.Errorf("UploadObject error %s %s", image.ImageId, err)
return "", err
return "", errors.Wrap(err, "bucket.PutObject")
}
defer cos.Bucket(bucketName).DeleteObject(context.Background(), image.ImageId)
defer bucket.DeleteObject(context.Background(), image.ImageId)
// 腾讯云镜像名称需要小于20个字符
imageBaseName := image.ImageId[:10]

18
pkg/util/ucloud/region.go
View File

@@ -636,6 +636,7 @@ func (self *SRegion) GetClient() *SUcloudClient {
return self.client
}
// https://docs.ucloud.cn/api/ufile-api/describe_bucket
func (region *SRegion) listBuckets(name string, offset int, limit int) ([]SBucket, error) {
params := NewUcloudParams()
if len(name) > 0 {
@@ -645,7 +646,8 @@ func (region *SRegion) listBuckets(name string, offset int, limit int) ([]SBucke
params.Set("Offset", offset)
}
buckets := make([]SBucket, 0)
err := region.DoAction("DescribeBucket", params, &buckets)
// request without RegionId
err := region.client.DoAction("DescribeBucket", params, &buckets)
if err != nil {
return nil, errors.Wrap(err, "DoAction DescribeBucket")
}
@@ -673,6 +675,7 @@ func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
ret := make([]cloudprovider.ICloudBucket, len(buckets))
for i := range buckets {
buckets[i].region = region
buckets[i].projectId = region.client.projectId
ret[i] = &buckets[i]
}
return ret, nil
@@ -707,15 +710,6 @@ func (region *SRegion) IBucketExist(name string) (bool, error) {
return true, nil
}
func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
parts, err := region.listBuckets(name, 0, 1)
if err != nil {
return nil, errors.Wrap(err, "region.listBuckets")
}
if len(parts) == 0 {
return nil, cloudprovider.ErrNotFound
}
bucket := parts[0]
bucket.region = region
return &bucket, nil
func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
return cloudprovider.GetIBucketById(region, name)
}

38
pkg/util/ucloud/shell/ufile.go
View File

@@ -1,43 +1,9 @@
package shell
import (
"yunion.io/x/onecloud/pkg/util/printutils"
"yunion.io/x/onecloud/pkg/util/shellutils"
"yunion.io/x/onecloud/pkg/util/ucloud"
"yunion.io/x/onecloud/pkg/multicloud/objectstore"
)
func init() {
type UFileBucketListOptions struct {
}
shellutils.R(&UFileBucketListOptions{}, "bucket-list", "List buckets", func(cli *ucloud.SRegion, args *UFileBucketListOptions) error {
buckets, err := cli.GetIBuckets()
if err != nil {
return err
}
printutils.PrintGetterList(buckets, nil)
return nil
})
type UFileBucketCreateOptions struct {
BUCKET string `help:"Name of bucket"`
Acl string `help:"Acl" choices:"private|public"`
}
shellutils.R(&UFileBucketCreateOptions{}, "bucket-create", "create a bucket", func(cli *ucloud.SRegion, args *UFileBucketCreateOptions) error {
err := cli.CreateIBucket(args.BUCKET, "", args.Acl)
if err != nil {
return err
}
return nil
})
type UFileBucketDeleteOptions struct {
BUCKET string `help:"Name of bucket"`
}
shellutils.R(&UFileBucketDeleteOptions{}, "bucket-delete", "delete a bucket", func(cli *ucloud.SRegion, args *UFileBucketDeleteOptions) error {
err := cli.DeleteIBucket(args.BUCKET)
if err != nil {
return err
}
return nil
})
objectstore.S3Shell()
}

30
pkg/util/ucloud/storagecache.go
View File

@@ -143,10 +143,14 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To
bucketName := GetBucketName(self.region.GetId(), image.ImageId)
// create bucket
if _, err := self.region.GetBucketDomain(bucketName); err != nil {
exist, err := self.region.IBucketExist(bucketName)
if err != nil {
return "", errors.Wrap(err, "self.region.IBucketExist")
}
if !exist {
err = self.region.CreateBucket(bucketName, "private")
if err != nil {
return "", err
return "", errors.Wrap(err, "CreateBucket")
}
}
defer func() {
@@ -165,7 +169,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To
log.Debugf("Images meta data %s", meta)
minDiskMB, _ := meta.Int("min_disk")
minDiskGB := int64(math.Ceil(float64(minDiskMB) / 1024))
// 在使用OBS桶的外部镜像文件制作镜像时生效且为必选字段。取值为40～1024GB。
if minDiskGB < 40 {
minDiskGB = 40
} else if minDiskGB > 1024 {
@@ -175,13 +179,16 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To
md5, _ := meta.GetString("checksum")
diskFormat, _ := meta.GetString("disk_format")
// upload to ucloud
bucket, err := self.region.GetIBucketById(bucketName)
if err != nil {
return "", errors.Wrap(err, "GetIBucketByName")
}
file := SFile{
region: self.region,
BucketName: bucketName,
File: reader,
FileSize: size,
FileName: image.ImageId,
FileMD5: md5,
bucket: bucket.(*SBucket),
File: reader,
FileSize: size,
FileName: image.ImageId,
FileMD5: md5,
}
err = file.Upload()
@@ -245,8 +252,7 @@ func (self *SRegion) createIImage(snapshotId, imageName, imageDesc string) (stri
return "", cloudprovider.ErrNotSupported
}
// https://docs.ucloud.cn/api/ufile-api/describe_bucket
func (self *SRegion) GetBucketDomain(name string) (string, error) {
/*func (self *SRegion) GetBucketDomain(name string) (string, error) {
params := NewUcloudParams()
params.Set("BucketName", name)
@@ -261,7 +267,7 @@ func (self *SRegion) GetBucketDomain(name string) (string, error) {
} else {
return "", fmt.Errorf("GetBucketDomain failed. %v", res)
}
}
}*/
// https://docs.ucloud.cn/api/ufile-api/create_bucket
func (self *SRegion) CreateBucket(name, bucketType string) error {

93
pkg/util/ucloud/ufile.go
View File

@@ -27,6 +27,7 @@ import (
"yunion.io/x/log"
"context"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/util/httputils"
)
@@ -34,6 +35,8 @@ import (
type SBucket struct {
region *SRegion
projectId string
Domain Domain `json:"Domain"`
BucketID string `json:"BucketId"`
Region string `json:"Region"`
@@ -55,13 +58,13 @@ type Domain struct {
}
type SFile struct {
region *SRegion
bucket *SBucket
BucketName string
File io.Reader
FileSize int64
FileName string
FileMD5 string
// BucketName string
File io.Reader
FileSize int64
FileName string
FileMD5 string
}
func (self *SFile) signHeader(httpMethod string) string {
@@ -76,24 +79,25 @@ func (self *SFile) signHeader(httpMethod string) string {
data += md5 + "\n"
data += contentType + "\n"
data += "\n"
data += "/" + self.BucketName + "/" + self.FileName
data += "/" + self.bucket.BucketName + "/" + self.FileName
h := hmac.New(sha1.New, []byte(self.region.client.accessKeySecret))
h := hmac.New(sha1.New, []byte(self.bucket.region.client.accessKeySecret))
h.Write([]byte(data))
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func (self *SFile) auth(httpMethod string) string {
return "UCloud" + " " + self.region.client.accessKeyId + ":" + self.signHeader(httpMethod)
return "UCloud" + " " + self.bucket.region.client.accessKeyId + ":" + self.signHeader(httpMethod)
}
func (self *SFile) GetHost() string {
host, err := self.region.GetBucketDomain(self.BucketName)
return self.bucket.Domain.Src[0]
/*host, err := self.bucket.region.GetBucketDomain(self.BucketName)
if err != nil {
log.Errorf("SFile GetHost %s", err)
return ""
}
return host
return host*/
}
func (self *SFile) GetUrl() string {
@@ -105,13 +109,13 @@ func (self *SFile) FetchFileUrl() string {
expired := strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10)
// sign
data := "GET\n\n\n" + expired + "\n"
data += "/" + self.BucketName + "/" + self.FileName
h := hmac.New(sha1.New, []byte(self.region.client.accessKeySecret))
data += "/" + self.bucket.BucketName + "/" + self.FileName
h := hmac.New(sha1.New, []byte(self.bucket.region.client.accessKeySecret))
h.Write([]byte(data))
sign := base64.StdEncoding.EncodeToString(h.Sum(nil))
urlEncoder := url.Values{}
urlEncoder.Add("UCloudPublicKey", self.region.client.accessKeyId)
urlEncoder.Add("UCloudPublicKey", self.bucket.region.client.accessKeyId)
urlEncoder.Add("Signature", sign)
urlEncoder.Add("Expires", expired)
querys := urlEncoder.Encode()
@@ -135,7 +139,7 @@ func (self *SFile) Delete() error {
}
func (self *SFile) request(req *http.Request) error {
res, err := http.DefaultClient.Do(req)
res, err := httputils.GetDefaultClient().Do(req)
if err != nil {
return err
}
@@ -150,11 +154,11 @@ func (self *SFile) request(req *http.Request) error {
}
func (b *SBucket) GetProjectId() string {
return ""
return b.projectId
}
func (b *SBucket) GetGlobalId() string {
return b.BucketName
return b.BucketID
}
func (b *SBucket) GetName() string {
@@ -181,20 +185,51 @@ func (b *SBucket) GetAcl() string {
return b.Type
}
func (b *SBucket) getSrcUrl() string {
if len(b.Domain.Src) > 0 {
return b.Domain.Src[0]
}
return ""
}
func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
ret := make([]cloudprovider.SBucketAccessUrl, 0)
regionId := b.region.GetId()
// hack, remove trailing digits
for len(regionId) > 0 {
lastDigit := regionId[len(regionId)-1]
if lastDigit >= '0' && lastDigit <= '9' {
regionId = regionId[:len(regionId)-1]
} else {
break
}
for i, u := range b.Domain.Src {
ret = append(ret, cloudprovider.SBucketAccessUrl{
Url: u,
Description: fmt.Sprintf("src%d", i),
})
}
for i, u := range b.Domain.CDN {
ret = append(ret, cloudprovider.SBucketAccessUrl{
Url: u,
Description: fmt.Sprintf("cdn%d", i),
})
}
ret = append(ret, cloudprovider.SBucketAccessUrl{
Url: fmt.Sprintf("https://%s.%s.ufileos.com", b.BucketName, regionId),
})
return ret
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}
return result, cloudprovider.ErrNotSupported
}
func (b *SBucket) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error {
return cloudprovider.ErrNotSupported
}
func (b *SBucket) DeleteObject(ctx context.Context, key string) error {
file := SFile{
bucket: b,
FileName: key,
}
return file.Delete()
}
func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
return "", cloudprovider.ErrNotSupported
}

27
vendor/github.com/google/go-querystring/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,27 @@
Copyright (c) 2013 Google. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

320
vendor/github.com/google/go-querystring/query/encode.go generated vendored Normal file
View File

@@ -0,0 +1,320 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package query implements encoding of structs into URL query parameters.
//
// As a simple example:
//
// type Options struct {
// Query string `url:"q"`
// ShowAll bool `url:"all"`
// Page int `url:"page"`
// }
//
// opt := Options{ "foo", true, 2 }
// v, _ := query.Values(opt)
// fmt.Print(v.Encode()) // will output: "q=foo&all=true&page=2"
//
// The exact mapping between Go values and url.Values is described in the
// documentation for the Values() function.
package query
import (
"bytes"
"fmt"
"net/url"
"reflect"
"strconv"
"strings"
"time"
)
var timeType = reflect.TypeOf(time.Time{})
var encoderType = reflect.TypeOf(new(Encoder)).Elem()
// Encoder is an interface implemented by any type that wishes to encode
// itself into URL values in a non-standard way.
type Encoder interface {
EncodeValues(key string, v *url.Values) error
}
// Values returns the url.Values encoding of v.
//
// Values expects to be passed a struct, and traverses it recursively using the
// following encoding rules.
//
// Each exported struct field is encoded as a URL parameter unless
//
// - the field's tag is "-", or
// - the field is empty and its tag specifies the "omitempty" option
//
// The empty values are false, 0, any nil pointer or interface value, any array
// slice, map, or string of length zero, and any time.Time that returns true
// for IsZero().
//
// The URL parameter name defaults to the struct field name but can be
// specified in the struct field's tag value. The "url" key in the struct
// field's tag value is the key name, followed by an optional comma and
// options. For example:
//
// // Field is ignored by this package.
// Field int `url:"-"`
//
// // Field appears as URL parameter "myName".
// Field int `url:"myName"`
//
// // Field appears as URL parameter "myName" and the field is omitted if
// // its value is empty
// Field int `url:"myName,omitempty"`
//
// // Field appears as URL parameter "Field" (the default), but the field
// // is skipped if empty. Note the leading comma.
// Field int `url:",omitempty"`
//
// For encoding individual field values, the following type-dependent rules
// apply:
//
// Boolean values default to encoding as the strings "true" or "false".
// Including the "int" option signals that the field should be encoded as the
// strings "1" or "0".
//
// time.Time values default to encoding as RFC3339 timestamps. Including the
// "unix" option signals that the field should be encoded as a Unix time (see
// time.Unix())
//
// Slice and Array values default to encoding as multiple URL values of the
// same name. Including the "comma" option signals that the field should be
// encoded as a single comma-delimited value. Including the "space" option
// similarly encodes the value as a single space-delimited string. Including
// the "semicolon" option will encode the value as a semicolon-delimited string.
// Including the "brackets" option signals that the multiple URL values should
// have "[]" appended to the value name. "numbered" will append a number to
// the end of each incidence of the value name, example:
// name0=value0&name1=value1, etc.
//
// Anonymous struct fields are usually encoded as if their inner exported
// fields were fields in the outer struct, subject to the standard Go
// visibility rules. An anonymous struct field with a name given in its URL
// tag is treated as having that name, rather than being anonymous.
//
// Non-nil pointer values are encoded as the value pointed to.
//
// Nested structs are encoded including parent fields in value names for
// scoping. e.g:
//
// "user[name]=acme&user[addr][postcode]=1234&user[addr][city]=SFO"
//
// All other values are encoded using their default string representation.
//
// Multiple fields that encode to the same URL parameter name will be included
// as multiple URL values of the same name.
func Values(v interface{}) (url.Values, error) {
values := make(url.Values)
val := reflect.ValueOf(v)
for val.Kind() == reflect.Ptr {
if val.IsNil() {
return values, nil
}
val = val.Elem()
}
if v == nil {
return values, nil
}
if val.Kind() != reflect.Struct {
return nil, fmt.Errorf("query: Values() expects struct input. Got %v", val.Kind())
}
err := reflectValue(values, val, "")
return values, err
}
// reflectValue populates the values parameter from the struct fields in val.
// Embedded structs are followed recursively (using the rules defined in the
// Values function documentation) breadth-first.
func reflectValue(values url.Values, val reflect.Value, scope string) error {
var embedded []reflect.Value
typ := val.Type()
for i := 0; i < typ.NumField(); i++ {
sf := typ.Field(i)
if sf.PkgPath != "" && !sf.Anonymous { // unexported
continue
}
sv := val.Field(i)
tag := sf.Tag.Get("url")
if tag == "-" {
continue
}
name, opts := parseTag(tag)
if name == "" {
if sf.Anonymous && sv.Kind() == reflect.Struct {
// save embedded struct for later processing
embedded = append(embedded, sv)
continue
}
name = sf.Name
}
if scope != "" {
name = scope + "[" + name + "]"
}
if opts.Contains("omitempty") && isEmptyValue(sv) {
continue
}
if sv.Type().Implements(encoderType) {
if !reflect.Indirect(sv).IsValid() {
sv = reflect.New(sv.Type().Elem())
}
m := sv.Interface().(Encoder)
if err := m.EncodeValues(name, &values); err != nil {
return err
}
continue
}
if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array {
var del byte
if opts.Contains("comma") {
del = ','
} else if opts.Contains("space") {
del = ' '
} else if opts.Contains("semicolon") {
del = ';'
} else if opts.Contains("brackets") {
name = name + "[]"
}
if del != 0 {
s := new(bytes.Buffer)
first := true
for i := 0; i < sv.Len(); i++ {
if first {
first = false
} else {
s.WriteByte(del)
}
s.WriteString(valueString(sv.Index(i), opts))
}
values.Add(name, s.String())
} else {
for i := 0; i < sv.Len(); i++ {
k := name
if opts.Contains("numbered") {
k = fmt.Sprintf("%s%d", name, i)
}
values.Add(k, valueString(sv.Index(i), opts))
}
}
continue
}
for sv.Kind() == reflect.Ptr {
if sv.IsNil() {
break
}
sv = sv.Elem()
}
if sv.Type() == timeType {
values.Add(name, valueString(sv, opts))
continue
}
if sv.Kind() == reflect.Struct {
reflectValue(values, sv, name)
continue
}
values.Add(name, valueString(sv, opts))
}
for _, f := range embedded {
if err := reflectValue(values, f, scope); err != nil {
return err
}
}
return nil
}
// valueString returns the string representation of a value.
func valueString(v reflect.Value, opts tagOptions) string {
for v.Kind() == reflect.Ptr {
if v.IsNil() {
return ""
}
v = v.Elem()
}
if v.Kind() == reflect.Bool && opts.Contains("int") {
if v.Bool() {
return "1"
}
return "0"
}
if v.Type() == timeType {
t := v.Interface().(time.Time)
if opts.Contains("unix") {
return strconv.FormatInt(t.Unix(), 10)
}
return t.Format(time.RFC3339)
}
return fmt.Sprint(v.Interface())
}
// isEmptyValue checks if a value should be considered empty for the purposes
// of omitting fields with the "omitempty" option.
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
}
if v.Type() == timeType {
return v.Interface().(time.Time).IsZero()
}
return false
}
// tagOptions is the string following a comma in a struct field's "url" tag, or
// the empty string. It does not include the leading comma.
type tagOptions []string
// parseTag splits a struct field's url tag into its name and comma-separated
// options.
func parseTag(tag string) (string, tagOptions) {
s := strings.Split(tag, ",")
return s[0], s[1:]
}
// Contains checks whether the tagOptions contains the specified option.
func (o tagOptions) Contains(option string) bool {
for _, s := range o {
if s == option {
return true
}
}
return false
}

7
vendor/github.com/mozillazg/go-httpheader/.bumpversion.cfg generated vendored Normal file
View File

@@ -0,0 +1,7 @@
[bumpversion]
commit = True
tag = True
current_version = 0.2.1
[bumpversion:file:encode.go]

27
vendor/github.com/mozillazg/go-httpheader/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,27 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
dist/
cover.html
cover.out

25
vendor/github.com/mozillazg/go-httpheader/.travis.yml generated vendored Normal file
View File

@@ -0,0 +1,25 @@
language: go
go:
- 1.6
- 1.7
- 1.8
- tip
sudo: false
before_install:
- go get github.com/mattn/goveralls
install:
- go get
- go build
script:
- make test
- $HOME/gopath/bin/goveralls -service=travis-ci -ignore=vendor/
matrix:
allow_failures:
- go: 1.6
- go: 1.7
- go: tip

15
vendor/github.com/mozillazg/go-httpheader/CHANGELOG.md generated vendored Normal file
View File

@@ -0,0 +1,15 @@
# Changelog
## 0.2.1 (2018-11-03)
* add go.mod file to identify as a module
## 0.2.0 (2017-06-24)
* support http.Header field.
## 0.1.0 (2017-06-10)
* Initial Release

21
vendor/github.com/mozillazg/go-httpheader/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2017 mozillazg
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

15
vendor/github.com/mozillazg/go-httpheader/Makefile generated vendored Normal file
View File

@@ -0,0 +1,15 @@
help:
@echo "test run test"
@echo "lint run lint"
.PHONY: test
test:
go test -v -cover -coverprofile cover.out
go tool cover -html=cover.out -o cover.html
.PHONY: lint
lint:
gofmt -s -w .
goimports -w .
golint .
go vet

63
vendor/github.com/mozillazg/go-httpheader/README.md generated vendored Normal file
View File

@@ -0,0 +1,63 @@
# go-httpheader
go-httpheader is a Go library for encoding structs into Header fields.
[![Build Status](https://img.shields.io/travis/mozillazg/go-httpheader/master.svg)](https://travis-ci.org/mozillazg/go-httpheader)
[![Coverage Status](https://img.shields.io/coveralls/mozillazg/go-httpheader/master.svg)](https://coveralls.io/r/mozillazg/go-httpheader?branch=master)
[![Go Report Card](https://goreportcard.com/badge/github.com/mozillazg/go-httpheader)](https://goreportcard.com/report/github.com/mozillazg/go-httpheader)
[![GoDoc](https://godoc.org/github.com/mozillazg/go-httpheader?status.svg)](https://godoc.org/github.com/mozillazg/go-httpheader)
## install
`go get -u github.com/mozillazg/go-httpheader`
## usage
```go
package main
import (
"fmt"
"net/http"
"github.com/mozillazg/go-httpheader"
)
type Options struct {
hide string
ContentType string `header:"Content-Type"`
Length int
XArray []string `header:"X-Array"`
TestHide string `header:"-"`
IgnoreEmpty string `header:"X-Empty,omitempty"`
IgnoreEmptyN string `header:"X-Empty-N,omitempty"`
CustomHeader http.Header
}
func main() {
opt := Options{
hide: "hide",
ContentType: "application/json",
Length: 2,
XArray: []string{"test1", "test2"},
TestHide: "hide",
IgnoreEmptyN: "n",
CustomHeader: http.Header{
"X-Test-1": []string{"233"},
"X-Test-2": []string{"666"},
},
}
h, _ := httpheader.Header(opt)
fmt.Printf("%#v", h)
// h:
// http.Header{
// "X-Test-1": []string{"233"},
// "X-Test-2": []string{"666"},
// "Content-Type": []string{"application/json"},
// "Length": []string{"2"},
// "X-Array": []string{"test1", "test2"},
// "X-Empty-N": []string{"n"},
//}
}
```

290
vendor/github.com/mozillazg/go-httpheader/encode.go generated vendored Normal file
View File

@@ -0,0 +1,290 @@
// Package query implements encoding of structs into http.Header fields.
//
// As a simple example:
//
// type Options struct {
// ContentType string `header:"Content-Type"`
// Length int
// }
//
// opt := Options{"application/json", 2}
// h, _ := httpheader.Header(opt)
// fmt.Printf("%#v", h)
// // will output:
// // http.Header{"Content-Type":[]string{"application/json"},"Length":[]string{"2"}}
//
// The exact mapping between Go values and http.Header is described in the
// documentation for the Header() function.
package httpheader
import (
"fmt"
"net/http"
"reflect"
"strconv"
"strings"
"time"
)
const tagName = "header"
// Version ...
const Version = "0.2.1"
var timeType = reflect.TypeOf(time.Time{})
var headerType = reflect.TypeOf(http.Header{})
var encoderType = reflect.TypeOf(new(Encoder)).Elem()
// Encoder is an interface implemented by any type that wishes to encode
// itself into Header fields in a non-standard way.
type Encoder interface {
EncodeHeader(key string, v *http.Header) error
}
// Header returns the http.Header encoding of v.
//
// Header expects to be passed a struct, and traverses it recursively using the
// following encoding rules.
//
// Each exported struct field is encoded as a Header field unless
//
// - the field's tag is "-", or
// - the field is empty and its tag specifies the "omitempty" option
//
// The empty values are false, 0, any nil pointer or interface value, any array
// slice, map, or string of length zero, and any time.Time that returns true
// for IsZero().
//
// The Header field name defaults to the struct field name but can be
// specified in the struct field's tag value. The "header" key in the struct
// field's tag value is the key name, followed by an optional comma and
// options. For example:
//
// // Field is ignored by this package.
// Field int `header:"-"`
//
// // Field appears as Header field "X-Name".
// Field int `header:"X-Name"`
//
// // Field appears as Header field "X-Name" and the field is omitted if
// // its value is empty
// Field int `header:"X-Name,omitempty"`
//
// // Field appears as Header field "Field" (the default), but the field
// // is skipped if empty. Note the leading comma.
// Field int `header:",omitempty"`
//
// For encoding individual field values, the following type-dependent rules
// apply:
//
// Boolean values default to encoding as the strings "true" or "false".
// Including the "int" option signals that the field should be encoded as the
// strings "1" or "0".
//
// time.Time values default to encoding as RFC1123("Mon, 02 Jan 2006 15:04:05 GMT")
// timestamps. Including the "unix" option signals that the field should be
// encoded as a Unix time (see time.Unix())
//
// Slice and Array values default to encoding as multiple Header values of the
// same name. example:
// X-Name: []string{"Tom", "Jim"}, etc.
//
// http.Header values will be used to extend the Header fields.
//
// Anonymous struct fields are usually encoded as if their inner exported
// fields were fields in the outer struct, subject to the standard Go
// visibility rules. An anonymous struct field with a name given in its Header
// tag is treated as having that name, rather than being anonymous.
//
// Non-nil pointer values are encoded as the value pointed to.
//
// All other values are encoded using their default string representation.
//
// Multiple fields that encode to the same Header filed name will be included
// as multiple Header values of the same name.
func Header(v interface{}) (http.Header, error) {
h := make(http.Header)
val := reflect.ValueOf(v)
for val.Kind() == reflect.Ptr {
if val.IsNil() {
return h, nil
}
val = val.Elem()
}
if v == nil {
return h, nil
}
if val.Kind() != reflect.Struct {
return nil, fmt.Errorf("httpheader: Header() expects struct input. Got %v", val.Kind())
}
err := reflectValue(h, val)
return h, err
}
// reflectValue populates the header fields from the struct fields in val.
// Embedded structs are followed recursively (using the rules defined in the
// Values function documentation) breadth-first.
func reflectValue(header http.Header, val reflect.Value) error {
var embedded []reflect.Value
typ := val.Type()
for i := 0; i < typ.NumField(); i++ {
sf := typ.Field(i)
if sf.PkgPath != "" && !sf.Anonymous { // unexported
continue
}
sv := val.Field(i)
tag := sf.Tag.Get(tagName)
if tag == "-" {
continue
}
name, opts := parseTag(tag)
if name == "" {
if sf.Anonymous && sv.Kind() == reflect.Struct {
// save embedded struct for later processing
embedded = append(embedded, sv)
continue
}
name = sf.Name
}
if opts.Contains("omitempty") && isEmptyValue(sv) {
continue
}
if sv.Type().Implements(encoderType) {
if !reflect.Indirect(sv).IsValid() {
sv = reflect.New(sv.Type().Elem())
}
m := sv.Interface().(Encoder)
if err := m.EncodeHeader(name, &header); err != nil {
return err
}
continue
}
if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array {
for i := 0; i < sv.Len(); i++ {
k := name
header.Add(k, valueString(sv.Index(i), opts))
}
continue
}
for sv.Kind() == reflect.Ptr {
if sv.IsNil() {
break
}
sv = sv.Elem()
}
if sv.Type() == timeType {
header.Add(name, valueString(sv, opts))
continue
}
if sv.Type() == headerType {
h := sv.Interface().(http.Header)
for k, vs := range h {
for _, v := range vs {
header.Add(k, v)
}
}
continue
}
if sv.Kind() == reflect.Struct {
reflectValue(header, sv)
continue
}
header.Add(name, valueString(sv, opts))
}
for _, f := range embedded {
if err := reflectValue(header, f); err != nil {
return err
}
}
return nil
}
// valueString returns the string representation of a value.
func valueString(v reflect.Value, opts tagOptions) string {
for v.Kind() == reflect.Ptr {
if v.IsNil() {
return ""
}
v = v.Elem()
}
if v.Kind() == reflect.Bool && opts.Contains("int") {
if v.Bool() {
return "1"
}
return "0"
}
if v.Type() == timeType {
t := v.Interface().(time.Time)
if opts.Contains("unix") {
return strconv.FormatInt(t.Unix(), 10)
}
return t.Format(http.TimeFormat)
}
return fmt.Sprint(v.Interface())
}
// isEmptyValue checks if a value should be considered empty for the purposes
// of omitting fields with the "omitempty" option.
func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
}
if v.Type() == timeType {
return v.Interface().(time.Time).IsZero()
}
return false
}
// tagOptions is the string following a comma in a struct field's "header" tag, or
// the empty string. It does not include the leading comma.
type tagOptions []string
// parseTag splits a struct field's header tag into its name and comma-separated
// options.
func parseTag(tag string) (string, tagOptions) {
s := strings.Split(tag, ",")
return s[0], s[1:]
}
// Contains checks whether the tagOptions contains the specified option.
func (o tagOptions) Contains(option string) bool {
for _, s := range o {
if s == option {
return true
}
}
return false
}

1
vendor/github.com/mozillazg/go-httpheader/go.mod generated vendored Normal file
View File

@@ -0,0 +1 @@
module github.com/mozillazg/go-httpheader

128
vendor/github.com/nelsonken/cos-go-sdk-v5/cos/auth.go generated vendored
View File

@@ -1,128 +0,0 @@
package cos
import (
"crypto/hmac"
"crypto/sha1"
"encoding/hex"
"fmt"
"net/http"
"sort"
"strconv"
"strings"
"time"
)
func (conn *Conn) signHeader(req *http.Request, params map[string]interface{}, headers map[string]string) {
signTime := getSignTime()
signature := conn.getSignature(req, params, headers, signTime)
authStr := fmt.Sprintf("q-sign-algorithm=sha1&q-ak=%s&q-sign-time=%s&q-key-time=%s&q-header-list=%s&q-url-param-list=%s&q-signature=%s",
conn.conf.SecretID, signTime, signTime, getHeadKeys(headers), getParamKeys(params), signature)
req.Header.Set("Authorization", authStr)
}
func getSignTime() string {
now := time.Now()
expired := now.Add(time.Second * 1800)
return fmt.Sprintf("%d;%d", now.Unix(), expired.Unix())
}
func getHeadKeys(headers map[string]string) string {
if headers == nil || len(headers) == 0 {
return ""
}
tmp := []string{}
for k := range headers {
tmp = append(tmp, strings.ToLower(k))
}
sort.Strings(tmp)
return strings.Join(tmp, ";")
}
func getParamKeys(params map[string]interface{}) string {
if params == nil || len(params) == 0 {
return ""
}
tmp := []string{}
for k := range params {
tmp = append(tmp, strings.ToLower(k))
}
sort.Strings(tmp)
return strings.Join(tmp, ";")
}
func (conn *Conn) getSignature(req *http.Request, params map[string]interface{}, headers map[string]string, signTime string) string {
httpString := fmt.Sprintf("%s\n%s\n%s\n%s\n", strings.ToLower(req.Method),
req.URL.Path, getParamStr(params), getHeadStr(headers))
httpString = sha(httpString)
signKey := hmacSha(conn.conf.SecretKey, signTime)
signStr := fmt.Sprintf("sha1\n%s\n%s\n", signTime, httpString)
return hmacSha(signKey, signStr)
}
func interfaceToString(i interface{}) string {
switch x := i.(type) {
case string:
return x
case int:
return strconv.Itoa(x)
case int64:
return strconv.FormatInt(x, 10)
case uint64:
return strconv.FormatUint(x, 10)
default:
return ""
}
}
func getParamStr(params map[string]interface{}) string {
if params == nil || len(params) == 0 {
return ""
}
tmp := []string{}
for k, v := range params {
str := strings.ToLower(fmt.Sprintf("%s=%s", k, interfaceToString(v)))
tmp = append(tmp, str)
}
sort.Strings(tmp)
return strings.Join(tmp, "&")
}
func getHeadStr(headers map[string]string) string {
if headers == nil || len(headers) == 0 {
return ""
}
tmp := []string{}
for k, v := range headers {
str := fmt.Sprintf("%s=%s", strings.ToLower(k), escape(v))
tmp = append(tmp, str)
}
sort.Strings(tmp)
return strings.Join(tmp, "&")
}
func sha(s string) string {
sha := sha1.New()
sha.Write([]byte(s))
b := sha.Sum(nil)
return hex.EncodeToString(b)
}
func hmacSha(k, s string) string {
enc := hmac.New(sha1.New, []byte(k))
enc.Write([]byte(s))
b := enc.Sum(nil)
return hex.EncodeToString(b)
}

315
vendor/github.com/nelsonken/cos-go-sdk-v5/cos/bucket.go generated vendored
View File

@@ -1,315 +0,0 @@
package cos
import (
"bytes"
"context"
"crypto/md5"
"encoding/hex"
"encoding/xml"
"fmt"
"io"
"net/http"
"os"
"strings"
)
// Bucket bucket
type Bucket struct {
Name string
conn *Conn
}
// ObjectSlice object slice
type ObjectSlice struct {
UploadID string
Size int64
Offset int64
Number int
MD5 string
Dst string
Result bool
}
// 获得云存储上文件信息
func (b *Bucket) HeadObject(ctx context.Context, object string) error {
resq, err := b.conn.Do(ctx, http.MethodHead, b.Name, object, nil, nil, nil)
if err == nil {
defer resq.Body.Close()
} else {
for k, v := range resq.Header {
value := fmt.Sprintf("%s", v)
fmt.Printf("%-18s: %s\n", k, strings.Replace(strings.Replace(value, "[", "", -1), "]", "", -1))
}
}
return err
}
func (b *Bucket) UploadObject(ctx context.Context, object string, content io.Reader, acl *AccessControl) error {
res, err := b.conn.Do(ctx, http.MethodPut, b.Name, object, nil, acl.GenHead(), content)
if err == nil {
defer res.Body.Close()
}
return err
}
func (b *Bucket) CopyObject(ctx context.Context, src, dst string, acl *AccessControl) error {
srcURL := fmt.Sprintf("%s-%s.cos.%s.%s/%s", b.Name, b.conn.conf.AppID, b.conn.conf.Region, b.conn.conf.Domain, dst)
header := map[string]string{
"x-cos-source-url": srcURL,
}
res, err := b.conn.Do(ctx, http.MethodPut, b.Name, dst, nil, header, nil)
if err == nil {
defer res.Body.Close()
}
return err
}
func (b *Bucket) DeleteObject(ctx context.Context, obj string) error {
res, err := b.conn.Do(ctx, http.MethodDelete, b.Name, obj, nil, nil, nil)
if err == nil {
defer res.Body.Close()
}
return err
}
func (b *Bucket) DownloadObject(ctx context.Context, object string, w io.Writer) error {
res, err := b.conn.Do(ctx, http.MethodGet, b.Name, object, nil, nil, nil)
if err != nil {
return err
}
_, err = io.Copy(w, res.Body)
return err
}
// UploadObjectBySlice upload by slice
func (b *Bucket) UploadObjectBySlice(ctx context.Context, dst, src string, taskNum int, headers map[string]string) error {
if taskNum < 1 {
return ParamError{"taskNum 必须大于1"}
}
uploadID, err := b.InitSliceUpload(ctx, dst, headers)
if err != nil {
return err
}
fd, err := os.Open(src)
if err != nil {
return err
}
defer fd.Close()
slices, err := b.PerformSliceUpload(ctx, dst, uploadID, fd, taskNum)
if err != nil {
return err
}
err = b.CompleteSliceUpload(ctx, dst, uploadID, fd, slices)
return err
}
// InitSliceUpload init upload by slice
func (b *Bucket) InitSliceUpload(ctx context.Context, obj string, headers map[string]string) (string, error) {
param := map[string]interface{}{
"uploads": "",
}
res, err := b.conn.Do(ctx, http.MethodPost, b.Name, obj, param, headers, nil)
if err != nil {
return "", err
}
defer res.Body.Close()
imur := &InitiateMultipartUploadResult{}
err = XMLDecode(res.Body, imur)
if err != nil {
return "", err
}
return imur.UploadID, nil
}
// CompleteSliceUpload finish slice Upload
func (b *Bucket) CompleteSliceUpload(ctx context.Context, dst, uploadID string, fd *os.File, slice []*ObjectSlice) error {
cmu := &CompleteMultipartUpload{}
cmu.Part = []struct {
PartNumber int
ETag string
}{}
for _, osl := range slice {
cmu.Part = append(cmu.Part, struct {
PartNumber int
ETag string
}{PartNumber: osl.Number, ETag: osl.MD5})
}
cmuXML, err := xml.Marshal(cmu)
if err != nil {
return err
}
param := map[string]interface{}{
"uploadId": uploadID,
}
res, err := b.conn.Do(ctx, http.MethodPost, b.Name, dst, param, nil, bytes.NewReader(cmuXML))
if err == nil {
defer res.Body.Close()
}
return err
}
// PerformSliceUpload perform slice upload
func (b *Bucket) PerformSliceUpload(ctx context.Context, dst, uploadID string, fd *os.File, taskNum int) ([]*ObjectSlice, error) {
oss, err := b.getFileSlices(fd, uploadID, dst)
if err != nil {
return nil, err
}
jobNum := len(oss)
jobs := make(chan *ObjectSlice, jobNum)
result := make(chan *ObjectSlice, jobNum)
for i := 0; i < taskNum; i++ {
go b.Worker(ctx, fd, jobs, result)
}
for _, osl := range oss {
jobs <- osl
}
close(jobs)
for i := 0; i < jobNum; i++ {
res := <-result
if !res.Result {
return nil, SliceError{fmt.Sprintf("part info : num:%d, md5:%s", res.Number, res.MD5)}
}
}
return oss, nil
}
// Worker woker for slice upload
func (b *Bucket) Worker(ctx context.Context, fd *os.File, jobs <-chan *ObjectSlice, result chan<- *ObjectSlice) {
for job := range jobs {
content, err := getFilePartContent(fd, job.Offset, job.Size)
if err != nil {
continue
}
err = b.UploadSlice(ctx, job.UploadID, job.Dst, job.Number, job.MD5, content)
if err == nil {
job.Result = true
} else {
job.Result = false
}
result <- job
}
}
// UploadSlice upload one slice
func (b *Bucket) UploadSlice(ctx context.Context, uploadID, dst string, number int, etag string, content io.Reader) error {
param := map[string]interface{}{
"PartNumber": number,
"uploadId": uploadID,
}
res, err := b.conn.Do(ctx, http.MethodPut, b.Name, dst, param, nil, content)
if err != nil {
return FileError{"PUT数据错误:" + err.Error()}
}
defer res.Body.Close()
if strings.Trim(res.Header.Get("Etag"), "\"") != etag {
return FileError{"cos-etag与文件MD5不匹配"}
}
return err
}
func (b *Bucket) getFileSlices(fd *os.File, uploadID, dst string) ([]*ObjectSlice, error) {
sliceSize := b.conn.conf.PartSize
fi, err := fd.Stat()
if err != nil {
return nil, err
}
fileSize := fi.Size()
oss := []*ObjectSlice{}
var i int
var offset int64
for fileSize > 0 {
var size int64
if fileSize > sliceSize {
size = sliceSize
} else {
size = fileSize
}
i++
md5, err := getFileMD5(fd, offset, size)
if err != nil {
return nil, err
}
osl := &ObjectSlice{}
osl.Size = size
osl.Number = i
osl.Offset = offset
osl.UploadID = uploadID
osl.MD5 = md5
osl.Dst = dst
oss = append(oss, osl)
fileSize -= sliceSize
offset += sliceSize
}
return oss, nil
}
func getFileMD5(fd *os.File, offset, size int64) (string, error) {
buf := make([]byte, size)
_, err := fd.ReadAt(buf, offset)
if err != nil {
return "", err
}
encoder := md5.New()
encoder.Write(buf)
b := encoder.Sum(nil)
return hex.EncodeToString(b), nil
}
func getFilePartContent(fd *os.File, offset, size int64) (io.Reader, error) {
buf := make([]byte, size)
_, err := fd.ReadAt(buf, offset)
if err != nil {
return nil, err
}
return bytes.NewReader(buf), nil
}
func (b *Bucket) AbortUpload(ctx context.Context, obj, uploadID string) error {
param := map[string]interface{}{
"uploadId": uploadID,
}
_, err := b.conn.Do(ctx, http.MethodDelete, b.Name, obj, param, nil, nil)
return err
}
// ObjectExists object exists
func (b *Bucket) ObjectExists(ctx context.Context, obj string) error {
_, err := b.conn.Do(ctx, http.MethodHead, b.Name, obj, nil, nil, nil)
return err
}

164
vendor/github.com/nelsonken/cos-go-sdk-v5/cos/client.go generated vendored
View File

@@ -1,164 +0,0 @@
package cos
import (
"context"
"net/http"
"time"
)
// Client 客户端， cos的句柄
type Client struct {
conn *Conn
}
// New cos包的入口
func New(o *Option) *Client {
client := Client{}
conf := getDefaultConf()
conf.AppID = o.AppID
conf.SecretID = o.SecretID
conf.SecretKey = o.SecretKey
conf.Region = o.Region
if o.Domain != "" {
conf.Domain = o.Domain
}
conn := Conn{&http.Client{}, conf}
client.conn = &conn
return &client
}
// GetTimeoutCtx 获取一个带超时的context
func GetTimeoutCtx(timeout time.Duration) context.Context {
ctx, _ := context.WithTimeout(context.Background(), timeout)
return ctx
}
// Bucket get bucket
func (c *Client) Bucket(name string) *Bucket {
return &Bucket{name, c.conn}
}
// GetBucketList 获取bucketlist
func (c *Client) GetBucketList(ctx context.Context) (*ListAllMyBucketsResult, error) {
req, err := http.NewRequest(http.MethodGet, "http://service.cos.myqcloud.com/", nil)
if err != nil {
return nil, err
}
req = req.WithContext(ctx)
c.conn.signHeader(req, nil, nil)
res, err := c.conn.c.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
res, err = checkHTTPErr(res)
if err != nil {
return nil, err
}
labr := &ListAllMyBucketsResult{}
err = XMLDecode(res.Body, labr)
if err != nil {
return nil, err
}
return labr, err
}
// CreateBucket 建立bucket
func (c *Client) CreateBucket(ctx context.Context, name string, acl *AccessControl) error {
res, err := c.conn.Do(ctx, http.MethodPut, name, "", nil, acl.GenHead(), nil)
if err == nil {
defer res.Body.Close()
}
return err
}
// DeleteBucket delete a bucket
func (c *Client) DeleteBucket(ctx context.Context, name string) error {
_, err := c.conn.Do(ctx, http.MethodDelete, name, "", nil, nil, nil)
return err
}
// GetBucketACL get bucket's acl
func (c *Client) GetBucketACL(ctx context.Context, name string) (*AccessControlPolicy, error) {
params := map[string]interface{}{"acl": ""}
res, err := c.conn.Do(ctx, http.MethodGet, name, "", params, nil, nil)
if err != nil {
return nil, err
}
defer res.Body.Close()
aclp := &AccessControlPolicy{}
err = XMLDecode(res.Body, aclp)
if err != nil {
return nil, err
}
return aclp, nil
}
// SetBucketACL set bucket's acl
func (c *Client) SetBucketACL(ctx context.Context, name string, acl *AccessControl) error {
params := map[string]interface{}{"acl": ""}
res, err := c.conn.Do(ctx, http.MethodPut, name, "", params, acl.GenHead(), nil)
if err == nil {
defer res.Body.Close()
}
return err
}
// BucketExists bucket exists?
func (c *Client) BucketExists(ctx context.Context, name string) error {
res, err := c.conn.Do(ctx, http.MethodHead, name, "", nil, nil, nil)
if err == nil {
defer res.Body.Close()
}
return err
}
// ListBucketContents list
func (c *Client) ListBucketContents(ctx context.Context, name string, qc *QueryCondition) (*ListBucketResult, error) {
resp, err := c.conn.Do(ctx, http.MethodGet, name, "", qc.GenParams(), nil, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
lbr := &ListBucketResult{}
err = XMLDecode(resp.Body, lbr)
if err != nil {
return nil, err
}
return lbr, nil
}
// ListUploading list uploading task
func (c *Client) ListUploading(ctx context.Context, bucket string, lu *ListUploadParam) (*ListMultipartUploadsResult, error) {
res, err := c.conn.Do(ctx, http.MethodGet, bucket, "", lu.GenParams(), nil, nil)
if err != nil {
return nil, err
}
defer res.Body.Close()
lmur := &ListMultipartUploadsResult{}
err = XMLDecode(res.Body, lmur)
if err != nil {
return nil, err
}
return lmur, nil
}

31
vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conf.go generated vendored
View File

@@ -1,31 +0,0 @@
package cos
const (
defaultPartSize = 80 * 1024 * 1024
defaultRetryTimes = 3
defaultUA = "cos-go-sdk-v5.2.9"
defaultDomain = "myqcloud.com"
)
// Conf config struct
type Conf struct {
AppID string
SecretID string
SecretKey string
Region string
PartSize int64
RetryTimes int
UA string
Domain string
Bucket string
}
func getDefaultConf() *Conf {
conf := Conf{}
conf.PartSize = defaultPartSize
conf.RetryTimes = defaultRetryTimes
conf.UA = defaultUA
conf.Domain = defaultDomain
return &conf
}

141
vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conn.go generated vendored
View File

@@ -1,141 +0,0 @@
package cos
import (
"bytes"
"context"
"encoding/xml"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
// Conn http 请求类
type Conn struct {
c *http.Client
conf *Conf
}
func (conn *Conn) Do(ctx context.Context, method, bucket, object string, params map[string]interface{}, headers map[string]string, body io.Reader) (*http.Response, error) {
queryStr := getQueryStr(params)
url := conn.buildURL(bucket, object, queryStr)
switch body.(type) {
case *bytes.Buffer, *bytes.Reader, *strings.Reader:
default:
if body != nil {
b, err := ioutil.ReadAll(body)
if err != nil {
return nil, err
}
body = bytes.NewReader(b)
}
}
req, err := http.NewRequest(method, url, body)
if err != nil {
return nil, err
}
conn.signHeader(req, params, headers)
req.Header.Set("User-Agent", conn.conf.UA)
setHeader(req, headers)
res, err := conn.c.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode < 200 || res.StatusCode >= 300 {
defer res.Body.Close()
return checkHTTPErr(res)
}
return res, nil
}
func getQueryStr(params map[string]interface{}) string {
if params == nil || len(params) == 0 {
return ""
}
buf := new(bytes.Buffer)
buf.WriteString("?")
for k, v := range params {
buf.WriteString(k)
vs := interfaceToString(v)
if vs == "" {
buf.WriteString("&")
continue
}
buf.WriteString("=")
buf.WriteString(vs)
buf.WriteString("&")
}
return strings.Trim(buf.String(), "&")
}
func (conn *Conn) buildURL(bucket, object, queryStr string) string {
domain := fmt.Sprintf("%s-%s.cos.%s.%s", bucket, conn.conf.AppID, conn.conf.Region, conn.conf.Domain)
url := fmt.Sprintf("http://%s/%s%s", domain, escape(object), queryStr)
return url
}
func escape(str string) string {
//go语言中将空格编码为+，需要改为%20
return strings.Replace(url.QueryEscape(str), "+", "%20", -1)
}
func setHeader(req *http.Request, headers map[string]string) {
if headers == nil {
return
}
for k, v := range headers {
req.Header.Set(k, v)
}
}
func checkHTTPErr(res *http.Response) (*http.Response, error) {
if res.StatusCode >= 200 && res.StatusCode < 300 {
return res, nil
}
err := HTTPError{}
err.Code = res.StatusCode
if res.StatusCode >= 300 && res.StatusCode < 400 {
err.Message = "资源被重定向"
}
if res.StatusCode >= 400 && res.StatusCode < 500 {
err.Message = "请求被拒绝"
}
if res.StatusCode >= 500 {
err.Message = "cos服务器错误"
}
if res.ContentLength > 0 {
resErr := &Error{}
e := XMLDecode(res.Body, resErr)
if e != nil {
return nil, err
}
err.Message += resErr.Message
}
return res, err
}
// XMLDecode xml解析方法
func XMLDecode(r io.Reader, i interface{}) error {
jd := xml.NewDecoder(r)
err := jd.Decode(i)
return err
}

10
vendor/github.com/nelsonken/cos-go-sdk-v5/cos/option.go generated vendored
View File

@@ -1,10 +0,0 @@
package cos
type Option struct {
AppID string `mapstructure:"app_id" json:"app_id"`
SecretID string `mapstructure:"secret_id" json:"secret_id"`
SecretKey string `mapstructure:"secret_key" json:"secret_key"`
Region string `mapstructure:"region" json:"region"`
Domain string `mapstructure:"domain" json:"domain"`
Bucket string `mapstructure:"bucket" json:"bucket"`
}

102
vendor/github.com/nelsonken/cos-go-sdk-v5/cos/requests.go generated vendored
View File

@@ -1,102 +0,0 @@
package cos
// AccessControl privilige
type AccessControl struct {
ACL string
GrantRead string
GrantWrite string
FullControl string
}
// GenHead 生成http head
func (acl *AccessControl) GenHead() map[string]string {
header := map[string]string{
"x-cos-acl": acl.ACL,
"x-cos-grant-read": acl.GrantRead,
"x-cos-grant-write": acl.GrantWrite,
"x-cos-grant-full-control": acl.FullControl,
}
for k, v := range header {
if v == "" {
delete(header, k)
}
}
return header
}
// QueryCondition query condition
type QueryCondition struct {
Prefix string
Delimiter string
EncodingType string
Marker string
MaxKeys int
}
// GenParams generate params:map[string]interface{}
func (qc *QueryCondition) GenParams() map[string]interface{} {
params := map[string]interface{}{
"prefix": qc.Prefix,
"delimiter": qc.Delimiter,
"encoding-type": qc.EncodingType,
"marker": qc.Marker,
"max-keys": qc.MaxKeys,
}
for k, v := range params {
if v == "" {
delete(params, k)
}
if v == 0 {
delete(params, k)
}
}
return params
}
// ListUploadParam list upload param
type ListUploadParam struct {
Prefix string
Delimiter string
EncodingType string
MaxUploads int
KeyMarker string
UploadIDMarker string
}
// GenParams generate params for request
func (lup *ListUploadParam) GenParams() map[string]interface{} {
params := map[string]interface{}{
"prefix": lup.Prefix,
"delimiter": lup.Delimiter,
"encoding-type": lup.EncodingType,
"max-uploads": lup.MaxUploads,
"key-marker": lup.KeyMarker,
"upload-id-marker": lup.UploadIDMarker,
}
for k, v := range params {
if v == "" {
delete(params, k)
}
if v == 0 {
delete(params, k)
}
}
params["uploads"] = ""
return params
}
// CompleteMultipartUpload compelete slice upload
type CompleteMultipartUpload struct {
Part []struct {
PartNumber int
ETag string
}
}

156
vendor/github.com/nelsonken/cos-go-sdk-v5/cos/response.go generated vendored
View File

@@ -1,156 +0,0 @@
package cos
import (
"fmt"
)
// ListAllMyBucketsResult 获取bucket列表的结果
type ListAllMyBucketsResult struct {
Owner struct {
ID string
DisplayName string
}
Buckets struct {
Bucket []struct {
Name string
Location string
CreateDate string
}
}
}
// Error 错误消息
type Error struct {
Code string
Message string
Resource string
RequestID string `xml:"RequestId"`
TraceID string `xml:"TaceId"`
}
// HTTPError http error struct
type HTTPError struct {
Code int
Message string
}
// Error error interface
func (he HTTPError) Error() string {
return fmt.Sprintf("%d:%s", he.Code, he.Message)
}
// AccessControlPolicy acl return
type AccessControlPolicy struct {
Owner struct {
ID string
DisplayName string
}
AccessControlList struct {
Grant []struct {
Grantee struct {
ID string
DisplayName string
}
Permission string
}
}
}
// ListBucketResult list bucket contents result
type ListBucketResult struct {
Name string
EncodingType string `xml:"Encoding-Type"`
Prefix string
Marker string
MaxKeys int
IsTruncated bool
NextMarker string
Contents []struct {
Key string
LastModified string
ETag string
Size int64
Owner struct {
ID string
}
StorageClass string
}
CommonPrefixes []struct {
Prefix string
}
}
// ListMultipartUploadsResult list uploading task
type ListMultipartUploadsResult struct {
Bucket string
EncodingType string `xml:"Encoding-Type"`
KeyMarker string
UploadIDMarker string `xml:"UploadIdMarker"`
NextKeyMarker string
NextUploadIDMarker string `xml:"NextUploadIdMarker"`
MaxUploads int
IsTruncated bool
Prefix string
Delimiter string
Upload []struct {
Key string
UploadID string
StorageClass string
Initiator struct {
UIN string
}
Owner struct {
UID string
}
Initiated string
}
CommonPrefixes []struct {
Prefix string
}
}
// InitiateMultipartUploadResult init slice upload
type InitiateMultipartUploadResult struct {
Bucket string
Key string
UploadID string `xml:"UploadId"`
}
// CompleteMultipartUploadResult compeleted slice upload
type CompleteMultipartUploadResult struct {
Location string
Bucket string
Key string
ETag string
}
// SliceError slice upload err
type SliceError struct {
Message string
}
// Error implements error
func (se SliceError) Error() string {
return fmt.Sprintf("上传分片失败:%s", se.Message)
}
// ParamError slice upload err
type ParamError struct {
Message string
}
// Error implements error
func (pe ParamError) Error() string {
return fmt.Sprintf("参数错误:%s", pe.Message)
}
// FileError slice upload err
type FileError struct {
Message string
}
// Error implements error
func (fe FileError) Error() string {
return fmt.Sprintf("文件错误:%s", fe.Message)
}

7
vendor/github.com/tencentyun/cos-go-sdk-v5/.bumpversion.cfg generated vendored Normal file
View File

@@ -0,0 +1,7 @@
[bumpversion]
commit = True
tag = True
current_version = 0.7.0
[bumpversion:file:cos.go]

29
vendor/github.com/tencentyun/cos-go-sdk-v5/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,29 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
dist/
cover.html
cover.out
covprofile
coverage.html

29
vendor/github.com/tencentyun/cos-go-sdk-v5/.travis.yml generated vendored Normal file
View File

@@ -0,0 +1,29 @@
language: go
go:
- '1.7'
- '1.8'
- '1.9'
- 1.10.x
- master
sudo: false
before_install:
- go get github.com/mattn/goveralls
- go get github.com/stretchr/testify
install:
- go get
- go build
- go build github.com/mattn/goveralls
script:
- make test
- make ci-test
- go test -coverprofile=cover.out github.com/toranger/cos-go-sdk-v5
- "${TRAVIS_HOME}/gopath/bin/goveralls -service=travis-ci -coverprofile=cover.out"
matrix:
allow_failures:
- go: 1.7
- go: master
env:
global:
- secure: XXB/cFVnJcAzhOZ2/zplwjhhhireQQGGRbNscPgQ0kpUQCyPZ6oIHvJMafuP4TVTJHEdMiaDxm0HNvgARuopXVaQNmK2UZj6xw40Ud7OT7ZUnw88xkQkXOI5GwG8oz9LqxIXUSItHegKXRLW0e1PoBdjZNv6lxGFAtuOcl9ekAg/q2lGIIQFefz6NK7gCmGYULKe+4J15VFldoYNM0JesxxxArTvtv8+k+U53oUwy9dex6z5oIA1zGIeKLcOD2xXgbjid/Ett3t0B2w3GfJWoM9rGV0eHgveOAUGe5tQkMKvl5LK1hj+93ZmU0MAG7x7t9jYKrFPqU/eDNJRMb4Ro6L7lIXVEKaBUkLx28PnwFQ5D043GBVtQGqYNcldZXIfbyYEHQZlD/BWFOt5YqTpGg+7Wm4NC3Yffqsurzk54juT7FftzVy0A8MFkqO+c5RHrOSUlm01pWXkGLHgZhUP5gEZEuUaoluSQTZksmAUJZ7F8DxwpE4SYBqfN27PZ87rWDNyOqNv1w1trzwx2IfdHHA+vfCZ7UM5e85gxFWUO2tJCUai2q21v3gBrcAgBOb6BwVzbWAorM2zY20f0l21XxOWMakA+r4JJA3s3EmcczcQeeL6pkFIAh+qKdFEPuyQTjH1mGpPzYFNbWtvPXijQo5PqyGrKL8W1t3ovwXMXoE=
- secure: bep0PPD/oYW5zY0QpeeC+WgFIya5DNRVmR92MO+e5BdFlSJPhstoG8bRh91EeftzC/Hyd3PUEIglPqTgZPxwysqW/81plsU95wV3qJi9gPi7+ZtYXH4xZTnaqgZsTr7jsKSVoKHSu7XqCtbSytW8YMN9wRWzG19/9hX2Z79Q6yNy5l9856Oyj1E2IXDjdZLPsWDhnZ8Vvk1wAVy2fc2esqKzHAZwm8n9vee2yR8vz7GXUszzpKvn4R43eNzdlFEHCmN0ANmxLJZmnYDpZHHfNf4slts+0S6I7awFXppuXUDaJPBRCia4XoFeSw+01IW1Vi0kAwvGLhxjJCWc4M/4ZU0byXDT11tDFvWa19NmnbYiizWiXNVecn1oNWYJqIKe7TTAMAtHSXAPmLX0rXuXKzwM09W6yrLFufCxyix9IOnenEbe9WwSdBbhmeLF3Wu/uVGkDog/FsXJM75sk956vV9UKh9zF4B9/NR8szJMF7shEs0Fbru5UUWheqg4AadPl3dhAWuj2+6NANa1LpH3JVD3II9dlXeMmMvsSwDvrYUaX/S8tf6JwZG0zCJK0TYp05rjxH+NIzWaMUTY7+HwYqqK3pOW3San0SlZiMq8N7GSnKUZ7WRQXYSB4gXHrg+mWyeVC7XnqiRtCwVi+LtPMu+YUbg7dwVi0vtKjYZYIUY=
- secure: Ob28vrOuHMKNKEtChkWbsaVv2SwLhcxXMnvGe4XN+y3mFvdhYnwpt6NdgThF8OCZ0761tvTRmvALfiZnO0uORjTtoHKkVPrnVIxlCcode0NVJZNHGn2fqjemdLKCnSeX7hm+9zeLpCnIvC+Sp3iZ3t2AH4AzgFx6nirWO3HwT5l9rNL9Q1CfwlOpNJJ36r9JTHwQnXmOfOmszUNoZ3rtiFXJ8dCi+BgY0lsiIRSiDkAH7KAPf86REM+ww81AaXG4/RuYx1Vj5zQCtZN7XEOViSXEbqqb8SrIFOccDu5FV12djg+4QS7FSjLVGrdIUcn4oI6pS24Et3oXf8xFx6JLYyGGhgZ2BsyJEx5vLQvkTWnMTrwZVRtCQ+g6lMUQpJhL2rBrmVBUqBFb5IH69O7corQm53n5qLM8IiosAQLfbOtML/1PyEpKCG2aOx1377Fx2yzxXW3ucP1PBqCzli0oCM2T52LfiNvZTzkIU6XJebBnzkZXepzOIFSur86kxgvQFElw9ro2X6XXPKU5S25xVaUSvaN1kmqLSkToJ9S1rmDYXnJR4aH0R2GcLw+EkMHFJJoAjnRHxrB4/1vOJbzmfS+qy6ShRhUMSD8gk4YJ6Y7o9h7oekuWOEn+XGhl29U9T5OApzHfoPEGZwLnpHxAiKJtQtv/TNhBIOFCjigsF7U=

21
vendor/github.com/tencentyun/cos-go-sdk-v5/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2017 mozillazg
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

22
vendor/github.com/tencentyun/cos-go-sdk-v5/Makefile generated vendored Normal file
View File

@@ -0,0 +1,22 @@
help:
@echo "test run test"
@echo "lint run lint"
@echo "example run examples"
.PHONY: test
test:
go test -v -cover -coverprofile cover.out
go tool cover -html=cover.out -o cover.html
.PHONY: lint
lint:
gofmt -s -w .
goimports -w .
golint .
go vet
.PHONY: example
example:
cd example && sh test.sh
ci-test:
cd costesting && go test -v

95
vendor/github.com/tencentyun/cos-go-sdk-v5/README.md generated vendored Normal file
View File

@@ -0,0 +1,95 @@
# cos-go-sdk-v5
腾讯云对象存储服务 COS(Cloud Object Storage) Go SDK（API 版本：V5 版本的 XML API）。
## Install
`go get -u github.com/tencentyun/cos-go-sdk-v5`
## Usage
```go
package main
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"time"
"github.com/tencentyun/cos-go-sdk-v5"
)
func main() {
//将<bucket>和<region>修改为真实的信息
//bucket的命名规则为{name}-{appid} ，此处填写的存储桶名称必须为此格式
u, _ := url.Parse("https://<bucket>.cos.<region>.myqcloud.com")
b := &cos.BaseURL{BucketURL: u}
c := cos.NewClient(b, &http.Client{
//设置超时时间
Timeout: 100 * time.Second,
Transport: &cos.AuthorizationTransport{
//如实填写账号和密钥，也可以设置为环境变量
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
},
})
name := "test/hello.txt"
resp, err := c.Object.Get(context.Background(), name, nil)
if err != nil {
panic(err)
}
bs, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
fmt.Printf("%s\n", string(bs))
}
```
所有的 API 在 [example](./example/) 目录下都有对应的使用示例。
Service API:
* [x] Get Service（使用示例：[service/get.go](./example/service/get.go)）
Bucket API:
* [x] Get Bucket（使用示例：[bucket/get.go](./example/bucket/get.go)）
* [x] Get Bucket ACL（使用示例：[bucket/getACL.go](./example/bucket/getACL.go)）
* [x] Get Bucket CORS（使用示例：[bucket/getCORS.go](./example/bucket/getCORS.go)）
* [x] Get Bucket Location（使用示例：[bucket/getLocation.go](./example/bucket/getLocation.go)）
* [x] Get Buket Lifecycle（使用示例：[bucket/getLifecycle.go](./example/bucket/getLifecycle.go)）
* [x] Get Bucket Tagging（使用示例：[bucket/getTagging.go](./example/bucket/getTagging.go)）
* [x] Put Bucket（使用示例：[bucket/put.go](./example/bucket/put.go)）
* [x] Put Bucket ACL（使用示例：[bucket/putACL.go](./example/bucket/putACL.go)）
* [x] Put Bucket CORS（使用示例：[bucket/putCORS.go](./example/bucket/putCORS.go)）
* [x] Put Bucket Lifecycle（使用示例：[bucket/putLifecycle.go](./example/bucket/putLifecycle.go)）
* [x] Put Bucket Tagging（使用示例：[bucket/putTagging.go](./example/bucket/putTagging.go)）
* [x] Delete Bucket（使用示例：[bucket/delete.go](./example/bucket/delete.go)）
* [x] Delete Bucket CORS（使用示例：[bucket/deleteCORS.go](./example/bucket/deleteCORS.go)）
* [x] Delete Bucket Lifecycle（使用示例：[bucket/deleteLifecycle.go](./example/bucket/deleteLifecycle.go)）
* [x] Delete Bucket Tagging（使用示例：[bucket/deleteTagging.go](./example/bucket/deleteTagging.go)）
* [x] Head Bucket（使用示例：[bucket/head.go](./example/bucket/head.go)）
* [x] List Multipart Uploads（使用示例：[bucket/listMultipartUploads.go](./example/bucket/listMultipartUploads.go)）
Object API:
* [x] Get Object（使用示例：[object/get.go](./example/object/get.go)）
* [x] Get Object ACL（使用示例：[object/getACL.go](./example/object/getACL.go)）
* [x] Put Object（使用示例：[object/put.go](./example/object/put.go)）
* [x] Put Object ACL（使用示例：[object/putACL.go](./example/object/putACL.go)）
* [x] Put Object Copy（使用示例：[object/copy.go](./example/object/copy.go)）
* [x] Delete Object（使用示例：[object/delete.go](./example/object/delete.go)）
* [x] Delete Multiple Object（使用示例：[object/deleteMultiple.go](./example/object/deleteMultiple.go)）
* [x] Head Object（使用示例：[object/head.go](./example/object/head.go)）
* [x] Options Object（使用示例：[object/options.go](./example/object/options.go)）
* [x] Initiate Multipart Upload（使用示例：[object/initiateMultipartUpload.go](./example/object/initiateMultipartUpload.go)）
* [x] Upload Part（使用示例：[object/uploadPart.go](./example/object/uploadPart.go)）
* [x] List Parts（使用示例：[object/listParts.go](./example/object/listParts.go)）
* [x] Complete Multipart Upload（使用示例：[object/completeMultipartUpload.go](./example/object/completeMultipartUpload.go)）
* [x] Abort Multipart Upload（使用示例：[object/abortMultipartUpload.go](./example/object/abortMultipartUpload.go)）
* [x] Mutipart Upload（使用示例：[object/MutiUpload.go](./example/object/MutiUpload.go)）

305
vendor/github.com/tencentyun/cos-go-sdk-v5/auth.go generated vendored Normal file
View File

@@ -0,0 +1,305 @@
package cos
import (
"crypto/hmac"
"crypto/sha1"
"fmt"
"hash"
"net/http"
"net/url"
"sort"
"strings"
"sync"
"time"
)
const sha1SignAlgorithm = "sha1"
const privateHeaderPrefix = "x-cos-"
const defaultAuthExpire = time.Hour
// 需要校验的 Headers 列表
var needSignHeaders = map[string]bool{
"host": true,
"range": true,
"x-cos-acl": true,
"x-cos-grant-read": true,
"x-cos-grant-write": true,
"x-cos-grant-full-control": true,
"response-content-type": true,
"response-content-language": true,
"response-expires": true,
"response-cache-control": true,
"response-content-disposition": true,
"response-content-encoding": true,
"cache-control": true,
"content-disposition": true,
"content-encoding": true,
"content-type": true,
"content-length": true,
"content-md5": true,
"expect": true,
"expires": true,
"x-cos-content-sha1": true,
"x-cos-storage-class": true,
"if-modified-since": true,
"origin": true,
"access-control-request-method": true,
"access-control-request-headers": true,
"x-cos-object-type": true,
}
func safeURLEncode(s string) string {
s = encodeURIComponent(s)
s = strings.Replace(s, "!", "%21", -1)
s = strings.Replace(s, "'", "%27", -1)
s = strings.Replace(s, "(", "%28", -1)
s = strings.Replace(s, ")", "%29", -1)
s = strings.Replace(s, "*", "%2A", -1)
return s
}
type valuesSignMap map[string][]string
func (vs valuesSignMap) Add(key, value string) {
key = strings.ToLower(key)
vs[key] = append(vs[key], value)
}
func (vs valuesSignMap) Encode() string {
var keys []string
for k := range vs {
keys = append(keys, k)
}
sort.Strings(keys)
var pairs []string
for _, k := range keys {
items := vs[k]
sort.Strings(items)
for _, val := range items {
pairs = append(
pairs,
fmt.Sprintf("%s=%s", safeURLEncode(k), safeURLEncode(val)))
}
}
return strings.Join(pairs, "&")
}
// AuthTime 用于生成签名所需的 q-sign-time 和 q-key-time 相关参数
type AuthTime struct {
SignStartTime time.Time
SignEndTime time.Time
KeyStartTime time.Time
KeyEndTime time.Time
}
// NewAuthTime 生成 AuthTime 的便捷函数
//
// expire: 从现在开始多久过期.
func NewAuthTime(expire time.Duration) *AuthTime {
signStartTime := time.Now()
keyStartTime := signStartTime
signEndTime := signStartTime.Add(expire)
keyEndTime := signEndTime
return &AuthTime{
SignStartTime: signStartTime,
SignEndTime: signEndTime,
KeyStartTime: keyStartTime,
KeyEndTime: keyEndTime,
}
}
// signString return q-sign-time string
func (a *AuthTime) signString() string {
return fmt.Sprintf("%d;%d", a.SignStartTime.Unix(), a.SignEndTime.Unix())
}
// keyString return q-key-time string
func (a *AuthTime) keyString() string {
return fmt.Sprintf("%d;%d", a.KeyStartTime.Unix(), a.KeyEndTime.Unix())
}
// newAuthorization 通过一系列步骤生成最终需要的 Authorization 字符串
func newAuthorization(secretID, secretKey string, req *http.Request, authTime *AuthTime) string {
signTime := authTime.signString()
keyTime := authTime.keyString()
signKey := calSignKey(secretKey, keyTime)
formatHeaders := *new(string)
signedHeaderList := *new([]string)
formatHeaders, signedHeaderList = genFormatHeaders(req.Header)
formatParameters, signedParameterList := genFormatParameters(req.URL.Query())
formatString := genFormatString(req.Method, *req.URL, formatParameters, formatHeaders)
stringToSign := calStringToSign(sha1SignAlgorithm, keyTime, formatString)
signature := calSignature(signKey, stringToSign)
return genAuthorization(
secretID, signTime, keyTime, signature, signedHeaderList,
signedParameterList,
)
}
// AddAuthorizationHeader 给 req 增加签名信息
func AddAuthorizationHeader(secretID, secretKey string, sessionToken string, req *http.Request, authTime *AuthTime) {
if secretID == "" {
return
}
auth := newAuthorization(secretID, secretKey, req,
authTime,
)
if len(sessionToken) > 0 {
req.Header.Set("x-cos-security-token", sessionToken)
}
req.Header.Set("Authorization", auth)
}
// calSignKey 计算 SignKey
func calSignKey(secretKey, keyTime string) string {
digest := calHMACDigest(secretKey, keyTime, sha1SignAlgorithm)
return fmt.Sprintf("%x", digest)
}
// calStringToSign 计算 StringToSign
func calStringToSign(signAlgorithm, signTime, formatString string) string {
h := sha1.New()
h.Write([]byte(formatString))
return fmt.Sprintf("%s\n%s\n%x\n", signAlgorithm, signTime, h.Sum(nil))
}
// calSignature 计算 Signature
func calSignature(signKey, stringToSign string) string {
digest := calHMACDigest(signKey, stringToSign, sha1SignAlgorithm)
return fmt.Sprintf("%x", digest)
}
// genAuthorization 生成 Authorization
func genAuthorization(secretID, signTime, keyTime, signature string, signedHeaderList, signedParameterList []string) string {
return strings.Join([]string{
"q-sign-algorithm=" + sha1SignAlgorithm,
"q-ak=" + secretID,
"q-sign-time=" + signTime,
"q-key-time=" + keyTime,
"q-header-list=" + strings.Join(signedHeaderList, ";"),
"q-url-param-list=" + strings.Join(signedParameterList, ";"),
"q-signature=" + signature,
}, "&")
}
// genFormatString 生成 FormatString
func genFormatString(method string, uri url.URL, formatParameters, formatHeaders string) string {
formatMethod := strings.ToLower(method)
formatURI := uri.Path
return fmt.Sprintf("%s\n%s\n%s\n%s\n", formatMethod, formatURI,
formatParameters, formatHeaders,
)
}
// genFormatParameters 生成 FormatParameters 和 SignedParameterList
// instead of the url.Values{}
func genFormatParameters(parameters url.Values) (formatParameters string, signedParameterList []string) {
ps := valuesSignMap{}
for key, values := range parameters {
key = strings.ToLower(key)
for _, value := range values {
ps.Add(key, value)
signedParameterList = append(signedParameterList, key)
}
}
//formatParameters = strings.ToLower(ps.Encode())
formatParameters = ps.Encode()
sort.Strings(signedParameterList)
return
}
// genFormatHeaders 生成 FormatHeaders 和 SignedHeaderList
func genFormatHeaders(headers http.Header) (formatHeaders string, signedHeaderList []string) {
hs := valuesSignMap{}
for key, values := range headers {
key = strings.ToLower(key)
for _, value := range values {
if isSignHeader(key) {
hs.Add(key, value)
signedHeaderList = append(signedHeaderList, key)
}
}
}
formatHeaders = hs.Encode()
sort.Strings(signedHeaderList)
return
}
// HMAC 签名
func calHMACDigest(key, msg, signMethod string) []byte {
var hashFunc func() hash.Hash
switch signMethod {
case "sha1":
hashFunc = sha1.New
default:
hashFunc = sha1.New
}
h := hmac.New(hashFunc, []byte(key))
h.Write([]byte(msg))
return h.Sum(nil)
}
func isSignHeader(key string) bool {
for k, v := range needSignHeaders {
if key == k && v {
return true
}
}
return strings.HasPrefix(key, privateHeaderPrefix)
}
// AuthorizationTransport 给请求增加 Authorization header
type AuthorizationTransport struct {
SecretID string
SecretKey string
SessionToken string
rwLocker sync.RWMutex
// 签名多久过期
Expire time.Duration
Transport http.RoundTripper
}
// SetCredential update the SecretID(ak), SercretKey(sk), sessiontoken
func (t *AuthorizationTransport) SetCredential(ak, sk, token string) {
t.rwLocker.Lock()
defer t.rwLocker.Unlock()
t.SecretID = ak
t.SecretKey = sk
t.SessionToken = token
}
// GetCredential get the ak, sk, token
func (t *AuthorizationTransport) GetCredential() (string, string, string) {
t.rwLocker.RLock()
defer t.rwLocker.RUnlock()
return t.SecretID, t.SecretKey, t.SessionToken
}
// RoundTrip implements the RoundTripper interface.
func (t *AuthorizationTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req = cloneRequest(req) // per RoundTrip contract
if t.Expire == time.Duration(0) {
t.Expire = defaultAuthExpire
}
ak, sk, token := t.GetCredential()
// 增加 Authorization header
authTime := NewAuthTime(t.Expire)
AddAuthorizationHeader(ak, sk, token, req, authTime)
resp, err := t.transport().RoundTrip(req)
return resp, err
}
func (t *AuthorizationTransport) transport() http.RoundTripper {
if t.Transport != nil {
return t.Transport
}
return http.DefaultTransport
}

104
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket.go generated vendored Normal file
View File

@@ -0,0 +1,104 @@
package cos
import (
"context"
"encoding/xml"
"net/http"
)
// BucketService 相关 API
type BucketService service
// BucketGetResult is the result of GetBucket
type BucketGetResult struct {
XMLName xml.Name `xml:"ListBucketResult"`
Name string
Prefix string `xml:"Prefix,omitempty"`
Marker string `xml:"Marker,omitempty"`
NextMarker string `xml:"NextMarker,omitempty"`
Delimiter string `xml:"Delimiter,omitempty"`
MaxKeys int
IsTruncated bool
Contents []Object `xml:"Contents,omitempty"`
CommonPrefixes []string `xml:"CommonPrefixes>Prefix,omitempty"`
EncodingType string `xml:"Encoding-Type,omitempty"`
}
// BucketGetOptions is the option of GetBucket
type BucketGetOptions struct {
Prefix string `url:"prefix,omitempty"`
Delimiter string `url:"delimiter,omitempty"`
EncodingType string `url:"encoding-type,omitempty"`
Marker string `url:"marker,omitempty"`
MaxKeys int `url:"max-keys,omitempty"`
}
// Get Bucket请求等同于 List Object请求，可以列出该Bucket下部分或者所有Object，发起该请求需要拥有Read权限。
//
// https://www.qcloud.com/document/product/436/7734
func (s *BucketService) Get(ctx context.Context, opt *BucketGetOptions) (*BucketGetResult, *Response, error) {
var res BucketGetResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/",
method: http.MethodGet,
optQuery: opt,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// BucketPutOptions is same to the ACLHeaderOptions
type BucketPutOptions ACLHeaderOptions
// Put Bucket请求可以在指定账号下创建一个Bucket。
//
// https://www.qcloud.com/document/product/436/7738
func (s *BucketService) Put(ctx context.Context, opt *BucketPutOptions) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/",
method: http.MethodPut,
optHeader: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// Delete Bucket请求可以在指定账号下删除Bucket，删除之前要求Bucket为空。
//
// https://www.qcloud.com/document/product/436/7732
func (s *BucketService) Delete(ctx context.Context) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/",
method: http.MethodDelete,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// Head Bucket请求可以确认是否存在该Bucket，是否有权限访问，Head的权限与Read一致。
//
// 当其存在时，返回 HTTP 状态码200；
// 当无权限时，返回 HTTP 状态码403；
// 当不存在时，返回 HTTP 状态码404。
//
// https://www.qcloud.com/document/product/436/7735
func (s *BucketService) Head(ctx context.Context) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/",
method: http.MethodHead,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// Bucket is the meta info of Bucket
type Bucket struct {
Name string
Region string `xml:"Location,omitempty"`
CreationDate string `xml:",omitempty"`
}

62
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_acl.go generated vendored Normal file
View File

@@ -0,0 +1,62 @@
package cos
import (
"context"
"net/http"
)
// BucketGetACLResult is same to the ACLXml
type BucketGetACLResult ACLXml
// GetACL 使用API读取Bucket的ACL表，只有所有者有权操作。
//
// https://www.qcloud.com/document/product/436/7733
func (s *BucketService) GetACL(ctx context.Context) (*BucketGetACLResult, *Response, error) {
var res BucketGetACLResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?acl",
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// BucketPutACLOptions is the option of PutBucketACL
type BucketPutACLOptions struct {
Header *ACLHeaderOptions `url:"-" xml:"-"`
Body *ACLXml `url:"-" header:"-"`
}
// PutACL 使用API写入Bucket的ACL表，您可以通过Header："x-cos-acl","x-cos-grant-read",
// "x-cos-grant-write","x-cos-grant-full-control"传入ACL信息，也可以通过body以XML格式传入ACL信息，
//
// 但是只能选择Header和Body其中一种，否则返回冲突。
//
// Put Bucket ACL是一个覆盖操作，传入新的ACL将覆盖原有ACL。只有所有者有权操作。
//
// "x-cos-acl"：枚举值为public-read，private；public-read意味这个Bucket有公有读私有写的权限，
// private意味这个Bucket有私有读写的权限。
//
// "x-cos-grant-read"：意味被赋予权限的用户拥有该Bucket的读权限
// "x-cos-grant-write"：意味被赋予权限的用户拥有该Bucket的写权限
// "x-cos-grant-full-control"：意味被赋予权限的用户拥有该Bucket的读写权限
//
// https://www.qcloud.com/document/product/436/7737
func (s *BucketService) PutACL(ctx context.Context, opt *BucketPutACLOptions) (*Response, error) {
header := opt.Header
body := opt.Body
if body != nil {
header = nil
}
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?acl",
method: http.MethodPut,
body: body,
optHeader: header,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}

71
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_cors.go generated vendored Normal file
View File

@@ -0,0 +1,71 @@
package cos
import (
"context"
"encoding/xml"
"net/http"
)
// BucketCORSRule is the rule of BucketCORS
type BucketCORSRule struct {
ID string `xml:"ID,omitempty"`
AllowedMethods []string `xml:"AllowedMethod"`
AllowedOrigins []string `xml:"AllowedOrigin"`
AllowedHeaders []string `xml:"AllowedHeader,omitempty"`
MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty"`
ExposeHeaders []string `xml:"ExposeHeader,omitempty"`
}
// BucketGetCORSResult is the result of GetBucketCORS
type BucketGetCORSResult struct {
XMLName xml.Name `xml:"CORSConfiguration"`
Rules []BucketCORSRule `xml:"CORSRule,omitempty"`
}
// GetCORS 实现 Bucket 跨域访问配置读取。
//
// https://www.qcloud.com/document/product/436/8274
func (s *BucketService) GetCORS(ctx context.Context) (*BucketGetCORSResult, *Response, error) {
var res BucketGetCORSResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?cors",
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// BucketPutCORSOptions is the option of PutBucketCORS
type BucketPutCORSOptions struct {
XMLName xml.Name `xml:"CORSConfiguration"`
Rules []BucketCORSRule `xml:"CORSRule,omitempty"`
}
// PutCORS 实现 Bucket 跨域访问设置，您可以通过传入XML格式的配置文件实现配置，文件大小限制为64 KB。
//
// https://www.qcloud.com/document/product/436/8279
func (s *BucketService) PutCORS(ctx context.Context, opt *BucketPutCORSOptions) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?cors",
method: http.MethodPut,
body: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// DeleteCORS 实现 Bucket 跨域访问配置删除。
//
// https://www.qcloud.com/document/product/436/8283
func (s *BucketService) DeleteCORS(ctx context.Context) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?cors",
method: http.MethodDelete,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}

134
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_inventory.go generated vendored Normal file
View File

@@ -0,0 +1,134 @@
package cos
import (
"context"
"encoding/xml"
"fmt"
"net/http"
)
// Notice bucket_inventory only for test. can not use
// BucketGetInventoryResult same struct to options
type BucketGetInventoryResult BucketPutInventoryOptions
// BucketListInventoryConfiguartion same struct to options
type BucketListInventoryConfiguartion BucketPutInventoryOptions
// BucketInventoryFilter ...
type BucketInventoryFilter struct {
Prefix string `xml:"Prefix,omitempty"`
}
// BucketInventoryOptionalFields ...
type BucketInventoryOptionalFields struct {
XMLName xml.Name `xml:"OptionalFields,omitempty"`
BucketInventoryFields []string `xml:"Field,omitempty"`
}
// BucketInventorySchedule ...
type BucketInventorySchedule struct {
Frequency string `xml:"Frequency"`
}
// BucketInventoryEncryption ...
type BucketInventoryEncryption struct {
XMLName xml.Name `xml:"Encryption"`
SSECOS string `xml:"SSE-COS,omitempty"`
}
// BucketInventoryDestinationContent ...
type BucketInventoryDestinationContent struct {
Bucket string `xml:"Bucket"`
AccountId string `xml:"AccountId,omitempty"`
Prefix string `xml:"Prefix,omitempty"`
Format string `xml:"Format"`
Encryption *BucketInventoryEncryption `xml:"Encryption,omitempty"`
}
// BucketInventoryDestination ...
type BucketInventoryDestination struct {
XMLName xml.Name `xml:"Destination"`
BucketDestination *BucketInventoryDestinationContent `xml:"COSBucketDestination"`
}
// BucketPutInventoryOptions ...
type BucketPutInventoryOptions struct {
XMLName xml.Name `xml:"InventoryConfiguration"`
ID string `xml:"Id"`
IsEnabled string `xml:"IsEnabled"`
IncludedObjectVersions string `xml:"IncludedObjectVersions"`
Filter *BucketInventoryFilter `xml:"Filter,omitempty"`
OptionalFields *BucketInventoryOptionalFields `xml:"OptionalFields,omitempty"`
Schedule *BucketInventorySchedule `xml:"Schedule"`
Destination *BucketInventoryDestination `xml:"Destination"`
}
// ListBucketInventoryConfigResult result of ListBucketInventoryConfiguration
type ListBucketInventoryConfigResult struct {
XMLName xml.Name `xml:"ListInventoryConfigurationResult"`
InventoryConfigurations []BucketListInventoryConfiguartion `xml:"InventoryConfiguration,omitempty"`
IsTruncated bool `xml:"IsTruncated,omitempty"`
ContinuationToken string `xml:"ContinuationToken,omitempty"`
NextContinuationToken string `xml:"NextContinuationToken,omitempty"`
}
// PutBucketInventory https://cloud.tencent.com/document/product/436/33707
func (s *BucketService) PutBucketInventoryTest(ctx context.Context, id string, opt *BucketPutInventoryOptions) (*Response, error) {
u := fmt.Sprintf("/?inventory&id=%s", id)
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodPut,
body: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// GetBucketInventory https://cloud.tencent.com/document/product/436/33705
func (s *BucketService) GetBucketInventoryTest(ctx context.Context, id string) (*BucketGetInventoryResult, *Response, error) {
u := fmt.Sprintf("/?inventory&id=%s", id)
var res BucketGetInventoryResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// DeleteBucketInventory https://cloud.tencent.com/document/product/436/33704
func (s *BucketService) DeleteBucketInventoryTest(ctx context.Context, id string) (*Response, error) {
u := fmt.Sprintf("/?inventory&id=%s", id)
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodDelete,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// ListBucketInventoryConfigurations https://cloud.tencent.com/document/product/436/33706
func (s *BucketService) ListBucketInventoryConfigurationsTest(ctx context.Context, token string) (*ListBucketInventoryConfigResult, *Response, error) {
var res ListBucketInventoryConfigResult
var u string
if token == "" {
u = "/?inventory"
} else {
u = fmt.Sprintf("/?inventory&continuation-token=%s", encodeURIComponent(token))
}
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}

92
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_lifecycle.go generated vendored Normal file
View File

@@ -0,0 +1,92 @@
package cos
import (
"context"
"encoding/xml"
"net/http"
)
// BucketLifecycleFilter is the param of BucketLifecycleRule
type BucketLifecycleFilter struct {
Prefix string `xml:"Prefix,omitempty"`
}
// BucketLifecycleExpiration is the param of BucketLifecycleRule
type BucketLifecycleExpiration struct {
Date string `xml:"Date,omitempty"`
Days int `xml:"Days,omitempty"`
}
// BucketLifecycleTransition is the param of BucketLifecycleRule
type BucketLifecycleTransition struct {
Date string `xml:"Date,omitempty"`
Days int `xml:"Days,omitempty"`
StorageClass string
}
// BucketLifecycleAbortIncompleteMultipartUpload is the param of BucketLifecycleRule
type BucketLifecycleAbortIncompleteMultipartUpload struct {
DaysAfterInitiation string `xml:"DaysAfterInititation,omitempty"`
}
// BucketLifecycleRule is the rule of BucketLifecycle
type BucketLifecycleRule struct {
ID string `xml:"ID,omitempty"`
Status string
Filter *BucketLifecycleFilter `xml:"Filter,omitempty"`
Transition *BucketLifecycleTransition `xml:"Transition,omitempty"`
Expiration *BucketLifecycleExpiration `xml:"Expiration,omitempty"`
AbortIncompleteMultipartUpload *BucketLifecycleAbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
}
// BucketGetLifecycleResult is the result of BucketGetLifecycle
type BucketGetLifecycleResult struct {
XMLName xml.Name `xml:"LifecycleConfiguration"`
Rules []BucketLifecycleRule `xml:"Rule,omitempty"`
}
// GetLifecycle 请求实现读取生命周期管理的配置。当配置不存在时，返回404 Not Found。
// https://www.qcloud.com/document/product/436/8278
func (s *BucketService) GetLifecycle(ctx context.Context) (*BucketGetLifecycleResult, *Response, error) {
var res BucketGetLifecycleResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?lifecycle",
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// BucketPutLifecycleOptions is the option of PutBucketLifecycle
type BucketPutLifecycleOptions struct {
XMLName xml.Name `xml:"LifecycleConfiguration"`
Rules []BucketLifecycleRule `xml:"Rule,omitempty"`
}
// PutLifecycle 请求实现设置生命周期管理的功能。您可以通过该请求实现数据的生命周期管理配置和定期删除。
// 此请求为覆盖操作，上传新的配置文件将覆盖之前的配置文件。生命周期管理对文件和文件夹同时生效。
// https://www.qcloud.com/document/product/436/8280
func (s *BucketService) PutLifecycle(ctx context.Context, opt *BucketPutLifecycleOptions) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?lifecycle",
method: http.MethodPut,
body: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// DeleteLifecycle 请求实现删除生命周期管理。
// https://www.qcloud.com/document/product/436/8284
func (s *BucketService) DeleteLifecycle(ctx context.Context) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?lifecycle",
method: http.MethodDelete,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}

28
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_location.go generated vendored Normal file
View File

@@ -0,0 +1,28 @@
package cos
import (
"context"
"encoding/xml"
"net/http"
)
// BucketGetLocationResult is the result of BucketGetLocation
type BucketGetLocationResult struct {
XMLName xml.Name `xml:"LocationConstraint"`
Location string `xml:",chardata"`
}
// GetLocation 接口获取Bucket所在地域信息，只有Bucket所有者有权限读取信息。
//
// https://www.qcloud.com/document/product/436/8275
func (s *BucketService) GetLocation(ctx context.Context) (*BucketGetLocationResult, *Response, error) {
var res BucketGetLocationResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?location",
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}

53
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_logging.go generated vendored Normal file
View File

@@ -0,0 +1,53 @@
package cos
import (
"context"
"encoding/xml"
"net/http"
)
// Notice bucket logging function is testing, can not use.
// BucketLoggingEnabled main struct of logging
type BucketLoggingEnabled struct {
TargetBucket string `xml:"TargetBucket"`
TargetPrefix string `xml:"TargetPrefix"`
}
// BucketPutLoggingOptions is the options of PutBucketLogging
type BucketPutLoggingOptions struct {
XMLName xml.Name `xml:"BucketLoggingStatus"`
LoggingEnabled *BucketLoggingEnabled `xml:"LoggingEnabled"`
}
// BucketGetLoggingResult is the result of GetBucketLogging
type BucketGetLoggingResult struct {
XMLName xml.Name `xml:"BucketLoggingStatus"`
LoggingEnabled *BucketLoggingEnabled `xml:"LoggingEnabled"`
}
// PutBucketLogging https://cloud.tencent.com/document/product/436/17054
func (s *BucketService) PutBucketLoggingTest(ctx context.Context, opt *BucketPutLoggingOptions) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?logging",
method: http.MethodPut,
body: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// GetBucketLogging https://cloud.tencent.com/document/product/436/17053
func (s *BucketService) GetBucketLoggingTest(ctx context.Context) (*BucketGetLoggingResult, *Response, error) {
var res BucketGetLoggingResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?logging",
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}

57
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_part.go generated vendored Normal file
View File

@@ -0,0 +1,57 @@
package cos
import (
"context"
"encoding/xml"
"net/http"
)
// ListMultipartUploadsResult is the result of ListMultipartUploads
type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"ListMultipartUploadsResult"`
Bucket string `xml:"Bucket"`
EncodingType string `xml:"Encoding-Type"`
KeyMarker string
UploadIDMarker string `xml:"UploadIdMarker"`
NextKeyMarker string
NextUploadIDMarker string `xml:"NextUploadIdMarker"`
MaxUploads int
IsTruncated bool
Uploads []struct {
Key string
UploadID string `xml:"UploadId"`
StorageClass string
Initiator *Initiator
Owner *Owner
Initiated string
} `xml:"Upload,omitempty"`
Prefix string
Delimiter string `xml:"delimiter,omitempty"`
CommonPrefixes []string `xml:"CommonPrefixs>Prefix,omitempty"`
}
// ListMultipartUploadsOptions is the option of ListMultipartUploads
type ListMultipartUploadsOptions struct {
Delimiter string `url:"delimiter,omitempty"`
EncodingType string `url:"encoding-type,omitempty"`
Prefix string `url:"prefix,omitempty"`
MaxUploads int `url:"max-uploads,omitempty"`
KeyMarker string `url:"key-marker,omitempty"`
UploadIDMarker string `url:"upload-id-marker,omitempty"`
}
// ListMultipartUploads 用来查询正在进行中的分块上传。单次最多列出1000个正在进行中的分块上传。
//
// https://www.qcloud.com/document/product/436/7736
func (s *BucketService) ListMultipartUploads(ctx context.Context, opt *ListMultipartUploadsOptions) (*ListMultipartUploadsResult, *Response, error) {
var res ListMultipartUploadsResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?uploads",
method: http.MethodGet,
result: &res,
optQuery: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}

73
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_replication.go generated vendored Normal file
View File

@@ -0,0 +1,73 @@
package cos
import (
"context"
"encoding/xml"
"net/http"
)
// ReplicationDestination is the sub struct of BucketReplicationRule
type ReplicationDestination struct {
Bucket string `xml:"Bucket"`
StorageClass string `xml:"StorageClass,omitempty"`
}
// BucketReplicationRule is the main param of replication
type BucketReplicationRule struct {
ID string `xml:"ID,omitempty"`
Status string `xml:"Status"`
Prefix string `xml:"Prefix"`
Destination *ReplicationDestination `xml:"Destination"`
}
// PutBucketReplicationOptions is the options of PutBucketReplication
type PutBucketReplicationOptions struct {
XMLName xml.Name `xml:"ReplicationConfiguration"`
Role string `xml:"Role"`
Rule []BucketReplicationRule `xml:"Rule"`
}
// GetBucketReplicationResult is the result of GetBucketReplication
type GetBucketReplicationResult struct {
XMLName xml.Name `xml:"ReplicationConfiguration"`
Role string `xml:"Role"`
Rule []BucketReplicationRule `xml:"Rule"`
}
// PutBucketReplication https://cloud.tencent.com/document/product/436/19223
func (s *BucketService) PutBucketReplication(ctx context.Context, opt *PutBucketReplicationOptions) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?replication",
method: http.MethodPut,
body: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// GetBucketReplication https://cloud.tencent.com/document/product/436/19222
func (s *BucketService) GetBucketReplication(ctx context.Context) (*GetBucketReplicationResult, *Response, error) {
var res GetBucketReplicationResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?replication",
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// DeleteBucketReplication https://cloud.tencent.com/document/product/436/19221
func (s *BucketService) DeleteBucketReplication(ctx context.Context) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?replication",
method: http.MethodDelete,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}

69
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_tagging.go generated vendored Normal file
View File

@@ -0,0 +1,69 @@
package cos
import (
"context"
"encoding/xml"
"net/http"
)
// BucketTaggingTag is the tag of BucketTagging
type BucketTaggingTag struct {
Key string
Value string
}
// BucketGetTaggingResult is the result of BucketGetTagging
type BucketGetTaggingResult struct {
XMLName xml.Name `xml:"Tagging"`
TagSet []BucketTaggingTag `xml:"TagSet>Tag,omitempty"`
}
// GetTagging 接口实现获取指定Bucket的标签。
//
// https://www.qcloud.com/document/product/436/8277
func (s *BucketService) GetTagging(ctx context.Context) (*BucketGetTaggingResult, *Response, error) {
var res BucketGetTaggingResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?tagging",
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// BucketPutTaggingOptions is the option of BucketPutTagging
type BucketPutTaggingOptions struct {
XMLName xml.Name `xml:"Tagging"`
TagSet []BucketTaggingTag `xml:"TagSet>Tag,omitempty"`
}
// PutTagging 接口实现给用指定Bucket打标签。用来组织和管理相关Bucket。
//
// 当该请求设置相同Key名称，不同Value时，会返回400。请求成功，则返回204。
//
// https://www.qcloud.com/document/product/436/8281
func (s *BucketService) PutTagging(ctx context.Context, opt *BucketPutTaggingOptions) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?tagging",
method: http.MethodPut,
body: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// DeleteTagging 接口实现删除指定Bucket的标签。
//
// https://www.qcloud.com/document/product/436/8286
func (s *BucketService) DeleteTagging(ctx context.Context) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?tagging",
method: http.MethodDelete,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}

45
vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_version.go generated vendored Normal file
View File

@@ -0,0 +1,45 @@
package cos
import (
"context"
"encoding/xml"
"net/http"
)
// BucketPutVersionOptions is the options of PutBucketVersioning
type BucketPutVersionOptions struct {
XMLName xml.Name `xml:"VersioningConfiguration"`
Status string `xml:"Status"`
}
// BucketGetVersionResult is the result of GetBucketVersioning
type BucketGetVersionResult struct {
XMLName xml.Name `xml:"VersioningConfiguration"`
Status string `xml:"Status"`
}
// PutVersion https://cloud.tencent.com/document/product/436/19889
// Status has Suspended\Enabled
func (s *BucketService) PutVersioning(ctx context.Context, opt *BucketPutVersionOptions) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?versioning",
method: http.MethodPut,
body: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// GetVersion https://cloud.tencent.com/document/product/436/19888
func (s *BucketService) GetVersioning(ctx context.Context) (*BucketGetVersionResult, *Response, error) {
var res BucketGetVersionResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?versioning",
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}

346
vendor/github.com/tencentyun/cos-go-sdk-v5/cos.go generated vendored Normal file
View File

@@ -0,0 +1,346 @@
package cos
import (
"bytes"
"context"
"encoding/base64"
"encoding/xml"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"reflect"
"text/template"
"strconv"
"github.com/google/go-querystring/query"
"github.com/mozillazg/go-httpheader"
)
const (
// Version current go sdk version
Version = "0.7.3"
userAgent = "cos-go-sdk-v5/" + Version
contentTypeXML = "application/xml"
defaultServiceBaseURL = "http://service.cos.myqcloud.com"
)
var bucketURLTemplate = template.Must(
template.New("bucketURLFormat").Parse(
"{{.Schema}}://{{.BucketName}}.cos.{{.Region}}.myqcloud.com",
),
)
// BaseURL 访问各 API 所需的基础 URL
type BaseURL struct {
// 访问 bucket, object 相关 API 的基础 URL（不包含 path 部分）: http://example.com
BucketURL *url.URL
// 访问 service API 的基础 URL（不包含 path 部分）: http://example.com
ServiceURL *url.URL
}
// NewBucketURL 生成 BaseURL 所需的 BucketURL
//
// bucketName: bucket名称, bucket的命名规则为{name}-{appid} ，此处填写的存储桶名称必须为此格式
// Region: 区域代码: ap-beijing-1,ap-beijing,ap-shanghai,ap-guangzhou...
// secure: 是否使用 https
func NewBucketURL(bucketName, region string, secure bool) *url.URL {
schema := "https"
if !secure {
schema = "http"
}
w := bytes.NewBuffer(nil)
bucketURLTemplate.Execute(w, struct {
Schema string
BucketName string
Region string
}{
schema, bucketName, region,
})
u, _ := url.Parse(w.String())
return u
}
// Client is a client manages communication with the COS API.
type Client struct {
client *http.Client
UserAgent string
BaseURL *BaseURL
common service
Service *ServiceService
Bucket *BucketService
Object *ObjectService
}
type service struct {
client *Client
}
// NewClient returns a new COS API client.
func NewClient(uri *BaseURL, httpClient *http.Client) *Client {
if httpClient == nil {
httpClient = &http.Client{}
}
baseURL := &BaseURL{}
if uri != nil {
baseURL.BucketURL = uri.BucketURL
baseURL.ServiceURL = uri.ServiceURL
}
if baseURL.ServiceURL == nil {
baseURL.ServiceURL, _ = url.Parse(defaultServiceBaseURL)
}
c := &Client{
client: httpClient,
UserAgent: userAgent,
BaseURL: baseURL,
}
c.common.client = c
c.Service = (*ServiceService)(&c.common)
c.Bucket = (*BucketService)(&c.common)
c.Object = (*ObjectService)(&c.common)
return c
}
func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) {
uri, err = addURLOptions(uri, optQuery)
if err != nil {
return
}
u, _ := url.Parse(uri)
urlStr := baseURL.ResolveReference(u).String()
var reader io.Reader
contentType := ""
contentMD5 := ""
if body != nil {
// 上传文件
if r, ok := body.(io.Reader); ok {
reader = r
} else {
b, err := xml.Marshal(body)
if err != nil {
return nil, err
}
contentType = contentTypeXML
reader = bytes.NewReader(b)
contentMD5 = base64.StdEncoding.EncodeToString(calMD5Digest(b))
}
}
req, err = http.NewRequest(method, urlStr, reader)
if err != nil {
return
}
req.Header, err = addHeaderOptions(req.Header, optHeader)
if err != nil {
return
}
if v := req.Header.Get("Content-Length"); req.ContentLength == 0 && v != "" && v != "0" {
req.ContentLength, _ = strconv.ParseInt(v, 10, 64)
}
if contentMD5 != "" {
req.Header["Content-MD5"] = []string{contentMD5}
}
if c.UserAgent != "" {
req.Header.Set("User-Agent", c.UserAgent)
}
if req.Header.Get("Content-Type") == "" && contentType != "" {
req.Header.Set("Content-Type", contentType)
}
return
}
func (c *Client) doAPI(ctx context.Context, req *http.Request, result interface{}, closeBody bool) (*Response, error) {
req = req.WithContext(ctx)
resp, err := c.client.Do(req)
if err != nil {
// If we got an error, and the context has been canceled,
// the context's error is probably more useful.
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
return nil, err
}
defer func() {
if closeBody {
// Close the body to let the Transport reuse the connection
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}
}()
response := newResponse(resp)
err = checkResponse(resp)
if err != nil {
// even though there was an error, we still return the response
// in case the caller wants to inspect it further
return response, err
}
if result != nil {
if w, ok := result.(io.Writer); ok {
io.Copy(w, resp.Body)
} else {
err = xml.NewDecoder(resp.Body).Decode(result)
if err == io.EOF {
err = nil // ignore EOF errors caused by empty response body
}
}
}
return response, err
}
type sendOptions struct {
// 基础 URL
baseURL *url.URL
// URL 中除基础 URL 外的剩余部分
uri string
// 请求方法
method string
body interface{}
// url 查询参数
optQuery interface{}
// http header 参数
optHeader interface{}
// 用 result 反序列化 resp.Body
result interface{}
// 是否禁用自动调用 resp.Body.Close()
// 自动调用 Close() 是为了能够重用连接
disableCloseBody bool
}
func (c *Client) send(ctx context.Context, opt *sendOptions) (resp *Response, err error) {
req, err := c.newRequest(ctx, opt.baseURL, opt.uri, opt.method, opt.body, opt.optQuery, opt.optHeader)
if err != nil {
return
}
resp, err = c.doAPI(ctx, req, opt.result, !opt.disableCloseBody)
if err != nil {
return
}
return
}
// addURLOptions adds the parameters in opt as URL query parameters to s. opt
// must be a struct whose fields may contain "url" tags.
func addURLOptions(s string, opt interface{}) (string, error) {
v := reflect.ValueOf(opt)
if v.Kind() == reflect.Ptr && v.IsNil() {
return s, nil
}
u, err := url.Parse(s)
if err != nil {
return s, err
}
qs, err := query.Values(opt)
if err != nil {
return s, err
}
// 保留原有的参数，并且放在前面。因为 cos 的 url 路由是以第一个参数作为路由的
// e.g. /?uploads
q := u.RawQuery
rq := qs.Encode()
if q != "" {
if rq != "" {
u.RawQuery = fmt.Sprintf("%s&%s", q, qs.Encode())
}
} else {
u.RawQuery = rq
}
return u.String(), nil
}
// addHeaderOptions adds the parameters in opt as Header fields to req. opt
// must be a struct whose fields may contain "header" tags.
func addHeaderOptions(header http.Header, opt interface{}) (http.Header, error) {
v := reflect.ValueOf(opt)
if v.Kind() == reflect.Ptr && v.IsNil() {
return header, nil
}
h, err := httpheader.Header(opt)
if err != nil {
return nil, err
}
for key, values := range h {
for _, value := range values {
header.Add(key, value)
}
}
return header, nil
}
// Owner defines Bucket/Object's owner
type Owner struct {
UIN string `xml:"uin,omitempty"`
ID string `xml:",omitempty"`
DisplayName string `xml:",omitempty"`
}
// Initiator same to the Owner struct
type Initiator Owner
// Response API 响应
type Response struct {
*http.Response
}
func newResponse(resp *http.Response) *Response {
return &Response{
Response: resp,
}
}
// ACLHeaderOptions is the option of ACLHeader
type ACLHeaderOptions struct {
XCosACL string `header:"x-cos-acl,omitempty" url:"-" xml:"-"`
XCosGrantRead string `header:"x-cos-grant-read,omitempty" url:"-" xml:"-"`
XCosGrantWrite string `header:"x-cos-grant-write,omitempty" url:"-" xml:"-"`
XCosGrantFullControl string `header:"x-cos-grant-full-control,omitempty" url:"-" xml:"-"`
}
// ACLGrantee is the param of ACLGrant
type ACLGrantee struct {
Type string `xml:"type,attr"`
UIN string `xml:"uin,omitempty"`
URI string `xml:"URI,omitempty"`
ID string `xml:",omitempty"`
DisplayName string `xml:",omitempty"`
SubAccount string `xml:"Subaccount,omitempty"`
}
// ACLGrant is the param of ACLXml
type ACLGrant struct {
Grantee *ACLGrantee
Permission string
}
// ACLXml is the ACL body struct
type ACLXml struct {
XMLName xml.Name `xml:"AccessControlPolicy"`
Owner *Owner
AccessControlList []ACLGrant `xml:"AccessControlList>Grant,omitempty"`
}

70
vendor/github.com/tencentyun/cos-go-sdk-v5/debug/http.go generated vendored Normal file
View File

@@ -0,0 +1,70 @@
package debug
import (
"fmt"
"io"
"net/http"
"net/http/httputil"
"os"
)
// DebugRequestTransport 会打印请求和响应信息, 方便调试.
type DebugRequestTransport struct {
RequestHeader bool
RequestBody bool // RequestHeader 为 true 时,这个选项才会生效
ResponseHeader bool
ResponseBody bool // ResponseHeader 为 true 时,这个选项才会生效
// debug 信息输出到 Writer 中, 默认是 os.Stderr
Writer io.Writer
Transport http.RoundTripper
}
// RoundTrip implements the RoundTripper interface.
func (t *DebugRequestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req = cloneRequest(req) // per RoundTrip contract
w := t.Writer
if w == nil {
w = os.Stderr
}
if t.RequestHeader {
a, _ := httputil.DumpRequest(req, t.RequestBody)
fmt.Fprintf(w, "%s\n\n", string(a))
}
resp, err := t.transport().RoundTrip(req)
if err != nil {
return resp, err
}
if t.ResponseHeader {
b, _ := httputil.DumpResponse(resp, t.ResponseBody)
fmt.Fprintf(w, "%s\n", string(b))
}
return resp, err
}
func (t *DebugRequestTransport) transport() http.RoundTripper {
if t.Transport != nil {
return t.Transport
}
return http.DefaultTransport
}
// cloneRequest returns a clone of the provided *http.Request. The clone is a
// shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
// shallow copy of the struct
r2 := new(http.Request)
*r2 = *r
// deep copy of the Header
r2.Header = make(http.Header, len(r.Header))
for k, s := range r.Header {
r2.Header[k] = append([]string(nil), s...)
}
return r2
}

3
vendor/github.com/tencentyun/cos-go-sdk-v5/doc.go generated vendored Normal file
View File

@@ -0,0 +1,3 @@
// Package cos is COS(Cloud Object Storage) Go SDK. The V5 version(XML API).
// There are examples of using each API in the project's 'example' directory.
package cos

49
vendor/github.com/tencentyun/cos-go-sdk-v5/error.go generated vendored Normal file
View File

@@ -0,0 +1,49 @@
package cos
import (
"encoding/xml"
"fmt"
"io/ioutil"
"net/http"
)
// ErrorResponse 包含 API 返回的错误信息
//
// https://www.qcloud.com/document/product/436/7730
type ErrorResponse struct {
XMLName xml.Name `xml:"Error"`
Response *http.Response `xml:"-"`
Code string
Message string
Resource string
RequestID string `header:"x-cos-request-id,omitempty" url:"-" xml:"-"`
TraceID string `xml:"TraceId,omitempty"`
}
// Error returns the error msg
func (r *ErrorResponse) Error() string {
RequestID := r.RequestID
if RequestID == "" {
RequestID = r.Response.Header.Get("X-Cos-Request-Id")
}
TraceID := r.TraceID
if TraceID == "" {
TraceID = r.Response.Header.Get("X-Cos-Trace-Id")
}
return fmt.Sprintf("%v %v: %d %v(Message: %v, RequestId: %v, TraceId: %v)",
r.Response.Request.Method, r.Response.Request.URL,
r.Response.StatusCode, r.Code, r.Message, RequestID, TraceID)
}
// 检查 response 是否是出错时的返回的 response
func checkResponse(r *http.Response) error {
if c := r.StatusCode; 200 <= c && c <= 299 {
return nil
}
errorResponse := &ErrorResponse{Response: r}
data, err := ioutil.ReadAll(r.Body)
if err == nil && data != nil {
xml.Unmarshal(data, errorResponse)
}
return errorResponse
}

85
vendor/github.com/tencentyun/cos-go-sdk-v5/helper.go generated vendored Normal file
View File

@@ -0,0 +1,85 @@
package cos
import (
"bytes"
"crypto/md5"
"crypto/sha1"
"fmt"
"net/http"
)
// 计算 md5 或 sha1 时的分块大小
const calDigestBlockSize = 1024 * 1024 * 10
func calMD5Digest(msg []byte) []byte {
// TODO: 分块计算,减少内存消耗
m := md5.New()
m.Write(msg)
return m.Sum(nil)
}
func calSHA1Digest(msg []byte) []byte {
// TODO: 分块计算,减少内存消耗
m := sha1.New()
m.Write(msg)
return m.Sum(nil)
}
// cloneRequest returns a clone of the provided *http.Request. The clone is a
// shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
// shallow copy of the struct
r2 := new(http.Request)
*r2 = *r
// deep copy of the Header
r2.Header = make(http.Header, len(r.Header))
for k, s := range r.Header {
r2.Header[k] = append([]string(nil), s...)
}
return r2
}
// encodeURIComponent like same function in javascript
//
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent
//
// http://www.ecma-international.org/ecma-262/6.0/#sec-uri-syntax-and-semantics
func encodeURIComponent(s string) string {
var b bytes.Buffer
written := 0
for i, n := 0, len(s); i < n; i++ {
c := s[i]
switch c {
case '-', '_', '.', '!', '~', '*', '\'', '(', ')':
continue
default:
// Unreserved according to RFC 3986 sec 2.3
if 'a' <= c && c <= 'z' {
continue
}
if 'A' <= c && c <= 'Z' {
continue
}
if '0' <= c && c <= '9' {
continue
}
}
b.WriteString(s[written:i])
fmt.Fprintf(&b, "%%%02X", c)
written = i + 1
}
if written == 0 {
return s
}
b.WriteString(s[written:])
return b.String()
}

483
vendor/github.com/tencentyun/cos-go-sdk-v5/object.go generated vendored Normal file
View File

@@ -0,0 +1,483 @@
package cos
import (
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
)
// ObjectService 相关 API
type ObjectService service
// ObjectGetOptions is the option of GetObject
type ObjectGetOptions struct {
ResponseContentType string `url:"response-content-type,omitempty" header:"-"`
ResponseContentLanguage string `url:"response-content-language,omitempty" header:"-"`
ResponseExpires string `url:"response-expires,omitempty" header:"-"`
ResponseCacheControl string `url:"response-cache-control,omitempty" header:"-"`
ResponseContentDisposition string `url:"response-content-disposition,omitempty" header:"-"`
ResponseContentEncoding string `url:"response-content-encoding,omitempty" header:"-"`
Range string `url:"-" header:"Range,omitempty"`
IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"`
}
// presignedURLTestingOptions is the opt of presigned url
type presignedURLTestingOptions struct {
authTime *AuthTime
}
// Get Object 请求可以将一个文件（Object）下载至本地。
// 该操作需要对目标 Object 具有读权限或目标 Object 对所有人都开放了读权限（公有读）。
//
// https://www.qcloud.com/document/product/436/7753
func (s *ObjectService) Get(ctx context.Context, name string, opt *ObjectGetOptions, id ...string) (*Response, error) {
var u string
if len(id) == 1 {
u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0])
} else if len(id) == 0 {
u = "/" + encodeURIComponent(name)
} else {
return nil, errors.New("wrong params")
}
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodGet,
optQuery: opt,
optHeader: opt,
disableCloseBody: true,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// GetToFile download the object to local file
func (s *ObjectService) GetToFile(ctx context.Context, name, localpath string, opt *ObjectGetOptions, id ...string) (*Response, error) {
resp, err := s.Get(ctx, name, opt, id...)
if err != nil {
return resp, err
}
defer resp.Body.Close()
// If file exist, overwrite it
fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
if err != nil {
return resp, err
}
_, err = io.Copy(fd, resp.Body)
fd.Close()
if err != nil {
return resp, err
}
return resp, nil
}
// GetPresignedURL get the object presigned to down or upload file by url
func (s *ObjectService) GetPresignedURL(ctx context.Context, httpMethod, name, ak, sk string, expired time.Duration, opt interface{}) (*url.URL, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/" + encodeURIComponent(name),
method: httpMethod,
optQuery: opt,
optHeader: opt,
}
req, err := s.client.newRequest(ctx, sendOpt.baseURL, sendOpt.uri, sendOpt.method, sendOpt.body, sendOpt.optQuery, sendOpt.optHeader)
if err != nil {
return nil, err
}
var authTime *AuthTime
if opt != nil {
if opt, ok := opt.(*presignedURLTestingOptions); ok {
authTime = opt.authTime
}
}
if authTime == nil {
authTime = NewAuthTime(expired)
}
authorization := newAuthorization(ak, sk, req, authTime)
sign := encodeURIComponent(authorization)
if req.URL.RawQuery == "" {
req.URL.RawQuery = fmt.Sprintf("sign=%s", sign)
} else {
req.URL.RawQuery = fmt.Sprintf("%s&sign=%s", req.URL.RawQuery, sign)
}
return req.URL, nil
}
// ObjectPutHeaderOptions the options of header of the put object
type ObjectPutHeaderOptions struct {
CacheControl string `header:"Cache-Control,omitempty" url:"-"`
ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"`
ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
ContentType string `header:"Content-Type,omitempty" url:"-"`
ContentMD5 string `header:"Content-MD5,omitempty" url:"-"`
ContentLength int `header:"Content-Length,omitempty" url:"-"`
Expect string `header:"Expect,omitempty" url:"-"`
Expires string `header:"Expires,omitempty" url:"-"`
XCosContentSHA1 string `header:"x-cos-content-sha1,omitempty" url:"-"`
// 自定义的 x-cos-meta-* header
XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"`
XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-"`
// 可选值: Normal, Appendable
//XCosObjectType string `header:"x-cos-object-type,omitempty" url:"-"`
}
// ObjectPutOptions the options of put object
type ObjectPutOptions struct {
*ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"`
*ObjectPutHeaderOptions `header:",omitempty" url:"-" xml:"-"`
}
// Put Object请求可以将一个文件（Oject）上传至指定Bucket。
//
// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ObjectPutHeaderOptions.ContentLength
//
// https://www.qcloud.com/document/product/436/7749
func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, opt *ObjectPutOptions) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/" + encodeURIComponent(name),
method: http.MethodPut,
body: r,
optHeader: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// PutFromFile put object from local file
// Notice that when use this put large file need set non-body of debug req/resp, otherwise will out of memory
func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (*Response, error) {
fd, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer fd.Close()
return s.Put(ctx, name, fd, opt)
}
// ObjectCopyHeaderOptions is the head option of the Copy
type ObjectCopyHeaderOptions struct {
// When use replace directive to update meta infos
CacheControl string `header:"Cache-Control,omitempty" url:"-"`
ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"`
ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
ContentType string `header:"Content-Type,omitempty" url:"-"`
Expires string `header:"Expires,omitempty" url:"-"`
Expect string `header:"Expect,omitempty" url:"-"`
XCosMetadataDirective string `header:"x-cos-metadata-directive,omitempty" url:"-" xml:"-"`
XCosCopySourceIfModifiedSince string `header:"x-cos-copy-source-If-Modified-Since,omitempty" url:"-" xml:"-"`
XCosCopySourceIfUnmodifiedSince string `header:"x-cos-copy-source-If-Unmodified-Since,omitempty" url:"-" xml:"-"`
XCosCopySourceIfMatch string `header:"x-cos-copy-source-If-Match,omitempty" url:"-" xml:"-"`
XCosCopySourceIfNoneMatch string `header:"x-cos-copy-source-If-None-Match,omitempty" url:"-" xml:"-"`
XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-" xml:"-"`
// 自定义的 x-cos-meta-* header
XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"`
XCosCopySource string `header:"x-cos-copy-source" url:"-" xml:"-"`
XCosServerSideEncryption string `header:"x-cos-server-side-encryption,omitempty" url:"-" xml:"-"`
}
// ObjectCopyOptions is the option of Copy, choose header or body
type ObjectCopyOptions struct {
*ObjectCopyHeaderOptions `header:",omitempty" url:"-" xml:"-"`
*ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"`
}
// ObjectCopyResult is the result of Copy
type ObjectCopyResult struct {
XMLName xml.Name `xml:"CopyObjectResult"`
ETag string `xml:"ETag,omitempty"`
LastModified string `xml:"LastModified,omitempty"`
}
// Copy 调用 PutObjectCopy 请求实现将一个文件从源路径复制到目标路径。建议文件大小 1M 到 5G，
// 超过 5G 的文件请使用分块上传 Upload - Copy。在拷贝的过程中，文件元属性和 ACL 可以被修改。
//
// 用户可以通过该接口实现文件移动，文件重命名，修改文件属性和创建副本。
//
// 注意：在跨帐号复制的时候，需要先设置被复制文件的权限为公有读，或者对目标帐号赋权，同帐号则不需要。
//
// https://cloud.tencent.com/document/product/436/10881
func (s *ObjectService) Copy(ctx context.Context, name, sourceURL string, opt *ObjectCopyOptions) (*ObjectCopyResult, *Response, error) {
var res ObjectCopyResult
if opt == nil {
opt = new(ObjectCopyOptions)
}
if opt.ObjectCopyHeaderOptions == nil {
opt.ObjectCopyHeaderOptions = new(ObjectCopyHeaderOptions)
}
opt.XCosCopySource = encodeURIComponent(sourceURL)
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/" + encodeURIComponent(name),
method: http.MethodPut,
body: nil,
optHeader: opt,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
// If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error.
if err == nil && resp.StatusCode == 200 {
if res.ETag == "" {
return &res, resp, errors.New("response 200 OK, but body contains an error")
}
}
return &res, resp, err
}
// Delete Object请求可以将一个文件（Object）删除。
//
// https://www.qcloud.com/document/product/436/7743
func (s *ObjectService) Delete(ctx context.Context, name string) (*Response, error) {
// When use "" string might call the delete bucket interface
if len(name) == 0 {
return nil, errors.New("empty object name")
}
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/" + encodeURIComponent(name),
method: http.MethodDelete,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// ObjectHeadOptions is the option of HeadObject
type ObjectHeadOptions struct {
IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"`
}
// Head Object请求可以取回对应Object的元数据，Head的权限与Get的权限一致
//
// https://www.qcloud.com/document/product/436/7745
func (s *ObjectService) Head(ctx context.Context, name string, opt *ObjectHeadOptions, id ...string) (*Response, error) {
var u string
if len(id) == 1 {
u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0])
} else if len(id) == 0 {
u = "/" + encodeURIComponent(name)
} else {
return nil, errors.New("wrong params")
}
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodHead,
optHeader: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
if resp != nil && resp.Header["X-Cos-Object-Type"] != nil && resp.Header["X-Cos-Object-Type"][0] == "appendable" {
resp.Header.Add("x-cos-next-append-position", resp.Header["Content-Length"][0])
}
return resp, err
}
// ObjectOptionsOptions is the option of object options
type ObjectOptionsOptions struct {
Origin string `url:"-" header:"Origin"`
AccessControlRequestMethod string `url:"-" header:"Access-Control-Request-Method"`
AccessControlRequestHeaders string `url:"-" header:"Access-Control-Request-Headers,omitempty"`
}
// Options Object请求实现跨域访问的预请求。即发出一个 OPTIONS 请求给服务器以确认是否可以进行跨域操作。
//
// 当CORS配置不存在时，请求返回403 Forbidden。
//
// https://www.qcloud.com/document/product/436/8288
func (s *ObjectService) Options(ctx context.Context, name string, opt *ObjectOptionsOptions) (*Response, error) {
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/" + encodeURIComponent(name),
method: http.MethodOptions,
optHeader: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// CASJobParameters support three way: Standard(in 35 hours), Expedited(quick way, in 15 mins), Bulk(in 5-12 hours_
type CASJobParameters struct {
Tier string `xml:"Tier"`
}
// ObjectRestoreOptions is the option of object restore
type ObjectRestoreOptions struct {
XMLName xml.Name `xml:"RestoreRequest"`
Days int `xml:"Days"`
Tier *CASJobParameters `xml:"CASJobParameters"`
}
// PutRestore API can recover an object of type archived by COS archive.
//
// https://cloud.tencent.com/document/product/436/12633
func (s *ObjectService) PostRestore(ctx context.Context, name string, opt *ObjectRestoreOptions) (*Response, error) {
u := fmt.Sprintf("/%s?restore", encodeURIComponent(name))
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodPost,
body: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// TODO Append 接口在优化未开放使用
//
// Append请求可以将一个文件（Object）以分块追加的方式上传至 Bucket 中。使用Append Upload的文件必须事前被设定为Appendable。
// 当Appendable的文件被执行Put Object的操作以后，文件被覆盖，属性改变为Normal。
//
// 文件属性可以在Head Object操作中被查询到，当您发起Head Object请求时，会返回自定义Header『x-cos-object-type』，该Header只有两个枚举值：Normal或者Appendable。
//
// 追加上传建议文件大小1M - 5G。如果position的值和当前Object的长度不致，COS会返回409错误。
// 如果Append一个Normal的Object，COS会返回409 ObjectNotAppendable。
//
// Appendable的文件不可以被复制，不参与版本管理，不参与生命周期管理，不可跨区域复制。
//
// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ObjectPutHeaderOptions.ContentLength
//
// https://www.qcloud.com/document/product/436/7741
// func (s *ObjectService) Append(ctx context.Context, name string, position int, r io.Reader, opt *ObjectPutOptions) (*Response, error) {
// u := fmt.Sprintf("/%s?append&position=%d", encodeURIComponent(name), position)
// if position != 0{
// opt = nil
// }
// sendOpt := sendOptions{
// baseURL: s.client.BaseURL.BucketURL,
// uri: u,
// method: http.MethodPost,
// optHeader: opt,
// body: r,
// }
// resp, err := s.client.send(ctx, &sendOpt)
// return resp, err
// }
// ObjectDeleteMultiOptions is the option of DeleteMulti
type ObjectDeleteMultiOptions struct {
XMLName xml.Name `xml:"Delete" header:"-"`
Quiet bool `xml:"Quiet" header:"-"`
Objects []Object `xml:"Object" header:"-"`
//XCosSha1 string `xml:"-" header:"x-cos-sha1"`
}
// ObjectDeleteMultiResult is the result of DeleteMulti
type ObjectDeleteMultiResult struct {
XMLName xml.Name `xml:"DeleteResult"`
DeletedObjects []Object `xml:"Deleted,omitempty"`
Errors []struct {
Key string
Code string
Message string
} `xml:"Error,omitempty"`
}
// DeleteMulti 请求实现批量删除文件，最大支持单次删除1000个文件。
// 对于返回结果，COS提供Verbose和Quiet两种结果模式。Verbose模式将返回每个Object的删除结果；
// Quiet模式只返回报错的Object信息。
// https://www.qcloud.com/document/product/436/8289
func (s *ObjectService) DeleteMulti(ctx context.Context, opt *ObjectDeleteMultiOptions) (*ObjectDeleteMultiResult, *Response, error) {
var res ObjectDeleteMultiResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/?delete",
method: http.MethodPost,
body: opt,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// Object is the meta info of the object
type Object struct {
Key string `xml:",omitempty"`
ETag string `xml:",omitempty"`
Size int `xml:",omitempty"`
PartNumber int `xml:",omitempty"`
LastModified string `xml:",omitempty"`
StorageClass string `xml:",omitempty"`
Owner *Owner `xml:",omitempty"`
}
type MultiUploadOptions struct {
OptIni *InitiateMultipartUploadOptions
PartSize int
}
// MultiUpload 为高级upload接口，并发分块上传
//
// 需要指定分块大小 partSize >= 1 ,单位为MB
// 同时请确认分块数量不超过10000
//
func (s *ObjectService) MultiUpload(ctx context.Context, name string, r io.Reader, opt *MultiUploadOptions) (*CompleteMultipartUploadResult, *Response, error) {
optini := opt.OptIni
res, _, err := s.InitiateMultipartUpload(ctx, name, optini)
if err != nil {
return nil, nil, err
}
uploadID := res.UploadID
bufSize := opt.PartSize * 1024 * 1024
buffer := make([]byte, bufSize)
optcom := &CompleteMultipartUploadOptions{}
PartUpload := func(ch chan *Response, ctx context.Context, name string, uploadId string, partNumber int, data io.Reader, opt *ObjectUploadPartOptions) {
defer func() {
if err := recover(); err != nil {
fmt.Println(err)
}
}()
resp, _ := s.UploadPart(context.Background(), name, uploadId, partNumber, data, nil)
ch <- resp
}
chs := make([]chan *Response, 10000)
PartNumber := 0
for i := 1; true; i++ {
bytesread, err := r.Read(buffer)
if err != nil {
if err != io.EOF {
return nil, nil, err
}
PartNumber = i
break
}
chs[i] = make(chan *Response)
go PartUpload(chs[i], context.Background(), name, uploadID, i, strings.NewReader(string(buffer[:bytesread])), nil)
}
for i := 1; i < PartNumber; i++ {
resp := <-chs[i]
// Notice one part fail can not get the etag according.
etag := resp.Header.Get("ETag")
optcom.Parts = append(optcom.Parts, Object{
PartNumber: i, ETag: etag},
)
}
v, resp, err := s.CompleteMultipartUpload(context.Background(), name, uploadID, optcom)
return v, resp, err
}

63
vendor/github.com/tencentyun/cos-go-sdk-v5/object_acl.go generated vendored Normal file
View File

@@ -0,0 +1,63 @@
package cos
import (
"context"
"net/http"
)
// ObjectGetACLResult is the result of GetObjectACL
type ObjectGetACLResult ACLXml
// GetACL Get Object ACL接口实现使用API读取Object的ACL表，只有所有者有权操作。
//
// https://www.qcloud.com/document/product/436/7744
func (s *ObjectService) GetACL(ctx context.Context, name string) (*ObjectGetACLResult, *Response, error) {
var res ObjectGetACLResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/" + encodeURIComponent(name) + "?acl",
method: http.MethodGet,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// ObjectPutACLOptions the options of put object acl
type ObjectPutACLOptions struct {
Header *ACLHeaderOptions `url:"-" xml:"-"`
Body *ACLXml `url:"-" header:"-"`
}
// PutACL 使用API写入Object的ACL表，您可以通过Header："x-cos-acl", "x-cos-grant-read" ,
// "x-cos-grant-write" ,"x-cos-grant-full-control"传入ACL信息，
// 也可以通过body以XML格式传入ACL信息，但是只能选择Header和Body其中一种，否则，返回冲突。
//
// Put Object ACL是一个覆盖操作，传入新的ACL将覆盖原有ACL。只有所有者有权操作。
//
// "x-cos-acl"：枚举值为public-read，private；public-read意味这个Object有公有读私有写的权限，
// private意味这个Object有私有读写的权限。
//
// "x-cos-grant-read"：意味被赋予权限的用户拥有该Object的读权限
//
// "x-cos-grant-write"：意味被赋予权限的用户拥有该Object的写权限
//
// "x-cos-grant-full-control"：意味被赋予权限的用户拥有该Object的读写权限
//
// https://www.qcloud.com/document/product/436/7748
func (s *ObjectService) PutACL(ctx context.Context, name string, opt *ObjectPutACLOptions) (*Response, error) {
header := opt.Header
body := opt.Body
if body != nil {
header = nil
}
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/" + encodeURIComponent(name) + "?acl",
method: http.MethodPut,
optHeader: header,
body: body,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}

191
vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go generated vendored Normal file
View File

@@ -0,0 +1,191 @@
package cos
import (
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
)
// InitiateMultipartUploadOptions is the option of InitateMultipartUpload
type InitiateMultipartUploadOptions struct {
*ACLHeaderOptions
*ObjectPutHeaderOptions
}
// InitiateMultipartUploadResult is the result of InitateMultipartUpload
type InitiateMultipartUploadResult struct {
XMLName xml.Name `xml:"InitiateMultipartUploadResult"`
Bucket string
Key string
UploadID string `xml:"UploadId"`
}
// InitiateMultipartUpload 请求实现初始化分片上传，成功执行此请求以后会返回Upload ID用于后续的Upload Part请求。
//
// https://www.qcloud.com/document/product/436/7746
func (s *ObjectService) InitiateMultipartUpload(ctx context.Context, name string, opt *InitiateMultipartUploadOptions) (*InitiateMultipartUploadResult, *Response, error) {
var res InitiateMultipartUploadResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: "/" + encodeURIComponent(name) + "?uploads",
method: http.MethodPost,
optHeader: opt,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// ObjectUploadPartOptions is the options of upload-part
type ObjectUploadPartOptions struct {
Expect string `header:"Expect,omitempty" url:"-"`
XCosContentSHA1 string `header:"x-cos-content-sha1" url:"-"`
ContentLength int `header:"Content-Length,omitempty" url:"-"`
}
// UploadPart 请求实现在初始化以后的分块上传，支持的块的数量为1到10000，块的大小为1 MB 到5 GB。
// 在每次请求Upload Part时候，需要携带partNumber和uploadID，partNumber为块的编号，支持乱序上传。
//
// 当传入uploadID和partNumber都相同的时候，后传入的块将覆盖之前传入的块。当uploadID不存在时会返回404错误，NoSuchUpload.
//
// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ContentLength
//
// https://www.qcloud.com/document/product/436/7750
func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *ObjectUploadPartOptions) (*Response, error) {
u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID)
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodPut,
optHeader: opt,
body: r,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}
// ObjectListPartsOptions is the option of ListParts
type ObjectListPartsOptions struct {
EncodingType string `url:"Encoding-type,omitempty"`
MaxParts string `url:"max-parts,omitempty"`
PartNumberMarker string `url:"part-number-marker,omitempty"`
}
// ObjectListPartsResult is the result of ListParts
type ObjectListPartsResult struct {
XMLName xml.Name `xml:"ListPartsResult"`
Bucket string
EncodingType string `xml:"Encoding-type,omitempty"`
Key string
UploadID string `xml:"UploadId"`
Initiator *Initiator `xml:"Initiator,omitempty"`
Owner *Owner `xml:"Owner,omitempty"`
StorageClass string
PartNumberMarker string
NextPartNumberMarker string `xml:"NextPartNumberMarker,omitempty"`
MaxParts string
IsTruncated bool
Parts []Object `xml:"Part,omitempty"`
}
// ListParts 用来查询特定分块上传中的已上传的块。
//
// https://www.qcloud.com/document/product/436/7747
func (s *ObjectService) ListParts(ctx context.Context, name, uploadID string, opt *ObjectListPartsOptions) (*ObjectListPartsResult, *Response, error) {
u := fmt.Sprintf("/%s?uploadId=%s", encodeURIComponent(name), uploadID)
var res ObjectListPartsResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodGet,
result: &res,
optQuery: opt,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
// CompleteMultipartUploadOptions is the option of CompleteMultipartUpload
type CompleteMultipartUploadOptions struct {
XMLName xml.Name `xml:"CompleteMultipartUpload"`
Parts []Object `xml:"Part"`
}
// CompleteMultipartUploadResult is the result CompleteMultipartUpload
type CompleteMultipartUploadResult struct {
XMLName xml.Name `xml:"CompleteMultipartUploadResult"`
Location string
Bucket string
Key string
ETag string
}
// ObjectList can used for sort the parts which needs in complete upload part
// sort.Sort(cos.ObjectList(opt.Parts))
type ObjectList []Object
func (o ObjectList) Len() int {
return len(o)
}
func (o ObjectList) Swap(i, j int) {
o[i], o[j] = o[j], o[i]
}
func (o ObjectList) Less(i, j int) bool { // rewrite the Less method from small to big
return o[i].PartNumber < o[j].PartNumber
}
// CompleteMultipartUpload 用来实现完成整个分块上传。当您已经使用Upload Parts上传所有块以后，你可以用该API完成上传。
// 在使用该API时，您必须在Body中给出每一个块的PartNumber和ETag，用来校验块的准确性。
//
// 由于分块上传的合并需要数分钟时间，因而当合并分块开始的时候，COS就立即返回200的状态码，在合并的过程中，
// COS会周期性的返回空格信息来保持连接活跃，直到合并完成，COS会在Body中返回合并后块的内容。
//
// 当上传块小于1 MB的时候，在调用该请求时，会返回400 EntityTooSmall；
// 当上传块编号不连续的时候，在调用该请求时，会返回400 InvalidPart；
// 当请求Body中的块信息没有按序号从小到大排列的时候，在调用该请求时，会返回400 InvalidPartOrder；
// 当UploadId不存在的时候，在调用该请求时，会返回404 NoSuchUpload。
//
// 建议您及时完成分块上传或者舍弃分块上传，因为已上传但是未终止的块会占用存储空间进而产生存储费用。
//
// https://www.qcloud.com/document/product/436/7742
func (s *ObjectService) CompleteMultipartUpload(ctx context.Context, name, uploadID string, opt *CompleteMultipartUploadOptions) (*CompleteMultipartUploadResult, *Response, error) {
u := fmt.Sprintf("/%s?uploadId=%s", encodeURIComponent(name), uploadID)
var res CompleteMultipartUploadResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodPost,
body: opt,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
// If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error.
if err == nil && resp.StatusCode == 200 {
if res.ETag == "" {
return &res, resp, errors.New("response 200 OK, but body contains an error")
}
}
return &res, resp, err
}
// AbortMultipartUpload 用来实现舍弃一个分块上传并删除已上传的块。当您调用Abort Multipart Upload时，
// 如果有正在使用这个Upload Parts上传块的请求，则Upload Parts会返回失败。当该UploadID不存在时，会返回404 NoSuchUpload。
//
// 建议您及时完成分块上传或者舍弃分块上传，因为已上传但是未终止的块会占用存储空间进而产生存储费用。
//
// https://www.qcloud.com/document/product/436/7740
func (s *ObjectService) AbortMultipartUpload(ctx context.Context, name, uploadID string) (*Response, error) {
u := fmt.Sprintf("/%s?uploadId=%s", encodeURIComponent(name), uploadID)
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BucketURL,
uri: u,
method: http.MethodDelete,
}
resp, err := s.client.send(ctx, &sendOpt)
return resp, err
}

Some files were not shown because too many files have changed in this diff Show More