fanhuadesenlinnn/cloudpods
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git synced 2026-09-20 08:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

httputils, aliyun: respects env proxy settings (#4084)

Browse Source 
* vendor: update aliyun-oss-go-sdk to v2.0.4

To use ClientOption oss.HTTPClient

* aliyun: use a http client that respects env proxy settings

* httputils: use http.ProxyFromEnvironment by default
This commit is contained in:
Yousong Zhou
2019-12-10 21:15:09 +08:00
committed by Jian Qiu
parent 64bbf1c518
commit fb1725be32
29 changed files with 3872 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -13,7 +13,7 @@ require (
github.com/Shopify/sarama v1.20.0 // indirect
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20180112035716-b5829a0278c8
github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20181116160301-c6838fdc33ed
github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible
github.com/anacrolix/dht v0.0.0-20181129074040-b09db78595aa // indirect
github.com/anacrolix/go-libutp v0.0.0-20180808010927-aebbeb60ea05 // indirect
github.com/anacrolix/log v0.0.0-20180808012509-286fcf906b48 // indirect

4
go.sum
View File

@@ -33,8 +33,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20180112035716-b5829a0278c8 h1:mb7rn6WX8viizO36Nv6VVCBrgwaq01HPrZv1qq07/Yg=
github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20180112035716-b5829a0278c8/go.mod h1:T9M45xf79ahXVelWoOBmH0y4aC1t5kXO5BxwyakgIGA=
github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20181116160301-c6838fdc33ed h1:8Nj1ihALo/UVM2EnOVQ1EY/kwNKRwLx1GW/z3Y65czY=
github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20181116160301-c6838fdc33ed/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible h1:EaK5256H3ELiyaq5O/Zwd6fnghD6DqmZDQmmzzJklUU=
github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/anacrolix/dht v0.0.0-20180412060941-24cbf25b72a4/go.mod h1:hQfX2BrtuQsLQMYQwsypFAab/GvHg8qxwVi4OJdR1WI=
github.com/anacrolix/dht v0.0.0-20181127122252-acaa69c368bb/go.mod h1:Z/mP+H70xwnUv957UXVkTrzYRhKs0mGjS0lyqS331g4=
github.com/anacrolix/dht v0.0.0-20181129074040-b09db78595aa h1:5RyX9Tvqa9oewlJhFihkxe8biS0ojTlrNU3ryqm5g2s=

14
pkg/multicloud/aliyun/aliyun.go
View File

@@ -31,6 +31,7 @@ import (
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/util/httputils"
)
const (
@@ -227,8 +228,19 @@ func getOSSInternalDomain(regionId string) string {
// https://help.aliyun.com/document_detail/31837.html?spm=a2c4g.11186623.2.6.XqEgD1
func (client *SAliyunClient) getOssClient(regionId string) (*oss.Client, error) {
// NOTE
//
// oss package as of version 20181116160301-c6838fdc33ed does not
// respect http.ProxyFromEnvironment.
//
// The ClientOption Proxy, AuthProxy lacks the feature NO_PROXY has
// which can be used to whitelist ips, domains from http_proxy,
// https_proxy setting
cliOpts := []oss.ClientOption{
oss.HTTPClient(httputils.GetDefaultClient()),
}
ep := getOSSExternalDomain(regionId)
cli, err := oss.New(ep, client.accessKey, client.secret)
cli, err := oss.New(ep, client.accessKey, client.secret, cliOpts...)
if err != nil {
return nil, errors.Wrap(err, "oss.New")
}

1
pkg/util/httputils/httputils.go
View File

@@ -144,6 +144,7 @@ func GetAddrPort(urlStr string) (string, int, error) {
func GetTransport(insecure bool, timeout time.Duration) *http.Transport {
return &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: timeout,
KeepAlive: timeout,

53
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/auth.go generated vendored
View File

@@ -5,10 +5,12 @@ import (
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"fmt"
"hash"
"io"
"net/http"
"sort"
"strconv"
"strings"
)
@@ -20,14 +22,17 @@ type headerSorter struct {
// signHeader signs the header and sets it as the authorization header.
func (conn Conn) signHeader(req *http.Request, canonicalizedResource string) {
akIf := conn.config.GetCredentials()
// Get the final authorization string
authorizationStr := "OSS " + conn.config.AccessKeyID + ":" + conn.getSignedStr(req, canonicalizedResource)
authorizationStr := "OSS " + akIf.GetAccessKeyID() + ":" + conn.getSignedStr(req, canonicalizedResource, akIf.GetAccessKeySecret())
// Give the parameter "Authorization" value
req.Header.Set(HTTPHeaderAuthorization, authorizationStr)
}
func (conn Conn) getSignedStr(req *http.Request, canonicalizedResource string) string {
func (conn Conn) getSignedStr(req *http.Request, canonicalizedResource string, keySecret string) string {
// Find out the "x-oss-"'s address in header of the request
temp := make(map[string]string)
@@ -54,13 +59,55 @@ func (conn Conn) getSignedStr(req *http.Request, canonicalizedResource string) s
contentMd5 := req.Header.Get(HTTPHeaderContentMD5)
signStr := req.Method + "\n" + contentMd5 + "\n" + contentType + "\n" + date + "\n" + canonicalizedOSSHeaders + canonicalizedResource
h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(conn.config.AccessKeySecret))
// convert sign to log for easy to view
if conn.config.LogLevel >= Debug {
var signBuf bytes.Buffer
for i := 0; i < len(signStr); i++ {
if signStr[i] != '\n' {
signBuf.WriteByte(signStr[i])
} else {
signBuf.WriteString("\\n")
}
}
conn.config.WriteLog(Debug, "[Req:%p]signStr:%s\n", req, signBuf.String())
}
h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(keySecret))
io.WriteString(h, signStr)
signedStr := base64.StdEncoding.EncodeToString(h.Sum(nil))
return signedStr
}
func (conn Conn) getRtmpSignedStr(bucketName, channelName, playlistName string, expiration int64, keySecret string, params map[string]interface{}) string {
if params[HTTPParamAccessKeyID] == nil {
return ""
}
canonResource := fmt.Sprintf("/%s/%s", bucketName, channelName)
canonParamsKeys := []string{}
for key := range params {
if key != HTTPParamAccessKeyID && key != HTTPParamSignature && key != HTTPParamExpires && key != HTTPParamSecurityToken {
canonParamsKeys = append(canonParamsKeys, key)
}
}
sort.Strings(canonParamsKeys)
canonParamsStr := ""
for _, key := range canonParamsKeys {
canonParamsStr = fmt.Sprintf("%s%s:%s\n", canonParamsStr, key, params[key].(string))
}
expireStr := strconv.FormatInt(expiration, 10)
signStr := expireStr + "\n" + canonParamsStr + canonResource
h := hmac.New(func() hash.Hash { return sha1.New() }, []byte(keySecret))
io.WriteString(h, signStr)
signedStr := base64.StdEncoding.EncodeToString(h.Sum(nil))
return signedStr
}
// newHeaderSorter is an additional function for function SignHeader.
func newHeaderSorter(m map[string]string) *headerSorter {
hs := &headerSorter{

286
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/bucket.go generated vendored
View File

@@ -233,7 +233,17 @@ func (bucket Bucket) DoGetObject(request *GetObjectRequest, options []Option) (*
//
func (bucket Bucket) CopyObject(srcObjectKey, destObjectKey string, options ...Option) (CopyObjectResult, error) {
var out CopyObjectResult
options = append(options, CopySource(bucket.BucketName, url.QueryEscape(srcObjectKey)))
//first find version id
versionIdKey := "versionId"
versionId, _ := findOption(options, versionIdKey, nil)
if versionId == nil {
options = append(options, CopySource(bucket.BucketName, url.QueryEscape(srcObjectKey)))
} else {
options = deleteOption(options, versionIdKey)
options = append(options, CopySourceVersion(bucket.BucketName, url.QueryEscape(srcObjectKey), versionId.(string)))
}
params := map[string]interface{}{}
resp, err := bucket.do("PUT", destObjectKey, params, options, nil, nil)
if err != nil {
@@ -281,7 +291,17 @@ func (bucket Bucket) CopyObjectFrom(srcBucketName, srcObjectKey, destObjectKey s
func (bucket Bucket) copy(srcObjectKey, destBucketName, destObjectKey string, options ...Option) (CopyObjectResult, error) {
var out CopyObjectResult
options = append(options, CopySource(bucket.BucketName, url.QueryEscape(srcObjectKey)))
//first find version id
versionIdKey := "versionId"
versionId, _ := findOption(options, versionIdKey, nil)
if versionId == nil {
options = append(options, CopySource(bucket.BucketName, url.QueryEscape(srcObjectKey)))
} else {
options = deleteOption(options, versionIdKey)
options = append(options, CopySourceVersion(bucket.BucketName, url.QueryEscape(srcObjectKey), versionId.(string)))
}
headers := make(map[string]string)
err := handleOptions(headers, options)
if err != nil {
@@ -289,6 +309,14 @@ func (bucket Bucket) copy(srcObjectKey, destBucketName, destObjectKey string, op
}
params := map[string]interface{}{}
resp, err := bucket.Client.Conn.Do("PUT", destBucketName, destObjectKey, params, headers, nil, 0, nil)
// get response header
respHeader, _ := findOption(options, responseHeader, nil)
if respHeader != nil {
pRespHeader := respHeader.(*http.Header)
*pRespHeader = resp.Headers
}
if err != nil {
return out, err
}
@@ -357,6 +385,14 @@ func (bucket Bucket) DoAppendObject(request *AppendObjectRequest, options []Opti
handleOptions(headers, opts)
resp, err := bucket.Client.Conn.Do("POST", bucket.BucketName, request.ObjectKey, params, headers,
request.Reader, initCRC, listener)
// get response header
respHeader, _ := findOption(options, responseHeader, nil)
if respHeader != nil {
pRespHeader := respHeader.(*http.Header)
*pRespHeader = resp.Headers
}
if err != nil {
return nil, err
}
@@ -384,9 +420,9 @@ func (bucket Bucket) DoAppendObject(request *AppendObjectRequest, options []Opti
//
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) DeleteObject(objectKey string) error {
params := map[string]interface{}{}
resp, err := bucket.do("DELETE", objectKey, params, nil, nil, nil)
func (bucket Bucket) DeleteObject(objectKey string, options ...Option) error {
params, _ := getRawParams(options)
resp, err := bucket.do("DELETE", objectKey, params, options, nil, nil)
if err != nil {
return err
}
@@ -409,6 +445,63 @@ func (bucket Bucket) DeleteObjects(objectKeys []string, options ...Option) (Dele
for _, key := range objectKeys {
dxml.Objects = append(dxml.Objects, DeleteObject{Key: key})
}
isQuiet, _ := findOption(options, deleteObjectsQuiet, false)
dxml.Quiet = isQuiet.(bool)
bs, err := xml.Marshal(dxml)
if err != nil {
return out, err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
contentType := http.DetectContentType(buffer.Bytes())
options = append(options, ContentType(contentType))
sum := md5.Sum(bs)
b64 := base64.StdEncoding.EncodeToString(sum[:])
options = append(options, ContentMD5(b64))
params := map[string]interface{}{}
params["delete"] = nil
params["encoding-type"] = "url"
resp, err := bucket.do("POST", "", params, options, buffer, nil)
if err != nil {
return out, err
}
defer resp.Body.Close()
deletedResult := DeleteObjectVersionsResult{}
if !dxml.Quiet {
if err = xmlUnmarshal(resp.Body, &deletedResult); err == nil {
err = decodeDeleteObjectsResult(&deletedResult)
}
}
// Keep compatibility:need convert to struct DeleteObjectsResult
out.XMLName = deletedResult.XMLName
for _, v := range deletedResult.DeletedObjectsDetail {
out.DeletedObjects = append(out.DeletedObjects, v.Key)
}
return out, err
}
// DeleteObjectVersions deletes multiple object versions.
//
// objectVersions the object keys and versions to delete.
// options the options for deleting objects.
// Supported option is DeleteObjectsQuiet which means it will not return error even deletion failed (not recommended). By default it's not used.
//
// DeleteObjectVersionsResult the result object.
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) DeleteObjectVersions(objectVersions []DeleteObject, options ...Option) (DeleteObjectVersionsResult, error) {
out := DeleteObjectVersionsResult{}
dxml := deleteXML{}
dxml.Objects = objectVersions
isQuiet, _ := findOption(options, deleteObjectsQuiet, false)
dxml.Quiet = isQuiet.(bool)
@@ -449,15 +542,15 @@ func (bucket Bucket) DeleteObjects(objectKeys []string, options ...Option) (Dele
//
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) IsObjectExist(objectKey string) (bool, error) {
_, err := bucket.GetObjectMeta(objectKey)
func (bucket Bucket) IsObjectExist(objectKey string, options ...Option) (bool, error) {
_, err := bucket.GetObjectMeta(objectKey, options...)
if err == nil {
return true, nil
}
switch err.(type) {
case ServiceError:
if err.(ServiceError).StatusCode == 404 && err.(ServiceError).Code == "NoSuchKey" {
if err.(ServiceError).StatusCode == 404 {
return false, nil
}
}
@@ -509,6 +602,32 @@ func (bucket Bucket) ListObjects(options ...Option) (ListObjectsResult, error) {
return out, err
}
// ListObjectVersions lists objects of all versions under the current bucket.
func (bucket Bucket) ListObjectVersions(options ...Option) (ListObjectVersionsResult, error) {
var out ListObjectVersionsResult
options = append(options, EncodingType("url"))
params, err := getRawParams(options)
if err != nil {
return out, err
}
params["versions"] = nil
resp, err := bucket.do("GET", "", params, options, nil, nil)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
if err != nil {
return out, err
}
err = decodeListObjectVersionsResult(&out)
return out, err
}
// SetObjectMeta sets the metadata of the Object.
//
// objectKey object
@@ -533,7 +652,7 @@ func (bucket Bucket) SetObjectMeta(objectKey string, options ...Option) error {
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) GetObjectDetailedMeta(objectKey string, options ...Option) (http.Header, error) {
params := map[string]interface{}{}
params, _ := getRawParams(options)
resp, err := bucket.do("HEAD", objectKey, params, options, nil, nil)
if err != nil {
return nil, err
@@ -554,10 +673,10 @@ func (bucket Bucket) GetObjectDetailedMeta(objectKey string, options ...Option)
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) GetObjectMeta(objectKey string, options ...Option) (http.Header, error) {
params := map[string]interface{}{}
params, _ := getRawParams(options)
params["objectMeta"] = nil
//resp, err := bucket.do("GET", objectKey, "?objectMeta", "", nil, nil, nil)
resp, err := bucket.do("GET", objectKey, params, options, nil, nil)
resp, err := bucket.do("HEAD", objectKey, params, options, nil, nil)
if err != nil {
return nil, err
}
@@ -582,9 +701,9 @@ func (bucket Bucket) GetObjectMeta(objectKey string, options ...Option) (http.He
//
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) SetObjectACL(objectKey string, objectACL ACLType) error {
options := []Option{ObjectACL(objectACL)}
params := map[string]interface{}{}
func (bucket Bucket) SetObjectACL(objectKey string, objectACL ACLType, options ...Option) error {
options = append(options, ObjectACL(objectACL))
params, _ := getRawParams(options)
params["acl"] = nil
resp, err := bucket.do("PUT", objectKey, params, options, nil, nil)
if err != nil {
@@ -601,11 +720,11 @@ func (bucket Bucket) SetObjectACL(objectKey string, objectACL ACLType) error {
// GetObjectACLResult the result object when error is nil. GetObjectACLResult.Acl is the object ACL.
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) GetObjectACL(objectKey string) (GetObjectACLResult, error) {
func (bucket Bucket) GetObjectACL(objectKey string, options ...Option) (GetObjectACLResult, error) {
var out GetObjectACLResult
params := map[string]interface{}{}
params, _ := getRawParams(options)
params["acl"] = nil
resp, err := bucket.do("GET", objectKey, params, nil, nil, nil)
resp, err := bucket.do("GET", objectKey, params, options, nil, nil)
if err != nil {
return out, err
}
@@ -630,7 +749,7 @@ func (bucket Bucket) GetObjectACL(objectKey string) (GetObjectACLResult, error)
//
func (bucket Bucket) PutSymlink(symObjectKey string, targetObjectKey string, options ...Option) error {
options = append(options, symlinkTarget(url.QueryEscape(targetObjectKey)))
params := map[string]interface{}{}
params, _ := getRawParams(options)
params["symlink"] = nil
resp, err := bucket.do("PUT", symObjectKey, params, options, nil, nil)
if err != nil {
@@ -648,10 +767,10 @@ func (bucket Bucket) PutSymlink(symObjectKey string, targetObjectKey string, opt
// error it's nil if no error, otherwise it's an error object.
// When error is nil, the target file key is in the X-Oss-Symlink-Target header of the returned object.
//
func (bucket Bucket) GetSymlink(objectKey string) (http.Header, error) {
params := map[string]interface{}{}
func (bucket Bucket) GetSymlink(objectKey string, options ...Option) (http.Header, error) {
params, _ := getRawParams(options)
params["symlink"] = nil
resp, err := bucket.do("GET", objectKey, params, nil, nil, nil)
resp, err := bucket.do("GET", objectKey, params, options, nil, nil)
if err != nil {
return nil, err
}
@@ -678,10 +797,10 @@ func (bucket Bucket) GetSymlink(objectKey string) (http.Header, error) {
//
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) RestoreObject(objectKey string) error {
params := map[string]interface{}{}
func (bucket Bucket) RestoreObject(objectKey string, options ...Option) error {
params, _ := getRawParams(options)
params["restore"] = nil
resp, err := bucket.do("POST", objectKey, params, nil, nil, nil)
resp, err := bucket.do("POST", objectKey, params, options, nil, nil)
if err != nil {
return err
}
@@ -911,9 +1030,9 @@ func (bucket Bucket) DoGetObjectWithURL(signedURL string, options []Option) (*Ge
//
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) ProcessObject(objectKey string, process string) (ProcessObjectResult, error) {
func (bucket Bucket) ProcessObject(objectKey string, process string, options ...Option) (ProcessObjectResult, error) {
var out ProcessObjectResult
params := map[string]interface{}{}
params, _ := getRawParams(options)
params["x-oss-process"] = nil
processData := fmt.Sprintf("%v=%v", "x-oss-process", process)
data := strings.NewReader(processData)
@@ -927,6 +1046,92 @@ func (bucket Bucket) ProcessObject(objectKey string, process string) (ProcessObj
return out, err
}
//
// PutObjectTagging add tagging to object
//
// objectKey object key to add tagging
// tagging tagging to be added
//
// error nil if success, otherwise error
//
func (bucket Bucket) PutObjectTagging(objectKey string, tagging Tagging, options ...Option) error {
bs, err := xml.Marshal(tagging)
if err != nil {
return err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
params, _ := getRawParams(options)
params["tagging"] = nil
resp, err := bucket.do("PUT", objectKey, params, options, buffer, nil)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
//
// GetObjectTagging get tagging of the object
//
// objectKey object key to get tagging
//
// Tagging
// error nil if success, otherwise error
func (bucket Bucket) GetObjectTagging(objectKey string, options ...Option) (GetObjectTaggingResult, error) {
var out GetObjectTaggingResult
params, _ := getRawParams(options)
params["tagging"] = nil
resp, err := bucket.do("GET", objectKey, params, options, nil, nil)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
//
// DeleteObjectTagging delete object taggging
//
// objectKey object key to delete tagging
//
// error nil if success, otherwise error
//
func (bucket Bucket) DeleteObjectTagging(objectKey string, options ...Option) error {
params, _ := getRawParams(options)
params["tagging"] = nil
if objectKey == "" {
return fmt.Errorf("invalid argument: object name is empty")
}
resp, err := bucket.do("DELETE", objectKey, params, options, nil, nil)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusNoContent})
}
func (bucket Bucket) OptionsMethod(objectKey string, options ...Option) (http.Header, error) {
var out http.Header
resp, err := bucket.do("OPTIONS", objectKey, nil, options, nil, nil)
if err != nil {
return out, err
}
defer resp.Body.Close()
out = resp.Headers
return out, nil
}
// Private
func (bucket Bucket) do(method, objectName string, params map[string]interface{}, options []Option,
data io.Reader, listener ProgressListener) (*Response, error) {
@@ -935,8 +1140,23 @@ func (bucket Bucket) do(method, objectName string, params map[string]interface{}
if err != nil {
return nil, err
}
return bucket.Client.Conn.Do(method, bucket.BucketName, objectName,
err = CheckBucketName(bucket.BucketName)
if len(bucket.BucketName) > 0 && err != nil {
return nil, err
}
resp, err := bucket.Client.Conn.Do(method, bucket.BucketName, objectName,
params, headers, data, 0, listener)
// get response header
respHeader, _ := findOption(options, responseHeader, nil)
if respHeader != nil {
pRespHeader := respHeader.(*http.Header)
*pRespHeader = resp.Headers
}
return resp, err
}
func (bucket Bucket) doURL(method HTTPMethod, signedURL string, params map[string]interface{}, options []Option,
@@ -946,7 +1166,17 @@ func (bucket Bucket) doURL(method HTTPMethod, signedURL string, params map[strin
if err != nil {
return nil, err
}
return bucket.Client.Conn.DoURL(method, signedURL, headers, data, 0, listener)
resp, err := bucket.Client.Conn.DoURL(method, signedURL, headers, data, 0, listener)
// get response header
respHeader, _ := findOption(options, responseHeader, nil)
if respHeader != nil {
pRespHeader := respHeader.(*http.Header)
*pRespHeader = resp.Headers
}
return resp, err
}
func (bucket Bucket) getConfig() *Config {

622
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/client.go generated vendored
View File

@@ -5,7 +5,11 @@ package oss
import (
"bytes"
"encoding/xml"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"strings"
"time"
@@ -18,8 +22,9 @@ import (
type (
// Client OSS client
Client struct {
Config *Config // OSS client configuration
Conn *Conn // Send HTTP request
Config *Config // OSS client configuration
Conn *Conn // Send HTTP request
HTTPClient *http.Client //http.Client to use - if nil will make its own
}
// ClientOption client option such as UseCname, Timeout, SecurityToken.
@@ -44,15 +49,18 @@ func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption)
// URL parse
url := &urlMaker{}
url.Init(config.Endpoint, config.IsCname, config.IsUseProxy)
err := url.Init(config.Endpoint, config.IsCname, config.IsUseProxy)
if err != nil {
return nil, err
}
// HTTP connect
conn := &Conn{config: config, url: url}
// OSS client
client := &Client{
config,
conn,
Config: config,
Conn: conn,
}
// Client options parse
@@ -61,7 +69,7 @@ func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption)
}
// Create HTTP connection
err := conn.init(config, url)
err = conn.init(config, url, client.HTTPClient)
return client, err
}
@@ -74,6 +82,11 @@ func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption)
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) Bucket(bucketName string) (*Bucket, error) {
err := CheckBucketName(bucketName)
if err != nil {
return nil, err
}
return &Bucket{
client,
bucketName,
@@ -95,21 +108,29 @@ func (client Client) CreateBucket(bucketName string, options ...Option) error {
buffer := new(bytes.Buffer)
isOptSet, val, _ := isOptionSet(options, storageClass)
if isOptSet {
cbConfig := createBucketConfiguration{StorageClass: val.(StorageClassType)}
bs, err := xml.Marshal(cbConfig)
if err != nil {
return err
}
buffer.Write(bs)
var cbConfig createBucketConfiguration
cbConfig.StorageClass = StorageStandard
contentType := http.DetectContentType(buffer.Bytes())
headers[HTTPHeaderContentType] = contentType
isStorageSet, valStroage, _ := isOptionSet(options, storageClass)
isRedundancySet, valRedundancy, _ := isOptionSet(options, redundancyType)
if isStorageSet {
cbConfig.StorageClass = valStroage.(StorageClassType)
}
if isRedundancySet {
cbConfig.DataRedundancyType = valRedundancy.(DataRedundancyType)
}
bs, err := xml.Marshal(cbConfig)
if err != nil {
return err
}
buffer.Write(bs)
contentType := http.DetectContentType(buffer.Bytes())
headers[HTTPHeaderContentType] = contentType
params := map[string]interface{}{}
resp, err := client.do("PUT", bucketName, params, headers, buffer)
resp, err := client.do("PUT", bucketName, params, headers, buffer, options...)
if err != nil {
return err
}
@@ -136,7 +157,7 @@ func (client Client) ListBuckets(options ...Option) (ListBucketsResult, error) {
return out, err
}
resp, err := client.do("GET", "", params, nil, nil)
resp, err := client.do("GET", "", params, nil, nil, options...)
if err != nil {
return out, err
}
@@ -149,7 +170,7 @@ func (client Client) ListBuckets(options ...Option) (ListBucketsResult, error) {
// IsBucketExist checks if the bucket exists
//
// bucketName the bucket name.
//
//
// bool true if it exists, and it's only valid when error is nil.
// error it's nil if no error, otherwise it's an error object.
//
@@ -171,9 +192,9 @@ func (client Client) IsBucketExist(bucketName string) (bool, error) {
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) DeleteBucket(bucketName string) error {
func (client Client) DeleteBucket(bucketName string, options ...Option) error {
params := map[string]interface{}{}
resp, err := client.do("DELETE", bucketName, params, nil, nil)
resp, err := client.do("DELETE", bucketName, params, nil, nil, options...)
if err != nil {
return err
}
@@ -184,7 +205,7 @@ func (client Client) DeleteBucket(bucketName string) error {
// GetBucketLocation gets the bucket location.
//
// Checks out the following link for more information :
// Checks out the following link for more information :
// https://help.aliyun.com/document_detail/oss/user_guide/oss_concept/endpoint.html
//
// bucketName the bucket name
@@ -216,6 +237,7 @@ func (client Client) GetBucketLocation(bucketName string) (string, error) {
func (client Client) SetBucketACL(bucketName string, bucketACL ACLType) error {
headers := map[string]string{HTTPHeaderOssACL: string(bucketACL)}
params := map[string]interface{}{}
params["acl"] = nil
resp, err := client.do("PUT", bucketName, params, headers, nil)
if err != nil {
return err
@@ -253,12 +275,16 @@ func (client Client) GetBucketACL(bucketName string) (GetBucketACLResult, error)
// bucketName the bucket name.
// rules the lifecycle rules. There're two kind of rules: absolute time expiration and relative time expiration in days and day/month/year respectively.
// Check out sample/bucket_lifecycle.go for more details.
//
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) SetBucketLifecycle(bucketName string, rules []LifecycleRule) error {
lxml := lifecycleXML{Rules: convLifecycleRule(rules)}
bs, err := xml.Marshal(lxml)
err := verifyLifecycleRules(rules)
if err != nil {
return err
}
lifecycleCfg := LifecycleConfiguration{Rules: rules}
bs, err := xml.Marshal(lifecycleCfg)
if err != nil {
return err
}
@@ -300,7 +326,7 @@ func (client Client) DeleteBucketLifecycle(bucketName string) error {
// GetBucketLifecycle gets the bucket's lifecycle settings.
//
// bucketName the bucket name.
//
//
// GetBucketLifecycleResponse the result object upon successful request. It's only valid when error is nil.
// error it's nil if no error, otherwise it's an error object.
//
@@ -508,6 +534,39 @@ func (client Client) SetBucketWebsite(bucketName, indexDocument, errorDocument s
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// SetBucketWebsiteDetail sets the bucket's static website's detail
//
// OSS supports static web site hosting for the bucket data. When the bucket is enabled with that, you can access the file in the bucket like the way to access a static website.
// For more information, please check out: https://help.aliyun.com/document_detail/oss/user_guide/static_host_website.html
//
// bucketName the bucket name to enable static web site.
//
// wxml the website's detail
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) SetBucketWebsiteDetail(bucketName string, wxml WebsiteXML, options ...Option) error {
bs, err := xml.Marshal(wxml)
if err != nil {
return err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
contentType := http.DetectContentType(buffer.Bytes())
headers := make(map[string]string)
headers[HTTPHeaderContentType] = contentType
params := map[string]interface{}{}
params["website"] = nil
resp, err := client.do("PUT", bucketName, params, headers, buffer, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// DeleteBucketWebsite deletes the bucket's static web site settings.
//
// bucketName the bucket name.
@@ -633,10 +692,213 @@ func (client Client) GetBucketCORS(bucketName string) (GetBucketCORSResult, erro
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) GetBucketInfo(bucketName string) (GetBucketInfoResult, error) {
func (client Client) GetBucketInfo(bucketName string, options ...Option) (GetBucketInfoResult, error) {
var out GetBucketInfoResult
params := map[string]interface{}{}
params["bucketInfo"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
// convert None to ""
if err == nil {
if out.BucketInfo.SseRule.KMSMasterKeyID == "None" {
out.BucketInfo.SseRule.KMSMasterKeyID = ""
}
if out.BucketInfo.SseRule.SSEAlgorithm == "None" {
out.BucketInfo.SseRule.SSEAlgorithm = ""
}
}
return out, err
}
// SetBucketVersioning set bucket versioning:Enabled、Suspended
// bucketName the bucket name.
// error it's nil if no error, otherwise it's an error object.
func (client Client) SetBucketVersioning(bucketName string, versioningConfig VersioningConfig, options ...Option) error {
var err error
var bs []byte
bs, err = xml.Marshal(versioningConfig)
if err != nil {
return err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
contentType := http.DetectContentType(buffer.Bytes())
headers := map[string]string{}
headers[HTTPHeaderContentType] = contentType
params := map[string]interface{}{}
params["versioning"] = nil
resp, err := client.do("PUT", bucketName, params, headers, buffer, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// GetBucketVersioning get bucket versioning status:Enabled、Suspended
// bucketName the bucket name.
// error it's nil if no error, otherwise it's an error object.
func (client Client) GetBucketVersioning(bucketName string, options ...Option) (GetBucketVersioningResult, error) {
var out GetBucketVersioningResult
params := map[string]interface{}{}
params["versioning"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
// SetBucketEncryption set bucket encryption config
// bucketName the bucket name.
// error it's nil if no error, otherwise it's an error object.
func (client Client) SetBucketEncryption(bucketName string, encryptionRule ServerEncryptionRule, options ...Option) error {
var err error
var bs []byte
bs, err = xml.Marshal(encryptionRule)
if err != nil {
return err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
contentType := http.DetectContentType(buffer.Bytes())
headers := map[string]string{}
headers[HTTPHeaderContentType] = contentType
params := map[string]interface{}{}
params["encryption"] = nil
resp, err := client.do("PUT", bucketName, params, headers, buffer, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// GetBucketEncryption get bucket encryption
// bucketName the bucket name.
// error it's nil if no error, otherwise it's an error object.
func (client Client) GetBucketEncryption(bucketName string, options ...Option) (GetBucketEncryptionResult, error) {
var out GetBucketEncryptionResult
params := map[string]interface{}{}
params["encryption"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
// DeleteBucketEncryption delete bucket encryption config
// bucketName the bucket name.
// error it's nil if no error, otherwise it's an error bucket
func (client Client) DeleteBucketEncryption(bucketName string, options ...Option) error {
params := map[string]interface{}{}
params["encryption"] = nil
resp, err := client.do("DELETE", bucketName, params, nil, nil, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusNoContent})
}
//
// SetBucketTagging add tagging to bucket
// bucketName name of bucket
// tagging tagging to be added
// error nil if success, otherwise error
func (client Client) SetBucketTagging(bucketName string, tagging Tagging, options ...Option) error {
var err error
var bs []byte
bs, err = xml.Marshal(tagging)
if err != nil {
return err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
contentType := http.DetectContentType(buffer.Bytes())
headers := map[string]string{}
headers[HTTPHeaderContentType] = contentType
params := map[string]interface{}{}
params["tagging"] = nil
resp, err := client.do("PUT", bucketName, params, headers, buffer, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// GetBucketTagging get tagging of the bucket
// bucketName name of bucket
// error nil if success, otherwise error
func (client Client) GetBucketTagging(bucketName string, options ...Option) (GetBucketTaggingResult, error) {
var out GetBucketTaggingResult
params := map[string]interface{}{}
params["tagging"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
//
// DeleteBucketTagging delete bucket tagging
// bucketName name of bucket
// error nil if success, otherwise error
//
func (client Client) DeleteBucketTagging(bucketName string, options ...Option) error {
params := map[string]interface{}{}
params["tagging"] = nil
resp, err := client.do("DELETE", bucketName, params, nil, nil, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusNoContent})
}
// GetBucketStat get bucket stat
// bucketName the bucket name.
// error it's nil if no error, otherwise it's an error object.
func (client Client) GetBucketStat(bucketName string) (GetBucketStatResult, error) {
var out GetBucketStatResult
params := map[string]interface{}{}
params["stat"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil)
if err != nil {
return out, err
@@ -647,6 +909,252 @@ func (client Client) GetBucketInfo(bucketName string) (GetBucketInfoResult, erro
return out, err
}
// GetBucketPolicy API operation for Object Storage Service.
//
// Get the policy from the bucket.
//
// bucketName the bucket name.
//
// string return the bucket's policy, and it's only valid when error is nil.
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) GetBucketPolicy(bucketName string, options ...Option) (string, error) {
params := map[string]interface{}{}
params["policy"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil, options...)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
out := string(body)
return out, err
}
// SetBucketPolicy API operation for Object Storage Service.
//
// Set the policy from the bucket.
//
// bucketName the bucket name.
//
// policy the bucket policy.
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) SetBucketPolicy(bucketName string, policy string, options ...Option) error {
params := map[string]interface{}{}
params["policy"] = nil
buffer := strings.NewReader(policy)
resp, err := client.do("PUT", bucketName, params, nil, buffer, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// DeleteBucketPolicy API operation for Object Storage Service.
//
// Deletes the policy from the bucket.
//
// bucketName the bucket name.
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) DeleteBucketPolicy(bucketName string, options ...Option) error {
params := map[string]interface{}{}
params["policy"] = nil
resp, err := client.do("DELETE", bucketName, params, nil, nil, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusNoContent})
}
// SetBucketRequestPayment API operation for Object Storage Service.
//
// Set the requestPayment of bucket
//
// bucketName the bucket name.
//
// paymentConfig the payment configuration
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) SetBucketRequestPayment(bucketName string, paymentConfig RequestPaymentConfiguration, options ...Option) error {
params := map[string]interface{}{}
params["requestPayment"] = nil
var bs []byte
bs, err := xml.Marshal(paymentConfig)
if err != nil {
return err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
contentType := http.DetectContentType(buffer.Bytes())
headers := map[string]string{}
headers[HTTPHeaderContentType] = contentType
resp, err := client.do("PUT", bucketName, params, headers, buffer, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// GetBucketRequestPayment API operation for Object Storage Service.
//
// Get bucket requestPayment
//
// bucketName the bucket name.
//
// RequestPaymentConfiguration the payment configuration
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) GetBucketRequestPayment(bucketName string, options ...Option) (RequestPaymentConfiguration, error) {
var out RequestPaymentConfiguration
params := map[string]interface{}{}
params["requestPayment"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
// GetUserQoSInfo API operation for Object Storage Service.
//
// Get user qos.
//
// UserQoSConfiguration the User Qos and range Information.
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) GetUserQoSInfo(options ...Option) (UserQoSConfiguration, error) {
var out UserQoSConfiguration
params := map[string]interface{}{}
params["qosInfo"] = nil
resp, err := client.do("GET", "", params, nil, nil, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
// SetBucketQoSInfo API operation for Object Storage Service.
//
// Set Bucket Qos information.
//
// bucketName tht bucket name.
//
// qosConf the qos configuration.
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) SetBucketQoSInfo(bucketName string, qosConf BucketQoSConfiguration, options ...Option) error {
params := map[string]interface{}{}
params["qosInfo"] = nil
var bs []byte
bs, err := xml.Marshal(qosConf)
if err != nil {
return err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
contentTpye := http.DetectContentType(buffer.Bytes())
headers := map[string]string{}
headers[HTTPHeaderContentType] = contentTpye
resp, err := client.do("PUT", bucketName, params, headers, buffer, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// GetBucketQosInfo API operation for Object Storage Service.
//
// Get Bucket Qos information.
//
// bucketName tht bucket name.
//
// BucketQoSConfiguration the return qos configuration.
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) GetBucketQosInfo(bucketName string, options ...Option) (BucketQoSConfiguration, error) {
var out BucketQoSConfiguration
params := map[string]interface{}{}
params["qosInfo"] = nil
resp, err := client.do("GET", bucketName, params, nil, nil, options...)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
// DeleteBucketQosInfo API operation for Object Storage Service.
//
// Delete Bucket QoS information.
//
// bucketName tht bucket name.
//
// error it's nil if no error, otherwise it's an error object.
//
func (client Client) DeleteBucketQosInfo(bucketName string, options ...Option) error {
params := map[string]interface{}{}
params["qosInfo"] = nil
resp, err := client.do("DELETE", bucketName, params, nil, nil, options...)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusNoContent})
}
// LimitUploadSpeed set upload bandwidth limit speed,default is 0,unlimited
// upSpeed KB/s, 0 is unlimited,default is 0
// error it's nil if success, otherwise failure
func (client Client) LimitUploadSpeed(upSpeed int) error {
if client.Config == nil {
return fmt.Errorf("client config is nil")
}
return client.Config.LimitUploadSpeed(upSpeed)
}
// UseCname sets the flag of using CName. By default it's false.
//
// isUseCname true: the endpoint has the CName, false: the endpoint does not have cname. Default is false.
@@ -757,9 +1265,63 @@ func AuthProxy(proxyHost, proxyUser, proxyPassword string) ClientOption {
}
}
//
// HTTPClient sets the http.Client in use to the one passed in
//
func HTTPClient(HTTPClient *http.Client) ClientOption {
return func(client *Client) {
client.HTTPClient = HTTPClient
}
}
//
// SetLogLevel sets the oss sdk log level
//
func SetLogLevel(LogLevel int) ClientOption {
return func(client *Client) {
client.Config.LogLevel = LogLevel
}
}
//
// SetLogger sets the oss sdk logger
//
func SetLogger(Logger *log.Logger) ClientOption {
return func(client *Client) {
client.Config.Logger = Logger
}
}
// SetCredentialsProvider sets funciton for get the user's ak
func SetCredentialsProvider(provider CredentialsProvider) ClientOption {
return func(client *Client) {
client.Config.CredentialsProvider = provider
}
}
// SetLocalAddr sets funciton for local addr
func SetLocalAddr(localAddr net.Addr) ClientOption {
return func(client *Client) {
client.Config.LocalAddr = localAddr
}
}
// Private
func (client Client) do(method, bucketName string, params map[string]interface{},
headers map[string]string, data io.Reader) (*Response, error) {
return client.Conn.Do(method, bucketName, "", params,
headers, data, 0, nil)
headers map[string]string, data io.Reader, options ...Option) (*Response, error) {
err := CheckBucketName(bucketName)
if len(bucketName) > 0 && err != nil {
return nil, err
}
resp, err := client.Conn.Do(method, bucketName, "", params, headers, data, 0, nil)
// get response header
respHeader, _ := findOption(options, responseHeader, nil)
if respHeader != nil {
pRespHeader := respHeader.(*http.Header)
*pRespHeader = resp.Headers
}
return resp, err
}

149
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/conf.go generated vendored
View File

@@ -1,9 +1,26 @@
package oss
import (
"bytes"
"fmt"
"log"
"net"
"os"
"time"
)
// Define the level of the output log
const (
LogOff = iota
Error
Warn
Info
Debug
)
// LogTag Tag for each level of log
var LogTag = []string{"[error]", "[warn]", "[info]", "[debug]"}
// HTTPTimeout defines HTTP timeout.
type HTTPTimeout struct {
ConnectTimeout time.Duration
@@ -13,26 +30,110 @@ type HTTPTimeout struct {
IdleConnTimeout time.Duration
}
// HTTPMaxConns defines max idle connections and max idle connections per host
type HTTPMaxConns struct {
MaxIdleConns int
MaxIdleConnsPerHost int
}
// CredentialInf is interface for get AccessKeyID,AccessKeySecret,SecurityToken
type Credentials interface {
GetAccessKeyID() string
GetAccessKeySecret() string
GetSecurityToken() string
}
// CredentialInfBuild is interface for get CredentialInf
type CredentialsProvider interface {
GetCredentials() Credentials
}
type defaultCredentials struct {
config *Config
}
func (defCre *defaultCredentials) GetAccessKeyID() string {
return defCre.config.AccessKeyID
}
func (defCre *defaultCredentials) GetAccessKeySecret() string {
return defCre.config.AccessKeySecret
}
func (defCre *defaultCredentials) GetSecurityToken() string {
return defCre.config.SecurityToken
}
type defaultCredentialsProvider struct {
config *Config
}
func (defBuild *defaultCredentialsProvider) GetCredentials() Credentials {
return &defaultCredentials{config: defBuild.config}
}
// Config defines oss configuration
type Config struct {
Endpoint string // OSS endpoint
AccessKeyID string // AccessId
AccessKeySecret string // AccessKey
RetryTimes uint // Retry count by default it's 5.
UserAgent string // SDK name/version/system information
IsDebug bool // Enable debug mode. Default is false.
Timeout uint // Timeout in seconds. By default it's 60.
SecurityToken string // STS Token
IsCname bool // If cname is in the endpoint.
HTTPTimeout HTTPTimeout // HTTP timeout
IsUseProxy bool // Flag of using proxy.
ProxyHost string // Flag of using proxy host.
IsAuthProxy bool // Flag of needing authentication.
ProxyUser string // Proxy user
ProxyPassword string // Proxy password
IsEnableMD5 bool // Flag of enabling MD5 for upload.
MD5Threshold int64 // Memory footprint threshold for each MD5 computation (16MB is the default), in byte. When the data is more than that, temp file is used.
IsEnableCRC bool // Flag of enabling CRC for upload.
Endpoint string // OSS endpoint
AccessKeyID string // AccessId
AccessKeySecret string // AccessKey
RetryTimes uint // Retry count by default it's 5.
UserAgent string // SDK name/version/system information
IsDebug bool // Enable debug mode. Default is false.
Timeout uint // Timeout in seconds. By default it's 60.
SecurityToken string // STS Token
IsCname bool // If cname is in the endpoint.
HTTPTimeout HTTPTimeout // HTTP timeout
HTTPMaxConns HTTPMaxConns // Http max connections
IsUseProxy bool // Flag of using proxy.
ProxyHost string // Flag of using proxy host.
IsAuthProxy bool // Flag of needing authentication.
ProxyUser string // Proxy user
ProxyPassword string // Proxy password
IsEnableMD5 bool // Flag of enabling MD5 for upload.
MD5Threshold int64 // Memory footprint threshold for each MD5 computation (16MB is the default), in byte. When the data is more than that, temp file is used.
IsEnableCRC bool // Flag of enabling CRC for upload.
LogLevel int // Log level
Logger *log.Logger // For write log
UploadLimitSpeed int // Upload limit speed:KB/s, 0 is unlimited
UploadLimiter *OssLimiter // Bandwidth limit reader for upload
CredentialsProvider CredentialsProvider // User provides interface to get AccessKeyID, AccessKeySecret, SecurityToken
LocalAddr net.Addr // local client host info
}
// LimitUploadSpeed uploadSpeed:KB/s, 0 is unlimited,default is 0
func (config *Config) LimitUploadSpeed(uploadSpeed int) error {
if uploadSpeed < 0 {
return fmt.Errorf("invalid argument, the value of uploadSpeed is less than 0")
} else if uploadSpeed == 0 {
config.UploadLimitSpeed = 0
config.UploadLimiter = nil
return nil
}
var err error
config.UploadLimiter, err = GetOssLimiter(uploadSpeed)
if err == nil {
config.UploadLimitSpeed = uploadSpeed
}
return err
}
// WriteLog output log function
func (config *Config) WriteLog(LogLevel int, format string, a ...interface{}) {
if config.LogLevel < LogLevel || config.Logger == nil {
return
}
var logBuffer bytes.Buffer
logBuffer.WriteString(LogTag[LogLevel-1])
logBuffer.WriteString(fmt.Sprintf(format, a...))
config.Logger.Printf("%s", logBuffer.String())
}
// for get Credentials
func (config *Config) GetCredentials() Credentials {
return config.CredentialsProvider.GetCredentials()
}
// getDefaultOssConfig gets the default configuration.
@@ -44,8 +145,8 @@ func getDefaultOssConfig() *Config {
config.AccessKeySecret = ""
config.RetryTimes = 5
config.IsDebug = false
config.UserAgent = userAgent
config.Timeout = 60 // Seconds
config.UserAgent = userAgent()
config.Timeout = 60 // Seconds
config.SecurityToken = ""
config.IsCname = false
@@ -54,6 +155,8 @@ func getDefaultOssConfig() *Config {
config.HTTPTimeout.HeaderTimeout = time.Second * 60 // 60s
config.HTTPTimeout.LongTimeout = time.Second * 300 // 300s
config.HTTPTimeout.IdleConnTimeout = time.Second * 50 // 50s
config.HTTPMaxConns.MaxIdleConns = 100
config.HTTPMaxConns.MaxIdleConnsPerHost = 100
config.IsUseProxy = false
config.ProxyHost = ""
@@ -65,5 +168,11 @@ func getDefaultOssConfig() *Config {
config.IsEnableMD5 = false
config.IsEnableCRC = true
config.LogLevel = LogOff
config.Logger = log.New(os.Stdout, "", log.LstdFlags)
provider := &defaultCredentialsProvider{config: &config}
config.CredentialsProvider = provider
return &config
}

209
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/conn.go generated vendored
View File

@@ -27,25 +27,50 @@ type Conn struct {
client *http.Client
}
var signKeyList = []string{"acl", "uploads", "location", "cors", "logging", "website", "referer", "lifecycle", "delete", "append", "tagging", "objectMeta", "uploadId", "partNumber", "security-token", "position", "img", "style", "styleName", "replication", "replicationProgress", "replicationLocation", "cname", "bucketInfo", "comp", "qos", "live", "status", "vod", "startTime", "endTime", "symlink", "x-oss-process", "response-content-type", "response-content-language", "response-expires", "response-cache-control", "response-content-disposition", "response-content-encoding", "udf", "udfName", "udfImage", "udfId", "udfImageDesc", "udfApplication", "comp", "udfApplicationLog", "restore", "callback", "callback-var"}
var signKeyList = []string{"acl", "uploads", "location", "cors",
"logging", "website", "referer", "lifecycle",
"delete", "append", "tagging", "objectMeta",
"uploadId", "partNumber", "security-token",
"position", "img", "style", "styleName",
"replication", "replicationProgress",
"replicationLocation", "cname", "bucketInfo",
"comp", "qos", "live", "status", "vod",
"startTime", "endTime", "symlink",
"x-oss-process", "response-content-type", "x-oss-traffic-limit",
"response-content-language", "response-expires",
"response-cache-control", "response-content-disposition",
"response-content-encoding", "udf", "udfName", "udfImage",
"udfId", "udfImageDesc", "udfApplication", "comp",
"udfApplicationLog", "restore", "callback", "callback-var", "qosInfo",
"policy", "stat", "encryption", "versions", "versioning", "versionId", "requestPayment"}
// init initializes Conn
func (conn *Conn) init(config *Config, urlMaker *urlMaker) error {
// New transport
transport := newTransport(conn, config)
func (conn *Conn) init(config *Config, urlMaker *urlMaker, client *http.Client) error {
if client == nil {
// New transport
transport := newTransport(conn, config)
// Proxy
if conn.config.IsUseProxy {
proxyURL, err := url.Parse(config.ProxyHost)
if err != nil {
return err
// Proxy
if conn.config.IsUseProxy {
proxyURL, err := url.Parse(config.ProxyHost)
if err != nil {
return err
}
if config.IsAuthProxy {
if config.ProxyPassword != "" {
proxyURL.User = url.UserPassword(config.ProxyUser, config.ProxyPassword)
} else {
proxyURL.User = url.User(config.ProxyUser)
}
}
transport.Proxy = http.ProxyURL(proxyURL)
}
transport.Proxy = http.ProxyURL(proxyURL)
client = &http.Client{Transport: transport}
}
conn.config = config
conn.url = urlMaker
conn.client = &http.Client{Transport: transport}
conn.client = client
return nil
}
@@ -105,19 +130,29 @@ func (conn Conn) DoURL(method HTTPMethod, signedURL string, headers map[string]s
}
// Transfer started
event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength)
event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength, 0)
publishProgress(listener, event)
if conn.config.LogLevel >= Debug {
conn.LoggerHTTPReq(req)
}
resp, err := conn.client.Do(req)
if err != nil {
// Transfer failed
event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength)
event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0)
publishProgress(listener, event)
conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error())
return nil, err
}
if conn.config.LogLevel >= Debug {
//print out http resp
conn.LoggerHTTPResp(req, resp)
}
// Transfer completed
event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength)
event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength, 0)
publishProgress(listener, event)
return conn.handleResponse(resp, crc)
@@ -212,8 +247,10 @@ func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource st
req.Header.Set(HTTPHeaderDate, date)
req.Header.Set(HTTPHeaderHost, conn.config.Endpoint)
req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
if conn.config.SecurityToken != "" {
req.Header.Set(HTTPHeaderOssSecurityToken, conn.config.SecurityToken)
akIf := conn.config.GetCredentials()
if akIf.GetSecurityToken() != "" {
req.Header.Set(HTTPHeaderOssSecurityToken, akIf.GetSecurityToken())
}
if headers != nil {
@@ -225,27 +262,39 @@ func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource st
conn.signHeader(req, canonicalizedResource)
// Transfer started
event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength)
event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength, 0)
publishProgress(listener, event)
if conn.config.LogLevel >= Debug {
conn.LoggerHTTPReq(req)
}
resp, err := conn.client.Do(req)
if err != nil {
// Transfer failed
event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength)
event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0)
publishProgress(listener, event)
conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error())
return nil, err
}
if conn.config.LogLevel >= Debug {
//print out http resp
conn.LoggerHTTPResp(req, resp)
}
// Transfer completed
event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength)
event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength, 0)
publishProgress(listener, event)
return conn.handleResponse(resp, crc)
}
func (conn Conn) signURL(method HTTPMethod, bucketName, objectName string, expiration int64, params map[string]interface{}, headers map[string]string) string {
if conn.config.SecurityToken != "" {
params[HTTPParamSecurityToken] = conn.config.SecurityToken
akIf := conn.config.GetCredentials()
if akIf.GetSecurityToken() != "" {
params[HTTPParamSecurityToken] = akIf.GetSecurityToken()
}
subResource := conn.getSubResource(params)
canonicalizedResource := conn.url.getResource(bucketName, objectName, subResource)
@@ -272,16 +321,38 @@ func (conn Conn) signURL(method HTTPMethod, bucketName, objectName string, expir
}
}
signedStr := conn.getSignedStr(req, canonicalizedResource)
signedStr := conn.getSignedStr(req, canonicalizedResource, akIf.GetAccessKeySecret())
params[HTTPParamExpires] = strconv.FormatInt(expiration, 10)
params[HTTPParamAccessKeyID] = conn.config.AccessKeyID
params[HTTPParamAccessKeyID] = akIf.GetAccessKeyID()
params[HTTPParamSignature] = signedStr
urlParams := conn.getURLParams(params)
return conn.url.getSignURL(bucketName, objectName, urlParams)
}
func (conn Conn) signRtmpURL(bucketName, channelName, playlistName string, expiration int64) string {
params := map[string]interface{}{}
if playlistName != "" {
params[HTTPParamPlaylistName] = playlistName
}
expireStr := strconv.FormatInt(expiration, 10)
params[HTTPParamExpires] = expireStr
akIf := conn.config.GetCredentials()
if akIf.GetAccessKeyID() != "" {
params[HTTPParamAccessKeyID] = akIf.GetAccessKeyID()
if akIf.GetSecurityToken() != "" {
params[HTTPParamSecurityToken] = akIf.GetSecurityToken()
}
signedStr := conn.getRtmpSignedStr(bucketName, channelName, playlistName, expiration, akIf.GetAccessKeySecret(), params)
params[HTTPParamSignature] = signedStr
}
urlParams := conn.getURLParams(params)
return conn.url.getSignRtmpURL(bucketName, channelName, urlParams)
}
// handleBody handles request body
func (conn Conn) handleBody(req *http.Request, body io.Reader, initCRC uint64,
listener ProgressListener, tracker *readerTracker) (*os.File, hash.Hash64) {
@@ -322,11 +393,33 @@ func (conn Conn) handleBody(req *http.Request, body io.Reader, initCRC uint64,
if !ok && reader != nil {
rc = ioutil.NopCloser(reader)
}
req.Body = rc
if conn.isUploadLimitReq(req) {
limitReader := &LimitSpeedReader{
reader: rc,
ossLimiter: conn.config.UploadLimiter,
}
req.Body = limitReader
} else {
req.Body = rc
}
return file, crc
}
// isUploadLimitReq: judge limit upload speed or not
func (conn Conn) isUploadLimitReq(req *http.Request) bool {
if conn.config.UploadLimitSpeed == 0 || conn.config.UploadLimiter == nil {
return false
}
if req.Method != "GET" && req.Method != "DELETE" && req.Method != "HEAD" {
if req.ContentLength > 0 {
return true
}
}
return false
}
func tryGetFileSize(f *os.File) int64 {
fInfo, _ := f.Stat()
return fInfo.Size()
@@ -347,8 +440,10 @@ func (conn Conn) handleResponse(resp *http.Response, crc hash.Hash64) (*Response
}
if len(respBody) == 0 {
// No error in response body
err = fmt.Errorf("oss: service returned empty response body, status = %s, RequestId = %s", resp.Status, resp.Header.Get(HTTPHeaderOssRequestID))
err = ServiceError{
StatusCode: statusCode,
RequestID: resp.Header.Get(HTTPHeaderOssRequestID),
}
} else {
// Response contains storage service error object, unmarshal
srvErr, errIn := serviceErrFromXML(respBody, resp.StatusCode,
@@ -390,6 +485,46 @@ func (conn Conn) handleResponse(resp *http.Response, crc hash.Hash64) (*Response
}, nil
}
// LoggerHTTPReq Print the header information of the http request
func (conn Conn) LoggerHTTPReq(req *http.Request) {
var logBuffer bytes.Buffer
logBuffer.WriteString(fmt.Sprintf("[Req:%p]Method:%s\t", req, req.Method))
logBuffer.WriteString(fmt.Sprintf("Host:%s\t", req.URL.Host))
logBuffer.WriteString(fmt.Sprintf("Path:%s\t", req.URL.Path))
logBuffer.WriteString(fmt.Sprintf("Query:%s\t", req.URL.RawQuery))
logBuffer.WriteString(fmt.Sprintf("Header info:"))
for k, v := range req.Header {
var valueBuffer bytes.Buffer
for j := 0; j < len(v); j++ {
if j > 0 {
valueBuffer.WriteString(" ")
}
valueBuffer.WriteString(v[j])
}
logBuffer.WriteString(fmt.Sprintf("\t%s:%s", k, valueBuffer.String()))
}
conn.config.WriteLog(Debug, "%s\n", logBuffer.String())
}
// LoggerHTTPResp Print Response to http request
func (conn Conn) LoggerHTTPResp(req *http.Request, resp *http.Response) {
var logBuffer bytes.Buffer
logBuffer.WriteString(fmt.Sprintf("[Resp:%p]StatusCode:%d\t", req, resp.StatusCode))
logBuffer.WriteString(fmt.Sprintf("Header info:"))
for k, v := range resp.Header {
var valueBuffer bytes.Buffer
for j := 0; j < len(v); j++ {
if j > 0 {
valueBuffer.WriteString(" ")
}
valueBuffer.WriteString(v[j])
}
logBuffer.WriteString(fmt.Sprintf("\t%s:%s", k, valueBuffer.String()))
}
conn.config.WriteLog(Debug, "%s\n", logBuffer.String())
}
func calcMD5(body io.Reader, contentLen, md5Threshold int64) (reader io.Reader, b64 string, tempFile *os.File, err error) {
if contentLen == 0 || contentLen > md5Threshold {
// Huge body, use temporary file
@@ -521,7 +656,7 @@ type urlMaker struct {
}
// Init parses endpoint
func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) {
func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) error {
if strings.HasPrefix(endpoint, "http://") {
um.Scheme = "http"
um.NetLoc = endpoint[len("http://"):]
@@ -533,6 +668,14 @@ func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) {
um.NetLoc = endpoint
}
//use url.Parse() to get real host
strUrl := um.Scheme + "://" + um.NetLoc
url, err := url.Parse(strUrl)
if err != nil {
return err
}
um.NetLoc = url.Host
host, _, err := net.SplitHostPort(um.NetLoc)
if err != nil {
host = um.NetLoc
@@ -550,6 +693,8 @@ func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) {
um.Type = urlTypeAliyun
}
um.IsProxy = isProxy
return nil
}
// getURL gets URL
@@ -571,6 +716,16 @@ func (um urlMaker) getSignURL(bucket, object, params string) string {
return fmt.Sprintf("%s://%s%s?%s", um.Scheme, host, path, params)
}
// getSignRtmpURL Build Sign Rtmp URL
func (um urlMaker) getSignRtmpURL(bucket, channelName, params string) string {
host, path := um.buildURL(bucket, "live")
channelName = url.QueryEscape(channelName)
channelName = strings.Replace(channelName, "+", "%20", -1)
return fmt.Sprintf("rtmp://%s%s/%s?%s", host, path, channelName, params)
}
// buildURL builds URL
func (um urlMaker) buildURL(bucket, object string) (string, string) {
var host = ""

68
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/const.go generated vendored
View File

@@ -19,6 +19,17 @@ const (
ACLDefault ACLType = "default"
)
// bucket versioning status
type VersioningStatus string
const (
// Versioning Status definition: Enabled
VersionEnabled VersioningStatus = "Enabled"
// Versioning Status definition: Suspended
VersionSuspended VersioningStatus = "Suspended"
)
// MetadataDirectiveType specifying whether use the metadata of source object when copying object.
type MetadataDirectiveType string
@@ -30,6 +41,25 @@ const (
MetaReplace MetadataDirectiveType = "REPLACE"
)
// TaggingDirectiveType specifying whether use the tagging of source object when copying object.
type TaggingDirectiveType string
const (
// TaggingCopy the target object's tagging is copied from the source one
TaggingCopy TaggingDirectiveType = "COPY"
// TaggingReplace the target object's tagging is created as part of the copy request (not same as the source one)
TaggingReplace TaggingDirectiveType = "REPLACE"
)
// AlgorithmType specifying the server side encryption algorithm name
type AlgorithmType string
const (
KMSAlgorithm AlgorithmType = "KMS"
AESAlgorithm AlgorithmType = "AES256"
)
// StorageClassType bucket storage type
type StorageClassType string
@@ -44,12 +74,26 @@ const (
StorageArchive StorageClassType = "Archive"
)
// RedundancyType bucket data Redundancy type
type DataRedundancyType string
const (
// RedundancyLRS Local redundancy, default value
RedundancyLRS DataRedundancyType = "LRS"
// RedundancyZRS Same city redundancy
RedundancyZRS DataRedundancyType = "ZRS"
)
// PayerType the type of request payer
type PayerType string
const (
// Requester the requester who send the request
Requester PayerType = "requester"
Requester PayerType = "Requester"
// BucketOwner the requester who send the request
BucketOwner PayerType = "BucketOwner"
)
// HTTPMethod HTTP request method
@@ -97,12 +141,15 @@ const (
HTTPHeaderIfUnmodifiedSince = "If-Unmodified-Since"
HTTPHeaderIfMatch = "If-Match"
HTTPHeaderIfNoneMatch = "If-None-Match"
HTTPHeaderACReqMethod = "Access-Control-Request-Method"
HTTPHeaderACReqHeaders = "Access-Control-Request-Headers"
HTTPHeaderOssACL = "X-Oss-Acl"
HTTPHeaderOssMetaPrefix = "X-Oss-Meta-"
HTTPHeaderOssObjectACL = "X-Oss-Object-Acl"
HTTPHeaderOssSecurityToken = "X-Oss-Security-Token"
HTTPHeaderOssServerSideEncryption = "X-Oss-Server-Side-Encryption"
HTTPHeaderOssServerSideEncryptionKeyID = "X-Oss-Server-Side-Encryption-Key-Id"
HTTPHeaderOssCopySource = "X-Oss-Copy-Source"
HTTPHeaderOssCopySourceRange = "X-Oss-Copy-Source-Range"
HTTPHeaderOssCopySourceIfMatch = "X-Oss-Copy-Source-If-Match"
@@ -117,7 +164,10 @@ const (
HTTPHeaderOssStorageClass = "X-Oss-Storage-Class"
HTTPHeaderOssCallback = "X-Oss-Callback"
HTTPHeaderOssCallbackVar = "X-Oss-Callback-Var"
HTTPHeaderOSSRequester = "X-Oss-Request-Payer"
HTTPHeaderOssRequester = "X-Oss-Request-Payer"
HTTPHeaderOssTagging = "X-Oss-Tagging"
HTTPHeaderOssTaggingDirective = "X-Oss-Tagging-Directive"
HTTPHeaderOssTrafficLimit = "X-Oss-Traffic-Limit"
)
// HTTP Param
@@ -126,6 +176,7 @@ const (
HTTPParamAccessKeyID = "OSSAccessKeyId"
HTTPParamSignature = "Signature"
HTTPParamSecurityToken = "security-token"
HTTPParamPlaylistName = "playlistName"
)
// Other constants
@@ -140,5 +191,16 @@ const (
CheckpointFileSuffix = ".cp" // Checkpoint file suffix
Version = "1.9.2" // Go SDK version
NullVersion = "null"
Version = "v2.0.4" // Go SDK version
)
// FrameType
const (
DataFrameType = 8388609
ContinuousFrameType = 8388612
EndFrameType = 8388613
MetaEndFrameCSVType = 8388614
MetaEndFrameJSONType = 8388615
)

57
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/download.go generated vendored
View File

@@ -38,8 +38,14 @@ func (bucket Bucket) DownloadFile(objectKey, filePath string, partSize int64, op
cpConf := getCpConfig(options)
routines := getRoutines(options)
var strVersionId string
versionId, _ := findOption(options, "versionId", nil)
if versionId != nil {
strVersionId = versionId.(string)
}
if cpConf != nil && cpConf.IsEnable {
cpFilePath := getDownloadCpFilePath(cpConf, bucket.BucketName, objectKey, filePath)
cpFilePath := getDownloadCpFilePath(cpConf, bucket.BucketName, objectKey, strVersionId, filePath)
if cpFilePath != "" {
return bucket.downloadFileWithCp(objectKey, filePath, partSize, options, cpFilePath, routines, uRange)
}
@@ -48,11 +54,11 @@ func (bucket Bucket) DownloadFile(objectKey, filePath string, partSize int64, op
return bucket.downloadFile(objectKey, filePath, partSize, options, routines, uRange)
}
func getDownloadCpFilePath(cpConf *cpConfig, srcBucket, srcObject, destFile string) string {
func getDownloadCpFilePath(cpConf *cpConfig, srcBucket, srcObject, versionId, destFile string) string {
if cpConf.FilePath == "" && cpConf.DirPath != "" {
src := fmt.Sprintf("oss://%v/%v", srcBucket, srcObject)
absPath, _ := filepath.Abs(destFile)
cpFileName := getCpFileName(src, absPath)
cpFileName := getCpFileName(src, absPath, versionId)
cpConf.FilePath = cpConf.DirPath + string(os.PathSeparator) + cpFileName
}
return cpConf.FilePath
@@ -225,12 +231,6 @@ func (bucket Bucket) downloadFile(objectKey, filePath string, partSize int64, op
tempFilePath := filePath + TempFileSuffix
listener := getProgressListener(options)
payerOptions := []Option{}
payer := getPayer(options)
if payer != "" {
payerOptions = append(payerOptions, RequestPayer(PayerType(payer)))
}
// If the file does not exist, create one. If exists, the download will overwrite it.
fd, err := os.OpenFile(tempFilePath, os.O_WRONLY|os.O_CREATE, FilePermMode)
if err != nil {
@@ -238,7 +238,10 @@ func (bucket Bucket) downloadFile(objectKey, filePath string, partSize int64, op
}
fd.Close()
meta, err := bucket.GetObjectDetailedMeta(objectKey, payerOptions...)
// Get the object detailed meta for object whole size
// must delete header:range to get whole object size
skipOptions := deleteOption(options, HTTPHeaderRange)
meta, err := bucket.GetObjectDetailedMeta(objectKey, skipOptions...)
if err != nil {
return err
}
@@ -266,7 +269,7 @@ func (bucket Bucket) downloadFile(objectKey, filePath string, partSize int64, op
var completedBytes int64
totalBytes := getObjectBytes(parts)
event := newProgressEvent(TransferStartedEvent, 0, totalBytes)
event := newProgressEvent(TransferStartedEvent, 0, totalBytes, 0)
publishProgress(listener, event)
// Start the download workers
@@ -284,13 +287,14 @@ func (bucket Bucket) downloadFile(objectKey, filePath string, partSize int64, op
select {
case part := <-results:
completed++
completedBytes += (part.End - part.Start + 1)
downBytes := (part.End - part.Start + 1)
completedBytes += downBytes
parts[part.Index].CRC64 = part.CRC64
event = newProgressEvent(TransferDataEvent, completedBytes, totalBytes)
event = newProgressEvent(TransferDataEvent, completedBytes, totalBytes, downBytes)
publishProgress(listener, event)
case err := <-failed:
close(die)
event = newProgressEvent(TransferFailedEvent, completedBytes, totalBytes)
event = newProgressEvent(TransferFailedEvent, completedBytes, totalBytes, 0)
publishProgress(listener, event)
return err
}
@@ -300,7 +304,7 @@ func (bucket Bucket) downloadFile(objectKey, filePath string, partSize int64, op
}
}
event = newProgressEvent(TransferCompletedEvent, completedBytes, totalBytes)
event = newProgressEvent(TransferCompletedEvent, completedBytes, totalBytes, 0)
publishProgress(listener, event)
if enableCRC {
@@ -474,12 +478,6 @@ func (bucket Bucket) downloadFileWithCp(objectKey, filePath string, partSize int
tempFilePath := filePath + TempFileSuffix
listener := getProgressListener(options)
payerOptions := []Option{}
payer := getPayer(options)
if payer != "" {
payerOptions = append(payerOptions, RequestPayer(PayerType(payer)))
}
// Load checkpoint data.
dcp := downloadCheckpoint{}
err := dcp.load(cpFilePath)
@@ -487,8 +485,10 @@ func (bucket Bucket) downloadFileWithCp(objectKey, filePath string, partSize int
os.Remove(cpFilePath)
}
// Get the object detailed meta.
meta, err := bucket.GetObjectDetailedMeta(objectKey, payerOptions...)
// Get the object detailed meta for object whole size
// must delete header:range to get whole object size
skipOptions := deleteOption(options, HTTPHeaderRange)
meta, err := bucket.GetObjectDetailedMeta(objectKey, skipOptions...)
if err != nil {
return err
}
@@ -517,7 +517,7 @@ func (bucket Bucket) downloadFileWithCp(objectKey, filePath string, partSize int
die := make(chan bool)
completedBytes := dcp.getCompletedBytes()
event := newProgressEvent(TransferStartedEvent, completedBytes, dcp.ObjStat.Size)
event := newProgressEvent(TransferStartedEvent, completedBytes, dcp.ObjStat.Size, 0)
publishProgress(listener, event)
// Start the download workers routine
@@ -538,12 +538,13 @@ func (bucket Bucket) downloadFileWithCp(objectKey, filePath string, partSize int
dcp.PartStat[part.Index] = true
dcp.Parts[part.Index].CRC64 = part.CRC64
dcp.dump(cpFilePath)
completedBytes += (part.End - part.Start + 1)
event = newProgressEvent(TransferDataEvent, completedBytes, dcp.ObjStat.Size)
downBytes := (part.End - part.Start + 1)
completedBytes += downBytes
event = newProgressEvent(TransferDataEvent, completedBytes, dcp.ObjStat.Size, downBytes)
publishProgress(listener, event)
case err := <-failed:
close(die)
event = newProgressEvent(TransferFailedEvent, completedBytes, dcp.ObjStat.Size)
event = newProgressEvent(TransferFailedEvent, completedBytes, dcp.ObjStat.Size, 0)
publishProgress(listener, event)
return err
}
@@ -553,7 +554,7 @@ func (bucket Bucket) downloadFileWithCp(objectKey, filePath string, partSize int
}
}
event = newProgressEvent(TransferCompletedEvent, completedBytes, dcp.ObjStat.Size)
event = newProgressEvent(TransferCompletedEvent, completedBytes, dcp.ObjStat.Size, 0)
publishProgress(listener, event)
if dcp.enableCRC {

9
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/error.go generated vendored
View File

@@ -14,14 +14,19 @@ type ServiceError struct {
Message string `xml:"Message"` // The detail error message from OSS
RequestID string `xml:"RequestId"` // The UUID used to uniquely identify the request
HostID string `xml:"HostId"` // The OSS server cluster's Id
Endpoint string `xml:"Endpoint"`
RawMessage string // The raw messages from OSS
StatusCode int // HTTP status code
}
// Error implements interface error
func (e ServiceError) Error() string {
return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s",
e.StatusCode, e.Code, e.Message, e.RequestID)
if e.Endpoint == "" {
return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=\"%s\", RequestId=%s",
e.StatusCode, e.Code, e.Message, e.RequestID)
}
return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=\"%s\", RequestId=%s, Endpoint=%s",
e.StatusCode, e.Code, e.Message, e.RequestID, e.Endpoint)
}
// UnexpectedStatusCodeError is returned when a storage service responds with neither an error

28
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/limit_reader_1_6.go generated vendored Normal file
View File

@@ -0,0 +1,28 @@
// +build !go1.7
// "golang.org/x/time/rate" is depended on golang context package go1.7 onward
// this file is only for build,not supports limit upload speed
package oss
import (
"fmt"
"io"
)
const (
perTokenBandwidthSize int = 1024
)
type OssLimiter struct {
}
type LimitSpeedReader struct {
io.ReadCloser
reader io.Reader
ossLimiter *OssLimiter
}
func GetOssLimiter(uploadSpeed int) (ossLimiter *OssLimiter, err error) {
err = fmt.Errorf("rate.Limiter is not supported below version go1.7")
return nil, err
}

90
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/limit_reader_1_7.go generated vendored Normal file
View File

@@ -0,0 +1,90 @@
// +build go1.7
package oss
import (
"fmt"
"io"
"math"
"time"
"golang.org/x/time/rate"
)
const (
perTokenBandwidthSize int = 1024
)
// OssLimiter wrapper rate.Limiter
type OssLimiter struct {
limiter *rate.Limiter
}
// GetOssLimiter create OssLimiter
// uploadSpeed KB/s
func GetOssLimiter(uploadSpeed int) (ossLimiter *OssLimiter, err error) {
limiter := rate.NewLimiter(rate.Limit(uploadSpeed), uploadSpeed)
// first consume the initial full token,the limiter will behave more accurately
limiter.AllowN(time.Now(), uploadSpeed)
return &OssLimiter{
limiter: limiter,
}, nil
}
// LimitSpeedReader for limit bandwidth upload
type LimitSpeedReader struct {
io.ReadCloser
reader io.Reader
ossLimiter *OssLimiter
}
// Read
func (r *LimitSpeedReader) Read(p []byte) (n int, err error) {
n = 0
err = nil
start := 0
burst := r.ossLimiter.limiter.Burst()
var end int
var tmpN int
var tc int
for start < len(p) {
if start+burst*perTokenBandwidthSize < len(p) {
end = start + burst*perTokenBandwidthSize
} else {
end = len(p)
}
tmpN, err = r.reader.Read(p[start:end])
if tmpN > 0 {
n += tmpN
start = n
}
if err != nil {
return
}
tc = int(math.Ceil(float64(tmpN) / float64(perTokenBandwidthSize)))
now := time.Now()
re := r.ossLimiter.limiter.ReserveN(now, tc)
if !re.OK() {
err = fmt.Errorf("LimitSpeedReader.Read() failure,ReserveN error,start:%d,end:%d,burst:%d,perTokenBandwidthSize:%d",
start, end, burst, perTokenBandwidthSize)
return
}
timeDelay := re.Delay()
time.Sleep(timeDelay)
}
return
}
// Close ...
func (r *LimitSpeedReader) Close() error {
rc, ok := r.reader.(io.ReadCloser)
if ok {
return rc.Close()
}
return nil
}

257
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/livechannel.go generated vendored Normal file
View File

@@ -0,0 +1,257 @@
package oss
import (
"bytes"
"encoding/xml"
"fmt"
"io"
"net/http"
"strconv"
"time"
)
//
// CreateLiveChannel create a live-channel
//
// channelName the name of the channel
// config configuration of the channel
//
// CreateLiveChannelResult the result of create live-channel
// error nil if success, otherwise error
//
func (bucket Bucket) CreateLiveChannel(channelName string, config LiveChannelConfiguration) (CreateLiveChannelResult, error) {
var out CreateLiveChannelResult
bs, err := xml.Marshal(config)
if err != nil {
return out, err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
params := map[string]interface{}{}
params["live"] = nil
resp, err := bucket.do("PUT", channelName, params, nil, buffer, nil)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
//
// PutLiveChannelStatus Set the status of the live-channel: enabled/disabled
//
// channelName the name of the channel
// status enabled/disabled
//
// error nil if success, otherwise error
//
func (bucket Bucket) PutLiveChannelStatus(channelName, status string) error {
params := map[string]interface{}{}
params["live"] = nil
params["status"] = status
resp, err := bucket.do("PUT", channelName, params, nil, nil, nil)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// PostVodPlaylist create an playlist based on the specified playlist name, startTime and endTime
//
// channelName the name of the channel
// playlistName the name of the playlist, must end with ".m3u8"
// startTime the start time of the playlist
// endTime the endtime of the playlist
//
// error nil if success, otherwise error
//
func (bucket Bucket) PostVodPlaylist(channelName, playlistName string, startTime, endTime time.Time) error {
params := map[string]interface{}{}
params["vod"] = nil
params["startTime"] = strconv.FormatInt(startTime.Unix(), 10)
params["endTime"] = strconv.FormatInt(endTime.Unix(), 10)
key := fmt.Sprintf("%s/%s", channelName, playlistName)
resp, err := bucket.do("POST", key, params, nil, nil, nil)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusOK})
}
// GetVodPlaylist get the playlist based on the specified channelName, startTime and endTime
//
// channelName the name of the channel
// startTime the start time of the playlist
// endTime the endtime of the playlist
//
// io.ReadCloser reader instance for reading data from response. It must be called close() after the usage and only valid when error is nil.
// error nil if success, otherwise error
//
func (bucket Bucket) GetVodPlaylist(channelName string, startTime, endTime time.Time) (io.ReadCloser, error) {
params := map[string]interface{}{}
params["vod"] = nil
params["startTime"] = strconv.FormatInt(startTime.Unix(), 10)
params["endTime"] = strconv.FormatInt(endTime.Unix(), 10)
resp, err := bucket.do("GET", channelName, params, nil, nil, nil)
if err != nil {
return nil, err
}
return resp.Body, nil
}
//
// GetLiveChannelStat Get the state of the live-channel
//
// channelName the name of the channel
//
// LiveChannelStat the state of the live-channel
// error nil if success, otherwise error
//
func (bucket Bucket) GetLiveChannelStat(channelName string) (LiveChannelStat, error) {
var out LiveChannelStat
params := map[string]interface{}{}
params["live"] = nil
params["comp"] = "stat"
resp, err := bucket.do("GET", channelName, params, nil, nil, nil)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
//
// GetLiveChannelInfo Get the configuration info of the live-channel
//
// channelName the name of the channel
//
// LiveChannelConfiguration the configuration info of the live-channel
// error nil if success, otherwise error
//
func (bucket Bucket) GetLiveChannelInfo(channelName string) (LiveChannelConfiguration, error) {
var out LiveChannelConfiguration
params := map[string]interface{}{}
params["live"] = nil
resp, err := bucket.do("GET", channelName, params, nil, nil, nil)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
//
// GetLiveChannelHistory Get push records of live-channel
//
// channelName the name of the channel
//
// LiveChannelHistory push records
// error nil if success, otherwise error
//
func (bucket Bucket) GetLiveChannelHistory(channelName string) (LiveChannelHistory, error) {
var out LiveChannelHistory
params := map[string]interface{}{}
params["live"] = nil
params["comp"] = "history"
resp, err := bucket.do("GET", channelName, params, nil, nil, nil)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
//
// ListLiveChannel list the live-channels
//
// options Prefix: filter by the name start with the value of "Prefix"
// MaxKeys: the maximum count returned
// Marker: cursor from which starting list
//
// ListLiveChannelResult live-channel list
// error nil if success, otherwise error
//
func (bucket Bucket) ListLiveChannel(options ...Option) (ListLiveChannelResult, error) {
var out ListLiveChannelResult
params, err := getRawParams(options)
if err != nil {
return out, err
}
params["live"] = nil
resp, err := bucket.do("GET", "", params, nil, nil, nil)
if err != nil {
return out, err
}
defer resp.Body.Close()
err = xmlUnmarshal(resp.Body, &out)
return out, err
}
//
// DeleteLiveChannel Delete the live-channel. When a client trying to stream the live-channel, the operation will fail. it will only delete the live-channel itself and the object generated by the live-channel will not be deleted.
//
// channelName the name of the channel
//
// error nil if success, otherwise error
//
func (bucket Bucket) DeleteLiveChannel(channelName string) error {
params := map[string]interface{}{}
params["live"] = nil
if channelName == "" {
return fmt.Errorf("invalid argument: channel name is empty")
}
resp, err := bucket.do("DELETE", channelName, params, nil, nil, nil)
if err != nil {
return err
}
defer resp.Body.Close()
return checkRespCode(resp.StatusCode, []int{http.StatusNoContent})
}
//
// SignRtmpURL Generate a RTMP push-stream signature URL for the trusted user to push the RTMP stream to the live-channel.
//
// channelName the name of the channel
// playlistName the name of the playlist, must end with ".m3u8"
// expires expiration (in seconds)
//
// string singed rtmp push stream url
// error nil if success, otherwise error
//
func (bucket Bucket) SignRtmpURL(channelName, playlistName string, expires int64) (string, error) {
if expires <= 0 {
return "", fmt.Errorf("invalid argument: %d, expires must greater than 0", expires)
}
expiration := time.Now().Unix() + expires
return bucket.Client.Conn.signRtmpURL(bucket.BucketName, channelName, playlistName, expiration), nil
}

777
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/mime.go generated vendored
View File

@@ -7,231 +7,558 @@ import (
)
var extToMimeType = map[string]string{
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
".potx": "application/vnd.openxmlformats-officedocument.presentationml.template",
".ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
".sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide",
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
".xlam": "application/vnd.ms-excel.addin.macroEnabled.12",
".xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
".apk": "application/vnd.android.package-archive",
".hqx": "application/mac-binhex40",
".cpt": "application/mac-compactpro",
".doc": "application/msword",
".ogg": "application/ogg",
".pdf": "application/pdf",
".rtf": "text/rtf",
".mif": "application/vnd.mif",
".xls": "application/vnd.ms-excel",
".ppt": "application/vnd.ms-powerpoint",
".odc": "application/vnd.oasis.opendocument.chart",
".odb": "application/vnd.oasis.opendocument.database",
".odf": "application/vnd.oasis.opendocument.formula",
".odg": "application/vnd.oasis.opendocument.graphics",
".otg": "application/vnd.oasis.opendocument.graphics-template",
".odi": "application/vnd.oasis.opendocument.image",
".odp": "application/vnd.oasis.opendocument.presentation",
".otp": "application/vnd.oasis.opendocument.presentation-template",
".ods": "application/vnd.oasis.opendocument.spreadsheet",
".ots": "application/vnd.oasis.opendocument.spreadsheet-template",
".odt": "application/vnd.oasis.opendocument.text",
".odm": "application/vnd.oasis.opendocument.text-master",
".ott": "application/vnd.oasis.opendocument.text-template",
".oth": "application/vnd.oasis.opendocument.text-web",
".sxw": "application/vnd.sun.xml.writer",
".stw": "application/vnd.sun.xml.writer.template",
".sxc": "application/vnd.sun.xml.calc",
".stc": "application/vnd.sun.xml.calc.template",
".sxd": "application/vnd.sun.xml.draw",
".std": "application/vnd.sun.xml.draw.template",
".sxi": "application/vnd.sun.xml.impress",
".sti": "application/vnd.sun.xml.impress.template",
".sxg": "application/vnd.sun.xml.writer.global",
".sxm": "application/vnd.sun.xml.math",
".sis": "application/vnd.symbian.install",
".wbxml": "application/vnd.wap.wbxml",
".wmlc": "application/vnd.wap.wmlc",
".wmlsc": "application/vnd.wap.wmlscriptc",
".bcpio": "application/x-bcpio",
".torrent": "application/x-bittorrent",
".bz2": "application/x-bzip2",
".vcd": "application/x-cdlink",
".pgn": "application/x-chess-pgn",
".cpio": "application/x-cpio",
".csh": "application/x-csh",
".dvi": "application/x-dvi",
".spl": "application/x-futuresplash",
".gtar": "application/x-gtar",
".hdf": "application/x-hdf",
".jar": "application/x-java-archive",
".jnlp": "application/x-java-jnlp-file",
".js": "application/x-javascript",
".ksp": "application/x-kspread",
".chrt": "application/x-kchart",
".kil": "application/x-killustrator",
".latex": "application/x-latex",
".rpm": "application/x-rpm",
".sh": "application/x-sh",
".shar": "application/x-shar",
".swf": "application/x-shockwave-flash",
".sit": "application/x-stuffit",
".sv4cpio": "application/x-sv4cpio",
".sv4crc": "application/x-sv4crc",
".tar": "application/x-tar",
".tcl": "application/x-tcl",
".tex": "application/x-tex",
".man": "application/x-troff-man",
".me": "application/x-troff-me",
".ms": "application/x-troff-ms",
".ustar": "application/x-ustar",
".src": "application/x-wais-source",
".zip": "application/zip",
".m3u": "audio/x-mpegurl",
".ra": "audio/x-pn-realaudio",
".wav": "audio/x-wav",
".wma": "audio/x-ms-wma",
".wax": "audio/x-ms-wax",
".pdb": "chemical/x-pdb",
".xyz": "chemical/x-xyz",
".bmp": "image/bmp",
".gif": "image/gif",
".ief": "image/ief",
".png": "image/png",
".wbmp": "image/vnd.wap.wbmp",
".ras": "image/x-cmu-raster",
".pnm": "image/x-portable-anymap",
".pbm": "image/x-portable-bitmap",
".pgm": "image/x-portable-graymap",
".ppm": "image/x-portable-pixmap",
".rgb": "image/x-rgb",
".xbm": "image/x-xbitmap",
".xpm": "image/x-xpixmap",
".xwd": "image/x-xwindowdump",
".css": "text/css",
".rtx": "text/richtext",
".tsv": "text/tab-separated-values",
".jad": "text/vnd.sun.j2me.app-descriptor",
".wml": "text/vnd.wap.wml",
".wmls": "text/vnd.wap.wmlscript",
".etx": "text/x-setext",
".mxu": "video/vnd.mpegurl",
".flv": "video/x-flv",
".wm": "video/x-ms-wm",
".wmv": "video/x-ms-wmv",
".wmx": "video/x-ms-wmx",
".wvx": "video/x-ms-wvx",
".avi": "video/x-msvideo",
".movie": "video/x-sgi-movie",
".ice": "x-conference/x-cooltalk",
".3gp": "video/3gpp",
".ai": "application/postscript",
".aif": "audio/x-aiff",
".aifc": "audio/x-aiff",
".aiff": "audio/x-aiff",
".asc": "text/plain",
".atom": "application/atom+xml",
".au": "audio/basic",
".bin": "application/octet-stream",
".cdf": "application/x-netcdf",
".cgm": "image/cgm",
".class": "application/octet-stream",
".dcr": "application/x-director",
".dif": "video/x-dv",
".dir": "application/x-director",
".djv": "image/vnd.djvu",
".djvu": "image/vnd.djvu",
".dll": "application/octet-stream",
".dmg": "application/octet-stream",
".dms": "application/octet-stream",
".dtd": "application/xml-dtd",
".dv": "video/x-dv",
".dxr": "application/x-director",
".eps": "application/postscript",
".exe": "application/octet-stream",
".ez": "application/andrew-inset",
".gram": "application/srgs",
".grxml": "application/srgs+xml",
".gz": "application/x-gzip",
".htm": "text/html",
".html": "text/html",
".ico": "image/x-icon",
".ics": "text/calendar",
".ifb": "text/calendar",
".iges": "model/iges",
".igs": "model/iges",
".jp2": "image/jp2",
".jpe": "image/jpeg",
".jpeg": "image/jpeg",
".jpg": "image/jpeg",
".kar": "audio/midi",
".lha": "application/octet-stream",
".lzh": "application/octet-stream",
".m4a": "audio/mp4a-latm",
".m4p": "audio/mp4a-latm",
".m4u": "video/vnd.mpegurl",
".m4v": "video/x-m4v",
".mac": "image/x-macpaint",
".mathml": "application/mathml+xml",
".mesh": "model/mesh",
".mid": "audio/midi",
".midi": "audio/midi",
".mov": "video/quicktime",
".mp2": "audio/mpeg",
".mp3": "audio/mpeg",
".mp4": "video/mp4",
".mpe": "video/mpeg",
".mpeg": "video/mpeg",
".mpg": "video/mpeg",
".mpga": "audio/mpeg",
".msh": "model/mesh",
".nc": "application/x-netcdf",
".oda": "application/oda",
".ogv": "video/ogv",
".pct": "image/pict",
".pic": "image/pict",
".pict": "image/pict",
".pnt": "image/x-macpaint",
".pntg": "image/x-macpaint",
".ps": "application/postscript",
".qt": "video/quicktime",
".qti": "image/x-quicktime",
".qtif": "image/x-quicktime",
".ram": "audio/x-pn-realaudio",
".rdf": "application/rdf+xml",
".rm": "application/vnd.rn-realmedia",
".roff": "application/x-troff",
".sgm": "text/sgml",
".sgml": "text/sgml",
".silo": "model/mesh",
".skd": "application/x-koan",
".skm": "application/x-koan",
".skp": "application/x-koan",
".skt": "application/x-koan",
".smi": "application/smil",
".smil": "application/smil",
".snd": "audio/basic",
".so": "application/octet-stream",
".svg": "image/svg+xml",
".t": "application/x-troff",
".texi": "application/x-texinfo",
".texinfo": "application/x-texinfo",
".tif": "image/tiff",
".tiff": "image/tiff",
".tr": "application/x-troff",
".txt": "text/plain",
".vrml": "model/vrml",
".vxml": "application/voicexml+xml",
".webm": "video/webm",
".wrl": "model/vrml",
".xht": "application/xhtml+xml",
".xhtml": "application/xhtml+xml",
".xml": "application/xml",
".xsl": "application/xml",
".xslt": "application/xslt+xml",
".xul": "application/vnd.mozilla.xul+xml",
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
".potx": "application/vnd.openxmlformats-officedocument.presentationml.template",
".ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
".sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide",
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
".xlam": "application/vnd.ms-excel.addin.macroEnabled.12",
".xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
".apk": "application/vnd.android.package-archive",
".hqx": "application/mac-binhex40",
".cpt": "application/mac-compactpro",
".doc": "application/msword",
".ogg": "application/ogg",
".pdf": "application/pdf",
".rtf": "text/rtf",
".mif": "application/vnd.mif",
".xls": "application/vnd.ms-excel",
".ppt": "application/vnd.ms-powerpoint",
".odc": "application/vnd.oasis.opendocument.chart",
".odb": "application/vnd.oasis.opendocument.database",
".odf": "application/vnd.oasis.opendocument.formula",
".odg": "application/vnd.oasis.opendocument.graphics",
".otg": "application/vnd.oasis.opendocument.graphics-template",
".odi": "application/vnd.oasis.opendocument.image",
".odp": "application/vnd.oasis.opendocument.presentation",
".otp": "application/vnd.oasis.opendocument.presentation-template",
".ods": "application/vnd.oasis.opendocument.spreadsheet",
".ots": "application/vnd.oasis.opendocument.spreadsheet-template",
".odt": "application/vnd.oasis.opendocument.text",
".odm": "application/vnd.oasis.opendocument.text-master",
".ott": "application/vnd.oasis.opendocument.text-template",
".oth": "application/vnd.oasis.opendocument.text-web",
".sxw": "application/vnd.sun.xml.writer",
".stw": "application/vnd.sun.xml.writer.template",
".sxc": "application/vnd.sun.xml.calc",
".stc": "application/vnd.sun.xml.calc.template",
".sxd": "application/vnd.sun.xml.draw",
".std": "application/vnd.sun.xml.draw.template",
".sxi": "application/vnd.sun.xml.impress",
".sti": "application/vnd.sun.xml.impress.template",
".sxg": "application/vnd.sun.xml.writer.global",
".sxm": "application/vnd.sun.xml.math",
".sis": "application/vnd.symbian.install",
".wbxml": "application/vnd.wap.wbxml",
".wmlc": "application/vnd.wap.wmlc",
".wmlsc": "application/vnd.wap.wmlscriptc",
".bcpio": "application/x-bcpio",
".torrent": "application/x-bittorrent",
".bz2": "application/x-bzip2",
".vcd": "application/x-cdlink",
".pgn": "application/x-chess-pgn",
".cpio": "application/x-cpio",
".csh": "application/x-csh",
".dvi": "application/x-dvi",
".spl": "application/x-futuresplash",
".gtar": "application/x-gtar",
".hdf": "application/x-hdf",
".jar": "application/x-java-archive",
".jnlp": "application/x-java-jnlp-file",
".js": "application/x-javascript",
".ksp": "application/x-kspread",
".chrt": "application/x-kchart",
".kil": "application/x-killustrator",
".latex": "application/x-latex",
".rpm": "application/x-rpm",
".sh": "application/x-sh",
".shar": "application/x-shar",
".swf": "application/x-shockwave-flash",
".sit": "application/x-stuffit",
".sv4cpio": "application/x-sv4cpio",
".sv4crc": "application/x-sv4crc",
".tar": "application/x-tar",
".tcl": "application/x-tcl",
".tex": "application/x-tex",
".man": "application/x-troff-man",
".me": "application/x-troff-me",
".ms": "application/x-troff-ms",
".ustar": "application/x-ustar",
".src": "application/x-wais-source",
".zip": "application/zip",
".m3u": "audio/x-mpegurl",
".ra": "audio/x-pn-realaudio",
".wav": "audio/x-wav",
".wma": "audio/x-ms-wma",
".wax": "audio/x-ms-wax",
".pdb": "chemical/x-pdb",
".xyz": "chemical/x-xyz",
".bmp": "image/bmp",
".gif": "image/gif",
".ief": "image/ief",
".png": "image/png",
".wbmp": "image/vnd.wap.wbmp",
".ras": "image/x-cmu-raster",
".pnm": "image/x-portable-anymap",
".pbm": "image/x-portable-bitmap",
".pgm": "image/x-portable-graymap",
".ppm": "image/x-portable-pixmap",
".rgb": "image/x-rgb",
".xbm": "image/x-xbitmap",
".xpm": "image/x-xpixmap",
".xwd": "image/x-xwindowdump",
".css": "text/css",
".rtx": "text/richtext",
".tsv": "text/tab-separated-values",
".jad": "text/vnd.sun.j2me.app-descriptor",
".wml": "text/vnd.wap.wml",
".wmls": "text/vnd.wap.wmlscript",
".etx": "text/x-setext",
".mxu": "video/vnd.mpegurl",
".flv": "video/x-flv",
".wm": "video/x-ms-wm",
".wmv": "video/x-ms-wmv",
".wmx": "video/x-ms-wmx",
".wvx": "video/x-ms-wvx",
".avi": "video/x-msvideo",
".movie": "video/x-sgi-movie",
".ice": "x-conference/x-cooltalk",
".3gp": "video/3gpp",
".ai": "application/postscript",
".aif": "audio/x-aiff",
".aifc": "audio/x-aiff",
".aiff": "audio/x-aiff",
".asc": "text/plain",
".atom": "application/atom+xml",
".au": "audio/basic",
".bin": "application/octet-stream",
".cdf": "application/x-netcdf",
".cgm": "image/cgm",
".class": "application/octet-stream",
".dcr": "application/x-director",
".dif": "video/x-dv",
".dir": "application/x-director",
".djv": "image/vnd.djvu",
".djvu": "image/vnd.djvu",
".dll": "application/octet-stream",
".dmg": "application/octet-stream",
".dms": "application/octet-stream",
".dtd": "application/xml-dtd",
".dv": "video/x-dv",
".dxr": "application/x-director",
".eps": "application/postscript",
".exe": "application/octet-stream",
".ez": "application/andrew-inset",
".gram": "application/srgs",
".grxml": "application/srgs+xml",
".gz": "application/x-gzip",
".htm": "text/html",
".html": "text/html",
".ico": "image/x-icon",
".ics": "text/calendar",
".ifb": "text/calendar",
".iges": "model/iges",
".igs": "model/iges",
".jp2": "image/jp2",
".jpe": "image/jpeg",
".jpeg": "image/jpeg",
".jpg": "image/jpeg",
".kar": "audio/midi",
".lha": "application/octet-stream",
".lzh": "application/octet-stream",
".m4a": "audio/mp4a-latm",
".m4p": "audio/mp4a-latm",
".m4u": "video/vnd.mpegurl",
".m4v": "video/x-m4v",
".mac": "image/x-macpaint",
".mathml": "application/mathml+xml",
".mesh": "model/mesh",
".mid": "audio/midi",
".midi": "audio/midi",
".mov": "video/quicktime",
".mp2": "audio/mpeg",
".mp3": "audio/mpeg",
".mp4": "video/mp4",
".mpe": "video/mpeg",
".mpeg": "video/mpeg",
".mpg": "video/mpeg",
".mpga": "audio/mpeg",
".msh": "model/mesh",
".nc": "application/x-netcdf",
".oda": "application/oda",
".ogv": "video/ogv",
".pct": "image/pict",
".pic": "image/pict",
".pict": "image/pict",
".pnt": "image/x-macpaint",
".pntg": "image/x-macpaint",
".ps": "application/postscript",
".qt": "video/quicktime",
".qti": "image/x-quicktime",
".qtif": "image/x-quicktime",
".ram": "audio/x-pn-realaudio",
".rdf": "application/rdf+xml",
".rm": "application/vnd.rn-realmedia",
".roff": "application/x-troff",
".sgm": "text/sgml",
".sgml": "text/sgml",
".silo": "model/mesh",
".skd": "application/x-koan",
".skm": "application/x-koan",
".skp": "application/x-koan",
".skt": "application/x-koan",
".smi": "application/smil",
".smil": "application/smil",
".snd": "audio/basic",
".so": "application/octet-stream",
".svg": "image/svg+xml",
".t": "application/x-troff",
".texi": "application/x-texinfo",
".texinfo": "application/x-texinfo",
".tif": "image/tiff",
".tiff": "image/tiff",
".tr": "application/x-troff",
".txt": "text/plain",
".vrml": "model/vrml",
".vxml": "application/voicexml+xml",
".webm": "video/webm",
".wrl": "model/vrml",
".xht": "application/xhtml+xml",
".xhtml": "application/xhtml+xml",
".xml": "application/xml",
".xsl": "application/xml",
".xslt": "application/xslt+xml",
".xul": "application/vnd.mozilla.xul+xml",
".webp": "image/webp",
".323": "text/h323",
".aab": "application/x-authoware-bin",
".aam": "application/x-authoware-map",
".aas": "application/x-authoware-seg",
".acx": "application/internet-property-stream",
".als": "audio/X-Alpha5",
".amc": "application/x-mpeg",
".ani": "application/octet-stream",
".asd": "application/astound",
".asf": "video/x-ms-asf",
".asn": "application/astound",
".asp": "application/x-asap",
".asr": "video/x-ms-asf",
".asx": "video/x-ms-asf",
".avb": "application/octet-stream",
".awb": "audio/amr-wb",
".axs": "application/olescript",
".bas": "text/plain",
".bin ": "application/octet-stream",
".bld": "application/bld",
".bld2": "application/bld2",
".bpk": "application/octet-stream",
".c": "text/plain",
".cal": "image/x-cals",
".cat": "application/vnd.ms-pkiseccat",
".ccn": "application/x-cnc",
".cco": "application/x-cocoa",
".cer": "application/x-x509-ca-cert",
".cgi": "magnus-internal/cgi",
".chat": "application/x-chat",
".clp": "application/x-msclip",
".cmx": "image/x-cmx",
".co": "application/x-cult3d-object",
".cod": "image/cis-cod",
".conf": "text/plain",
".cpp": "text/plain",
".crd": "application/x-mscardfile",
".crl": "application/pkix-crl",
".crt": "application/x-x509-ca-cert",
".csm": "chemical/x-csml",
".csml": "chemical/x-csml",
".cur": "application/octet-stream",
".dcm": "x-lml/x-evm",
".dcx": "image/x-dcx",
".der": "application/x-x509-ca-cert",
".dhtml": "text/html",
".dot": "application/msword",
".dwf": "drawing/x-dwf",
".dwg": "application/x-autocad",
".dxf": "application/x-autocad",
".ebk": "application/x-expandedbook",
".emb": "chemical/x-embl-dl-nucleotide",
".embl": "chemical/x-embl-dl-nucleotide",
".epub": "application/epub+zip",
".eri": "image/x-eri",
".es": "audio/echospeech",
".esl": "audio/echospeech",
".etc": "application/x-earthtime",
".evm": "x-lml/x-evm",
".evy": "application/envoy",
".fh4": "image/x-freehand",
".fh5": "image/x-freehand",
".fhc": "image/x-freehand",
".fif": "application/fractals",
".flr": "x-world/x-vrml",
".fm": "application/x-maker",
".fpx": "image/x-fpx",
".fvi": "video/isivideo",
".gau": "chemical/x-gaussian-input",
".gca": "application/x-gca-compressed",
".gdb": "x-lml/x-gdb",
".gps": "application/x-gps",
".h": "text/plain",
".hdm": "text/x-hdml",
".hdml": "text/x-hdml",
".hlp": "application/winhlp",
".hta": "application/hta",
".htc": "text/x-component",
".hts": "text/html",
".htt": "text/webviewhtml",
".ifm": "image/gif",
".ifs": "image/ifs",
".iii": "application/x-iphone",
".imy": "audio/melody",
".ins": "application/x-internet-signup",
".ips": "application/x-ipscript",
".ipx": "application/x-ipix",
".isp": "application/x-internet-signup",
".it": "audio/x-mod",
".itz": "audio/x-mod",
".ivr": "i-world/i-vrml",
".j2k": "image/j2k",
".jam": "application/x-jam",
".java": "text/plain",
".jfif": "image/pipeg",
".jpz": "image/jpeg",
".jwc": "application/jwc",
".kjx": "application/x-kjx",
".lak": "x-lml/x-lak",
".lcc": "application/fastman",
".lcl": "application/x-digitalloca",
".lcr": "application/x-digitalloca",
".lgh": "application/lgh",
".lml": "x-lml/x-lml",
".lmlpack": "x-lml/x-lmlpack",
".log": "text/plain",
".lsf": "video/x-la-asf",
".lsx": "video/x-la-asf",
".m13": "application/x-msmediaview",
".m14": "application/x-msmediaview",
".m15": "audio/x-mod",
".m3url": "audio/x-mpegurl",
".m4b": "audio/mp4a-latm",
".ma1": "audio/ma1",
".ma2": "audio/ma2",
".ma3": "audio/ma3",
".ma5": "audio/ma5",
".map": "magnus-internal/imagemap",
".mbd": "application/mbedlet",
".mct": "application/x-mascot",
".mdb": "application/x-msaccess",
".mdz": "audio/x-mod",
".mel": "text/x-vmel",
".mht": "message/rfc822",
".mhtml": "message/rfc822",
".mi": "application/x-mif",
".mil": "image/x-cals",
".mio": "audio/x-mio",
".mmf": "application/x-skt-lbs",
".mng": "video/x-mng",
".mny": "application/x-msmoney",
".moc": "application/x-mocha",
".mocha": "application/x-mocha",
".mod": "audio/x-mod",
".mof": "application/x-yumekara",
".mol": "chemical/x-mdl-molfile",
".mop": "chemical/x-mopac-input",
".mpa": "video/mpeg",
".mpc": "application/vnd.mpohun.certificate",
".mpg4": "video/mp4",
".mpn": "application/vnd.mophun.application",
".mpp": "application/vnd.ms-project",
".mps": "application/x-mapserver",
".mpv2": "video/mpeg",
".mrl": "text/x-mrml",
".mrm": "application/x-mrm",
".msg": "application/vnd.ms-outlook",
".mts": "application/metastream",
".mtx": "application/metastream",
".mtz": "application/metastream",
".mvb": "application/x-msmediaview",
".mzv": "application/metastream",
".nar": "application/zip",
".nbmp": "image/nbmp",
".ndb": "x-lml/x-ndb",
".ndwn": "application/ndwn",
".nif": "application/x-nif",
".nmz": "application/x-scream",
".nokia-op-logo": "image/vnd.nok-oplogo-color",
".npx": "application/x-netfpx",
".nsnd": "audio/nsnd",
".nva": "application/x-neva1",
".nws": "message/rfc822",
".oom": "application/x-AtlasMate-Plugin",
".p10": "application/pkcs10",
".p12": "application/x-pkcs12",
".p7b": "application/x-pkcs7-certificates",
".p7c": "application/x-pkcs7-mime",
".p7m": "application/x-pkcs7-mime",
".p7r": "application/x-pkcs7-certreqresp",
".p7s": "application/x-pkcs7-signature",
".pac": "audio/x-pac",
".pae": "audio/x-epac",
".pan": "application/x-pan",
".pcx": "image/x-pcx",
".pda": "image/x-pda",
".pfr": "application/font-tdpfr",
".pfx": "application/x-pkcs12",
".pko": "application/ynd.ms-pkipko",
".pm": "application/x-perl",
".pma": "application/x-perfmon",
".pmc": "application/x-perfmon",
".pmd": "application/x-pmd",
".pml": "application/x-perfmon",
".pmr": "application/x-perfmon",
".pmw": "application/x-perfmon",
".pnz": "image/png",
".pot,": "application/vnd.ms-powerpoint",
".pps": "application/vnd.ms-powerpoint",
".pqf": "application/x-cprplayer",
".pqi": "application/cprplayer",
".prc": "application/x-prc",
".prf": "application/pics-rules",
".prop": "text/plain",
".proxy": "application/x-ns-proxy-autoconfig",
".ptlk": "application/listenup",
".pub": "application/x-mspublisher",
".pvx": "video/x-pv-pvx",
".qcp": "audio/vnd.qcelp",
".r3t": "text/vnd.rn-realtext3d",
".rar": "application/octet-stream",
".rc": "text/plain",
".rf": "image/vnd.rn-realflash",
".rlf": "application/x-richlink",
".rmf": "audio/x-rmf",
".rmi": "audio/mid",
".rmm": "audio/x-pn-realaudio",
".rmvb": "audio/x-pn-realaudio",
".rnx": "application/vnd.rn-realplayer",
".rp": "image/vnd.rn-realpix",
".rt": "text/vnd.rn-realtext",
".rte": "x-lml/x-gps",
".rtg": "application/metastream",
".rv": "video/vnd.rn-realvideo",
".rwc": "application/x-rogerwilco",
".s3m": "audio/x-mod",
".s3z": "audio/x-mod",
".sca": "application/x-supercard",
".scd": "application/x-msschedule",
".sct": "text/scriptlet",
".sdf": "application/e-score",
".sea": "application/x-stuffit",
".setpay": "application/set-payment-initiation",
".setreg": "application/set-registration-initiation",
".shtml": "text/html",
".shtm": "text/html",
".shw": "application/presentations",
".si6": "image/si6",
".si7": "image/vnd.stiwap.sis",
".si9": "image/vnd.lgtwap.sis",
".slc": "application/x-salsa",
".smd": "audio/x-smd",
".smp": "application/studiom",
".smz": "audio/x-smd",
".spc": "application/x-pkcs7-certificates",
".spr": "application/x-sprite",
".sprite": "application/x-sprite",
".sdp": "application/sdp",
".spt": "application/x-spt",
".sst": "application/vnd.ms-pkicertstore",
".stk": "application/hyperstudio",
".stl": "application/vnd.ms-pkistl",
".stm": "text/html",
".svf": "image/vnd",
".svh": "image/svh",
".svr": "x-world/x-svr",
".swfl": "application/x-shockwave-flash",
".tad": "application/octet-stream",
".talk": "text/x-speech",
".taz": "application/x-tar",
".tbp": "application/x-timbuktu",
".tbt": "application/x-timbuktu",
".tgz": "application/x-compressed",
".thm": "application/vnd.eri.thm",
".tki": "application/x-tkined",
".tkined": "application/x-tkined",
".toc": "application/toc",
".toy": "image/toy",
".trk": "x-lml/x-gps",
".trm": "application/x-msterminal",
".tsi": "audio/tsplayer",
".tsp": "application/dsptype",
".ttf": "application/octet-stream",
".ttz": "application/t-time",
".uls": "text/iuls",
".ult": "audio/x-mod",
".uu": "application/x-uuencode",
".uue": "application/x-uuencode",
".vcf": "text/x-vcard",
".vdo": "video/vdo",
".vib": "audio/vib",
".viv": "video/vivo",
".vivo": "video/vivo",
".vmd": "application/vocaltec-media-desc",
".vmf": "application/vocaltec-media-file",
".vmi": "application/x-dreamcast-vms-info",
".vms": "application/x-dreamcast-vms",
".vox": "audio/voxware",
".vqe": "audio/x-twinvq-plugin",
".vqf": "audio/x-twinvq",
".vql": "audio/x-twinvq",
".vre": "x-world/x-vream",
".vrt": "x-world/x-vrt",
".vrw": "x-world/x-vream",
".vts": "workbook/formulaone",
".wcm": "application/vnd.ms-works",
".wdb": "application/vnd.ms-works",
".web": "application/vnd.xara",
".wi": "image/wavelet",
".wis": "application/x-InstallShield",
".wks": "application/vnd.ms-works",
".wmd": "application/x-ms-wmd",
".wmf": "application/x-msmetafile",
".wmlscript": "text/vnd.wap.wmlscript",
".wmz": "application/x-ms-wmz",
".wpng": "image/x-up-wpng",
".wps": "application/vnd.ms-works",
".wpt": "x-lml/x-gps",
".wri": "application/x-mswrite",
".wrz": "x-world/x-vrml",
".ws": "text/vnd.wap.wmlscript",
".wsc": "application/vnd.wap.wmlscriptc",
".wv": "video/wavelet",
".wxl": "application/x-wxl",
".x-gzip": "application/x-gzip",
".xaf": "x-world/x-vrml",
".xar": "application/vnd.xara",
".xdm": "application/x-xdma",
".xdma": "application/x-xdma",
".xdw": "application/vnd.fujixerox.docuworks",
".xhtm": "application/xhtml+xml",
".xla": "application/vnd.ms-excel",
".xlc": "application/vnd.ms-excel",
".xll": "application/x-excel",
".xlm": "application/vnd.ms-excel",
".xlt": "application/vnd.ms-excel",
".xlw": "application/vnd.ms-excel",
".xm": "audio/x-mod",
".xmz": "audio/x-mod",
".xof": "x-world/x-vrml",
".xpi": "application/x-xpinstall",
".xsit": "text/xml",
".yz1": "application/x-yz1",
".z": "application/x-compress",
".zac": "application/x-zaurus-zac",
".json": "application/json",
}
// TypeByExtension returns the MIME type associated with the file extension ext.

1
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/model.go generated vendored
View File

@@ -19,6 +19,7 @@ func (r *Response) Read(p []byte) (n int, err error) {
return r.Body.Read(p)
}
// Close close http reponse body
func (r *Response) Close() error {
return r.Body.Close()
}

72
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/multicopy.go generated vendored
View File

@@ -31,8 +31,14 @@ func (bucket Bucket) CopyFile(srcBucketName, srcObjectKey, destObjectKey string,
cpConf := getCpConfig(options)
routines := getRoutines(options)
var strVersionId string
versionId, _ := findOption(options, "versionId", nil)
if versionId != nil {
strVersionId = versionId.(string)
}
if cpConf != nil && cpConf.IsEnable {
cpFilePath := getCopyCpFilePath(cpConf, srcBucketName, srcObjectKey, destBucketName, destObjectKey)
cpFilePath := getCopyCpFilePath(cpConf, srcBucketName, srcObjectKey, destBucketName, destObjectKey, strVersionId)
if cpFilePath != "" {
return bucket.copyFileWithCp(srcBucketName, srcObjectKey, destBucketName, destObjectKey, partSize, options, cpFilePath, routines)
}
@@ -42,11 +48,11 @@ func (bucket Bucket) CopyFile(srcBucketName, srcObjectKey, destObjectKey string,
partSize, options, routines)
}
func getCopyCpFilePath(cpConf *cpConfig, srcBucket, srcObject, destBucket, destObject string) string {
func getCopyCpFilePath(cpConf *cpConfig, srcBucket, srcObject, destBucket, destObject, versionId string) string {
if cpConf.FilePath == "" && cpConf.DirPath != "" {
dest := fmt.Sprintf("oss://%v/%v", destBucket, destObject)
src := fmt.Sprintf("oss://%v/%v", srcBucket, srcObject)
cpFileName := getCpFileName(src, dest)
cpFileName := getCpFileName(src, dest, versionId)
cpConf.FilePath = cpConf.DirPath + string(os.PathSeparator) + cpFileName
}
return cpConf.FilePath
@@ -142,13 +148,9 @@ func (bucket Bucket) copyFile(srcBucketName, srcObjectKey, destBucketName, destO
srcBucket, err := bucket.Client.Bucket(srcBucketName)
listener := getProgressListener(options)
payerOptions := []Option{}
payer := getPayer(options)
if payer != "" {
payerOptions = append(payerOptions, RequestPayer(PayerType(payer)))
}
meta, err := srcBucket.GetObjectDetailedMeta(srcObjectKey, payerOptions...)
// for get whole length
skipOptions := deleteOption(options, HTTPHeaderRange)
meta, err := srcBucket.GetObjectDetailedMeta(srcObjectKey, skipOptions...)
if err != nil {
return err
}
@@ -173,11 +175,14 @@ func (bucket Bucket) copyFile(srcBucketName, srcObjectKey, destBucketName, destO
var completedBytes int64
totalBytes := getSrcObjectBytes(parts)
event := newProgressEvent(TransferStartedEvent, 0, totalBytes)
event := newProgressEvent(TransferStartedEvent, 0, totalBytes, 0)
publishProgress(listener, event)
// oss server don't support x-oss-storage-class
options = deleteOption(options, HTTPHeaderOssStorageClass)
// Start to copy workers
arg := copyWorkerArg{descBucket, imur, srcBucketName, srcObjectKey, payerOptions, copyPartHooker}
arg := copyWorkerArg{descBucket, imur, srcBucketName, srcObjectKey, options, copyPartHooker}
for w := 1; w <= routines; w++ {
go copyWorker(w, arg, jobs, results, failed, die)
}
@@ -193,13 +198,14 @@ func (bucket Bucket) copyFile(srcBucketName, srcObjectKey, destBucketName, destO
case part := <-results:
completed++
ups[part.PartNumber-1] = part
completedBytes += (parts[part.PartNumber-1].End - parts[part.PartNumber-1].Start + 1)
event = newProgressEvent(TransferDataEvent, completedBytes, totalBytes)
copyBytes := (parts[part.PartNumber-1].End - parts[part.PartNumber-1].Start + 1)
completedBytes += copyBytes
event = newProgressEvent(TransferDataEvent, completedBytes, totalBytes, copyBytes)
publishProgress(listener, event)
case err := <-failed:
close(die)
descBucket.AbortMultipartUpload(imur, payerOptions...)
event = newProgressEvent(TransferFailedEvent, completedBytes, totalBytes)
descBucket.AbortMultipartUpload(imur, options...)
event = newProgressEvent(TransferFailedEvent, completedBytes, totalBytes, 0)
publishProgress(listener, event)
return err
}
@@ -209,13 +215,13 @@ func (bucket Bucket) copyFile(srcBucketName, srcObjectKey, destBucketName, destO
}
}
event = newProgressEvent(TransferCompletedEvent, completedBytes, totalBytes)
event = newProgressEvent(TransferCompletedEvent, completedBytes, totalBytes, 0)
publishProgress(listener, event)
// Complete the multipart upload
_, err = descBucket.CompleteMultipartUpload(imur, ups, payerOptions...)
_, err = descBucket.CompleteMultipartUpload(imur, ups, options...)
if err != nil {
bucket.AbortMultipartUpload(imur, payerOptions...)
bucket.AbortMultipartUpload(imur, options...)
return err
}
return nil
@@ -385,12 +391,6 @@ func (bucket Bucket) copyFileWithCp(srcBucketName, srcObjectKey, destBucketName,
srcBucket, err := bucket.Client.Bucket(srcBucketName)
listener := getProgressListener(options)
payerOptions := []Option{}
payer := getPayer(options)
if payer != "" {
payerOptions = append(payerOptions, RequestPayer(PayerType(payer)))
}
// Load CP data
ccp := copyCheckpoint{}
err = ccp.load(cpFilePath)
@@ -399,7 +399,9 @@ func (bucket Bucket) copyFileWithCp(srcBucketName, srcObjectKey, destBucketName,
}
// Make sure the object is not updated.
meta, err := srcBucket.GetObjectDetailedMeta(srcObjectKey, payerOptions...)
// get whole length
skipOptions := deleteOption(options, HTTPHeaderRange)
meta, err := srcBucket.GetObjectDetailedMeta(srcObjectKey, skipOptions...)
if err != nil {
return err
}
@@ -426,11 +428,14 @@ func (bucket Bucket) copyFileWithCp(srcBucketName, srcObjectKey, destBucketName,
die := make(chan bool)
completedBytes := ccp.getCompletedBytes()
event := newProgressEvent(TransferStartedEvent, completedBytes, ccp.ObjStat.Size)
event := newProgressEvent(TransferStartedEvent, completedBytes, ccp.ObjStat.Size, 0)
publishProgress(listener, event)
// oss server don't support x-oss-storage-class
options = deleteOption(options, HTTPHeaderOssStorageClass)
// Start the worker coroutines
arg := copyWorkerArg{descBucket, imur, srcBucketName, srcObjectKey, payerOptions, copyPartHooker}
arg := copyWorkerArg{descBucket, imur, srcBucketName, srcObjectKey, options, copyPartHooker}
for w := 1; w <= routines; w++ {
go copyWorker(w, arg, jobs, results, failed, die)
}
@@ -446,12 +451,13 @@ func (bucket Bucket) copyFileWithCp(srcBucketName, srcObjectKey, destBucketName,
completed++
ccp.update(part)
ccp.dump(cpFilePath)
completedBytes += (parts[part.PartNumber-1].End - parts[part.PartNumber-1].Start + 1)
event = newProgressEvent(TransferDataEvent, completedBytes, ccp.ObjStat.Size)
copyBytes := (parts[part.PartNumber-1].End - parts[part.PartNumber-1].Start + 1)
completedBytes += copyBytes
event = newProgressEvent(TransferDataEvent, completedBytes, ccp.ObjStat.Size, copyBytes)
publishProgress(listener, event)
case err := <-failed:
close(die)
event = newProgressEvent(TransferFailedEvent, completedBytes, ccp.ObjStat.Size)
event = newProgressEvent(TransferFailedEvent, completedBytes, ccp.ObjStat.Size, 0)
publishProgress(listener, event)
return err
}
@@ -461,8 +467,8 @@ func (bucket Bucket) copyFileWithCp(srcBucketName, srcObjectKey, destBucketName,
}
}
event = newProgressEvent(TransferCompletedEvent, completedBytes, ccp.ObjStat.Size)
event = newProgressEvent(TransferCompletedEvent, completedBytes, ccp.ObjStat.Size, 0)
publishProgress(listener, event)
return ccp.complete(descBucket, ccp.CopyParts, cpFilePath, payerOptions)
return ccp.complete(descBucket, ccp.CopyParts, cpFilePath, options)
}

18
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/multipart.go generated vendored
View File

@@ -151,10 +151,22 @@ func (bucket Bucket) UploadPartCopy(imur InitiateMultipartUploadResult, srcBucke
startPosition, partSize int64, partNumber int, options ...Option) (UploadPart, error) {
var out UploadPartCopyResult
var part UploadPart
var opts []Option
//first find version id
versionIdKey := "versionId"
versionId, _ := findOption(options, versionIdKey, nil)
if versionId == nil {
opts = []Option{CopySource(srcBucketName, url.QueryEscape(srcObjectKey)),
CopySourceRange(startPosition, partSize)}
} else {
opts = []Option{CopySourceVersion(srcBucketName, url.QueryEscape(srcObjectKey), versionId.(string)),
CopySourceRange(startPosition, partSize)}
options = deleteOption(options, versionIdKey)
}
opts := []Option{CopySource(srcBucketName, url.QueryEscape(srcObjectKey)),
CopySourceRange(startPosition, partSize)}
opts = append(opts, options...)
params := map[string]interface{}{}
params["partNumber"] = strconv.Itoa(partNumber)
params["uploadId"] = imur.UploadID
@@ -243,7 +255,7 @@ func (bucket Bucket) ListUploadedParts(imur InitiateMultipartUploadResult, optio
}
params["uploadId"] = imur.UploadID
resp, err := bucket.do("GET", imur.Key, params, nil, nil, nil)
resp, err := bucket.do("GET", imur.Key, params, options, nil, nil)
if err != nil {
return out, err
}

132
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/option.go generated vendored
View File

@@ -3,6 +3,7 @@ package oss
import (
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
@@ -23,6 +24,8 @@ const (
initCRC64 = "init-crc64"
progressListener = "x-progress-listener"
storageClass = "storage-class"
responseHeader = "x-response-header"
redundancyType = "redundancy-type"
)
type (
@@ -65,6 +68,11 @@ func ContentEncoding(value string) Option {
return setHeader(HTTPHeaderContentEncoding, value)
}
// ContentLanguage is an option to set Content-Language header
func ContentLanguage(value string) Option {
return setHeader(HTTPHeaderContentLanguage, value)
}
// ContentMD5 is an option to set Content-MD5 header
func ContentMD5(value string) Option {
return setHeader(HTTPHeaderContentMD5, value)
@@ -120,6 +128,11 @@ func CopySource(sourceBucket, sourceObject string) Option {
return setHeader(HTTPHeaderOssCopySource, "/"+sourceBucket+"/"+sourceObject)
}
// CopySourceVersion is an option to set X-Oss-Copy-Source header,include versionId
func CopySourceVersion(sourceBucket, sourceObject string, versionId string) Option {
return setHeader(HTTPHeaderOssCopySource, "/"+sourceBucket+"/"+sourceObject+"?"+"versionId="+versionId)
}
// CopySourceRange is an option to set X-Oss-Copy-Source header
func CopySourceRange(startPosition, partSize int64) Option {
val := "bytes=" + strconv.FormatInt(startPosition, 10) + "-" +
@@ -157,6 +170,11 @@ func ServerSideEncryption(value string) Option {
return setHeader(HTTPHeaderOssServerSideEncryption, value)
}
// ServerSideEncryptionKeyID is an option to set X-Oss-Server-Side-Encryption-Key-Id header
func ServerSideEncryptionKeyID(value string) Option {
return setHeader(HTTPHeaderOssServerSideEncryptionKeyID, value)
}
// ObjectACL is an option to set X-Oss-Object-Acl header
func ObjectACL(acl ACLType) Option {
return setHeader(HTTPHeaderOssObjectACL, string(acl))
@@ -189,7 +207,43 @@ func CallbackVar(callbackVar string) Option {
// RequestPayer is an option to set payer who pay for the request
func RequestPayer(payerType PayerType) Option {
return setHeader(HTTPHeaderOSSRequester, string(payerType))
return setHeader(HTTPHeaderOssRequester, strings.ToLower(string(payerType)))
}
// SetTagging is an option to set object tagging
func SetTagging(tagging Tagging) Option {
if len(tagging.Tags) == 0 {
return nil
}
taggingValue := ""
for index, tag := range tagging.Tags {
if index != 0 {
taggingValue += "&"
}
taggingValue += url.QueryEscape(tag.Key) + "=" + url.QueryEscape(tag.Value)
}
return setHeader(HTTPHeaderOssTagging, taggingValue)
}
// TaggingDirective is an option to set X-Oss-Metadata-Directive header
func TaggingDirective(directive TaggingDirectiveType) Option {
return setHeader(HTTPHeaderOssTaggingDirective, string(directive))
}
// ACReqMethod is an option to set Access-Control-Request-Method header
func ACReqMethod(value string) Option {
return setHeader(HTTPHeaderACReqMethod, value)
}
// ACReqHeaders is an option to set Access-Control-Request-Headers header
func ACReqHeaders(value string) Option {
return setHeader(HTTPHeaderACReqHeaders, value)
}
// TrafficLimitHeader is an option to set X-Oss-Traffic-Limit
func TrafficLimitHeader(value int64) Option {
return setHeader(HTTPHeaderOssTrafficLimit, strconv.FormatInt(value, 10))
}
// Delimiter is an option to set delimiler parameter
@@ -227,6 +281,26 @@ func KeyMarker(value string) Option {
return addParam("key-marker", value)
}
// VersionIdMarker is an option to set version-id-marker parameter
func VersionIdMarker(value string) Option {
return addParam("version-id-marker", value)
}
// VersionId is an option to set versionId parameter
func VersionId(value string) Option {
return addParam("versionId", value)
}
// TagKey is an option to set tag key parameter
func TagKey(value string) Option {
return addParam("tag-key", value)
}
// TagValue is an option to set tag value parameter
func TagValue(value string) Option {
return addParam("tag-value", value)
}
// UploadIDMarker is an option to set upload-id-marker parameter
func UploadIDMarker(value string) Option {
return addParam("upload-id-marker", value)
@@ -252,6 +326,11 @@ func StorageClass(value StorageClassType) Option {
return addArg(storageClass, value)
}
// RedundancyType bucket data redundancy type
func RedundancyType(value DataRedundancyType) Option {
return addArg(redundancyType, value)
}
// Checkpoint configuration
type cpConfig struct {
IsEnable bool
@@ -284,6 +363,11 @@ func Progress(listener ProgressListener) Option {
return addArg(progressListener, listener)
}
// GetResponseHeader for get response http header
func GetResponseHeader(respHeader *http.Header) Option {
return addArg(responseHeader, respHeader)
}
// ResponseContentType is an option to set response-content-type param
func ResponseContentType(value string) Option {
return addParam("response-content-type", value)
@@ -319,6 +403,11 @@ func Process(value string) Option {
return addParam("x-oss-process", value)
}
// TrafficLimitParam is a option to set x-oss-traffic-limit
func TrafficLimitParam(value int64) Option {
return addParam("x-oss-traffic-limit", strconv.FormatInt(value, 10))
}
func setHeader(key string, value interface{}) Option {
return func(params map[string]optionValue) error {
if value == nil {
@@ -421,3 +510,44 @@ func isOptionSet(options []Option, option string) (bool, interface{}, error) {
}
return false, nil, nil
}
func deleteOption(options []Option, strKey string) []Option {
var outOption []Option
params := map[string]optionValue{}
for _, option := range options {
if option != nil {
option(params)
_, exist := params[strKey]
if !exist {
outOption = append(outOption, option)
} else {
delete(params, strKey)
}
}
}
return outOption
}
func GetRequestId(header http.Header) string {
return header.Get("x-oss-request-id")
}
func GetVersionId(header http.Header) string {
return header.Get("x-oss-version-id")
}
func GetCopySrcVersionId(header http.Header) string {
return header.Get("x-oss-copy-source-version-id")
}
func GetDeleteMark(header http.Header) bool {
value := header.Get("x-oss-delete-marker")
if strings.ToUpper(value) == "TRUE" {
return true
}
return false
}
func GetQosDelayTime(header http.Header) string {
return header.Get("x-oss-qos-delay-time")
}

8
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/progress.go generated vendored
View File

@@ -20,6 +20,7 @@ const (
type ProgressEvent struct {
ConsumedBytes int64
TotalBytes int64
RwBytes int64
EventType ProgressEventType
}
@@ -30,10 +31,11 @@ type ProgressListener interface {
// -------------------- Private --------------------
func newProgressEvent(eventType ProgressEventType, consumed, total int64) *ProgressEvent {
func newProgressEvent(eventType ProgressEventType, consumed, total int64, rwBytes int64) *ProgressEvent {
return &ProgressEvent{
ConsumedBytes: consumed,
TotalBytes: total,
RwBytes: rwBytes,
EventType: eventType}
}
@@ -78,7 +80,7 @@ func (t *teeReader) Read(p []byte) (n int, err error) {
// Read encountered error
if err != nil && err != io.EOF {
event := newProgressEvent(TransferFailedEvent, t.consumedBytes, t.totalBytes)
event := newProgressEvent(TransferFailedEvent, t.consumedBytes, t.totalBytes, 0)
publishProgress(t.listener, event)
}
@@ -92,7 +94,7 @@ func (t *teeReader) Read(p []byte) (n int, err error) {
}
// Progress
if t.listener != nil {
event := newProgressEvent(TransferDataEvent, t.consumedBytes, t.totalBytes)
event := newProgressEvent(TransferDataEvent, t.consumedBytes, t.totalBytes, int64(n))
publishProgress(t.listener, event)
}
// Track

197
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/select_object.go generated vendored Normal file
View File

@@ -0,0 +1,197 @@
package oss
import (
"bytes"
"encoding/xml"
"hash/crc32"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
)
// CreateSelectCsvObjectMeta is Creating csv object meta
//
// key the object key.
// csvMeta the csv file meta
// options the options for create csv Meta of the object.
//
// MetaEndFrameCSV the csv file meta info
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) CreateSelectCsvObjectMeta(key string, csvMeta CsvMetaRequest, options ...Option) (MetaEndFrameCSV, error) {
var endFrame MetaEndFrameCSV
params := map[string]interface{}{}
params["x-oss-process"] = "csv/meta"
csvMeta.encodeBase64()
bs, err := xml.Marshal(csvMeta)
if err != nil {
return endFrame, err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
resp, err := bucket.DoPostSelectObject(key, params, buffer, options...)
if err != nil {
return endFrame, err
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp)
return resp.Frame.MetaEndFrameCSV, err
}
// CreateSelectJsonObjectMeta is Creating json object meta
//
// key the object key.
// csvMeta the json file meta
// options the options for create json Meta of the object.
//
// MetaEndFrameJSON the json file meta info
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) CreateSelectJsonObjectMeta(key string, jsonMeta JsonMetaRequest, options ...Option) (MetaEndFrameJSON, error) {
var endFrame MetaEndFrameJSON
params := map[string]interface{}{}
params["x-oss-process"] = "json/meta"
bs, err := xml.Marshal(jsonMeta)
if err != nil {
return endFrame, err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
resp, err := bucket.DoPostSelectObject(key, params, buffer, options...)
if err != nil {
return endFrame, err
}
defer resp.Body.Close()
_, err = ioutil.ReadAll(resp)
return resp.Frame.MetaEndFrameJSON, err
}
// SelectObject is the select object api, approve csv and json file.
//
// key the object key.
// selectReq the request data for select object
// options the options for select file of the object.
//
// o.ReadCloser reader instance for reading data from response. It must be called close() after the usage and only valid when error is nil.
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) SelectObject(key string, selectReq SelectRequest, options ...Option) (io.ReadCloser, error) {
params := map[string]interface{}{}
if selectReq.InputSerializationSelect.JsonBodyInput.JsonIsEmpty() {
params["x-oss-process"] = "csv/select" // default select csv file
} else {
params["x-oss-process"] = "json/select"
}
selectReq.encodeBase64()
bs, err := xml.Marshal(selectReq)
if err != nil {
return nil, err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
resp, err := bucket.DoPostSelectObject(key, params, buffer, options...)
if err != nil {
return nil, err
}
if selectReq.OutputSerializationSelect.EnablePayloadCrc != nil && *selectReq.OutputSerializationSelect.EnablePayloadCrc == true {
resp.Frame.EnablePayloadCrc = true
}
resp.Frame.OutputRawData = strings.ToUpper(resp.Headers.Get("x-oss-select-output-raw")) == "TRUE"
return resp, err
}
// DoPostSelectObject is the SelectObject/CreateMeta api, approve csv and json file.
//
// key the object key.
// params the resource of oss approve csv/meta, json/meta, csv/select, json/select.
// buf the request data trans to buffer.
// options the options for select file of the object.
//
// SelectObjectResponse the response of select object.
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) DoPostSelectObject(key string, params map[string]interface{}, buf *bytes.Buffer, options ...Option) (*SelectObjectResponse, error) {
resp, err := bucket.do("POST", key, params, options, buf, nil)
if err != nil {
return nil, err
}
result := &SelectObjectResponse{
Body: resp.Body,
StatusCode: resp.StatusCode,
Frame: SelectObjectResult{},
}
result.Headers = resp.Headers
// result.Frame = SelectObjectResult{}
result.ReadTimeOut = bucket.getConfig().Timeout
// Progress
listener := getProgressListener(options)
// CRC32
crcCalc := crc32.NewIEEE()
result.WriterForCheckCrc32 = crcCalc
result.Body = TeeReader(resp.Body, nil, 0, listener, nil)
err = checkRespCode(resp.StatusCode, []int{http.StatusPartialContent, http.StatusOK})
return result, err
}
// SelectObjectIntoFile is the selectObject to file api
//
// key the object key.
// fileName saving file's name to localstation.
// selectReq the request data for select object
// options the options for select file of the object.
//
// error it's nil if no error, otherwise it's an error object.
//
func (bucket Bucket) SelectObjectIntoFile(key, fileName string, selectReq SelectRequest, options ...Option) error {
tempFilePath := fileName + TempFileSuffix
params := map[string]interface{}{}
if selectReq.InputSerializationSelect.JsonBodyInput.JsonIsEmpty() {
params["x-oss-process"] = "csv/select" // default select csv file
} else {
params["x-oss-process"] = "json/select"
}
selectReq.encodeBase64()
bs, err := xml.Marshal(selectReq)
if err != nil {
return err
}
buffer := new(bytes.Buffer)
buffer.Write(bs)
resp, err := bucket.DoPostSelectObject(key, params, buffer, options...)
if err != nil {
return err
}
defer resp.Close()
// If the local file does not exist, create a new one. If it exists, overwrite it.
fd, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, FilePermMode)
if err != nil {
return err
}
// Copy the data to the local file path.
_, err = io.Copy(fd, resp)
fd.Close()
if err != nil {
return err
}
return os.Rename(tempFilePath, fileName)
}

364
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/select_object_type.go generated vendored Normal file
View File

@@ -0,0 +1,364 @@
package oss
import (
"bytes"
"encoding/binary"
"fmt"
"hash"
"hash/crc32"
"io"
"net/http"
"time"
)
// The adapter class for Select object's response.
// The response consists of frames. Each frame has the following format:
// Type | Payload Length | Header Checksum | Payload | Payload Checksum
// |<4-->| <--4 bytes------><---4 bytes-------><-n/a-----><--4 bytes--------->
// And we have three kind of frames.
// Data Frame:
// Type:8388609
// Payload: Offset | Data
// <-8 bytes>
// Continuous Frame
// Type:8388612
// Payload: Offset (8-bytes)
// End Frame
// Type:8388613
// Payload: Offset | total scanned bytes | http status code | error message
// <-- 8bytes--><-----8 bytes--------><---4 bytes-------><---variabe--->
// SelectObjectResponse defines HTTP response from OSS SelectObject
type SelectObjectResponse struct {
StatusCode int
Headers http.Header
Body io.ReadCloser
Frame SelectObjectResult
ReadTimeOut uint
ClientCRC32 uint32
ServerCRC32 uint32
WriterForCheckCrc32 hash.Hash32
Finish bool
}
func (sr *SelectObjectResponse) Read(p []byte) (n int, err error) {
n, err = sr.readFrames(p)
return
}
// Close http reponse body
func (sr *SelectObjectResponse) Close() error {
return sr.Body.Close()
}
// PostSelectResult is the request of SelectObject
type PostSelectResult struct {
Response *SelectObjectResponse
}
// readFrames is read Frame
func (sr *SelectObjectResponse) readFrames(p []byte) (int, error) {
var nn int
var err error
var checkValid bool
if sr.Frame.OutputRawData == true {
nn, err = sr.Body.Read(p)
return nn, err
}
if sr.Finish {
return 0, io.EOF
}
for {
// if this Frame is Readed, then not reading Header
if sr.Frame.OpenLine != true {
err = sr.analysisHeader()
if err != nil {
return nn, err
}
}
if sr.Frame.FrameType == DataFrameType {
n, err := sr.analysisData(p[nn:])
if err != nil {
return nn, err
}
nn += n
// if this Frame is readed all data, then empty the Frame to read it with next frame
if sr.Frame.ConsumedBytesLength == sr.Frame.PayloadLength-8 {
checkValid, err = sr.checkPayloadSum()
if err != nil || !checkValid {
return nn, fmt.Errorf("%s", err.Error())
}
sr.emptyFrame()
}
if nn == len(p) {
return nn, nil
}
} else if sr.Frame.FrameType == ContinuousFrameType {
checkValid, err = sr.checkPayloadSum()
if err != nil || !checkValid {
return nn, fmt.Errorf("%s", err.Error())
}
} else if sr.Frame.FrameType == EndFrameType {
err = sr.analysisEndFrame()
if err != nil {
return nn, err
}
checkValid, err = sr.checkPayloadSum()
if checkValid {
sr.Finish = true
}
return nn, err
} else if sr.Frame.FrameType == MetaEndFrameCSVType {
err = sr.analysisMetaEndFrameCSV()
if err != nil {
return nn, err
}
checkValid, err = sr.checkPayloadSum()
if checkValid {
sr.Finish = true
}
return nn, err
} else if sr.Frame.FrameType == MetaEndFrameJSONType {
err = sr.analysisMetaEndFrameJSON()
if err != nil {
return nn, err
}
checkValid, err = sr.checkPayloadSum()
if checkValid {
sr.Finish = true
}
return nn, err
}
}
return nn, nil
}
type chanReadIO struct {
readLen int
err error
}
func (sr *SelectObjectResponse) readLen(p []byte, timeOut time.Duration) (int, error) {
r := sr.Body
ch := make(chan chanReadIO, 1)
defer close(ch)
go func(p []byte) {
var needReadLength int
readChan := chanReadIO{}
needReadLength = len(p)
for {
n, err := r.Read(p[readChan.readLen:needReadLength])
readChan.readLen += n
if err != nil {
readChan.err = err
ch <- readChan
return
}
if readChan.readLen == needReadLength {
break
}
}
ch <- readChan
}(p)
select {
case <-time.After(time.Second * timeOut):
return 0, fmt.Errorf("requestId: %s, readLen timeout, timeout is %d(second),need read:%d", sr.Headers.Get(HTTPHeaderOssRequestID), timeOut, len(p))
case result := <-ch:
return result.readLen, result.err
}
}
// analysisHeader is reading selectObject response body's header
func (sr *SelectObjectResponse) analysisHeader() error {
headFrameByte := make([]byte, 20)
_, err := sr.readLen(headFrameByte, time.Duration(sr.ReadTimeOut))
if err != nil {
return fmt.Errorf("requestId: %s, Read response frame header failure,err:%s", sr.Headers.Get(HTTPHeaderOssRequestID), err.Error())
}
frameTypeByte := headFrameByte[0:4]
sr.Frame.Version = frameTypeByte[0]
frameTypeByte[0] = 0
bytesToInt(frameTypeByte, &sr.Frame.FrameType)
if sr.Frame.FrameType != DataFrameType && sr.Frame.FrameType != ContinuousFrameType &&
sr.Frame.FrameType != EndFrameType && sr.Frame.FrameType != MetaEndFrameCSVType && sr.Frame.FrameType != MetaEndFrameJSONType {
return fmt.Errorf("requestId: %s, Unexpected frame type: %d", sr.Headers.Get(HTTPHeaderOssRequestID), sr.Frame.FrameType)
}
payloadLengthByte := headFrameByte[4:8]
bytesToInt(payloadLengthByte, &sr.Frame.PayloadLength)
headCheckSumByte := headFrameByte[8:12]
bytesToInt(headCheckSumByte, &sr.Frame.HeaderCheckSum)
byteOffset := headFrameByte[12:20]
bytesToInt(byteOffset, &sr.Frame.Offset)
sr.Frame.OpenLine = true
err = sr.writerCheckCrc32(byteOffset)
return err
}
// analysisData is reading the DataFrameType data of selectObject response body
func (sr *SelectObjectResponse) analysisData(p []byte) (int, error) {
var needReadLength int32
lenP := int32(len(p))
restByteLength := sr.Frame.PayloadLength - 8 - sr.Frame.ConsumedBytesLength
if lenP <= restByteLength {
needReadLength = lenP
} else {
needReadLength = restByteLength
}
n, err := sr.readLen(p[:needReadLength], time.Duration(sr.ReadTimeOut))
if err != nil {
return n, fmt.Errorf("read frame data error,%s", err.Error())
}
sr.Frame.ConsumedBytesLength += int32(n)
err = sr.writerCheckCrc32(p[:n])
return n, err
}
// analysisEndFrame is reading the EndFrameType data of selectObject response body
func (sr *SelectObjectResponse) analysisEndFrame() error {
var eF EndFrame
payLoadBytes := make([]byte, sr.Frame.PayloadLength-8)
_, err := sr.readLen(payLoadBytes, time.Duration(sr.ReadTimeOut))
if err != nil {
return fmt.Errorf("read end frame error:%s", err.Error())
}
bytesToInt(payLoadBytes[0:8], &eF.TotalScanned)
bytesToInt(payLoadBytes[8:12], &eF.HTTPStatusCode)
errMsgLength := sr.Frame.PayloadLength - 20
eF.ErrorMsg = string(payLoadBytes[12 : errMsgLength+12])
sr.Frame.EndFrame.TotalScanned = eF.TotalScanned
sr.Frame.EndFrame.HTTPStatusCode = eF.HTTPStatusCode
sr.Frame.EndFrame.ErrorMsg = eF.ErrorMsg
err = sr.writerCheckCrc32(payLoadBytes)
return err
}
// analysisMetaEndFrameCSV is reading the MetaEndFrameCSVType data of selectObject response body
func (sr *SelectObjectResponse) analysisMetaEndFrameCSV() error {
var mCF MetaEndFrameCSV
payLoadBytes := make([]byte, sr.Frame.PayloadLength-8)
_, err := sr.readLen(payLoadBytes, time.Duration(sr.ReadTimeOut))
if err != nil {
return fmt.Errorf("read meta end csv frame error:%s", err.Error())
}
bytesToInt(payLoadBytes[0:8], &mCF.TotalScanned)
bytesToInt(payLoadBytes[8:12], &mCF.Status)
bytesToInt(payLoadBytes[12:16], &mCF.SplitsCount)
bytesToInt(payLoadBytes[16:24], &mCF.RowsCount)
bytesToInt(payLoadBytes[24:28], &mCF.ColumnsCount)
errMsgLength := sr.Frame.PayloadLength - 36
mCF.ErrorMsg = string(payLoadBytes[28 : errMsgLength+28])
sr.Frame.MetaEndFrameCSV.ErrorMsg = mCF.ErrorMsg
sr.Frame.MetaEndFrameCSV.TotalScanned = mCF.TotalScanned
sr.Frame.MetaEndFrameCSV.Status = mCF.Status
sr.Frame.MetaEndFrameCSV.SplitsCount = mCF.SplitsCount
sr.Frame.MetaEndFrameCSV.RowsCount = mCF.RowsCount
sr.Frame.MetaEndFrameCSV.ColumnsCount = mCF.ColumnsCount
err = sr.writerCheckCrc32(payLoadBytes)
return err
}
// analysisMetaEndFrameJSON is reading the MetaEndFrameJSONType data of selectObject response body
func (sr *SelectObjectResponse) analysisMetaEndFrameJSON() error {
var mJF MetaEndFrameJSON
payLoadBytes := make([]byte, sr.Frame.PayloadLength-8)
_, err := sr.readLen(payLoadBytes, time.Duration(sr.ReadTimeOut))
if err != nil {
return fmt.Errorf("read meta end json frame error:%s", err.Error())
}
bytesToInt(payLoadBytes[0:8], &mJF.TotalScanned)
bytesToInt(payLoadBytes[8:12], &mJF.Status)
bytesToInt(payLoadBytes[12:16], &mJF.SplitsCount)
bytesToInt(payLoadBytes[16:24], &mJF.RowsCount)
errMsgLength := sr.Frame.PayloadLength - 32
mJF.ErrorMsg = string(payLoadBytes[24 : errMsgLength+24])
sr.Frame.MetaEndFrameJSON.ErrorMsg = mJF.ErrorMsg
sr.Frame.MetaEndFrameJSON.TotalScanned = mJF.TotalScanned
sr.Frame.MetaEndFrameJSON.Status = mJF.Status
sr.Frame.MetaEndFrameJSON.SplitsCount = mJF.SplitsCount
sr.Frame.MetaEndFrameJSON.RowsCount = mJF.RowsCount
err = sr.writerCheckCrc32(payLoadBytes)
return err
}
func (sr *SelectObjectResponse) checkPayloadSum() (bool, error) {
payLoadChecksumByte := make([]byte, 4)
n, err := sr.readLen(payLoadChecksumByte, time.Duration(sr.ReadTimeOut))
if n == 4 {
bytesToInt(payLoadChecksumByte, &sr.Frame.PayloadChecksum)
sr.ServerCRC32 = sr.Frame.PayloadChecksum
sr.ClientCRC32 = sr.WriterForCheckCrc32.Sum32()
if sr.Frame.EnablePayloadCrc == true && sr.ServerCRC32 != 0 && sr.ServerCRC32 != sr.ClientCRC32 {
return false, fmt.Errorf("RequestId: %s, Unexpected frame type: %d, client %d but server %d",
sr.Headers.Get(HTTPHeaderOssRequestID), sr.Frame.FrameType, sr.ClientCRC32, sr.ServerCRC32)
}
return true, err
}
return false, fmt.Errorf("RequestId:%s, read checksum error:%s", sr.Headers.Get(HTTPHeaderOssRequestID), err.Error())
}
func (sr *SelectObjectResponse) writerCheckCrc32(p []byte) (err error) {
err = nil
if sr.Frame.EnablePayloadCrc == true {
_, err = sr.WriterForCheckCrc32.Write(p)
}
return err
}
// emptyFrame is emptying SelectObjectResponse Frame information
func (sr *SelectObjectResponse) emptyFrame() {
crcCalc := crc32.NewIEEE()
sr.WriterForCheckCrc32 = crcCalc
sr.Finish = false
sr.Frame.ConsumedBytesLength = 0
sr.Frame.OpenLine = false
sr.Frame.Version = byte(0)
sr.Frame.FrameType = 0
sr.Frame.PayloadLength = 0
sr.Frame.HeaderCheckSum = 0
sr.Frame.Offset = 0
sr.Frame.Data = ""
sr.Frame.EndFrame.TotalScanned = 0
sr.Frame.EndFrame.HTTPStatusCode = 0
sr.Frame.EndFrame.ErrorMsg = ""
sr.Frame.MetaEndFrameCSV.TotalScanned = 0
sr.Frame.MetaEndFrameCSV.Status = 0
sr.Frame.MetaEndFrameCSV.SplitsCount = 0
sr.Frame.MetaEndFrameCSV.RowsCount = 0
sr.Frame.MetaEndFrameCSV.ColumnsCount = 0
sr.Frame.MetaEndFrameCSV.ErrorMsg = ""
sr.Frame.MetaEndFrameJSON.TotalScanned = 0
sr.Frame.MetaEndFrameJSON.Status = 0
sr.Frame.MetaEndFrameJSON.SplitsCount = 0
sr.Frame.MetaEndFrameJSON.RowsCount = 0
sr.Frame.MetaEndFrameJSON.ErrorMsg = ""
sr.Frame.PayloadChecksum = 0
}
// bytesToInt byte's array trans to int
func bytesToInt(b []byte, ret interface{}) {
binBuf := bytes.NewBuffer(b)
binary.Read(binBuf, binary.BigEndian, ret)
}

8
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/transport_1_6.go generated vendored
View File

@@ -9,15 +9,21 @@ import (
func newTransport(conn *Conn, config *Config) *http.Transport {
httpTimeOut := conn.config.HTTPTimeout
httpMaxConns := conn.config.HTTPMaxConns
// New Transport
transport := &http.Transport{
Dial: func(netw, addr string) (net.Conn, error) {
conn, err := net.DialTimeout(netw, addr, httpTimeOut.ConnectTimeout)
d := net.Dialer{Timeout: httpTimeOut.ConnectTimeout}
if config.LocalAddr != nil {
d.LocalAddr = config.LocalAddr
}
conn, err := d.Dial(netw, addr)
if err != nil {
return nil, err
}
return newTimeoutConn(conn, httpTimeOut.ReadWriteTimeout, httpTimeOut.LongTimeout), nil
},
MaxIdleConnsPerHost: httpMaxConns.MaxIdleConnsPerHost,
ResponseHeaderTimeout: httpTimeOut.HeaderTimeout,
}
return transport

9
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/transport_1_7.go generated vendored
View File

@@ -9,15 +9,22 @@ import (
func newTransport(conn *Conn, config *Config) *http.Transport {
httpTimeOut := conn.config.HTTPTimeout
httpMaxConns := conn.config.HTTPMaxConns
// New Transport
transport := &http.Transport{
Dial: func(netw, addr string) (net.Conn, error) {
conn, err := net.DialTimeout(netw, addr, httpTimeOut.ConnectTimeout)
d := net.Dialer{Timeout: httpTimeOut.ConnectTimeout}
if config.LocalAddr != nil {
d.LocalAddr = config.LocalAddr
}
conn, err := d.Dial(netw, addr)
if err != nil {
return nil, err
}
return newTimeoutConn(conn, httpTimeOut.ReadWriteTimeout, httpTimeOut.LongTimeout), nil
},
MaxIdleConns: httpMaxConns.MaxIdleConns,
MaxIdleConnsPerHost: httpMaxConns.MaxIdleConnsPerHost,
IdleConnTimeout: httpTimeOut.IdleConnTimeout,
ResponseHeaderTimeout: httpTimeOut.HeaderTimeout,
}

736
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/type.go generated vendored
View File

@@ -1,7 +1,9 @@
package oss
import (
"encoding/base64"
"encoding/xml"
"fmt"
"net/url"
"time"
)
@@ -42,77 +44,115 @@ type LifecycleConfiguration struct {
// LifecycleRule defines Lifecycle rules
type LifecycleRule struct {
XMLName xml.Name `xml:"Rule"`
ID string `xml:"ID"` // The rule ID
Prefix string `xml:"Prefix"` // The object key prefix
Status string `xml:"Status"` // The rule status (enabled or not)
Expiration LifecycleExpiration `xml:"Expiration"` // The expiration property
XMLName xml.Name `xml:"Rule"`
ID string `xml:"ID,omitempty"` // The rule ID
Prefix string `xml:"Prefix"` // The object key prefix
Status string `xml:"Status"` // The rule status (enabled or not)
Tags []Tag `xml:"Tag,omitempty"` // the tags property
Expiration *LifecycleExpiration `xml:"Expiration,omitempty"` // The expiration property
Transitions []LifecycleTransition `xml:"Transition,omitempty"` // The transition property
AbortMultipartUpload *LifecycleAbortMultipartUpload `xml:"AbortMultipartUpload,omitempty"` // The AbortMultipartUpload property
NonVersionExpiration *LifecycleVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
NonVersionTransition *LifecycleVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
}
// LifecycleExpiration defines the rule's expiration property
type LifecycleExpiration struct {
XMLName xml.Name `xml:"Expiration"`
Days int `xml:"Days,omitempty"` // Relative expiration time: The expiration time in days after the last modified time
Date time.Time `xml:"Date,omitempty"` // Absolute expiration time: The expiration time in date.
XMLName xml.Name `xml:"Expiration"`
Days int `xml:"Days,omitempty"` // Relative expiration time: The expiration time in days after the last modified time
Date string `xml:"Date,omitempty"` // Absolute expiration time: The expiration time in date, not recommended
CreatedBeforeDate string `xml:"CreatedBeforeDate,omitempty"` // objects created before the date will be expired
ExpiredObjectDeleteMarker *bool `xml:"ExpiredObjectDeleteMarker,omitempty"` // Specifies whether the expired delete tag is automatically deleted
}
type lifecycleXML struct {
XMLName xml.Name `xml:"LifecycleConfiguration"`
Rules []lifecycleRule `xml:"Rule"`
// LifecycleTransition defines the rule's transition propery
type LifecycleTransition struct {
XMLName xml.Name `xml:"Transition"`
Days int `xml:"Days,omitempty"` // Relative transition time: The transition time in days after the last modified time
CreatedBeforeDate string `xml:"CreatedBeforeDate,omitempty"` // objects created before the date will be expired
StorageClass StorageClassType `xml:"StorageClass,omitempty"` // Specifies the target storage type
}
type lifecycleRule struct {
XMLName xml.Name `xml:"Rule"`
ID string `xml:"ID"`
Prefix string `xml:"Prefix"`
Status string `xml:"Status"`
Expiration lifecycleExpiration `xml:"Expiration"`
// LifecycleAbortMultipartUpload defines the rule's abort multipart upload propery
type LifecycleAbortMultipartUpload struct {
XMLName xml.Name `xml:"AbortMultipartUpload"`
Days int `xml:"Days,omitempty"` // Relative expiration time: The expiration time in days after the last modified time
CreatedBeforeDate string `xml:"CreatedBeforeDate,omitempty"` // objects created before the date will be expired
}
type lifecycleExpiration struct {
XMLName xml.Name `xml:"Expiration"`
Days int `xml:"Days,omitempty"`
Date string `xml:"Date,omitempty"`
// LifecycleVersionExpiration defines the rule's NoncurrentVersionExpiration propery
type LifecycleVersionExpiration struct {
XMLName xml.Name `xml:"NoncurrentVersionExpiration"`
NoncurrentDays int `xml:"NoncurrentDays,omitempty"` // How many days after the Object becomes a non-current version
}
const expirationDateFormat = "2006-01-02T15:04:05.000Z"
func convLifecycleRule(rules []LifecycleRule) []lifecycleRule {
rs := []lifecycleRule{}
for _, rule := range rules {
r := lifecycleRule{}
r.ID = rule.ID
r.Prefix = rule.Prefix
r.Status = rule.Status
if rule.Expiration.Date.IsZero() {
r.Expiration.Days = rule.Expiration.Days
} else {
r.Expiration.Date = rule.Expiration.Date.Format(expirationDateFormat)
}
rs = append(rs, r)
}
return rs
// LifecycleVersionTransition defines the rule's NoncurrentVersionTransition propery
type LifecycleVersionTransition struct {
XMLName xml.Name `xml:"NoncurrentVersionTransition"`
NoncurrentDays int `xml:"NoncurrentDays,omitempty"` // How many days after the Object becomes a non-current version
StorageClass StorageClassType `xml:"StorageClass,omitempty"`
}
// BuildLifecycleRuleByDays builds a lifecycle rule with specified expiration days
const iso8601DateFormat = "2006-01-02T15:04:05.000Z"
// BuildLifecycleRuleByDays builds a lifecycle rule objects will expiration in days after the last modified time
func BuildLifecycleRuleByDays(id, prefix string, status bool, days int) LifecycleRule {
var statusStr = "Enabled"
if !status {
statusStr = "Disabled"
}
return LifecycleRule{ID: id, Prefix: prefix, Status: statusStr,
Expiration: LifecycleExpiration{Days: days}}
Expiration: &LifecycleExpiration{Days: days}}
}
// BuildLifecycleRuleByDate builds a lifecycle rule with specified expiration time.
// BuildLifecycleRuleByDate builds a lifecycle rule objects will expiration in specified date
func BuildLifecycleRuleByDate(id, prefix string, status bool, year, month, day int) LifecycleRule {
var statusStr = "Enabled"
if !status {
statusStr = "Disabled"
}
date := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC)
date := time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.UTC).Format(iso8601DateFormat)
return LifecycleRule{ID: id, Prefix: prefix, Status: statusStr,
Expiration: LifecycleExpiration{Date: date}}
Expiration: &LifecycleExpiration{Date: date}}
}
// ValidateLifecycleRule Determine if a lifecycle rule is valid, if it is invalid, it will return an error.
func verifyLifecycleRules(rules []LifecycleRule) error {
if len(rules) == 0 {
return fmt.Errorf("invalid rules, the length of rules is zero")
}
for _, rule := range rules {
if rule.Status != "Enabled" && rule.Status != "Disabled" {
return fmt.Errorf("invalid rule, the value of status must be Enabled or Disabled")
}
abortMPU := rule.AbortMultipartUpload
if abortMPU != nil {
if (abortMPU.Days != 0 && abortMPU.CreatedBeforeDate != "") || (abortMPU.Days == 0 && abortMPU.CreatedBeforeDate == "") {
return fmt.Errorf("invalid abort multipart upload lifecycle, must be set one of CreatedBeforeDate and Days")
}
}
transitions := rule.Transitions
if len(transitions) > 0 {
if len(transitions) > 2 {
return fmt.Errorf("invalid count of transition lifecycles, the count must than less than 3")
}
for _, transition := range transitions {
if (transition.Days != 0 && transition.CreatedBeforeDate != "") || (transition.Days == 0 && transition.CreatedBeforeDate == "") {
return fmt.Errorf("invalid transition lifecycle, must be set one of CreatedBeforeDate and Days")
}
if transition.StorageClass != StorageIA && transition.StorageClass != StorageArchive {
return fmt.Errorf("invalid transition lifecylce, the value of storage class must be IA or Archive")
}
}
} else if rule.Expiration == nil && abortMPU == nil && rule.NonVersionExpiration == nil && rule.NonVersionTransition == nil {
return fmt.Errorf("invalid rule, must set one of Expiration, AbortMultipartUplaod, NonVersionExpiration, NonVersionTransition and Transitions")
}
}
return nil
}
// GetBucketLifecycleResult defines GetBucketLifecycle's result object
@@ -151,8 +191,9 @@ type GetBucketLoggingResult LoggingXML
// WebsiteXML defines Website configuration
type WebsiteXML struct {
XMLName xml.Name `xml:"WebsiteConfiguration"`
IndexDocument IndexDocument `xml:"IndexDocument"` // The index page
ErrorDocument ErrorDocument `xml:"ErrorDocument"` // The error page
IndexDocument IndexDocument `xml:"IndexDocument,omitempty"` // The index page
ErrorDocument ErrorDocument `xml:"ErrorDocument,omitempty"` // The error page
RoutingRules []RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"` // The routing Rule list
}
// IndexDocument defines the index page info
@@ -167,6 +208,63 @@ type ErrorDocument struct {
Key string `xml:"Key"` // 404 error file name
}
// RoutingRule defines the routing rules
type RoutingRule struct {
XMLName xml.Name `xml:"RoutingRule"`
RuleNumber int `xml:"RuleNumber,omitempty"` // The routing number
Condition Condition `xml:"Condition,omitempty"` // The routing condition
Redirect Redirect `xml:"Redirect,omitempty"` // The routing redirect
}
// Condition defines codition in the RoutingRule
type Condition struct {
XMLName xml.Name `xml:"Condition"`
KeyPrefixEquals string `xml:"KeyPrefixEquals,omitempty"` // Matching objcet prefix
HTTPErrorCodeReturnedEquals int `xml:"HttpErrorCodeReturnedEquals,omitempty"` // The rule is for Accessing to the specified object
IncludeHeader []IncludeHeader `xml:"IncludeHeader"` // The rule is for request which include header
}
// IncludeHeader defines includeHeader in the RoutingRule's Condition
type IncludeHeader struct {
XMLName xml.Name `xml:"IncludeHeader"`
Key string `xml:"Key,omitempty"` // The Include header key
Equals string `xml:"Equals,omitempty"` // The Include header value
}
// Redirect defines redirect in the RoutingRule
type Redirect struct {
XMLName xml.Name `xml:"Redirect"`
RedirectType string `xml:"RedirectType,omitempty"` // The redirect type, it have Mirror,External,Internal,AliCDN
PassQueryString *bool `xml:"PassQueryString"` // Whether to send the specified request's parameters, true or false
MirrorURL string `xml:"MirrorURL,omitempty"` // Mirror of the website address back to the source.
MirrorPassQueryString *bool `xml:"MirrorPassQueryString"` // To Mirror of the website Whether to send the specified request's parameters, true or false
MirrorFollowRedirect *bool `xml:"MirrorFollowRedirect"` // Redirect the location, if the mirror return 3XX
MirrorCheckMd5 *bool `xml:"MirrorCheckMd5"` // Check the mirror is MD5.
MirrorHeaders MirrorHeaders `xml:"MirrorHeaders,omitempty"` // Mirror headers
Protocol string `xml:"Protocol,omitempty"` // The redirect Protocol
HostName string `xml:"HostName,omitempty"` // The redirect HostName
ReplaceKeyPrefixWith string `xml:"ReplaceKeyPrefixWith,omitempty"` // object name'Prefix replace the value
HttpRedirectCode int `xml:"HttpRedirectCode,omitempty"` // THe redirect http code
ReplaceKeyWith string `xml:"ReplaceKeyWith,omitempty"` // object name replace the value
}
// MirrorHeaders defines MirrorHeaders in the Redirect
type MirrorHeaders struct {
XMLName xml.Name `xml:"MirrorHeaders"`
PassAll *bool `xml:"PassAll"` // Penetrating all of headers to source website.
Pass []string `xml:"Pass"` // Penetrating some of headers to source website.
Remove []string `xml:"Remove"` // Prohibit passthrough some of headers to source website
Set []MirrorHeaderSet `xml:"Set"` // Setting some of headers send to source website
}
// MirrorHeaderSet defines Set for Redirect's MirrorHeaders
type MirrorHeaderSet struct {
XMLName xml.Name `xml:"Set"`
Key string `xml:"Key,omitempty"` // The mirror header key
Value string `xml:"Value,omitempty"` // The mirror header value
}
// GetBucketWebsiteResult defines the result from GetBucketWebsite request.
type GetBucketWebsiteResult WebsiteXML
@@ -198,14 +296,23 @@ type GetBucketInfoResult struct {
// BucketInfo defines Bucket information
type BucketInfo struct {
XMLName xml.Name `xml:"Bucket"`
Name string `xml:"Name"` // Bucket name
Location string `xml:"Location"` // Bucket datacenter
CreationDate time.Time `xml:"CreationDate"` // Bucket creation time
ExtranetEndpoint string `xml:"ExtranetEndpoint"` // Bucket external endpoint
IntranetEndpoint string `xml:"IntranetEndpoint"` // Bucket internal endpoint
ACL string `xml:"AccessControlList>Grant"` // Bucket ACL
Owner Owner `xml:"Owner"` // Bucket owner
StorageClass string `xml:"StorageClass"` // Bucket storage class
Name string `xml:"Name"` // Bucket name
Location string `xml:"Location"` // Bucket datacenter
CreationDate time.Time `xml:"CreationDate"` // Bucket creation time
ExtranetEndpoint string `xml:"ExtranetEndpoint"` // Bucket external endpoint
IntranetEndpoint string `xml:"IntranetEndpoint"` // Bucket internal endpoint
ACL string `xml:"AccessControlList>Grant"` // Bucket ACL
RedundancyType string `xml:"DataRedundancyType"` // Bucket DataRedundancyType
Owner Owner `xml:"Owner"` // Bucket owner
StorageClass string `xml:"StorageClass"` // Bucket storage class
SseRule SSERule `xml:"ServerSideEncryptionRule"` // Bucket ServerSideEncryptionRule
Versioning string `xml:"Versioning"` // Bucket Versioning
}
type SSERule struct {
XMLName xml.Name `xml:"ServerSideEncryptionRule"` // Bucket ServerSideEncryptionRule
KMSMasterKeyID string `xml:"KMSMasterKeyID"` // Bucket KMSMasterKeyID
SSEAlgorithm string `xml:"SSEAlgorithm"` // Bucket SSEAlgorithm
}
// ListObjectsResult defines the result from ListObjects request
@@ -233,6 +340,46 @@ type ObjectProperties struct {
StorageClass string `xml:"StorageClass"` // Object storage class (Standard, IA, Archive)
}
// ListObjectVersionsResult defines the result from ListObjectVersions request
type ListObjectVersionsResult struct {
XMLName xml.Name `xml:"ListVersionsResult"`
Name string `xml:"Name"` // The Bucket Name
Owner Owner `xml:"Owner"` // The owner of bucket
Prefix string `xml:"Prefix"` // The object prefix
KeyMarker string `xml:"KeyMarker"` // The start marker filter.
VersionIdMarker string `xml:"VersionIdMarker"` // The start VersionIdMarker filter.
MaxKeys int `xml:"MaxKeys"` // Max keys to return
Delimiter string `xml:"Delimiter"` // The delimiter for grouping objects' name
IsTruncated bool `xml:"IsTruncated"` // Flag indicates if all results are returned (when it's false)
NextKeyMarker string `xml:"NextKeyMarker"` // The start point of the next query
NextVersionIdMarker string `xml:"NextVersionIdMarker"` // The start point of the next query
CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` // You can think of commonprefixes as "folders" whose names end with the delimiter
ObjectDeleteMarkers []ObjectDeleteMarkerProperties `xml:"DeleteMarker"` // DeleteMarker list
ObjectVersions []ObjectVersionProperties `xml:"Version"` // version list
}
type ObjectDeleteMarkerProperties struct {
XMLName xml.Name `xml:"DeleteMarker"`
Key string `xml:"Key"` // The Object Key
VersionId string `xml:"VersionId"` // The Object VersionId
IsLatest bool `xml:"IsLatest"` // is current version or not
LastModified time.Time `xml:"LastModified"` // Object last modified time
Owner Owner `xml:"Owner"` // bucket owner element
}
type ObjectVersionProperties struct {
XMLName xml.Name `xml:"Version"`
Key string `xml:"Key"` // The Object Key
VersionId string `xml:"VersionId"` // The Object VersionId
IsLatest bool `xml:"IsLatest"` // is latest version or not
LastModified time.Time `xml:"LastModified"` // Object last modified time
Type string `xml:"Type"` // Object type
Size int64 `xml:"Size"` // Object size
ETag string `xml:"ETag"` // Object ETag
StorageClass string `xml:"StorageClass"` // Object storage class (Standard, IA, Archive)
Owner Owner `xml:"Owner"` // bucket owner element
}
// Owner defines Bucket/Object's owner
type Owner struct {
XMLName xml.Name `xml:"Owner"`
@@ -258,14 +405,30 @@ type deleteXML struct {
// DeleteObject defines the struct for deleting object
type DeleteObject struct {
XMLName xml.Name `xml:"Object"`
Key string `xml:"Key"` // Object name
XMLName xml.Name `xml:"Object"`
Key string `xml:"Key"` // Object name
VersionId string `xml:"VersionId,omitempty"` // Object VersionId
}
// DeleteObjectsResult defines result of DeleteObjects request
type DeleteObjectsResult struct {
XMLName xml.Name `xml:"DeleteResult"`
DeletedObjects []string `xml:"Deleted>Key"` // Deleted object list
XMLName xml.Name
DeletedObjects []string // Deleted object key list
}
// DeleteObjectsResult_inner defines result of DeleteObjects request
type DeleteObjectVersionsResult struct {
XMLName xml.Name `xml:"DeleteResult"`
DeletedObjectsDetail []DeletedKeyInfo `xml:"Deleted"` // Deleted object detail info
}
// DeleteKeyInfo defines object delete info
type DeletedKeyInfo struct {
XMLName xml.Name `xml:"Deleted"`
Key string `xml:"Key"` // Object key
VersionId string `xml:"VersionId"` // VersionId
DeleteMarker bool `xml:"DeleteMarker"` // Object DeleteMarker
DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId"` // Object DeleteMarkerVersionId
}
// InitiateMultipartUploadResult defines result of InitiateMultipartUpload request
@@ -372,10 +535,10 @@ type ProcessObjectResult struct {
}
// decodeDeleteObjectsResult decodes deleting objects result in URL encoding
func decodeDeleteObjectsResult(result *DeleteObjectsResult) error {
func decodeDeleteObjectsResult(result *DeleteObjectVersionsResult) error {
var err error
for i := 0; i < len(result.DeletedObjects); i++ {
result.DeletedObjects[i], err = url.QueryUnescape(result.DeletedObjects[i])
for i := 0; i < len(result.DeletedObjectsDetail); i++ {
result.DeletedObjectsDetail[i].Key, err = url.QueryUnescape(result.DeletedObjectsDetail[i].Key)
if err != nil {
return err
}
@@ -417,6 +580,73 @@ func decodeListObjectsResult(result *ListObjectsResult) error {
return nil
}
// decodeListObjectVersionsResult decodes list version objects result in URL encoding
func decodeListObjectVersionsResult(result *ListObjectVersionsResult) error {
var err error
// decode:Delimiter
result.Delimiter, err = url.QueryUnescape(result.Delimiter)
if err != nil {
return err
}
// decode Prefix
result.Prefix, err = url.QueryUnescape(result.Prefix)
if err != nil {
return err
}
// decode KeyMarker
result.KeyMarker, err = url.QueryUnescape(result.KeyMarker)
if err != nil {
return err
}
// decode VersionIdMarker
result.VersionIdMarker, err = url.QueryUnescape(result.VersionIdMarker)
if err != nil {
return err
}
// decode NextKeyMarker
result.NextKeyMarker, err = url.QueryUnescape(result.NextKeyMarker)
if err != nil {
return err
}
// decode NextVersionIdMarker
result.NextVersionIdMarker, err = url.QueryUnescape(result.NextVersionIdMarker)
if err != nil {
return err
}
// decode CommonPrefixes
for i := 0; i < len(result.CommonPrefixes); i++ {
result.CommonPrefixes[i], err = url.QueryUnescape(result.CommonPrefixes[i])
if err != nil {
return err
}
}
// decode deleteMarker
for i := 0; i < len(result.ObjectDeleteMarkers); i++ {
result.ObjectDeleteMarkers[i].Key, err = url.QueryUnescape(result.ObjectDeleteMarkers[i].Key)
if err != nil {
return err
}
}
// decode ObjectVersions
for i := 0; i < len(result.ObjectVersions); i++ {
result.ObjectVersions[i].Key, err = url.QueryUnescape(result.ObjectVersions[i].Key)
if err != nil {
return err
}
}
return nil
}
// decodeListUploadedPartsResult decodes
func decodeListUploadedPartsResult(result *ListUploadedPartsResult) error {
var err error
@@ -463,6 +693,388 @@ func decodeListMultipartUploadResult(result *ListMultipartUploadResult) error {
// createBucketConfiguration defines the configuration for creating a bucket.
type createBucketConfiguration struct {
XMLName xml.Name `xml:"CreateBucketConfiguration"`
StorageClass StorageClassType `xml:"StorageClass,omitempty"`
XMLName xml.Name `xml:"CreateBucketConfiguration"`
StorageClass StorageClassType `xml:"StorageClass,omitempty"`
DataRedundancyType DataRedundancyType `xml:"DataRedundancyType,omitempty"`
}
// LiveChannelConfiguration defines the configuration for live-channel
type LiveChannelConfiguration struct {
XMLName xml.Name `xml:"LiveChannelConfiguration"`
Description string `xml:"Description,omitempty"` //Description of live-channel, up to 128 bytes
Status string `xml:"Status,omitempty"` //Specify the status of livechannel
Target LiveChannelTarget `xml:"Target"` //target configuration of live-channel
// use point instead of struct to avoid omit empty snapshot
Snapshot *LiveChannelSnapshot `xml:"Snapshot,omitempty"` //snapshot configuration of live-channel
}
// LiveChannelTarget target configuration of live-channel
type LiveChannelTarget struct {
XMLName xml.Name `xml:"Target"`
Type string `xml:"Type"` //the type of object, only supports HLS
FragDuration int `xml:"FragDuration,omitempty"` //the length of each ts object (in seconds), in the range [1,100]
FragCount int `xml:"FragCount,omitempty"` //the number of ts objects in the m3u8 object, in the range of [1,100]
PlaylistName string `xml:"PlaylistName,omitempty"` //the name of m3u8 object, which must end with ".m3u8" and the length range is [6,128]
}
// LiveChannelSnapshot snapshot configuration of live-channel
type LiveChannelSnapshot struct {
XMLName xml.Name `xml:"Snapshot"`
RoleName string `xml:"RoleName,omitempty"` //The role of snapshot operations, it sholud has write permission of DestBucket and the permission to send messages to the NotifyTopic.
DestBucket string `xml:"DestBucket,omitempty"` //Bucket the snapshots will be written to. should be the same owner as the source bucket.
NotifyTopic string `xml:"NotifyTopic,omitempty"` //Topics of MNS for notifying users of high frequency screenshot operation results
Interval int `xml:"Interval,omitempty"` //interval of snapshots, threre is no snapshot if no I-frame during the interval time
}
// CreateLiveChannelResult the result of crete live-channel
type CreateLiveChannelResult struct {
XMLName xml.Name `xml:"CreateLiveChannelResult"`
PublishUrls []string `xml:"PublishUrls>Url"` //push urls list
PlayUrls []string `xml:"PlayUrls>Url"` //play urls list
}
// LiveChannelStat the result of get live-channel state
type LiveChannelStat struct {
XMLName xml.Name `xml:"LiveChannelStat"`
Status string `xml:"Status"` //Current push status of live-channel: Disabled,Live,Idle
ConnectedTime time.Time `xml:"ConnectedTime"` //The time when the client starts pushing, format: ISO8601
RemoteAddr string `xml:"RemoteAddr"` //The ip address of the client
Video LiveChannelVideo `xml:"Video"` //Video stream information
Audio LiveChannelAudio `xml:"Audio"` //Audio stream information
}
// LiveChannelVideo video stream information
type LiveChannelVideo struct {
XMLName xml.Name `xml:"Video"`
Width int `xml:"Width"` //Width (unit: pixels)
Height int `xml:"Height"` //Height (unit: pixels)
FrameRate int `xml:"FrameRate"` //FramRate
Bandwidth int `xml:"Bandwidth"` //Bandwidth (unit: B/s)
}
// LiveChannelAudio audio stream information
type LiveChannelAudio struct {
XMLName xml.Name `xml:"Audio"`
SampleRate int `xml:"SampleRate"` //SampleRate
Bandwidth int `xml:"Bandwidth"` //Bandwidth (unit: B/s)
Codec string `xml:"Codec"` //Encoding forma
}
// LiveChannelHistory the result of GetLiveChannelHistory, at most return up to lastest 10 push records
type LiveChannelHistory struct {
XMLName xml.Name `xml:"LiveChannelHistory"`
Record []LiveRecord `xml:"LiveRecord"` //push records list
}
// LiveRecord push recode
type LiveRecord struct {
XMLName xml.Name `xml:"LiveRecord"`
StartTime time.Time `xml:"StartTime"` //StartTime, format: ISO8601
EndTime time.Time `xml:"EndTime"` //EndTime, format: ISO8601
RemoteAddr string `xml:"RemoteAddr"` //The ip address of remote client
}
// ListLiveChannelResult the result of ListLiveChannel
type ListLiveChannelResult struct {
XMLName xml.Name `xml:"ListLiveChannelResult"`
Prefix string `xml:"Prefix"` //Filter by the name start with the value of "Prefix"
Marker string `xml:"Marker"` //cursor from which starting list
MaxKeys int `xml:"MaxKeys"` //The maximum count returned. the default value is 100. it cannot be greater than 1000.
IsTruncated bool `xml:"IsTruncated"` //Indicates whether all results have been returned, "true" indicates partial results returned while "false" indicates all results have been returned
NextMarker string `xml:"NextMarker"` //NextMarker indicate the Marker value of the next request
LiveChannel []LiveChannelInfo `xml:"LiveChannel"` //The infomation of live-channel
}
// LiveChannelInfo the infomation of live-channel
type LiveChannelInfo struct {
XMLName xml.Name `xml:"LiveChannel"`
Name string `xml:"Name"` //The name of live-channel
Description string `xml:"Description"` //Description of live-channel
Status string `xml:"Status"` //Status: disabled or enabled
LastModified time.Time `xml:"LastModified"` //Last modification time, format: ISO8601
PublishUrls []string `xml:"PublishUrls>Url"` //push urls list
PlayUrls []string `xml:"PlayUrls>Url"` //play urls list
}
// Tag a tag for the object
type Tag struct {
XMLName xml.Name `xml:"Tag"`
Key string `xml:"Key"`
Value string `xml:"Value"`
}
// Tagging tagset for the object
type Tagging struct {
XMLName xml.Name `xml:"Tagging"`
Tags []Tag `xml:"TagSet>Tag,omitempty"`
}
// for GetObjectTagging return value
type GetObjectTaggingResult Tagging
// VersioningConfig for the bucket
type VersioningConfig struct {
XMLName xml.Name `xml:"VersioningConfiguration"`
Status string `xml:"Status"`
}
type GetBucketVersioningResult VersioningConfig
// Server Encryption rule for the bucket
type ServerEncryptionRule struct {
XMLName xml.Name `xml:"ServerSideEncryptionRule"`
SSEDefault SSEDefaultRule `xml:"ApplyServerSideEncryptionByDefault"`
}
// Server Encryption deafult rule for the bucket
type SSEDefaultRule struct {
XMLName xml.Name `xml:"ApplyServerSideEncryptionByDefault"`
SSEAlgorithm string `xml:"SSEAlgorithm"`
KMSMasterKeyID string `xml:"KMSMasterKeyID"`
}
type GetBucketEncryptionResult ServerEncryptionRule
type GetBucketTaggingResult Tagging
type BucketStat struct {
XMLName xml.Name `xml:"BucketStat"`
Storage int64 `xml:"Storage"`
ObjectCount int64 `xml:"ObjectCount"`
MultipartUploadCount int64 `xml:"MultipartUploadCount"`
}
type GetBucketStatResult BucketStat
// RequestPaymentConfiguration define the request payment configuration
type RequestPaymentConfiguration struct {
XMLName xml.Name `xml:"RequestPaymentConfiguration"`
Payer string `xml:"Payer,omitempty"`
}
// BucketQoSConfiguration define QoS configuration
type BucketQoSConfiguration struct {
XMLName xml.Name `xml:"QoSConfiguration"`
TotalUploadBandwidth *int `xml:"TotalUploadBandwidth"` // Total upload bandwidth
IntranetUploadBandwidth *int `xml:"IntranetUploadBandwidth"` // Intranet upload bandwidth
ExtranetUploadBandwidth *int `xml:"ExtranetUploadBandwidth"` // Extranet upload bandwidth
TotalDownloadBandwidth *int `xml:"TotalDownloadBandwidth"` // Total download bandwidth
IntranetDownloadBandwidth *int `xml:"IntranetDownloadBandwidth"` // Intranet download bandwidth
ExtranetDownloadBandwidth *int `xml:"ExtranetDownloadBandwidth"` // Extranet download bandwidth
TotalQPS *int `xml:"TotalQps"` // Total Qps
IntranetQPS *int `xml:"IntranetQps"` // Intranet Qps
ExtranetQPS *int `xml:"ExtranetQps"` // Extranet Qps
}
// UserQoSConfiguration define QoS and Range configuration
type UserQoSConfiguration struct {
XMLName xml.Name `xml:"QoSConfiguration"`
Region string `xml:"Region,omitempty"` // Effective area of Qos configuration
BucketQoSConfiguration
}
//////////////////////////////////////////////////////////////
/////////////////// Select OBject ////////////////////////////
//////////////////////////////////////////////////////////////
type CsvMetaRequest struct {
XMLName xml.Name `xml:"CsvMetaRequest"`
InputSerialization InputSerialization `xml:"InputSerialization"`
OverwriteIfExists *bool `xml:"OverwriteIfExists,omitempty"`
}
// encodeBase64 encode base64 of the CreateSelectObjectMeta api request params
func (meta *CsvMetaRequest) encodeBase64() {
meta.InputSerialization.CSV.RecordDelimiter =
base64.StdEncoding.EncodeToString([]byte(meta.InputSerialization.CSV.RecordDelimiter))
meta.InputSerialization.CSV.FieldDelimiter =
base64.StdEncoding.EncodeToString([]byte(meta.InputSerialization.CSV.FieldDelimiter))
meta.InputSerialization.CSV.QuoteCharacter =
base64.StdEncoding.EncodeToString([]byte(meta.InputSerialization.CSV.QuoteCharacter))
}
type JsonMetaRequest struct {
XMLName xml.Name `xml:"JsonMetaRequest"`
InputSerialization InputSerialization `xml:"InputSerialization"`
OverwriteIfExists *bool `xml:"OverwriteIfExists,omitempty"`
}
type InputSerialization struct {
XMLName xml.Name `xml:"InputSerialization"`
CSV CSV `xml:CSV,omitempty`
JSON JSON `xml:JSON,omitempty`
CompressionType string `xml:"CompressionType,omitempty"`
}
type CSV struct {
XMLName xml.Name `xml:"CSV"`
RecordDelimiter string `xml:"RecordDelimiter,omitempty"`
FieldDelimiter string `xml:"FieldDelimiter,omitempty"`
QuoteCharacter string `xml:"QuoteCharacter,omitempty"`
}
type JSON struct {
XMLName xml.Name `xml:"JSON"`
JSONType string `xml:"Type,omitempty"`
}
// SelectRequest is for the SelectObject request params of json file
type SelectRequest struct {
XMLName xml.Name `xml:"SelectRequest"`
Expression string `xml:"Expression"`
InputSerializationSelect InputSerializationSelect `xml:"InputSerialization"`
OutputSerializationSelect OutputSerializationSelect `xml:"OutputSerialization"`
SelectOptions SelectOptions `xml:"Options,omitempty"`
}
type InputSerializationSelect struct {
XMLName xml.Name `xml:"InputSerialization"`
CsvBodyInput CSVSelectInput `xml:CSV,omitempty`
JsonBodyInput JSONSelectInput `xml:JSON,omitempty`
CompressionType string `xml:"CompressionType,omitempty"`
}
type CSVSelectInput struct {
XMLName xml.Name `xml:"CSV"`
FileHeaderInfo string `xml:"FileHeaderInfo,omitempty"`
RecordDelimiter string `xml:"RecordDelimiter,omitempty"`
FieldDelimiter string `xml:"FieldDelimiter,omitempty"`
QuoteCharacter string `xml:"QuoteCharacter,omitempty"`
CommentCharacter string `xml:"CommentCharacter,omitempty"`
Range string `xml:"Range,omitempty"`
SplitRange string
}
type JSONSelectInput struct {
XMLName xml.Name `xml:"JSON"`
JSONType string `xml:"Type,omitempty"`
Range string `xml:"Range,omitempty"`
ParseJSONNumberAsString *bool `xml:"ParseJsonNumberAsString"`
SplitRange string
}
func (jsonInput *JSONSelectInput) JsonIsEmpty() bool {
if jsonInput.JSONType != "" {
return false
}
return true
}
type OutputSerializationSelect struct {
XMLName xml.Name `xml:"OutputSerialization"`
CsvBodyOutput CSVSelectOutput `xml:CSV,omitempty`
JsonBodyOutput JSONSelectOutput `xml:JSON,omitempty`
OutputRawData *bool `xml:"OutputRawData,omitempty"`
KeepAllColumns *bool `xml:"KeepAllColumns,omitempty"`
EnablePayloadCrc *bool `xml:"EnablePayloadCrc,omitempty"`
OutputHeader *bool `xml:"OutputHeader,omitempty"`
}
type CSVSelectOutput struct {
XMLName xml.Name `xml:"CSV"`
RecordDelimiter string `xml:"RecordDelimiter,omitempty"`
FieldDelimiter string `xml:"FieldDelimiter,omitempty"`
}
type JSONSelectOutput struct {
XMLName xml.Name `xml:"JSON"`
RecordDelimiter string `xml:"RecordDelimiter,omitempty"`
}
func (selectReq *SelectRequest) encodeBase64() {
if selectReq.InputSerializationSelect.JsonBodyInput.JsonIsEmpty() {
selectReq.csvEncodeBase64()
} else {
selectReq.jsonEncodeBase64()
}
}
// csvEncodeBase64 encode base64 of the SelectObject api request params
func (selectReq *SelectRequest) csvEncodeBase64() {
selectReq.Expression = base64.StdEncoding.EncodeToString([]byte(selectReq.Expression))
selectReq.InputSerializationSelect.CsvBodyInput.RecordDelimiter =
base64.StdEncoding.EncodeToString([]byte(selectReq.InputSerializationSelect.CsvBodyInput.RecordDelimiter))
selectReq.InputSerializationSelect.CsvBodyInput.FieldDelimiter =
base64.StdEncoding.EncodeToString([]byte(selectReq.InputSerializationSelect.CsvBodyInput.FieldDelimiter))
selectReq.InputSerializationSelect.CsvBodyInput.QuoteCharacter =
base64.StdEncoding.EncodeToString([]byte(selectReq.InputSerializationSelect.CsvBodyInput.QuoteCharacter))
selectReq.InputSerializationSelect.CsvBodyInput.CommentCharacter =
base64.StdEncoding.EncodeToString([]byte(selectReq.InputSerializationSelect.CsvBodyInput.CommentCharacter))
selectReq.OutputSerializationSelect.CsvBodyOutput.FieldDelimiter =
base64.StdEncoding.EncodeToString([]byte(selectReq.OutputSerializationSelect.CsvBodyOutput.FieldDelimiter))
selectReq.OutputSerializationSelect.CsvBodyOutput.RecordDelimiter =
base64.StdEncoding.EncodeToString([]byte(selectReq.OutputSerializationSelect.CsvBodyOutput.RecordDelimiter))
// handle Range
if selectReq.InputSerializationSelect.CsvBodyInput.Range != "" {
selectReq.InputSerializationSelect.CsvBodyInput.Range = "line-range=" + selectReq.InputSerializationSelect.CsvBodyInput.Range
}
if selectReq.InputSerializationSelect.CsvBodyInput.SplitRange != "" {
selectReq.InputSerializationSelect.CsvBodyInput.Range = "split-range=" + selectReq.InputSerializationSelect.CsvBodyInput.SplitRange
}
}
// jsonEncodeBase64 encode base64 of the SelectObject api request params
func (selectReq *SelectRequest) jsonEncodeBase64() {
selectReq.Expression = base64.StdEncoding.EncodeToString([]byte(selectReq.Expression))
selectReq.OutputSerializationSelect.JsonBodyOutput.RecordDelimiter =
base64.StdEncoding.EncodeToString([]byte(selectReq.OutputSerializationSelect.JsonBodyOutput.RecordDelimiter))
// handle Range
if selectReq.InputSerializationSelect.JsonBodyInput.Range != "" {
selectReq.InputSerializationSelect.JsonBodyInput.Range = "line-range=" + selectReq.InputSerializationSelect.JsonBodyInput.Range
}
if selectReq.InputSerializationSelect.JsonBodyInput.SplitRange != "" {
selectReq.InputSerializationSelect.JsonBodyInput.Range = "split-range=" + selectReq.InputSerializationSelect.JsonBodyInput.SplitRange
}
}
// CsvOptions is a element in the SelectObject api request's params
type SelectOptions struct {
XMLName xml.Name `xml:"Options"`
SkipPartialDataRecord *bool `xml:"SkipPartialDataRecord,omitempty"`
MaxSkippedRecordsAllowed string `xml:"MaxSkippedRecordsAllowed,omitempty"`
}
// SelectObjectResult is the SelectObject api's return
type SelectObjectResult struct {
Version byte
FrameType int32
PayloadLength int32
HeaderCheckSum uint32
Offset uint64
Data string // DataFrame
EndFrame EndFrame // EndFrame
MetaEndFrameCSV MetaEndFrameCSV // MetaEndFrameCSV
MetaEndFrameJSON MetaEndFrameJSON // MetaEndFrameJSON
PayloadChecksum uint32
ReadFlagInfo
}
// ReadFlagInfo if reading the frame data, recode the reading status
type ReadFlagInfo struct {
OpenLine bool
ConsumedBytesLength int32
EnablePayloadCrc bool
OutputRawData bool
}
// EndFrame is EndFrameType of SelectObject api
type EndFrame struct {
TotalScanned int64
HTTPStatusCode int32
ErrorMsg string
}
// MetaEndFrameCSV is MetaEndFrameCSVType of CreateSelectObjectMeta
type MetaEndFrameCSV struct {
TotalScanned int64
Status int32
SplitsCount int32
RowsCount int64
ColumnsCount int32
ErrorMsg string
}
// MetaEndFrameJSON is MetaEndFrameJSON of CreateSelectObjectMeta
type MetaEndFrameJSON struct {
TotalScanned int64
Status int32
SplitsCount int32
RowsCount int64
ErrorMsg string
}

68
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/upload.go generated vendored
View File

@@ -24,7 +24,7 @@ import (
//
func (bucket Bucket) UploadFile(objectKey, filePath string, partSize int64, options ...Option) error {
if partSize < MinPartSize || partSize > MaxPartSize {
return errors.New("oss: part size invalid range (1024KB, 5GB]")
return errors.New("oss: part size invalid range (100KB, 5GB]")
}
cpConf := getCpConfig(options)
@@ -44,7 +44,7 @@ func getUploadCpFilePath(cpConf *cpConfig, srcFile, destBucket, destObject strin
if cpConf.FilePath == "" && cpConf.DirPath != "" {
dest := fmt.Sprintf("oss://%v/%v", destBucket, destObject)
absPath, _ := filepath.Abs(srcFile)
cpFileName := getCpFileName(absPath, dest)
cpFileName := getCpFileName(absPath, dest, "")
cpConf.FilePath = cpConf.DirPath + string(os.PathSeparator) + cpFileName
}
return cpConf.FilePath
@@ -63,7 +63,7 @@ func getCpConfig(options []Option) *cpConfig {
}
// getCpFileName return the name of the checkpoint file
func getCpFileName(src, dest string) string {
func getCpFileName(src, dest, versionId string) string {
md5Ctx := md5.New()
md5Ctx.Write([]byte(src))
srcCheckSum := hex.EncodeToString(md5Ctx.Sum(nil))
@@ -72,7 +72,14 @@ func getCpFileName(src, dest string) string {
md5Ctx.Write([]byte(dest))
destCheckSum := hex.EncodeToString(md5Ctx.Sum(nil))
return fmt.Sprintf("%v-%v.cp", srcCheckSum, destCheckSum)
if versionId == "" {
return fmt.Sprintf("%v-%v.cp", srcCheckSum, destCheckSum)
}
md5Ctx.Reset()
md5Ctx.Write([]byte(versionId))
versionCheckSum := hex.EncodeToString(md5Ctx.Sum(nil))
return fmt.Sprintf("%v-%v-%v.cp", srcCheckSum, destCheckSum, versionCheckSum)
}
// getRoutines gets the routine count. by default it's 1.
@@ -94,7 +101,7 @@ func getRoutines(options []Option) int {
// getPayer return the payer of the request
func getPayer(options []Option) string {
payerOpt, err := findOption(options, HTTPHeaderOSSRequester, nil)
payerOpt, err := findOption(options, HTTPHeaderOssRequester, nil)
if err != nil || payerOpt == nil {
return ""
}
@@ -175,11 +182,9 @@ func (bucket Bucket) uploadFile(objectKey, filePath string, partSize int64, opti
return err
}
payerOptions := []Option{}
payer := getPayer(options)
if payer != "" {
payerOptions = append(payerOptions, RequestPayer(PayerType(payer)))
}
partOptions := ChoiceTransferPartOption(options)
completeOptions := ChoiceCompletePartOption(options)
abortOptions := ChoiceAbortPartOption(options)
// Initialize the multipart upload
imur, err := bucket.InitiateMultipartUpload(objectKey, options...)
@@ -194,11 +199,11 @@ func (bucket Bucket) uploadFile(objectKey, filePath string, partSize int64, opti
var completedBytes int64
totalBytes := getTotalBytes(chunks)
event := newProgressEvent(TransferStartedEvent, 0, totalBytes)
event := newProgressEvent(TransferStartedEvent, 0, totalBytes, 0)
publishProgress(listener, event)
// Start the worker coroutine
arg := workerArg{&bucket, filePath, imur, payerOptions, uploadPartHooker}
arg := workerArg{&bucket, filePath, imur, partOptions, uploadPartHooker}
for w := 1; w <= routines; w++ {
go worker(w, arg, jobs, results, failed, die)
}
@@ -215,13 +220,16 @@ func (bucket Bucket) uploadFile(objectKey, filePath string, partSize int64, opti
completed++
parts[part.PartNumber-1] = part
completedBytes += chunks[part.PartNumber-1].Size
event = newProgressEvent(TransferDataEvent, completedBytes, totalBytes)
// why RwBytes in ProgressEvent is 0 ?
// because read or write event has been notified in teeReader.Read()
event = newProgressEvent(TransferDataEvent, completedBytes, totalBytes, 0)
publishProgress(listener, event)
case err := <-failed:
close(die)
event = newProgressEvent(TransferFailedEvent, completedBytes, totalBytes)
event = newProgressEvent(TransferFailedEvent, completedBytes, totalBytes, 0)
publishProgress(listener, event)
bucket.AbortMultipartUpload(imur, payerOptions...)
bucket.AbortMultipartUpload(imur, abortOptions...)
return err
}
@@ -230,13 +238,13 @@ func (bucket Bucket) uploadFile(objectKey, filePath string, partSize int64, opti
}
}
event = newProgressEvent(TransferStartedEvent, completedBytes, totalBytes)
event = newProgressEvent(TransferStartedEvent, completedBytes, totalBytes, 0)
publishProgress(listener, event)
// Complete the multpart upload
_, err = bucket.CompleteMultipartUpload(imur, parts, payerOptions...)
_, err = bucket.CompleteMultipartUpload(imur, parts, completeOptions...)
if err != nil {
bucket.AbortMultipartUpload(imur, payerOptions...)
bucket.AbortMultipartUpload(imur, abortOptions...)
return err
}
return nil
@@ -299,7 +307,7 @@ func (cp uploadCheckpoint) isValid(filePath string) (bool, error) {
// Compare the file size, file's last modified time and file's MD5
if cp.FileStat.Size != st.Size() ||
cp.FileStat.LastModified != st.ModTime() ||
!cp.FileStat.LastModified.Equal(st.ModTime()) ||
cp.FileStat.MD5 != md {
return false, nil
}
@@ -448,11 +456,8 @@ func complete(cp *uploadCheckpoint, bucket *Bucket, parts []UploadPart, cpFilePa
func (bucket Bucket) uploadFileWithCp(objectKey, filePath string, partSize int64, options []Option, cpFilePath string, routines int) error {
listener := getProgressListener(options)
payerOptions := []Option{}
payer := getPayer(options)
if payer != "" {
payerOptions = append(payerOptions, RequestPayer(PayerType(payer)))
}
partOptions := ChoiceTransferPartOption(options)
completeOptions := ChoiceCompletePartOption(options)
// Load CP data
ucp := uploadCheckpoint{}
@@ -482,11 +487,14 @@ func (bucket Bucket) uploadFileWithCp(objectKey, filePath string, partSize int64
die := make(chan bool)
completedBytes := ucp.getCompletedBytes()
event := newProgressEvent(TransferStartedEvent, completedBytes, ucp.FileStat.Size)
// why RwBytes in ProgressEvent is 0 ?
// because read or write event has been notified in teeReader.Read()
event := newProgressEvent(TransferStartedEvent, completedBytes, ucp.FileStat.Size, 0)
publishProgress(listener, event)
// Start the workers
arg := workerArg{&bucket, filePath, imur, payerOptions, uploadPartHooker}
arg := workerArg{&bucket, filePath, imur, partOptions, uploadPartHooker}
for w := 1; w <= routines; w++ {
go worker(w, arg, jobs, results, failed, die)
}
@@ -503,11 +511,11 @@ func (bucket Bucket) uploadFileWithCp(objectKey, filePath string, partSize int64
ucp.updatePart(part)
ucp.dump(cpFilePath)
completedBytes += ucp.Parts[part.PartNumber-1].Chunk.Size
event = newProgressEvent(TransferDataEvent, completedBytes, ucp.FileStat.Size)
event = newProgressEvent(TransferDataEvent, completedBytes, ucp.FileStat.Size, 0)
publishProgress(listener, event)
case err := <-failed:
close(die)
event = newProgressEvent(TransferFailedEvent, completedBytes, ucp.FileStat.Size)
event = newProgressEvent(TransferFailedEvent, completedBytes, ucp.FileStat.Size, 0)
publishProgress(listener, event)
return err
}
@@ -517,10 +525,10 @@ func (bucket Bucket) uploadFileWithCp(objectKey, filePath string, partSize int64
}
}
event = newProgressEvent(TransferCompletedEvent, completedBytes, ucp.FileStat.Size)
event = newProgressEvent(TransferCompletedEvent, completedBytes, ucp.FileStat.Size, 0)
publishProgress(listener, event)
// Complete the multipart upload
err = complete(&ucp, &bucket, ucp.allParts(), cpFilePath, payerOptions)
err = complete(&ucp, &bucket, ucp.allParts(), cpFilePath, completeOptions)
return err
}

138
vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/utils.go generated vendored
View File

@@ -4,6 +4,7 @@ import (
"bytes"
"errors"
"fmt"
"hash/crc32"
"hash/crc64"
"net/http"
"os"
@@ -16,11 +17,11 @@ import (
// userAgent gets user agent
// It has the SDK version information, OS information and GO version
var userAgent = func() string {
func userAgent() string {
sys := getSysInfo()
return fmt.Sprintf("aliyun-sdk-go/%s (%s/%s/%s;%s)", Version, sys.name,
sys.release, sys.machine, runtime.Version())
}()
}
type sysInfo struct {
name string // OS name such as windows/Linux
@@ -263,3 +264,136 @@ func GetPartEnd(begin int64, total int64, per int64) int64 {
var crcTable = func() *crc64.Table {
return crc64.MakeTable(crc64.ECMA)
}
// crcTable returns the table constructed from the specified polynomial
var crc32Table = func() *crc32.Table {
return crc32.MakeTable(crc32.IEEE)
}
// choiceTransferPartOption choices valid option supported by Uploadpart or DownloadPart
func ChoiceTransferPartOption(options []Option) []Option {
var outOption []Option
listener, _ := findOption(options, progressListener, nil)
if listener != nil {
outOption = append(outOption, Progress(listener.(ProgressListener)))
}
payer, _ := findOption(options, HTTPHeaderOssRequester, nil)
if payer != nil {
outOption = append(outOption, RequestPayer(PayerType(payer.(string))))
}
versionId, _ := findOption(options, "versionId", nil)
if versionId != nil {
outOption = append(outOption, VersionId(versionId.(string)))
}
trafficLimit, _ := findOption(options, HTTPHeaderOssTrafficLimit, nil)
if trafficLimit != nil {
speed, _ := strconv.ParseInt(trafficLimit.(string), 10, 64)
outOption = append(outOption, TrafficLimitHeader(speed))
}
respHeader, _ := findOption(options, responseHeader, nil)
if respHeader != nil {
outOption = append(outOption, GetResponseHeader(respHeader.(*http.Header)))
}
return outOption
}
// ChoiceCompletePartOption choices valid option supported by CompleteMulitiPart
func ChoiceCompletePartOption(options []Option) []Option {
var outOption []Option
listener, _ := findOption(options, progressListener, nil)
if listener != nil {
outOption = append(outOption, Progress(listener.(ProgressListener)))
}
payer, _ := findOption(options, HTTPHeaderOssRequester, nil)
if payer != nil {
outOption = append(outOption, RequestPayer(PayerType(payer.(string))))
}
acl, _ := findOption(options, HTTPHeaderOssObjectACL, nil)
if acl != nil {
outOption = append(outOption, ObjectACL(ACLType(acl.(string))))
}
callback, _ := findOption(options, HTTPHeaderOssCallback, nil)
if callback != nil {
outOption = append(outOption, Callback(callback.(string)))
}
callbackVar, _ := findOption(options, HTTPHeaderOssCallbackVar, nil)
if callbackVar != nil {
outOption = append(outOption, CallbackVar(callbackVar.(string)))
}
respHeader, _ := findOption(options, responseHeader, nil)
if respHeader != nil {
outOption = append(outOption, GetResponseHeader(respHeader.(*http.Header)))
}
return outOption
}
// ChoiceAbortPartOption choices valid option supported by AbortMultipartUpload
func ChoiceAbortPartOption(options []Option) []Option {
var outOption []Option
payer, _ := findOption(options, HTTPHeaderOssRequester, nil)
if payer != nil {
outOption = append(outOption, RequestPayer(PayerType(payer.(string))))
}
respHeader, _ := findOption(options, responseHeader, nil)
if respHeader != nil {
outOption = append(outOption, GetResponseHeader(respHeader.(*http.Header)))
}
return outOption
}
// ChoiceHeadObjectOption choices valid option supported by HeadObject
func ChoiceHeadObjectOption(options []Option) []Option {
var outOption []Option
payer, _ := findOption(options, HTTPHeaderOssRequester, nil)
if payer != nil {
outOption = append(outOption, RequestPayer(PayerType(payer.(string))))
}
versionId, _ := findOption(options, "versionId", nil)
if versionId != nil {
outOption = append(outOption, VersionId(versionId.(string)))
}
respHeader, _ := findOption(options, responseHeader, nil)
if respHeader != nil {
outOption = append(outOption, GetResponseHeader(respHeader.(*http.Header)))
}
return outOption
}
func CheckBucketName(bucketName string) error {
nameLen := len(bucketName)
if nameLen < 3 || nameLen > 63 {
return fmt.Errorf("bucket name %s len is between [3-63],now is %d", bucketName, nameLen)
}
for _, v := range bucketName {
if !(('a' <= v && v <= 'z') || ('0' <= v && v <= '9') || v == '-') {
return fmt.Errorf("bucket name %s can only include lowercase letters, numbers, and -", bucketName)
}
}
if bucketName[0] == '-' || bucketName[nameLen-1] == '-' {
return fmt.Errorf("bucket name %s must start and end with a lowercase letter or number", bucketName)
}
return nil
}

2
vendor/modules.txt vendored
View File

@@ -49,7 +49,7 @@ github.com/aliyun/alibaba-cloud-sdk-go/sdk/errors
github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests
github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses
github.com/aliyun/alibaba-cloud-sdk-go/sdk/utils
# github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20181116160301-c6838fdc33ed
# github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible
github.com/aliyun/aliyun-oss-go-sdk/oss
# github.com/anacrolix/dht v0.0.0-20181129074040-b09db78595aa
github.com/anacrolix/dht