From 9fff0bef22702c7976b2b41cbb474ba5860f9a6f Mon Sep 17 00:00:00 2001 From: LyndonKong Date: Wed, 18 Oct 2023 18:50:08 +0800 Subject: [PATCH] feature: add support for for VolcEngine (#18348) --- cmd/climc/shell/compute/cloudaccounts.go | 4 + cmd/climc/shell/compute/usages.go | 4 +- cmd/climc/shell/misc/feature.go | 1 + go.mod | 3 +- go.sum | 7 +- pkg/apis/compute/cloudaccount_const.go | 5 + pkg/apis/compute/guest_const.go | 5 + pkg/apis/compute/host_const.go | 2 + pkg/apis/compute/storage_const.go | 5 + pkg/compute/guestdrivers/volcengine.go | 157 ++ pkg/compute/hostdrivers/volcengine.go | 81 + pkg/compute/regiondrivers/volcengine.go | 68 + pkg/mcclient/options/base.go | 2 +- pkg/mcclient/options/cloudaccounts.go | 28 + pkg/mcclient/options/compute/servers.go | 4 +- .../volcengine/ve-tos-golang-sdk/v2/LICENSE | 490 ++++ .../ve-tos-golang-sdk/v2/tos/acl.go | 211 ++ .../ve-tos-golang-sdk/v2/tos/base_client.go | 124 + .../ve-tos-golang-sdk/v2/tos/bucket.go | 264 ++ .../ve-tos-golang-sdk/v2/tos/check.go | 126 + .../ve-tos-golang-sdk/v2/tos/checksum.go | 55 + .../ve-tos-golang-sdk/v2/tos/client.go | 740 ++++++ .../ve-tos-golang-sdk/v2/tos/config.go | 31 + .../ve-tos-golang-sdk/v2/tos/consts.go | 149 ++ .../ve-tos-golang-sdk/v2/tos/copy.go | 310 +++ .../ve-tos-golang-sdk/v2/tos/cors.go | 81 + .../ve-tos-golang-sdk/v2/tos/crc.go | 113 + .../ve-tos-golang-sdk/v2/tos/credential.go | 161 ++ .../ve-tos-golang-sdk/v2/tos/custom_domain.go | 76 + .../ve-tos-golang-sdk/v2/tos/dns_cache.go | 213 ++ .../ve-tos-golang-sdk/v2/tos/download_file.go | 388 +++ .../ve-tos-golang-sdk/v2/tos/enum/enum.go | 181 ++ .../ve-tos-golang-sdk/v2/tos/error.go | 323 +++ .../ve-tos-golang-sdk/v2/tos/fetch.go | 149 ++ .../ve-tos-golang-sdk/v2/tos/http_trace.go | 106 + .../ve-tos-golang-sdk/v2/tos/lifecycle.go | 116 + .../ve-tos-golang-sdk/v2/tos/logger.go | 9 + .../ve-tos-golang-sdk/v2/tos/metadata.go | 149 ++ .../ve-tos-golang-sdk/v2/tos/mime.go | 572 +++++ .../ve-tos-golang-sdk/v2/tos/mirror_back.go | 76 + .../ve-tos-golang-sdk/v2/tos/multipart.go | 513 ++++ .../ve-tos-golang-sdk/v2/tos/notification.go | 54 + .../ve-tos-golang-sdk/v2/tos/object.go | 1121 +++++++++ .../ve-tos-golang-sdk/v2/tos/options.go | 274 ++ .../ve-tos-golang-sdk/v2/tos/parse_output.go | 249 ++ .../ve-tos-golang-sdk/v2/tos/policy.go | 180 ++ .../ve-tos-golang-sdk/v2/tos/proxy.go | 55 + .../ve-tos-golang-sdk/v2/tos/rate_limiter.go | 58 + .../ve-tos-golang-sdk/v2/tos/realtime_log.go | 76 + .../ve-tos-golang-sdk/v2/tos/rename.go | 129 + .../ve-tos-golang-sdk/v2/tos/replication.go | 79 + .../ve-tos-golang-sdk/v2/tos/request.go | 425 ++++ .../v2/tos/resumable_copy.go | 289 +++ .../ve-tos-golang-sdk/v2/tos/sign_v4.go | 414 +++ .../ve-tos-golang-sdk/v2/tos/tagging.go | 17 + .../ve-tos-golang-sdk/v2/tos/transport.go | 254 ++ .../ve-tos-golang-sdk/v2/tos/type.go | 2221 +++++++++++++++++ .../ve-tos-golang-sdk/v2/tos/type_internal.go | 1009 ++++++++ .../ve-tos-golang-sdk/v2/tos/upload_file.go | 428 ++++ .../ve-tos-golang-sdk/v2/tos/util.go | 165 ++ .../ve-tos-golang-sdk/v2/tos/versioning.go | 38 + .../ve-tos-golang-sdk/v2/tos/website.go | 82 + .../volcengine/volc-sdk-golang/base/aes.go | 51 + .../volcengine/volc-sdk-golang/base/client.go | 290 +++ .../volcengine/volc-sdk-golang/base/model.go | 109 + .../volcengine/volc-sdk-golang/base/sign.go | 288 +++ .../volcengine/volc-sdk-golang/base/utils.go | 180 ++ .../x/sync/singleflight/singleflight.go | 205 ++ vendor/modules.txt | 15 +- .../pkg/apis/compute/cloudaccount_const.go | 2 + .../cloudmux/pkg/apis/compute/guest_const.go | 1 + .../x/cloudmux/pkg/apis/compute/host_const.go | 1 + .../pkg/apis/compute/storage_const.go | 5 + .../cloudmux/pkg/multicloud/loader/loader.go | 1 + .../pkg/multicloud/volcengine/bucket.go | 416 +++ .../pkg/multicloud/volcengine/charge.go | 48 + .../pkg/multicloud/volcengine/disk.go | 334 +++ .../cloudmux/pkg/multicloud/volcengine/eip.go | 388 +++ .../pkg/multicloud/volcengine/errors.go | 29 + .../pkg/multicloud/volcengine/host.go | 250 ++ .../pkg/multicloud/volcengine/image.go | 320 +++ .../pkg/multicloud/volcengine/instance.go | 832 ++++++ .../pkg/multicloud/volcengine/instancenic.go | 122 + .../pkg/multicloud/volcengine/keypairs.go | 143 ++ .../volcengine/latitude_and_longitude.go | 26 + .../pkg/multicloud/volcengine/natdtable.go | 176 ++ .../pkg/multicloud/volcengine/natgateway.go | 293 +++ .../pkg/multicloud/volcengine/natstable.go | 194 ++ .../pkg/multicloud/volcengine/network.go | 255 ++ .../volcengine/networkinterfaces.go | 174 ++ .../pkg/multicloud/volcengine/objects.go | 102 + .../pkg/multicloud/volcengine/project.go | 140 ++ .../multicloud/volcengine/provider}/doc.go | 4 +- .../volcengine/provider/provider.go | 189 ++ .../pkg/multicloud/volcengine/region.go | 680 +++++ .../pkg/multicloud/volcengine/routetable.go | 290 +++ .../multicloud/volcengine/securitygroup.go | 327 +++ .../pkg/multicloud/volcengine/storage.go | 155 ++ .../pkg/multicloud/volcengine/storagecache.go | 174 ++ .../pkg/multicloud/volcengine/tag_base.go | 62 + .../pkg/multicloud/volcengine/user.go | 63 + .../pkg/multicloud/volcengine/volcengine.go | 390 +++ .../cloudmux/pkg/multicloud/volcengine/vpc.go | 261 ++ .../pkg/multicloud/volcengine/wire.go | 142 ++ .../pkg/multicloud/volcengine/zone.go | 183 ++ 105 files changed, 22016 insertions(+), 13 deletions(-) create mode 100644 pkg/compute/guestdrivers/volcengine.go create mode 100644 pkg/compute/hostdrivers/volcengine.go create mode 100644 pkg/compute/regiondrivers/volcengine.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/LICENSE create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/acl.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/base_client.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/bucket.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/check.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/checksum.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/client.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/config.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/consts.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/copy.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/cors.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/crc.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/credential.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/custom_domain.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/dns_cache.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/download_file.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum/enum.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/error.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/fetch.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/http_trace.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/lifecycle.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/logger.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/metadata.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mime.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mirror_back.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/multipart.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/notification.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/object.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/options.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/parse_output.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/policy.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/proxy.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rate_limiter.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/realtime_log.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rename.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/replication.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/request.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/resumable_copy.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/sign_v4.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/tagging.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/transport.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type_internal.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/upload_file.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/util.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/versioning.go create mode 100644 vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/website.go create mode 100644 vendor/github.com/volcengine/volc-sdk-golang/base/aes.go create mode 100644 vendor/github.com/volcengine/volc-sdk-golang/base/client.go create mode 100644 vendor/github.com/volcengine/volc-sdk-golang/base/model.go create mode 100644 vendor/github.com/volcengine/volc-sdk-golang/base/sign.go create mode 100644 vendor/github.com/volcengine/volc-sdk-golang/base/utils.go create mode 100644 vendor/golang.org/x/sync/singleflight/singleflight.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/bucket.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/charge.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/disk.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/eip.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/errors.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/host.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/image.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instance.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instancenic.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/keypairs.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/latitude_and_longitude.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natdtable.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natgateway.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natstable.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/network.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/networkinterfaces.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/objects.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/project.go rename {pkg/keystone/cache => vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider}/doc.go (86%) create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/provider.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/region.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/routetable.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/securitygroup.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storage.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storagecache.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/tag_base.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/user.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/volcengine.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/vpc.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/wire.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/zone.go diff --git a/cmd/climc/shell/compute/cloudaccounts.go b/cmd/climc/shell/compute/cloudaccounts.go index 45cded5926..5e2f0f74aa 100644 --- a/cmd/climc/shell/compute/cloudaccounts.go +++ b/cmd/climc/shell/compute/cloudaccounts.go @@ -41,6 +41,7 @@ func init() { cmd.CreateWithKeyword("create-hcs", &options.SHCSAccountCreateOptions{}) cmd.CreateWithKeyword("create-hcsop", &options.SHcsOpCloudAccountCreateOptions{}) cmd.CreateWithKeyword("create-ucloud", &options.SUcloudCloudAccountCreateOptions{}) + cmd.CreateWithKeyword("create-volcengine", &options.SVolcengineCloudAccountCreateOptions{}) cmd.CreateWithKeyword("create-zstack", &options.SZStackCloudAccountCreateOptions{}) cmd.CreateWithKeyword("create-s3", &options.SS3CloudAccountCreateOptions{}) cmd.CreateWithKeyword("create-ceph", &options.SCephCloudAccountCreateOptions{}) @@ -71,6 +72,7 @@ func init() { cmd.UpdateWithKeyword("update-hcso", &options.SHCSOAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-hcs", &options.SHCSAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-ucloud", &options.SUcloudCloudAccountUpdateOptions{}) + cmd.UpdateWithKeyword("update-volcengine", &options.SVolcengineCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-zstack", &options.SZStackCloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-s3", &options.SS3CloudAccountUpdateOptions{}) cmd.UpdateWithKeyword("update-ctyun", &options.SCtyunCloudAccountUpdateOptions{}) @@ -99,6 +101,7 @@ func init() { cmd.PerformWithKeyword("update-credential-hcso", "update-credential", &options.SHCSOAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-hcs", "update-credential", &options.SHCSOAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-ucloud", "update-credential", &options.SUcloudCloudAccountUpdateCredentialOptions{}) + cmd.PerformWithKeyword("update-credential-volcengine", "update-credential", &options.SVolcengineCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-zstack", "update-credential", &options.SZStackCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-s3", "update-credential", &options.SS3CloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("update-credential-ctyun", "update-credential", &options.SCtyunCloudAccountUpdateCredentialOptions{}) @@ -122,6 +125,7 @@ func init() { cmd.PerformWithKeyword("test-connectivity-openstack", "test-connectivity", &options.SOpenStackCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("test-connectivity-huawei", "test-connectivity", &options.SHuaweiCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("test-connectivity-ucloud", "test-connectivity", &options.SUcloudCloudAccountUpdateCredentialOptions{}) + cmd.PerformWithKeyword("test-connectivity-volcengine", "test-connectivity", &options.SVolcengineCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("test-connectivity-zstack", "test-connectivity", &options.SZStackCloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("test-connectivity-s3", "test-connectivity", &options.SS3CloudAccountUpdateCredentialOptions{}) cmd.PerformWithKeyword("test-connectivity-ctyun", "test-connectivity", &options.SCtyunCloudAccountUpdateCredentialOptions{}) diff --git a/cmd/climc/shell/compute/usages.go b/cmd/climc/shell/compute/usages.go index c9b8168924..1b6274e2b5 100644 --- a/cmd/climc/shell/compute/usages.go +++ b/cmd/climc/shell/compute/usages.go @@ -32,8 +32,8 @@ import ( type GeneralUsageOptions struct { HostType []string `help:"Host types" choices:"hypervisor|baremetal|esxi|xen|kubelet|hyperv|aliyun|azure|aws|huawei|qcloud|openstack|ucloud|zstack|google|ctyun"` - Provider []string `help:"Provider" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|Google|Ctyun"` - Brand []string `help:"Brands" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|DStack|Google|Ctyun"` + Provider []string `help:"Provider" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|VolcEngine|ZStack|Google|Ctyun"` + Brand []string `help:"Brands" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|VolcEngine|ZStack|DStack|Google|Ctyun"` Project string `help:"show usage of specified project"` ProjectDomain string `help:"show usage of specified domain"` diff --git a/cmd/climc/shell/misc/feature.go b/cmd/climc/shell/misc/feature.go index bfea220943..060e3f9088 100644 --- a/cmd/climc/shell/misc/feature.go +++ b/cmd/climc/shell/misc/feature.go @@ -66,6 +66,7 @@ func init() { "baidu", "cucloud", "qingcloud", + "volcengine", } const ( diff --git a/go.mod b/go.mod index 5d6ab7de7c..3b96800657 100644 --- a/go.mod +++ b/go.mod @@ -233,6 +233,8 @@ require ( github.com/tklauser/numcpus v0.4.0 // indirect github.com/ugorji/go/codec v1.1.7 // indirect github.com/vmware/govmomi v0.20.1 // indirect + github.com/volcengine/ve-tos-golang-sdk/v2 v2.6.2 // indirect + github.com/volcengine/volc-sdk-golang v1.0.23 // indirect github.com/willf/bitset v1.1.9 // indirect github.com/willf/bloom v2.0.3+incompatible // indirect github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect @@ -253,7 +255,6 @@ require ( google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 6742a0ccd0..b120e710a2 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,7 @@ github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.39.0 h1:74BBwkEmiqBbi2CGflEh34l0YNtIibTjZsibGarkNjo= github.com/aws/aws-sdk-go v1.39.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= @@ -709,6 +710,10 @@ github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaU github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmware/govmomi v0.20.1 h1:7b/SeTUB3tER8ZLGLLLH3xcnB2xeuLULXmfPFqPSRZA= github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/volcengine/ve-tos-golang-sdk/v2 v2.6.2 h1:FZ7zrUf70YKVuES50mHO3z2VW0K+Arq4uGlDqb+1VA0= +github.com/volcengine/ve-tos-golang-sdk/v2 v2.6.2/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM= +github.com/volcengine/volc-sdk-golang v1.0.23 h1:anOslb2Qp6ywnsbyq9jqR0ljuO63kg9PY+4OehIk5R8= +github.com/volcengine/volc-sdk-golang v1.0.23/go.mod h1:AfG/PZRUkHJ9inETvbjNifTDgut25Wbkm2QoYBTbvyU= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.9 h1:GBtFynGY9ZWZmEC9sWuu41/7VBXPFCOAbCbqTflOg9c= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= @@ -1174,8 +1179,6 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231012115531-f16047235f44 h1:UWU0ISJM6AL7cvxzGT6cK4EPUX7VIPp+x0V6gWcX9fg= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231012115531-f16047235f44/go.mod h1:2sgCN7nRPQL3woLfdgqLDd92vwAHqtlz3KKiHxC5BAw= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/pkg/apis/compute/cloudaccount_const.go b/pkg/apis/compute/cloudaccount_const.go index 76be1b1588..8740f6a4e2 100644 --- a/pkg/apis/compute/cloudaccount_const.go +++ b/pkg/apis/compute/cloudaccount_const.go @@ -54,6 +54,7 @@ const ( CLOUD_PROVIDER_HCS = compute.CLOUD_PROVIDER_HCS CLOUD_PROVIDER_OPENSTACK = compute.CLOUD_PROVIDER_OPENSTACK CLOUD_PROVIDER_UCLOUD = compute.CLOUD_PROVIDER_UCLOUD + CLOUD_PROVIDER_VOLCENGINE = compute.CLOUD_PROVIDER_VOLCENGINE CLOUD_PROVIDER_ZSTACK = compute.CLOUD_PROVIDER_ZSTACK CLOUD_PROVIDER_GOOGLE = compute.CLOUD_PROVIDER_GOOGLE CLOUD_PROVIDER_CTYUN = compute.CLOUD_PROVIDER_CTYUN @@ -134,6 +135,7 @@ var ( CLOUD_PROVIDER_HCS, CLOUD_PROVIDER_OPENSTACK, CLOUD_PROVIDER_UCLOUD, + CLOUD_PROVIDER_VOLCENGINE, CLOUD_PROVIDER_ZSTACK, CLOUD_PROVIDER_GOOGLE, CLOUD_PROVIDER_CTYUN, @@ -194,6 +196,9 @@ var ( CLOUD_PROVIDER_UCLOUD: { HOST_TYPE_UCLOUD, }, + CLOUD_PROVIDER_VOLCENGINE: { + HOST_TYPE_VOLCENGINE, + }, CLOUD_PROVIDER_ZSTACK: { HOST_TYPE_ZSTACK, }, diff --git a/pkg/apis/compute/guest_const.go b/pkg/apis/compute/guest_const.go index edd84595e2..bb377d3b0a 100644 --- a/pkg/apis/compute/guest_const.go +++ b/pkg/apis/compute/guest_const.go @@ -191,6 +191,7 @@ const ( HYPERVISOR_HCSOP = compute.HYPERVISOR_HCSOP HYPERVISOR_OPENSTACK = compute.HYPERVISOR_OPENSTACK HYPERVISOR_UCLOUD = compute.HYPERVISOR_UCLOUD + HYPERVISOR_VOLCENGINE = compute.HYPERVISOR_VOLCENGINE HYPERVISOR_ZSTACK = compute.HYPERVISOR_ZSTACK HYPERVISOR_GOOGLE = compute.HYPERVISOR_GOOGLE HYPERVISOR_CTYUN = compute.HYPERVISOR_CTYUN @@ -268,6 +269,7 @@ var HYPERVISORS = []string{ HYPERVISOR_HCSOP, HYPERVISOR_OPENSTACK, HYPERVISOR_UCLOUD, + HYPERVISOR_VOLCENGINE, HYPERVISOR_ZSTACK, HYPERVISOR_GOOGLE, HYPERVISOR_CTYUN, @@ -299,6 +301,7 @@ var PUBLIC_CLOUD_HYPERVISORS = []string{ HYPERVISOR_QCLOUD, HYPERVISOR_HUAWEI, HYPERVISOR_UCLOUD, + HYPERVISOR_VOLCENGINE, HYPERVISOR_GOOGLE, HYPERVISOR_CTYUN, HYPERVISOR_ECLOUD, @@ -343,6 +346,7 @@ var HYPERVISOR_HOSTTYPE = map[string]string{ HYPERVISOR_HCS: HOST_TYPE_HCS, HYPERVISOR_OPENSTACK: HOST_TYPE_OPENSTACK, HYPERVISOR_UCLOUD: HOST_TYPE_UCLOUD, + HYPERVISOR_VOLCENGINE: HOST_TYPE_VOLCENGINE, HYPERVISOR_ZSTACK: HOST_TYPE_ZSTACK, HYPERVISOR_GOOGLE: HOST_TYPE_GOOGLE, HYPERVISOR_CTYUN: HOST_TYPE_CTYUN, @@ -377,6 +381,7 @@ var HOSTTYPE_HYPERVISOR = map[string]string{ HOST_TYPE_HCS: HYPERVISOR_HCS, HOST_TYPE_OPENSTACK: HYPERVISOR_OPENSTACK, HOST_TYPE_UCLOUD: HYPERVISOR_UCLOUD, + HOST_TYPE_VOLCENGINE: HYPERVISOR_VOLCENGINE, HOST_TYPE_ZSTACK: HYPERVISOR_ZSTACK, HOST_TYPE_GOOGLE: HYPERVISOR_GOOGLE, HOST_TYPE_CTYUN: HYPERVISOR_CTYUN, diff --git a/pkg/apis/compute/host_const.go b/pkg/apis/compute/host_const.go index 0e3e0fb57b..0f63679f8a 100644 --- a/pkg/apis/compute/host_const.go +++ b/pkg/apis/compute/host_const.go @@ -38,6 +38,7 @@ const ( HOST_TYPE_HCS = compute.HOST_TYPE_HCS HOST_TYPE_OPENSTACK = compute.HOST_TYPE_OPENSTACK HOST_TYPE_UCLOUD = compute.HOST_TYPE_UCLOUD + HOST_TYPE_VOLCENGINE = compute.HOST_TYPE_VOLCENGINE HOST_TYPE_ZSTACK = compute.HOST_TYPE_ZSTACK HOST_TYPE_GOOGLE = compute.HOST_TYPE_GOOGLE HOST_TYPE_CTYUN = compute.HOST_TYPE_CTYUN @@ -133,6 +134,7 @@ var HOST_TYPES = []string{ HOST_TYPE_HCSOP, HOST_TYPE_OPENSTACK, HOST_TYPE_UCLOUD, + HOST_TYPE_VOLCENGINE, HOST_TYPE_ZSTACK, HOST_TYPE_CTYUN, HOST_TYPE_GOOGLE, diff --git a/pkg/apis/compute/storage_const.go b/pkg/apis/compute/storage_const.go index 9966d07d45..c9eeee4ce4 100644 --- a/pkg/apis/compute/storage_const.go +++ b/pkg/apis/compute/storage_const.go @@ -90,6 +90,11 @@ const ( STORAGE_UCLOUD_LOCAL_SSD = compute.STORAGE_UCLOUD_LOCAL_SSD // SSD本地盘 STORAGE_UCLOUD_EXCLUSIVE_LOCAL_DISK = compute.STORAGE_UCLOUD_EXCLUSIVE_LOCAL_DISK // 独享本地盘 + // VolcEngine storage types + STORAGE_VOLC_CLOUD_FLEXPL = compute.STORAGE_VOLCENGINE_FlexPL // 极速型SSD云盘, FlexPL规格 + STORAGE_VOLC_CLOUD_PL0 = compute.STORAGE_VOLCENGINE_PL0 //极速型SSD云盘, PL0规格 + STORAGE_VOLC_CLOUD_PTSSD = compute.STORAGE_VOLCENGINE_PTSSD // 性能型SSD, 上一代产品 + // Zstack storage type STORAGE_ZSTACK_LOCAL_STORAGE = compute.STORAGE_ZSTACK_LOCAL_STORAGE STORAGE_ZSTACK_CEPH = compute.STORAGE_ZSTACK_CEPH diff --git a/pkg/compute/guestdrivers/volcengine.go b/pkg/compute/guestdrivers/volcengine.go new file mode 100644 index 0000000000..69a0c04cd7 --- /dev/null +++ b/pkg/compute/guestdrivers/volcengine.go @@ -0,0 +1,157 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guestdrivers + +import ( + "fmt" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/pkg/util/billing" + "yunion.io/x/pkg/util/rbacscope" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SVolcengineGuestDriver struct { + SManagedVirtualizedGuestDriver +} + +func init() { + driver := SVolcengineGuestDriver{} + models.RegisterGuestDriver(&driver) +} + +func (self *SVolcengineGuestDriver) GetHypervisor() string { + return api.HYPERVISOR_VOLCENGINE +} + +func (self *SVolcengineGuestDriver) GetProvider() string { + return api.CLOUD_PROVIDER_VOLCENGINE +} + +func (self *SVolcengineGuestDriver) GetComputeQuotaKeys(scope rbacscope.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys { + keys := models.SComputeResourceKeys{} + keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId) + keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD + keys.Provider = api.CLOUD_PROVIDER_VOLCENGINE + keys.Brand = api.CLOUD_PROVIDER_VOLCENGINE + keys.Hypervisor = api.HYPERVISOR_VOLCENGINE + return keys +} + +func (self *SVolcengineGuestDriver) GetDefaultSysDiskBackend() string { + return "" +} + +func (self *SVolcengineGuestDriver) GetMinimalSysDiskSizeGb() int { + return 40 +} + +func (self *SVolcengineGuestDriver) GetStorageTypes() []string { + return []string{} +} + +func (self *SVolcengineGuestDriver) ChooseHostStorage(host *models.SHost, guest *models.SGuest, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) { + return chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil +} + +func (self *SVolcengineGuestDriver) GetDetachDiskStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SVolcengineGuestDriver) GetAttachDiskStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SVolcengineGuestDriver) GetRebuildRootStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SVolcengineGuestDriver) GetChangeConfigStatus(guest *models.SGuest) ([]string, error) { + return []string{api.VM_READY}, nil +} + +func (self *SVolcengineGuestDriver) GetDeployStatus() ([]string, error) { + return []string{api.VM_READY, api.VM_RUNNING}, nil +} + +func (self *SVolcengineGuestDriver) GetGuestInitialStateAfterCreate() string { + return api.VM_RUNNING +} + +func (self *SVolcengineGuestDriver) GetGuestInitialStateAfterRebuild() string { + return api.VM_RUNNING +} + +func (self *SVolcengineGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability { + return cloudprovider.SInstanceCapability{ + Hypervisor: self.GetHypervisor(), + Provider: self.GetProvider(), + DefaultAccount: cloudprovider.SDefaultAccount{ + Linux: cloudprovider.SOsDefaultAccount{ + DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER, + Changeable: false, + }, + Windows: cloudprovider.SOsDefaultAccount{ + DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER, + Changeable: false, + }, + }, + Storages: cloudprovider.Storage{ + DataDisk: []cloudprovider.StorageInfo{ + cloudprovider.StorageInfo{ + {StorageType: api.STORAGE_VOLCENGINE_PTSSD, MaxSizeGb: 8192, MinSizeGb: 20, StepSizeGb: 1, Resizable: true}, + {StorageType: api.STORAGE_VOLCENGINE_PL0, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true}, + {StorageType: api.STORAGE_VOLCENGINE_FlexPL, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true}, + }, + }, + SysDisk: []cloudprovider.StorageInfo{ + cloudprovider.StorageInfo{ + {StorageType: api.STORAGE_VOLCENGINE_PTSSD, MaxSizeGb: 500, MinSizeGb: 40, StepSizeGb: 1, Resizable: true}, + {StorageType: api.STORAGE_VOLCENGINE_PL0, MaxSizeGb: 2048, MinSizeGb: 40, StepSizeGb: 1, Resizable: true}, + {StorageType: api.STORAGE_VOLCENGINE_FlexPL, MaxSizeGb: 2048, MinSizeGb: 40, StepSizeGb: 1, Resizable: true}, + }, + }, + }, + } +} + +func (self *SVolcengineGuestDriver) IsSupportedBillingCycle(bc billing.SBillingCycle) bool { + months := bc.GetMonths() + if (months >= 1 && months <= 9) || (months == 12) || (months == 24) || (months == 36) { + return true + } + + return false +} + +func (self *SVolcengineGuestDriver) IsNeedInjectPasswordByCloudInit() bool { + return true +} + +func (self *SVolcengineGuestDriver) IsSupportSetAutoRenew() bool { + return false +} + +func (self *SVolcengineGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error { + if !utils.IsInStringArray(guest.Status, []string{api.VM_RUNNING, api.VM_READY}) { + return fmt.Errorf("cannot resize disk when guest in status %s", guest.Status) + } + return nil +} diff --git a/pkg/compute/hostdrivers/volcengine.go b/pkg/compute/hostdrivers/volcengine.go new file mode 100644 index 0000000000..627feebf35 --- /dev/null +++ b/pkg/compute/hostdrivers/volcengine.go @@ -0,0 +1,81 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostdrivers + +import ( + "context" + "fmt" + + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SVolcengineHostDriver struct { + SManagedVirtualizationHostDriver +} + +func init() { + driver := SVolcengineHostDriver{} + models.RegisterHostDriver(&driver) +} + +func (self *SVolcengineHostDriver) GetHostType() string { + return api.HOST_TYPE_VOLCENGINE +} + +func (self *SVolcengineHostDriver) GetHypervisor() string { + return api.HYPERVISOR_VOLCENGINE +} + +func (self *SVolcengineHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb int) error { + if sizeGb%10 != 0 { + return fmt.Errorf("The disk size must be a multiple of 10Gb") + } + min, max := 0, 0 + switch storage.StorageType { + case api.STORAGE_VOLC_CLOUD_PL0: + min, max = 10, 32768 + case api.STORAGE_VOLC_CLOUD_FLEXPL: + min, max = 10, 32768 + default: + return fmt.Errorf("Not support create or resize %s disk", storage.StorageType) + } + if sizeGb < min || sizeGb > max { + return fmt.Errorf("The %s disk size must be in the range of %d ~ %dGB", storage.StorageType, min, max) + } + return nil +} + +func (self *SVolcengineHostDriver) ValidateResetDisk(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, snapshot *models.SSnapshot, guests []models.SGuest, input *api.DiskResetInput) (*api.DiskResetInput, error) { + for _, guest := range guests { + if !utils.IsInStringArray(guest.Status, []string{api.VM_RUNNING, api.VM_READY}) { + return nil, httperrors.NewBadGatewayError("Volcengine reset disk required guest status is running or read") + } + } + return input, nil +} + +func (self *SVolcengineHostDriver) RequestDeleteSnapshotWithStorage(ctx context.Context, host *models.SHost, snapshot *models.SSnapshot, task taskman.ITask) error { + return httperrors.NewNotImplementedError("not implement") +} + +func (driver *SVolcengineHostDriver) GetStoragecacheQuota(host *models.SHost) int { + return 10 +} diff --git a/pkg/compute/regiondrivers/volcengine.go b/pkg/compute/regiondrivers/volcengine.go new file mode 100644 index 0000000000..55ff05220c --- /dev/null +++ b/pkg/compute/regiondrivers/volcengine.go @@ -0,0 +1,68 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regiondrivers + +import ( + "context" + + "yunion.io/x/jsonutils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +type SVolcengineRegionDriver struct { + SManagedVirtualizationRegionDriver +} + +func init() { + driver := SVolcengineRegionDriver{} + models.RegisterRegionDriver(&driver) +} + +func (self *SVolcengineRegionDriver) IsSecurityGroupBelongVpc() bool { + return true +} + +func (self *SVolcengineRegionDriver) IsAllowSecurityGroupNameRepeat() bool { + return false +} + +func (self *SVolcengineRegionDriver) GenerateSecurityGroupName(name string) string { + return name +} + +func (self *SVolcengineRegionDriver) GetProvider() string { + return api.CLOUD_PROVIDER_VOLCENGINE +} + +func (self *SVolcengineRegionDriver) ValidateCreateVpcData(ctx context.Context, userCred mcclient.TokenCredential, input api.VpcCreateInput) (api.VpcCreateInput, error) { + var cidrV = validators.NewIPv4PrefixValidator("cidr_block") + if err := cidrV.Validate(jsonutils.Marshal(input).(*jsonutils.JSONDict)); err != nil { + return input, err + } + err := IsInPrivateIpRange(cidrV.Value.ToIPRange()) + if err != nil { + return input, err + } + + if cidrV.Value.MaskLen > 29 { + return input, httperrors.NewInputParameterError("%s request the mask range should be less than or equal to 29", self.GetProvider()) + } + return input, nil +} diff --git a/pkg/mcclient/options/base.go b/pkg/mcclient/options/base.go index 6a6e4e8714..9b68fcfde3 100644 --- a/pkg/mcclient/options/base.go +++ b/pkg/mcclient/options/base.go @@ -245,7 +245,7 @@ type BaseListOptions struct { Manager []string `help:"List objects belonging to the cloud provider" json:"manager,omitempty"` Account string `help:"List objects belonging to the cloud account" json:"account,omitempty"` - Provider []string `help:"List objects from the provider" choices:"OneCloud|VMware|Aliyun|Apsara|Qcloud|Azure|Aws|Huawei|OpenStack|Ucloud|ZStack|Google|Ctyun|Cloudpods|Nutanix|BingoCloud|IncloudSphere|JDcloud|Proxmox|Ceph|Ecloud|HCSO|HCS|HCSOP|H3C|S3|RemoteFile|Ksyun|Baidu|QingCloud" json:"provider,omitempty"` + Provider []string `help:"List objects from the provider" choices:"OneCloud|VMware|Aliyun|Apsara|Qcloud|Azure|Aws|Huawei|OpenStack|Ucloud|VolcEngine|ZStack|Google|Ctyun|Cloudpods|Nutanix|BingoCloud|IncloudSphere|JDcloud|Proxmox|Ceph|Ecloud|HCSO|HCS|HCSOP|H3C|S3|RemoteFile|Ksyun|Baidu|QingCloud" json:"provider,omitempty"` Brand []string `help:"List objects belonging to a special brand"` CloudEnv string `help:"Cloud environment" choices:"public|private|onpremise|private_or_onpremise" json:"cloud_env,omitempty"` PublicCloud *bool `help:"List objects belonging to public cloud" json:"public_cloud"` diff --git a/pkg/mcclient/options/cloudaccounts.go b/pkg/mcclient/options/cloudaccounts.go index 270a3e9c89..36ab747a23 100644 --- a/pkg/mcclient/options/cloudaccounts.go +++ b/pkg/mcclient/options/cloudaccounts.go @@ -346,6 +346,17 @@ func (opts *SUcloudCloudAccountCreateOptions) Params() (jsonutils.JSONObject, er return params, nil } +type SVolcengineCloudAccountCreateOptions struct { + SCloudAccountCreateBaseOptions + SAccessKeyCredential +} + +func (opts *SVolcengineCloudAccountCreateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(opts) + params.(*jsonutils.JSONDict).Add(jsonutils.NewString("VolcEngine"), "provider") + return params, nil +} + type SZStackCloudAccountCreateOptions struct { SCloudAccountCreateBaseOptions SUserPasswordCredential @@ -565,6 +576,15 @@ func (opts *SUcloudCloudAccountUpdateCredentialOptions) Params() (jsonutils.JSON return jsonutils.Marshal(opts), nil } +type SVolcengineCloudAccountUpdateCredentialOptions struct { + SCloudAccountIdOptions + SUserPasswordCredential +} + +func (opts *SVolcengineCloudAccountUpdateCredentialOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opts), nil +} + type SZStackCloudAccountUpdateCredentialOptions struct { SCloudAccountIdOptions SUserPasswordCredential @@ -940,6 +960,14 @@ func (opts *SUcloudCloudAccountUpdateOptions) Params() (jsonutils.JSONObject, er return jsonutils.Marshal(opts), nil } +type SVolcengineCloudAccountUpdateOptions struct { + SCloudAccountUpdateBaseOptions +} + +func (opts *SVolcengineCloudAccountUpdateOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opts), nil +} + type SZStackCloudAccountUpdateOptions struct { SCloudAccountUpdateBaseOptions } diff --git a/pkg/mcclient/options/compute/servers.go b/pkg/mcclient/options/compute/servers.go index 42d5de9a6b..d3e08e4fab 100644 --- a/pkg/mcclient/options/compute/servers.go +++ b/pkg/mcclient/options/compute/servers.go @@ -44,7 +44,7 @@ type ServerListOptions struct { Gpu *bool `help:"Show gpu servers"` Secgroup string `help:"Secgroup ID or Name"` AdminSecgroup string `help:"AdminSecgroup ID or Name"` - Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure|aws|huawei|ucloud|zstack|openstack|google|ctyun|incloudsphere|nutanix|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox|ksyun|baidu|cucloud|qingcloud"` + Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure|aws|huawei|ucloud|volcengine|zstack|openstack|google|ctyun|incloudsphere|nutanix|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox|ksyun|baidu|cucloud|qingcloud"` Region string `help:"Show servers in cloudregion"` WithEip *bool `help:"Show Servers with EIP"` WithoutEip *bool `help:"Show Servers without EIP"` @@ -249,7 +249,7 @@ type ServerConfigs struct { Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"` BackupHost string `help:"Perfered host where virtual backup server should be created"` - Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei|openstack|ucloud|zstack|google|ctyun|incloudsphere|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox"` + Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei|openstack|ucloud|volcengine|zstack|google|ctyun|incloudsphere|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox"` ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"` Backup bool `help:"Create server with backup server"` AutoSwitchToBackupOnHostDown bool `help:"Auto switch to backup server on host down"` diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/LICENSE b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/LICENSE new file mode 100644 index 0000000000..fe9f15ee67 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/LICENSE @@ -0,0 +1,490 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +Third-party dependencies listed below are subject to different license terms. + +yaml for Go +Apache License +Version 2.0, January 2004 +========================= +http://www.apache.org/licenses/ +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +1. Definitions. +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions +granted by this License. +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). +"Derivative Works" shall mean any work, whether in Source or Object form, that is +based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work by +the copyright owner or by an individual or Legal Entity authorized to submit on +behalf of the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent to the +Licensor or its representatives, including but not limited to communication on +electronic mailing lists, source code control systems, and issue tracking systems +that are managed by, or on behalf of, the Licensor for the purpose of discussing +and improving the Work, but excluding communication that is conspicuously marked +or otherwise designated in writing by the copyright owner as "Not a +Contribution." +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of +whom a Contribution has been received by Licensor and subsequently incorporated +within the Work. +2. Grant of Copyright License. Subject to the terms and conditions of this +License, each Contributor hereby grants to You a perpetual, worldwide, +non-exclusive, no-charge, royalty-free, irrevocable copyright license to +reproduce, prepare Derivative Works of, publicly display, publicly perform, +sublicense, and distribute the Work and such Derivative Works in Source or Object +form. +3. Grant of Patent License. Subject to the terms and conditions of this License, +each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer the Work, where such license applies only to those patent claims +licensable by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) with the Work to +which such Contribution(s) was submitted. If You institute patent litigation +against any entity (including a cross-claim or counterclaim in a lawsuit) +alleging that the Work or a Contribution incorporated within the Work constitutes +direct or contributory patent infringement, then any patent licenses granted to +You under this License for that Work shall terminate as of the date such +litigation is filed. +4. Redistribution. You may reproduce and distribute copies of the Work or +Derivative Works thereof in any medium, with or without modifications, and in +Source or Object form, provided that You meet the following conditions: + a. You must give any other recipients of the Work or Derivative Works a copy of + this License; and + b. You must cause any modified files to carry prominent notices stating that + You changed the files; and + c. You must retain, in the Source form of any Derivative Works that You + distribute, all copyright, patent, trademark, and attribution notices from + the Source form of the Work, excluding those notices that do not pertain to + any part of the Derivative Works; and + d. If the Work includes a "NOTICE" text file as part of its distribution, then + any Derivative Works that You distribute must include a readable copy of the + attribution notices contained within such NOTICE file, excluding those + notices that do not pertain to any part of the Derivative Works, in at least + one of the following places: within a NOTICE text file distributed as part of + the Derivative Works; within the Source form or documentation, if provided + along with the Derivative Works; or, within a display generated by the + Derivative Works, if and wherever such third-party notices normally appear. + The contents of the NOTICE file are for informational purposes only and do + not modify the License. You may add Your own attribution notices within + Derivative Works that You distribute, alongside or as an addendum to the + NOTICE text from the Work, provided that such additional attribution notices + cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. +5. Submission of Contributions. Unless You explicitly state otherwise, any +Contribution intentionally submitted for inclusion in the Work by You to the +Licensor shall be under the terms and conditions of this License, without any +additional terms or conditions. Notwithstanding the above, nothing herein shall +supersede or modify the terms of any separate license agreement you may have +executed with Licensor regarding such Contributions. +6. Trademarks. This License does not grant permission to use the trade names, +trademarks, service marks, or product names of the Licensor, except as required +for reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in +writing, Licensor provides the Work (and each Contributor provides its +Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +either express or implied, including, without limitation, any warranties or +conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any risks +associated with Your exercise of permissions under this License. +8. Limitation of Liability. In no event and under no legal theory, whether in +tort (including negligence), contract, or otherwise, unless required by +applicable law (such as deliberate and grossly negligent acts) or agreed to in +writing, shall any Contributor be liable to You for damages, including any +direct, indirect, special, incidental, or consequential damages of any character +arising as a result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, work stoppage, +computer failure or malfunction, or any and all other commercial damages or +losses), even if such Contributor has been advised of the possibility of such +damages. +9. Accepting Warranty or Additional Liability. While redistributing the Work or +Derivative Works thereof, You may choose to offer, and charge a fee for, +acceptance of support, warranty, indemnity, or other liability obligations and/or +rights consistent with this License. However, in accepting such obligations, You +may act only on Your own behalf and on Your sole responsibility, not on behalf of +any other Contributor, and only if You agree to indemnify, defend, and hold each +Contributor harmless for any liability incurred by, or claims asserted against, +such Contributor by reason of your accepting any such warranty or additional +liability. +END OF TERMS AND CONDITIONS +APPENDIX: How to apply the Apache License to your work +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also recommend +that a file or class name and description of purpose be included on the same +"printed page" as the copyright notice for easier identification within +third-party archives. + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law + or agreed to in writing, software distributed under the License is + distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the specific language + governing permissions and limitations under the License. +--- +go-check-check + +BSD 2-clause "Simplified" License +Copyright (c) 2010-2013 Gustavo Niemeyer +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE +--- +golang.org/x/sync +BSD 3-clause "New" or "Revised" License +Copyright (c) , +All rights reserved. +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +--- +pmezard-go-difflib +BSD 3-clause "New" or "Revised" License +Source: https://github.com/pmezard/go-difflib +Files: * +Copyright: 2013 Patrick Mézard +License: BSD-3-clause +Files: debian/* +Copyright: 2016 Dmitry Smirnov +License: BSD-3-clause +License: BSD-3-clause +Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. . + The names of its contributors may not be used to endorse or promote + products derived from this software without specific prior written + permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE +--- +go-spew +ISC License +Copyright (c) 2012-2016 Dave Collins +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE +--- +stretchr/objx +The MIT License +Copyright (c) 2014 Stretchr, Inc. +Copyright (c) 2017-2018 objx contributors +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE +--- +Go Testify +The MIT License +=============== +Copyright (c) +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in the +Software without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the +Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/acl.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/acl.go new file mode 100644 index 0000000000..a0d73077b5 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/acl.go @@ -0,0 +1,211 @@ +package tos + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" +) + +// PutObjectAcl AclGrant, AclRules can not set both. +// +// Deprecated: ues PutObjectACL of ClientV2 instead +func (bkt *Bucket) PutObjectAcl(ctx context.Context, input *PutObjectAclInput, options ...Option) (*PutObjectAclOutput, error) { + if err := isValidKey(input.Key); err != nil { + return nil, err + } + + var content io.Reader + if input.AclRules != nil { + data, err := json.Marshal(input.AclRules) + if err != nil { + return nil, fmt.Errorf("tos: marshal BucketAcl Ruels err: %s", err.Error()) + } + content = bytes.NewReader(data) + } + + builder := bkt.client.newBuilder(bkt.name, input.Key, options...). + WithQuery("acl", ""). + WithQuery("versionId", input.VersionID) + if grant := input.AclGrant; grant != nil { + builder.WithHeader(HeaderACL, grant.ACL). + WithHeader(HeaderGrantFullControl, grant.GrantFullControl). + WithHeader(HeaderGrantRead, grant.GrantRead). + WithHeader(HeaderGrantReadAcp, grant.GrantReadAcp). + WithHeader(HeaderGrantWriteAcp, grant.GrantWriteAcp) + } + + res, err := builder.WithRetry(OnRetryFromStart, StatusCodeClassifier{}).Request(ctx, http.MethodPut, content, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + return &PutObjectAclOutput{RequestInfo: res.RequestInfo()}, nil +} + +// PutObjectACL put object ACL +func (cli *ClientV2) PutObjectACL(ctx context.Context, input *PutObjectACLInput) (*PutObjectACLOutput, error) { + if err := isValidKey(input.Key); err != nil { + return nil, err + } + + if len(input.ACL) != 0 { + if err := isValidACL(input.ACL); err != nil { + return nil, err + } + } + + var content io.Reader + if len(input.Grants) != 0 { + + for _, grants := range input.Grants { + if err := isValidGrantee(grants.GranteeV2.Type); len(grants.GranteeV2.Type) != 0 && err != nil { + return nil, err + } + if err := isValidCannedType(grants.GranteeV2.Canned); len(grants.GranteeV2.Canned) != 0 && err != nil { + return nil, err + } + + if err := isValidPermission(grants.Permission); len(grants.Permission) != 0 && err != nil { + return nil, err + } + } + + data, err := json.Marshal(&accessControlList{ + Owner: input.Owner, + Grants: input.Grants, + BucketOwnerEntrusted: input.BucketOwnerEntrusted, + }) + if err != nil { + return nil, InvalidMarshal + } + content = bytes.NewReader(data) + } + builder := cli.newBuilder(input.Bucket, input.Key). + WithQuery("acl", ""). + WithParams(*input) + res, err := builder.WithRetry(OnRetryFromStart, StatusCodeClassifier{}).Request(ctx, http.MethodPut, content, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + return &PutObjectACLOutput{ + PutObjectAclOutput{RequestInfo: res.RequestInfo()}, + }, nil +} + +// GetObjectAcl get object ACL +// objectKey: the name of object +// Options: WithVersionID the version of the object +// +// Deprecated: use GetObjectACL of ClientV2 instead +func (bkt *Bucket) GetObjectAcl(ctx context.Context, objectKey string, options ...Option) (*GetObjectAclOutput, error) { + if err := isValidKey(objectKey); err != nil { + return nil, err + } + + res, err := bkt.client.newBuilder(bkt.name, objectKey, options...). + WithQuery("acl", ""). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + out := GetObjectAclOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(out.RequestID, res.Body, &out); err != nil { + return nil, err + } + + out.VersionID = res.Header.Get(HeaderVersionID) + return &out, nil +} + +// GetObjectACL get object ACL +func (cli *ClientV2) GetObjectACL(ctx context.Context, input *GetObjectACLInput) (*GetObjectACLOutput, error) { + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + if err := isValidKey(input.Key); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, input.Key). + WithQuery("acl", ""). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + out := GetObjectACLOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(out.RequestID, res.Body, &out); err != nil { + return nil, err + } + out.VersionID = res.Header.Get(HeaderVersionID) + return &out, nil +} + +func (cli *ClientV2) GetBucketACL(ctx context.Context, input *GetBucketACLInput) (*GetBucketACLOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("acl", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketACLOutput{RequestInfo: res.RequestInfo()} + marshalRes := bucketACL{} + if err = marshalOutput(output.RequestID, res.Body, &marshalRes); err != nil { + return nil, err + } + output.Grants = marshalRes.GrantList + output.Owner = marshalRes.Owner + return &output, nil +} + +func (cli *ClientV2) PutBucketACL(ctx context.Context, input *PutBucketACLInput) (*PutBucketACLOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + reqBuilder := cli.newBuilder(input.Bucket, ""). + WithQuery("acl", ""). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + WithParams(*input) + var reqData io.Reader + + if input.Owner.ID != "" && len(input.Grants) != 0 { + data, contentMD5, err := marshalInput("PutBucketACLInput", bucketACL{ + Owner: input.Owner, + GrantList: input.Grants, + }) + if err != nil { + return nil, err + } + _ = reqBuilder.WithHeader(HeaderContentMD5, contentMD5) + reqData = bytes.NewReader(data) + } + + res, err := reqBuilder.Request(ctx, http.MethodPut, reqData, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketACLOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/base_client.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/base_client.go new file mode 100644 index 0000000000..bab5e9233f --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/base_client.go @@ -0,0 +1,124 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +func newBaseClient(c *Client) *baseClient { + return &baseClient{Client: c} +} + +type baseClient struct { + *Client +} + +func (cli *baseClient) PutObjectTagging(ctx context.Context, input *PutObjectTaggingInput, option ...Option) (*PutObjectTaggingOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + data, contentMD5, err := marshalInput("PutObjectTaggingInput", putObjectTaggingInput{ + TagSet: input.TagSet, + }) + if err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, input.Key, option...). + WithQuery("tagging", ""). + WithParams(*input). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutObjectTaggingOutput{RequestInfo: res.RequestInfo()} + output.VersionID = res.Header.Get(HeaderVersionID) + return &output, nil +} + +func (cli *baseClient) GetObjectTagging(ctx context.Context, input *GetObjectTaggingInput, option ...Option) (*GetObjectTaggingOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, input.Key, option...). + WithQuery("tagging", ""). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetObjectTaggingOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + output.VersionID = res.Header.Get(HeaderVersionID) + return &output, nil +} + +func (cli *baseClient) DeleteObjectTagging(ctx context.Context, input *DeleteObjectTaggingInput, option ...Option) (*DeleteObjectTaggingOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, input.Key, option...). + WithQuery("tagging", ""). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteObjectTaggingOutput{RequestInfo: res.RequestInfo()} + output.VersionID = res.Header.Get(HeaderVersionID) + + return &output, nil + +} + +func (cli *baseClient) RestoreObject(ctx context.Context, input *RestoreObjectInput, option ...Option) (*RestoreObjectOutput, error) { + + if input == nil { + return nil, InputIsNilClientError + } + + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + data, contentMD5, err := marshalInput("RestoreObjectInput", restoreObjectInput{ + Days: input.Days, + RestoreJobParameters: input.RestoreJobParameters, + }) + if err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, input.Key, option...). + WithParams(*input). + WithQuery("restore", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPost, bytes.NewReader(data), cli.roundTripper(http.StatusOK, http.StatusAccepted)) + if err != nil { + return nil, err + } + defer res.Close() + output := RestoreObjectOutput{RequestInfo: res.RequestInfo()} + return &output, nil + +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/bucket.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/bucket.go new file mode 100644 index 0000000000..c31fffe0a1 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/bucket.go @@ -0,0 +1,264 @@ +package tos + +import ( + "bytes" + "context" + "net/http" + + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" +) + +// Bucket create a Bucket handle +// +// Deprecated: request with bucket handle is deprecated, use ClientV2 instead +func (cli *Client) Bucket(bucket string) (*Bucket, error) { + if err := isValidBucketName(bucket, false); err != nil { + return nil, err + } + return &Bucket{name: bucket, client: cli, baseClient: newBaseClient(cli)}, nil +} + +// CreateBucket create a bucket +// +// Deprecated: use CreateBucket of ClientV2 instead +func (cli *Client) CreateBucket(ctx context.Context, input *CreateBucketInput) (*CreateBucketOutput, error) { + if err := isValidBucketName(input.Bucket, false); err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, ""). + WithHeader(HeaderACL, input.ACL). + WithHeader(HeaderGrantFullControl, input.GrantFullControl). + WithHeader(HeaderGrantRead, input.GrantRead). + WithHeader(HeaderGrantReadAcp, input.GrantReadAcp). + WithHeader(HeaderGrantWrite, input.GrantWrite). + WithHeader(HeaderGrantWriteAcp, input.GrantWriteAcp). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + return &CreateBucketOutput{ + RequestInfo: res.RequestInfo(), + Location: res.Header.Get(HeaderLocation), + }, nil +} + +// CreateBucketV2 create a bucket +func (cli *ClientV2) CreateBucketV2(ctx context.Context, input *CreateBucketV2Input) (*CreateBucketV2Output, error) { + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + // TODO: ACL和Grant不能同时设置，可以在sdk校验 + if err := isValidACL(input.ACL); len(input.ACL) != 0 && err != nil { + return nil, err + } + + if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) != 0 && err != nil { + return nil, err + } + + if err := isValidAzRedundancy(input.AzRedundancy); len(input.AzRedundancy) != 0 && err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, ""). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + return &CreateBucketV2Output{ + CreateBucketOutput: CreateBucketOutput{ + RequestInfo: res.RequestInfo(), + Location: res.Header.Get(HeaderLocation)}}, nil +} + +// HeadBucket get some info of a bucket +// +// Deprecated: use HeadBucket of ClientV2 instead +func (cli *Client) HeadBucket(ctx context.Context, bucket string) (*HeadBucketOutput, error) { + if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(bucket, ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodHead, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + return &HeadBucketOutput{ + RequestInfo: res.RequestInfo(), + Region: res.Header.Get(HeaderBucketRegion), + StorageClass: enum.StorageClassType(res.Header.Get(HeaderStorageClass)), + AzRedundancy: enum.AzRedundancyType(res.Header.Get(HeaderAzRedundancy)), + }, nil +} + +// HeadBucket get some info of a bucket +func (cli *ClientV2) HeadBucket(ctx context.Context, input *HeadBucketInput) (*HeadBucketOutput, error) { + return cli.Client.HeadBucket(ctx, input.Bucket) +} + +// DeleteBucket delete a bucket +// +// Deprecated: use DeleteBucket of ClientV2 instead +func (cli *Client) DeleteBucket(ctx context.Context, bucket string) (*DeleteBucketOutput, error) { + if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + res, err := cli.newBuilder(bucket, ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + return &DeleteBucketOutput{RequestInfo: res.RequestInfo()}, nil +} + +// DeleteBucket delete a bucket.Deleting a non-empty bucket is not allowed. +// A bucket is empty only if there is no exist object and uncanceled segmented tasks. +func (cli *ClientV2) DeleteBucket(ctx context.Context, input *DeleteBucketInput) (*DeleteBucketOutput, error) { + return cli.Client.DeleteBucket(ctx, input.Bucket) +} + +// ListBuckets list the buckets that the AK can access +// +// Deprecated: use ListBuckets of ClientV2 instead +func (cli *Client) ListBuckets(ctx context.Context, _ *ListBucketsInput) (*ListBucketsOutput, error) { + res, err := cli.newBuilder("", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := ListBucketsOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +// ListBuckets list the buckets that the AK can access +func (cli *ClientV2) ListBuckets(ctx context.Context, _ *ListBucketsInput) (*ListBucketsOutput, error) { + res, err := cli.newBuilder("", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := ListBucketsOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) PutBucketStorageClass(ctx context.Context, input *PutBucketStorageClassInput) (*PutBucketStorageClassOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + if err := isValidStorageClass(input.StorageClass); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("storageClass", ""). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketStorageClassOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketLocation(ctx context.Context, input *GetBucketLocationInput) (*GetBucketLocationOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("location", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketLocationOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) PutBucketVersioning(ctx context.Context, input *PutBucketVersioningInput) (*PutBucketVersioningOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + data, contentMD5, err := marshalInput("PutBucketVersioning", putBucketVersioningInput{ + Status: input.Status, + }) + if err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("versioning", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketVersioningOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketVersioning(ctx context.Context, input *GetBucketVersioningInput) (*GetBucketVersioningOutputV2, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("versioning", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketVersioningOutputV2{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/check.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/check.go new file mode 100644 index 0000000000..25c9e04685 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/check.go @@ -0,0 +1,126 @@ +package tos + +import ( + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" +) + +func IsValidBucketName(name string) error { + if length := len(name); length < 3 || length > 63 { + return InvalidBucketNameLength + } + for i := range name { + if char := name[i]; !(('a' <= char && char <= 'z') || ('0' <= char && char <= '9') || char == '-') { + return InvalidBucketNameCharacter + } + } + if name[0] == '-' || name[len(name)-1] == '-' { + return InvalidBucketNameStartingOrEnding + } + return nil +} + +// isValidBucketName validate bucket name, return TosClientError if failed +func isValidBucketName(name string, isCustomDomain bool) error { + if isCustomDomain { + return nil + } + return IsValidBucketName(name) +} + +// isValidNames validate bucket name and keys, return TosClientError if failed +func isValidNames(bucket string, key string, isCustomDomain bool, keys ...string) error { + if err := isValidBucketName(bucket, isCustomDomain); err != nil { + return err + } + if err := isValidKey(key, keys...); err != nil { + return err + } + return nil +} + +// validKey validate single key, return TosClientError if failed +func validKey(key string) error { + if len(key) < 1 { + return InvalidObjectNameLength + } + return nil +} + +// isValidKey validate keys, return TosClientError if failed +func isValidKey(key string, keys ...string) error { + if err := validKey(key); err != nil { + return err + } + for _, k := range keys { + if err := validKey(k); err != nil { + return err + } + } + return nil +} + +// isValidACL validate aclType, return TosClientError if failed +func isValidACL(aclType enum.ACLType) error { + if aclType == enum.ACLPrivate || aclType == enum.ACLPublicRead || aclType == enum.ACLPublicReadWrite || + aclType == enum.ACLAuthRead || aclType == enum.ACLBucketOwnerRead || + aclType == enum.ACLBucketOwnerFullControl || aclType == enum.ACLLogDeliveryWrite || + aclType == enum.ACLBucketOwnerEntrusted { + return nil + } + + return InvalidACL +} + +// isValidStorageClass validate Storage Class, return TosClientError if failed +func isValidStorageClass(storageClass enum.StorageClassType) error { + + if storageClass == enum.StorageClassIa || storageClass == enum.StorageClassStandard || storageClass == enum.StorageClassArchiveFr || storageClass == enum.StorageClassColdArchive || storageClass == enum.StorageClassIntelligentTiering { + return nil + } + + return InvalidStorageClass +} + +func isValidGrantee(granteeType enum.GranteeType) error { + if granteeType == enum.GranteeUser || granteeType == enum.GranteeGroup { + return nil + } + return InvalidGrantee +} + +func isValidCannedType(cannedType enum.CannedType) error { + if cannedType == enum.CannedAllUsers || cannedType == enum.CannedAuthenticatedUsers { + return nil + } + return InvalidCanned +} + +func isValidAzRedundancy(redundancyType enum.AzRedundancyType) error { + if redundancyType == enum.AzRedundancySingleAz || redundancyType == enum.AzRedundancyMultiAz { + return nil + } + return InvalidAzRedundancy +} + +func isValidMetadataDirective(directiveType enum.MetadataDirectiveType) error { + if directiveType == enum.MetadataDirectiveCopy || directiveType == enum.MetadataDirectiveReplace { + return nil + } + return InvalidMetadataDirective +} + +func isValidPermission(permissionType enum.PermissionType) error { + if permissionType == enum.PermissionRead || permissionType == enum.PermissionReadAcp || + permissionType == enum.PermissionWriteAcp || permissionType == enum.PermissionWrite || + permissionType == enum.PermissionFullControl { + return nil + } + return InvalidPermission +} + +func isValidSSECAlgorithm(algorithm string) error { + if algorithm == enum.SSETosAlg || algorithm == enum.SSEKMS { + return nil + } + return InvalidSSECAlgorithm +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/checksum.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/checksum.go new file mode 100644 index 0000000000..3b13d02938 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/checksum.go @@ -0,0 +1,55 @@ +package tos + +import ( + "crypto/md5" + "encoding/hex" + "errors" + "hash" + "io" + "strings" +) + +var ( + ErrETagMissMatch = errors.New("tos: ETag miss match") +) + +// ETagCheckReadCloser checks ETag on read EOF +type ETagCheckReadCloser struct { + reader io.Reader + closer io.Closer + checksum hash.Hash + eTag string + requestID string +} + +func NewETagCheckReadCloser(reader io.ReadCloser, eTag, requestID string) *ETagCheckReadCloser { + checksum := md5.New() + + return &ETagCheckReadCloser{ + reader: io.TeeReader(reader, checksum), + closer: reader, + checksum: checksum, + eTag: strings.Trim(eTag, `"`), + requestID: requestID, + } +} + +func (ec *ETagCheckReadCloser) Read(p []byte) (n int, err error) { + n, err = ec.reader.Read(p) + if err == io.EOF && len(ec.eTag) > 0 { + sum := ec.checksum.Sum(nil) + if hexSum := hex.EncodeToString(sum); hexSum != ec.eTag { + return n, &ChecksumError{ + RequestID: ec.requestID, + ExpectedChecksum: ec.eTag, + ActualChecksum: hexSum, + } + } + } + + return n, err +} + +func (ec *ETagCheckReadCloser) Close() error { + return ec.closer.Close() +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/client.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/client.go new file mode 100644 index 0000000000..9a23f5c977 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/client.go @@ -0,0 +1,740 @@ +package tos + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "net" + "net/http" + "net/url" + "runtime" + "strconv" + "strings" + "time" +) + +const ( + signPolicyDate = "x-tos-date" + signPolicyCredential = "x-tos-credential" + signPolicyAlgorithm = "x-tos-algorithm" + signPolicySecurityToken = "x-tos-security-token" + signPolicyExpiration = "expiration" + signConditionContentLengthRange = "content-length-range" + signConditionBucket = "bucket" + signConditionKey = "key" + signConditions = "conditions" + maxPreSignExpires = 604800 // 7 day + defaultSignExpires = 3600 // 1 hour +) + +// Client TOS Client +// use NewClient to create a new Client +// +// example: +// client, err := NewClient(endpoint, WithCredentials(credentials), WithRegion(region)) +// if err != nil { +// // ... +// } +// // do something +// +// if you only access the public bucket: +// client, err := NewClient(endpoint) +// // do something +// +// Deprecated: use ClientV2 instead +type Client struct { + scheme string + host string + urlMode urlMode + userAgent string + credentials Credentials // nullable + signer Signer // nullable + transport Transport + recognizer ContentTypeRecognizer + config Config + retry *retryer + enableCRC bool + logger Logger + isCustomDomain bool +} + +// ClientV2 TOS ClientV2 +// use NewClientV2 to create a new ClientV2 +// +// example: +// client, err := NewClientV2(endpoint, WithCredentials(credentials), WithRegion(region)) +// if err != nil { +// // ... +// } +// // do something +// +// if you only access the public bucket: +// client, err := NewClientV2(endpoint) +// // do something +// +type ClientV2 struct { + Client + baseClient *baseClient +} + +func (cli *ClientV2) Close() { + if t, ok := cli.transport.(*DefaultTransport); ok { + if h, ok := t.client.Transport.(*http.Transport); ok { + h.CloseIdleConnections() + } + } +} + +func (cli *ClientV2) SetHTTPTransport(transport http.RoundTripper) { + cli.transport = newDefaultTranposrtWithHTTPTransport(transport) +} + +type ClientOption func(*Client) + +// WithCredentials set Credentials +// +// see StaticCredentials, WithoutSecretKeyCredentials and FederationCredentials +func WithCredentials(credentials Credentials) ClientOption { + return func(client *Client) { + client.credentials = credentials + } +} + +// WithEnableVerifySSL set whether a client verifies the server's certificate chain and host name. +func WithEnableVerifySSL(enable bool) ClientOption { + skip := !enable + return func(client *Client) { + client.config.TransportConfig.InsecureSkipVerify = skip + } +} + +// WithRequestTimeout set timeout for single http request +func WithRequestTimeout(timeout time.Duration) ClientOption { + return func(client *Client) { + client.config.TransportConfig.ResponseHeaderTimeout = timeout + + } +} + +// +// WithLogger sets the tos sdk logger +// +func WithLogger(logger Logger) ClientOption { + return func(client *Client) { + client.logger = logger + } +} + +// WithConnectionTimeout set timeout for constructing connection +func WithConnectionTimeout(timeout time.Duration) ClientOption { + return func(client *Client) { + client.config.TransportConfig.DialTimeout = timeout + } +} + +// WithProxy set http Proxy for tos client +func WithProxy(proxy *Proxy) ClientOption { + return func(client *Client) { + client.config.TransportConfig.Proxy = proxy + } +} + +// WithMaxConnections set maximum number of http connections +func WithMaxConnections(max int) ClientOption { + return func(client *Client) { + client.config.TransportConfig.MaxIdleConns = max + client.config.TransportConfig.MaxIdleConnsPerHost = max + client.config.TransportConfig.MaxConnsPerHost = max + } + +} + +// WithIdleConnTimeout set max idle time of a http connection +func WithIdleConnTimeout(timeout time.Duration) ClientOption { + return func(client *Client) { + client.config.TransportConfig.IdleConnTimeout = timeout + } +} + +// WithUserAgentSuffix set suffix of user-agent +func WithUserAgentSuffix(suffix string) ClientOption { + return func(client *Client) { + client.userAgent = strings.Join([]string{client.userAgent, suffix}, " ") + } +} + +// WithDNSCacheTime set dnsCacheTime in Minute +func WithDNSCacheTime(dnsCacheTime int) ClientOption { + return func(client *Client) { + client.config.TransportConfig.DNSCacheTime = time.Minute * time.Duration(dnsCacheTime) + } +} + +// WithEnableCRC set if check crc after uploading object. +// Checking crc is enabled by default. +func WithEnableCRC(enableCRC bool) ClientOption { + return func(client *Client) { + client.enableCRC = enableCRC + } +} + +// // WithMaxRetryCount set MaxRetryCount +func WithMaxRetryCount(retryCount int) ClientOption { + return func(client *Client) { + if client.retry != nil { + client.retry.SetBackoff(exponentialBackoff(retryCount, DefaultRetryBackoffBase)) + } + } +} + +func WithCustomDomain(isCustomDomain bool) ClientOption { + return func(client *Client) { + client.isCustomDomain = isCustomDomain + } +} + +// WithTransport set Transport +// +// Deprecated: this function is Deprecated. +// If you want to set http.Transport use WithHTTPTransport instead +func WithTransport(transport Transport) ClientOption { + return func(client *Client) { + client.transport = transport + } +} + +// WithHTTPTransport set Transport of http.Client +func WithHTTPTransport(transport http.RoundTripper) ClientOption { + return func(client *Client) { + client.transport = newDefaultTranposrtWithHTTPTransport(transport) + } +} + +// WithTransportConfig set TransportConfig +func WithTransportConfig(config *TransportConfig) ClientOption { + return func(client *Client) { + // client.config never be nil + client.config.TransportConfig = *config + } +} + +// WithSocketTimeout set read-write timeout +func WithSocketTimeout(readTimeout, writeTimeout time.Duration) ClientOption { + return func(client *Client) { + client.config.TransportConfig.ReadTimeout = readTimeout + client.config.TransportConfig.WriteTimeout = writeTimeout + } +} + +// WithRegion set region +func WithRegion(region string) ClientOption { + return func(client *Client) { + // client.config never be nil + client.config.Region = region + if endpoint, ok := SupportedRegion()[region]; ok { + if len(client.config.Endpoint) == 0 { + client.config.Endpoint = endpoint + } + } + } +} + +// WithSigner for self-defined Signer +func WithSigner(signer Signer) ClientOption { + return func(client *Client) { + client.signer = signer + } +} + +// WithPathAccessMode url mode is path model or default mode +// +// Deprecated: This option is deprecated. Setting PathAccessMode will be ignored silently. +func WithPathAccessMode(pathAccessMode bool) ClientOption { + return func(client *Client) { + } +} + +// WithAutoRecognizeContentType set to recognize Content-Type or not, the default is enabled. +func WithAutoRecognizeContentType(enable bool) ClientOption { + return func(client *Client) { + if enable { + client.recognizer = ExtensionBasedContentTypeRecognizer{} + } else { + client.recognizer = EmptyContentTypeRecognizer{} + } + } +} + +// WithContentTypeRecognizer set ContentTypeRecognizer to recognize Content-Type, +// the default is ExtensionBasedContentTypeRecognizer +func WithContentTypeRecognizer(recognizer ContentTypeRecognizer) ClientOption { + return func(client *Client) { + client.recognizer = recognizer + } +} + +func schemeHost(endpoint string) (scheme string, host string, urlMode urlMode) { + if strings.HasPrefix(endpoint, "https://") { + scheme = "https" + host = endpoint[len("https://"):] + } else if strings.HasPrefix(endpoint, "http://") { + scheme = "http" + host = endpoint[len("http://"):] + } else { + scheme = "https" + host = endpoint + } + urlMode = urlModeDefault + hostWithoutPort, _, _ := net.SplitHostPort(host) + if net.ParseIP(host) != nil || net.ParseIP(hostWithoutPort) != nil { + urlMode = urlModePath + } + + return scheme, host, urlMode +} + +func initClient(client *Client, endpoint string, options ...ClientOption) error { + client.config.Endpoint = endpoint + for _, option := range options { + option(client) + } + client.scheme, client.host, client.urlMode = schemeHost(client.config.Endpoint) + if client.transport == nil { + transport := NewDefaultTransport(&client.config.TransportConfig) + transport.WithDefaultTransportLogger(client.logger) + client.transport = transport + } + if cred := client.credentials; cred != nil && client.signer == nil { + if len(client.config.Region) == 0 { + if region, ok := SupportedEndpoint()[client.host]; ok { + client.config.Region = region + } else { + return newTosClientError("tos: missing Region option", nil) + } + } + signer := NewSignV4(cred, client.config.Region) + signer.WithSignLogger(client.logger) + client.signer = signer + } + return nil +} + +// NewClient create a new Tos Client +// endpoint: access endpoint +// options: WithCredentials set Credentials +// WithRegion set region, this is required if WithCredentials is used +// WithSocketTimeout set read-write timeout +// WithTransportConfig set TransportConfig +// WithTransport set self-defined Transport +func NewClient(endpoint string, options ...ClientOption) (*Client, error) { + client := Client{ + recognizer: ExtensionBasedContentTypeRecognizer{}, + config: defaultConfig(), + userAgent: fmt.Sprintf("tos-go-sdk/%s (%s/%s;%s)", Version, runtime.GOOS, runtime.GOARCH, runtime.Version()), + retry: newRetryer([]time.Duration{}), + } + client.retry.SetJitter(0.25) + err := initClient(&client, endpoint, options...) + if err != nil { + return nil, err + } + return &client, nil +} + +// NewClientV2 create a new Tos ClientV2 +// endpoint: access endpoint +// options: WithCredentials set Credentials +// WithRegion set region, this is required if WithCredentials is used. +// If Region is supported and the Endpoint parameter is not set, the Endpoint will be resolved automatically +// WithSocketTimeout set read-write timeout +// WithTransportConfig set TransportConfig +// WithTransport set self-defined Transport +// WithLogger set self-defined Logger +// WithEnableCRC set CRC switch. +// WithMaxRetryCount set Max Retry Count +func NewClientV2(endpoint string, options ...ClientOption) (*ClientV2, error) { + if strings.Contains(endpoint, "s3") { + return nil, InvalidS3Endpoint + } + client := ClientV2{ + Client: Client{ + recognizer: ExtensionBasedContentTypeRecognizer{}, + config: defaultConfig(), + retry: newRetryer(exponentialBackoff(DefaultRetryTime, DefaultRetryBackoffBase)), + userAgent: fmt.Sprintf("tos-go-sdk/%s (%s/%s;%s)", Version, runtime.GOOS, runtime.GOARCH, runtime.Version()), + enableCRC: true, + }, + } + client.retry.SetJitter(0.25) + err := initClient(&client.Client, endpoint, options...) + if err != nil { + return nil, err + } + client.baseClient = newBaseClient(&client.Client) + return &client, nil +} + +func (cli *Client) newBuilder(bucket, object string, options ...Option) *requestBuilder { + rb := &requestBuilder{ + Signer: cli.signer, + Scheme: cli.scheme, + Host: cli.host, + Bucket: bucket, + Object: object, + URLMode: cli.urlMode, + Query: make(url.Values), + Header: make(http.Header), + OnRetry: func(req *Request) error { return nil }, + Classifier: StatusCodeClassifier{}, + IsCustomDomain: cli.isCustomDomain, + } + rb.Header.Set(HeaderUserAgent, cli.userAgent) + if typ := cli.recognizer.ContentType(object); len(typ) > 0 { + rb.Header.Set(HeaderContentType, typ) + } + for _, option := range options { + option(rb) + } + rb.Retry = cli.retry + return rb +} + +func (cli *Client) roundTrip(ctx context.Context, req *Request, expectedCode int, expectedCodes ...int) (*Response, error) { + res, err := cli.transport.RoundTrip(ctx, req) + if err != nil { + return nil, err + } + readBody := req.Method != http.MethodHead + if err = checkError(res, readBody, expectedCode, expectedCodes...); err != nil { + return nil, err + } + return res, nil +} + +func (cli *Client) roundTripper(expectedCode int, expectedCodes ...int) roundTripper { + return func(ctx context.Context, req *Request) (*Response, error) { + start := time.Now() + resp, err := cli.roundTrip(ctx, req, expectedCode, expectedCodes...) + if cli.logger != nil { + if err != nil { + cli.logger.Info(fmt.Sprintf("[tos] http error:%s.", err.Error())) + } else { + cli.logger.Info(fmt.Sprintf("[tos] Response StatusCode:%d, RequestId:%s, Cost:%d ms", resp.StatusCode, resp.RequestInfo().RequestID, time.Since(start).Milliseconds())) + } + } + return resp, err + } +} + +// PreSignedURL return pre-signed url +// httpMethod: HTTP method, { +// PutObject: http.MethodPut +// GetObject: http.MethodGet +// HeadObject: http.MethodHead +// DeleteObject: http.MethodDelete +// }, +// bucket: the bucket name +// objectKey: the object name +// ttl: the time-to-live of signed URL +// options: WithVersionID the version id of the object +// Deprecated: use PreSignedURL of ClientV2 instead +func (cli *Client) PreSignedURL(httpMethod string, bucket, objectKey string, ttl time.Duration, options ...Option) (string, error) { + + return cli.newBuilder(bucket, objectKey, options...). + PreSignedURL(httpMethod, ttl) +} + +// PreSignedURL return pre-signed url +func (cli *ClientV2) PreSignedURL(input *PreSignedURLInput) (*PreSignedURLOutput, error) { + rb := cli.newBuilder(input.Bucket, input.Key) + if input.IsCustomDomain != nil { + rb.IsCustomDomain = *input.IsCustomDomain + } + if input.AlternativeEndpoint != "" { + schema, host, _ := schemeHost(input.AlternativeEndpoint) + rb.Host = host + rb.Scheme = schema + } + for k, v := range input.Header { + rb.WithHeader(k, v) + } + for k, v := range input.Query { + rb.WithQuery(k, v) + } + if input.Expires == 0 { + input.Expires = defaultPreSignedURLExpires + } + + signedURL, err := rb.PreSignedURL(string(input.HTTPMethod), time.Second*time.Duration(input.Expires)) + if err != nil { + return nil, err + } + signed := make(map[string]string) + for k := range rb.Header { + signed[k] = rb.Header.Get(k) + } + output := &PreSignedURLOutput{ + SignedUrl: signedURL, + SignedHeader: signed, + } + return output, nil +} + +func (cli *ClientV2) PreSignedPostSignature(ctx context.Context, input *PreSingedPostSignatureInput) (*PreSingedPostSignatureOutput, error) { + algorithm := signPrefix + postPolicy := make(map[string]interface{}) + cred := cli.credentials.Credential() + region := cli.config.Region + date := UTCNow() + if input.Expires == 0 { + input.Expires = defaultSignExpires + } + + postPolicy[signPolicyExpiration] = date.Add(time.Second * time.Duration(input.Expires)).Format(serverTimeFormat) + + cond := make([]interface{}, 0) + credential := fmt.Sprintf("%s/%s/%s/tos/request", cred.AccessKeyID, date.Format(yyMMdd), region) + cond = append(cond, map[string]string{signPolicyAlgorithm: algorithm}) + cond = append(cond, map[string]string{signPolicyCredential: credential}) + cond = append(cond, map[string]string{signPolicyDate: date.Format(iso8601Layout)}) + + if cred.SecurityToken != "" { + cond = append(cond, map[string]string{signPolicySecurityToken: cred.SecurityToken}) + } + + if input.Bucket != "" { + cond = append(cond, map[string]string{signConditionBucket: input.Bucket}) + } + + if input.Key != "" { + cond = append(cond, map[string]string{signConditionKey: input.Key}) + } + + for _, condition := range input.Conditions { + if condition.Operator != nil { + cond = append(cond, []string{*condition.Operator, "$" + condition.Key, condition.Value}) + } else { + cond = append(cond, map[string]string{condition.Key: condition.Value}) + + } + } + if input.ContentLengthRange != nil { + cond = append(cond, []interface{}{signConditionContentLengthRange, input.ContentLengthRange.RangeStart, input.ContentLengthRange.RangeEnd}) + } + postPolicy[signConditions] = cond + originPolicy, err := json.Marshal(postPolicy) + if err != nil { + return nil, InvalidMarshal + } + signK := SigningKey(&SigningKeyInfo{ + Date: date.Format(yyMMdd), + Region: region, + Credential: &cred, + }) + policy := base64.StdEncoding.EncodeToString(originPolicy) + return &PreSingedPostSignatureOutput{ + OriginPolicy: string(originPolicy), + Policy: policy, + Algorithm: signPrefix, + Credential: credential, + Date: date.Format(iso8601Layout), + Signature: hex.EncodeToString(hmacSHA256(signK, []byte(policy))), + }, nil + +} + +func (cli *ClientV2) FetchObjectV2(ctx context.Context, input *FetchObjectInputV2) (*FetchObjectOutputV2, error) { + if err := isValidKey(input.Key); err != nil { + return nil, err + } + if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) > 0 && err != nil { + return nil, InvalidStorageClass + } + + if err := isValidACL(input.ACL); len(input.ACL) > 0 && err != nil { + return nil, InvalidACL + } + + data, contentMD5, err := marshalInput("FetchObjectInputV2", &fetchObjectInput{ + URL: input.URL, + IgnoreSameKey: input.IgnoreSameKey, + ContentMD5: input.HexMD5, + }) + if err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, input.Key). + WithQuery("fetch", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithParams(*input). + WithRetry(OnRetryFromStart, ServerErrorClassifier{}). + Request(ctx, http.MethodPost, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := FetchObjectOutputV2{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + + output.VersionID = res.Header.Get(HeaderVersionID) + output.SSECAlgorithm = res.Header.Get(HeaderSSECustomerAlgorithm) + output.SSECKeyMD5 = res.Header.Get(HeaderCopySourceSSECKeyMD5) + return &output, nil +} + +func (cli *ClientV2) PutFetchTaskV2(ctx context.Context, input *PutFetchTaskInputV2) (*PutFetchTaskOutputV2, error) { + + if err := isValidKey(input.Key); err != nil { + return nil, err + } + + if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) > 0 && err != nil { + return nil, err + } + + if err := isValidACL(input.ACL); len(input.ACL) > 0 && err != nil { + return nil, err + } + + data, contentMD5, err := marshalInput("PutFetchTaskInputV2", putFetchTaskV2Input{ + URL: input.URL, + IgnoreSameKey: input.IgnoreSameKey, + HexMD5: input.HexMD5, + Object: input.Key, + }) + if err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("fetchTask", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithParams(*input). + WithRetry(OnRetryFromStart, ServerErrorClassifier{}). + Request(ctx, http.MethodPost, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutFetchTaskOutputV2{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) PreSignedPolicyURL(ctx context.Context, input *PreSingedPolicyURLInput) (*PreSingedPolicyURLOutput, error) { + + if err := isValidBucketName(input.Bucket, input.IsCustomDomain); err != nil { + return nil, err + } + if input.Expires == 0 { + input.Expires = defaultSignExpires + } + + policyConditions := make(map[string]interface{}) + cred := cli.credentials.Credential() + region := cli.config.Region + + query := make(url.Values) + date := UTCNow() + query.Add(v4Expires, strconv.FormatInt(input.Expires, 10)) + // algorithm + algorithm := signPrefix + query.Add(v4Algorithm, algorithm) + // data + dateQuery := date.Format(iso8601Layout) + query.Add(v4Date, dateQuery) + // credential + credential := fmt.Sprintf("%s/%s/%s/tos/request", cred.AccessKeyID, date.Format(yyMMdd), region) + query.Add(v4Credential, credential) + + if cred.SecurityToken != "" { + query.Add(v4SecurityToken, cred.SecurityToken) + } + + // input conditions + cond := make([]interface{}, 0) + for _, condition := range input.Conditions { + key := condition.Key + operator := condition.Operator + if key != "key" { + return nil, InvalidPreSignedConditions + } + if operator != nil && !(*operator == "eq" || *operator == "starts-with") { + return nil, InvalidPreSignedConditions + } + if operator != nil { + cond = append(cond, []string{*operator, "$" + key, condition.Value}) + } else { + cond = append(cond, map[string]string{key: condition.Value}) + } + } + cond = append(cond, map[string]string{"bucket": input.Bucket}) + policyConditions[signConditions] = cond + originPolicy, err := json.Marshal(policyConditions) + if err != nil { + return nil, InvalidMarshal + } + policy := base64.StdEncoding.EncodeToString(originPolicy) + query.Add("X-Tos-Policy", policy) + + // CanonicalRequest + const split = byte('\n') + var buf bytes.Buffer + var req bytes.Buffer + req.Grow(512) + var queryReq = make(KVs, 0, len(query)) + for key, values := range query { + queryReq = append(queryReq, KV{Key: key, Values: values}) + } + req.Write(encodeQuery(queryReq)) + req.WriteByte(split) + req.WriteString(unsignedPayload) + canonicalStr := req.String() + // StringToSign + buf.Grow(len(signPrefix) + 128) + + buf.WriteString(signPrefix) + buf.WriteByte(split) + + buf.WriteString(date.Format(iso8601Layout)) + buf.WriteByte(split) + + buf.WriteString(date.Format(yyMMdd)) // yyMMdd + '/' + region + '/' + service + '/' + request + buf.WriteByte('/') + buf.WriteString(region) + + buf.WriteString("/tos/request") + buf.WriteByte(split) + + sum := sha256.Sum256([]byte(canonicalStr)) + buf.WriteString(hex.EncodeToString(sum[:])) + + // SigningKey + signK := SigningKey(&SigningKeyInfo{Date: date.Format(yyMMdd), Region: region, Credential: &cred}) + // Signature + sign := hmacSHA256(signK, buf.Bytes()) + query.Add(v4Signature, hex.EncodeToString(sign)) + rawQuery := query.Encode() + // set scheme and host + resScheme := cli.scheme + resHost := cli.host + if input.AlternativeEndpoint != "" { + resScheme, resHost, _ = schemeHost(input.AlternativeEndpoint) + } + return &PreSingedPolicyURLOutput{ + bucket: input.Bucket, + SignatureQuery: rawQuery, + scheme: resScheme, + host: resHost, + isCustomDomain: input.IsCustomDomain, + }, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/config.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/config.go new file mode 100644 index 0000000000..8d259d8595 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/config.go @@ -0,0 +1,31 @@ +package tos + +import "time" + +type Config struct { + Endpoint string + Region string + TransportConfig TransportConfig +} + +func defaultConfig() Config { + return Config{ + TransportConfig: DefaultTransportConfig(), + } +} + +func DefaultTransportConfig() TransportConfig { + return TransportConfig{ + MaxIdleConns: 1024, + MaxIdleConnsPerHost: 1024, + MaxConnsPerHost: 1024, + DialTimeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + IdleConnTimeout: 60 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 60 * time.Second, + ExpectContinueTimeout: 3 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + } +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/consts.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/consts.go new file mode 100644 index 0000000000..f53e846129 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/consts.go @@ -0,0 +1,149 @@ +package tos + +import ( + "hash/crc64" + "os" +) + +const ( + // Version tos-go-sdk version + Version = "v2.6.2" +) + +const TempFileSuffix = ".temp" +const DefaultFilePerm = os.FileMode(0644) + +var DefaultCrcTable = func() *crc64.Table { + return crc64.MakeTable(crc64.ECMA) +} + +const DefaultTaskBufferSize = 100 +const DefaultListMaxKeys = 1000 + +func SupportedRegion() map[string]string { + return map[string]string{ + "cn-beijing": "tos-cn-beijing.volces.com", + "cn-guangzhou": "tos-cn-guangzhou.volces.com", + "cn-shanghai": "tos-cn-shanghai.volces.com", + } +} + +func SupportedEndpoint() map[string]string { + supportEndpoint := make(map[string]string) + for key, value := range SupportedRegion() { + supportEndpoint[value] = key + } + return supportEndpoint +} + +const ( + defaultPreSignedURLExpires = 3600 + maxPreSignedURLExpires = 604800 +) + +const ( + MaxPartSize = 5 * 1024 * 1024 * 1024 + MinPartSize = 5 * 1024 * 1024 + DefaultPartSize = 20 * 1024 * 1024 +) + +const ( + // Deprecated: use enum.ACLPrivate instead + ACLPrivate = "private" + // Deprecated: use enum.ACLPublicRead instead + ACLPublicRead = "public-read" + // Deprecated: use enum.ACLPublicReadWrite instead + ACLPublicReadWrite = "public-read-write" + // Deprecated: use enum.ACLAuthRead instead + ACLAuthRead = "authenticated-read" + // Deprecated: use enum.ACLBucketOwnerRead instead + ACLBucketOwnerRead = "bucket-owner-read" + // Deprecated: use enum.ACLBucketOwnerFullControl instead + ACLBucketOwnerFullControl = "bucket-owner-full-control" + // Deprecated: use enum.ACLLogDeliveryWrite instead + ACLLogDeliveryWrite = "log-delivery-write" + + // Deprecated: use enum.PermissionRead instead + PermissionRead = "READ" + // Deprecated: use enum.PermissionWrite instead + PermissionTypeWrite = "WRITE" + // Deprecated: use enum.PermissionReadAcp instead + PermissionTypeReadAcp = "READ_ACP" + // Deprecated: use enum.PermissionWriteAcp instead + PermissionTypeWriteAcp = "WRITE_ACP" + // Deprecated: use enum.PermissionFullControl instead + PermissionFullControl = "FULL_CONTROL" +) + +const ( + ISO8601TimeFormat = "2006-01-02T15:04:05.000Z07:00" +) + +const ( + // MetadataDirectiveReplace replace source object metadata when calling CopyObject + MetadataDirectiveReplace = "REPLACE" + + // MetadataDirectiveCopy copy source object metadata when calling CopyObject + MetadataDirectiveCopy = "COPY" +) +const ( + QueryPartNumber = "partNumber" +) +const ( + HeaderUserAgent = "User-Agent" + HeaderContentLength = "Content-Length" + HeaderContentType = "Content-Type" + HeaderContentMD5 = "Content-MD5" + HeaderContentSha256 = "X-Tos-Content-Sha256" + HeaderContentLanguage = "Content-Language" + HeaderContentEncoding = "Content-Encoding" + HeaderContentDisposition = "Content-Disposition" + HeaderLastModified = "Last-Modified" + HeaderCacheControl = "Cache-Control" + HeaderExpires = "Expires" + HeaderETag = "ETag" + HeaderVersionID = "X-Tos-Version-Id" + HeaderDeleteMarker = "X-Tos-Delete-Marker" + HeaderStorageClass = "X-Tos-Storage-Class" + HeaderAzRedundancy = "X-Tos-Az-Redundancy" + HeaderRestore = "X-Tos-Restore" + HeaderTag = "X-Tos-Tag" + HeaderSSECustomerAlgorithm = "X-Tos-Server-Side-Encryption-Customer-Algorithm" + HeaderSSECustomerKeyMD5 = "X-Tos-Server-Side-Encryption-Customer-Key-MD5" + HeaderSSECustomerKey = "X-Tos-Server-Side-Encryption-Customer-Key" + HeaderServerSideEncryption = "X-Tos-Server-Side-Encryption" + HeaderServerSideEncryptionKmsKeyID = "X-Tos-Server-Side-Encryption-Kms-Key-Id" + HeaderCopySourceSSECAlgorithm = "X-Tos-Server-Side-Encryption-Customer-Algorithm" + HeaderCopySourceSSECKeyMD5 = "X-Tos-Server-Side-Encryption-Customer-Key-MD5" + HeaderCopySourceSSECKey = "X-Tos-Server-Side-Encryption-Customer-Key" + HeaderIfModifiedSince = "If-Modified-Since" + HeaderIfUnmodifiedSince = "If-Unmodified-Since" + HeaderIfMatch = "If-Match" + HeaderIfNoneMatch = "If-None-Match" + HeaderRange = "Range" + HeaderContentRange = "Content-Range" + HeaderRequestID = "X-Tos-Request-Id" + HeaderID2 = "X-Tos-Id-2" + HeaderBucketRegion = "X-Tos-Bucket-Region" + HeaderLocation = "Location" + HeaderACL = "X-Tos-Acl" + HeaderGrantFullControl = "X-Tos-Grant-Full-Control" + HeaderGrantRead = "X-Tos-Grant-Read" + HeaderGrantReadAcp = "X-Tos-Grant-Read-Acp" + HeaderGrantWrite = "X-Tos-Grant-Write" + HeaderGrantWriteAcp = "X-Tos-Grant-Write-Acp" + HeaderNextAppendOffset = "X-Tos-Next-Append-Offset" + HeaderObjectType = "X-Tos-Object-Type" + HeaderHashCrc64ecma = "X-Tos-Hash-Crc64ecma" + HeaderMetadataDirective = "X-Tos-Metadata-Directive" + HeaderCopySource = "X-Tos-Copy-Source" + HeaderCopySourceIfMatch = "X-Tos-Copy-Source-If-Match" + HeaderCopySourceIfNoneMatch = "X-Tos-Copy-Source-If-None-Match" + HeaderCopySourceIfModifiedSince = "X-Tos-Copy-Source-If-Modified-Since" + HeaderCopySourceIfUnmodifiedSince = "X-Tos-Copy-Source-If-Unmodified-Since" + HeaderCopySourceRange = "X-Tos-Copy-Source-Range" + HeaderCopySourceVersionID = "X-Tos-Copy-Source-Version-Id" + HeaderWebsiteRedirectLocation = "X-Tos-Website-Redirect-Location" + HeaderCSType = "X-Tos-Cs-Type" + HeaderMetaPrefix = "X-Tos-Meta-" +) diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/copy.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/copy.go new file mode 100644 index 0000000000..c868304cee --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/copy.go @@ -0,0 +1,310 @@ +package tos + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "time" +) + +// CopyObject copy an object +// srcObjectKey: the source object name +// dstObjectKey: the destination object name. srcObjectKey and dstObjectKey belongs to the same bucket. +// options: WithVersionID the version id of source object, +// WithMetadataDirective copy source object metadata or replace with new object metadata, +// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl, +// WithCopySourceIfMatch WithCopySourceIfNoneMatch WithCopySourceIfModifiedSince WithCopySourceIfUnmodifiedSince set copy conditions +// if CopyObject called with WithMetadataDirective(tos.MetadataDirectiveReplace), these options can be used: +// WithContentType set Content-Type, +// WithContentDisposition set Content-Disposition, +// WithContentLanguage set Content-Language, +// WithContentEncoding set Content-Encoding, +// WithCacheControl set Cache-Control, +// WithExpires set Expires, +// WithMeta set meta header(s), +// +// Deprecated: use CopyObject of ClientV2 instead +func (bkt *Bucket) CopyObject(ctx context.Context, srcObjectKey, dstObjectKey string, options ...Option) (*CopyObjectOutput, error) { + if err := isValidKey(dstObjectKey, srcObjectKey); err != nil { + return nil, err + } + + return bkt.client.copyObject(ctx, bkt.name, dstObjectKey, bkt.name, srcObjectKey, options...) +} + +// CopyObjectTo copy an object to target bucket +// dstBucket: the destination bucket +// dstObjectKey: the destination object name +// srcObjectKey: the source object name +// options: WithVersionID the version id of source object, +// WithMetadataDirective copy source object metadata or replace with new object metadata. +// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl, +// WithCopySourceIfMatch WithCopySourceIfNoneMatch WithCopySourceIfModifiedSince WithCopySourceIfUnmodifiedSince set copy conditions +// if CopyObjectTo called with WithMetadataDirective(tos.MetadataDirectiveReplace), these options can be used: +// WithContentType set Content-Type, +// WithContentDisposition set Content-Disposition, +// WithContentLanguage set Content-Language, +// WithContentEncoding set Content-Encoding, +// WithCacheControl set Cache-Control, +// WithExpires set Expires, +// WithMeta set meta header(s), +// +// Deprecated: use CopyObject of ClientV2 instead +func (bkt *Bucket) CopyObjectTo(ctx context.Context, dstBucket, dstObjectKey, srcObjectKey string, options ...Option) (*CopyObjectOutput, error) { + if err := isValidNames(dstBucket, dstObjectKey, false, srcObjectKey); err != nil { + return nil, err + } + + return bkt.client.copyObject(ctx, dstBucket, dstObjectKey, bkt.name, srcObjectKey, options...) +} + +// CopyObjectFrom copy an object from target bucket +// srcBucket: the srcBucket bucket +// srcObjectKey: the source object name +// dstObjectKey: the destination object name +// options: WithVersionID the version id of source object, +// WithMetadataDirective copy source object metadata or replace with new object metadata +// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl, +// WithCopySourceIfMatch WithCopySourceIfNoneMatch WithCopySourceIfModifiedSince WithCopySourceIfUnmodifiedSince set copy conditions +// if CopyObjectFrom called with WithMetadataDirective(tos.MetadataDirectiveReplace), these options can be used: +// WithContentType set Content-Type, +// WithContentDisposition set Content-Disposition, +// WithContentLanguage set Content-Language, +// WithContentEncoding set Content-Encoding, +// WithCacheControl set Cache-Control, +// WithExpires set Expires, +// WithMeta set meta header(s), +// +// Deprecated: use CopyObject of ClientV2 instead +func (bkt *Bucket) CopyObjectFrom(ctx context.Context, srcBucket, srcObjectKey, dstObjectKey string, options ...Option) (*CopyObjectOutput, error) { + if err := isValidNames(srcBucket, srcObjectKey, false, dstObjectKey); err != nil { + return nil, err + } + + return bkt.client.copyObject(ctx, bkt.name, dstObjectKey, srcBucket, srcObjectKey, options...) +} + +func (cli *Client) copyObject(ctx context.Context, dstBucket, dstObject string, srcBucket, srcObject string, options ...Option) (*CopyObjectOutput, error) { + res, err := cli.newBuilder(dstBucket, dstObject, options...). + WithCopySource(srcBucket, srcObject). + WithRetry(nil, ServerErrorClassifier{}). + Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + marshalOut := copyObjectOutput{} + + if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &marshalOut); err != nil { + return nil, err + } + if marshalOut.ETag == "" { + return nil, &TosServerError{ + TosError: TosError{marshalOut.Message}, + RequestInfo: res.RequestInfo(), + Code: marshalOut.Code, + HostID: marshalOut.HostID, + Resource: marshalOut.Resource, + } + } + out := CopyObjectOutput{RequestInfo: res.RequestInfo(), ETag: marshalOut.ETag, LastModified: marshalOut.LastModified} + out.VersionID = res.Header.Get(HeaderVersionID) + out.SourceVersionID = res.Header.Get(HeaderCopySourceVersionID) + out.SSECAlgorithm = res.Header.Get(HeaderSSECustomerAlgorithm) + out.SSECKeyMD5 = res.Header.Get(HeaderSSECustomerKeyMD5) + out.ServerSideEncryption = res.Header.Get(HeaderServerSideEncryption) + out.ServerSideEncryptionKeyID = res.Header.Get(HeaderServerSideEncryptionKmsKeyID) + return &out, nil +} + +// CopyObject copy an object +func (cli *ClientV2) CopyObject(ctx context.Context, input *CopyObjectInput) (*CopyObjectOutput, error) { + if err := isValidBucketName(input.SrcBucket, false); err != nil { + return nil, err + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + if err := isValidKey(input.Key, input.SrcKey); err != nil { + return nil, err + } + if err := isValidMetadataDirective(input.MetadataDirective); len(input.MetadataDirective) != 0 && err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, input.Key). + WithParams(*input). + WithCopySource(input.SrcBucket, input.SrcKey). + WithRetry(nil, ServerErrorClassifier{}). + Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + marshalOut := copyObjectOutput{} + if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &marshalOut); err != nil { + return nil, err + } + // Body 的 Etag 存在复制成功 + if marshalOut.ETag == "" { + return nil, &TosServerError{ + TosError: TosError{marshalOut.Message}, + RequestInfo: res.RequestInfo(), + Code: marshalOut.Code, + HostID: marshalOut.HostID, + Resource: marshalOut.Resource, + } + } + out := CopyObjectOutput{RequestInfo: res.RequestInfo(), ETag: marshalOut.ETag, LastModified: marshalOut.LastModified} + out.VersionID = res.Header.Get(HeaderVersionID) + out.SourceVersionID = res.Header.Get(HeaderCopySourceVersionID) + return &out, nil +} + +type uploadPartCopyOutput struct { + ETag string `json:"ETag,omitempty"` + LastModified string `json:"LastModified,omitempty"` + Error +} + +func copyRange(startOffset, partSize *int64) string { + cr := "" + if startOffset != nil { + if partSize != nil { + cr = fmt.Sprintf("bytes=%d-%d", *startOffset, *startOffset+*partSize-1) + } else { + cr = fmt.Sprintf("bytes=%d-", *startOffset) + } + } else if partSize != nil { + cr = fmt.Sprintf("bytes=0-%d", *partSize-1) + } + return cr +} + +func copySource(bucket, object, versionID string) string { + if len(versionID) == 0 { + return "/" + bucket + "/" + url.QueryEscape(object) + } + return "/" + bucket + "/" + url.QueryEscape(object) + "?versionId=" + versionID +} + +func (up *UploadPartCopyOutput) uploadedPart() uploadedPart { + return uploadedPart{PartNumber: up.PartNumber, ETag: up.ETag} +} + +// UploadPartCopy copy a part of object as a part of a multipart upload operation +// input: uploadID, DestinationKey, SourceBucket, SourceKey and other parameters, +// options: WithCopySourceIfMatch WithCopySourceIfNoneMatch WithCopySourceIfModifiedSince WithCopySourceIfUnmodifiedSince set copy conditions +// +// Deprecated: use UploadPartCopy of ClientV2 instead +func (bkt *Bucket) UploadPartCopy(ctx context.Context, input *UploadPartCopyInput, options ...Option) (*UploadPartCopyOutput, error) { + if err := isValidNames(input.SourceBucket, input.DestinationKey, false); err != nil { + return nil, err + } + + res, err := bkt.client.newBuilder(bkt.name, input.DestinationKey, options...). + WithQuery("partNumber", strconv.Itoa(input.PartNumber)). + WithQuery("uploadId", input.UploadID). + WithQuery("versionId", input.SourceVersionID). + WithHeader(HeaderCopySourceRange, copyRange(input.StartOffset, input.PartSize)). + WithCopySource(input.SourceBucket, input.SourceKey). + WithRetry(nil, ServerErrorClassifier{}). + Request(ctx, http.MethodPut, nil, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + var out uploadPartCopyOutput + if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &out); err != nil { + return nil, err + } + if out.ETag == "" { + return nil, &TosServerError{ + TosError: TosError{out.Message}, + RequestInfo: res.RequestInfo(), + Code: out.Code, + HostID: out.HostID, + Resource: out.Resource, + } + } + return &UploadPartCopyOutput{ + RequestInfo: res.RequestInfo(), + VersionID: res.Header.Get(HeaderVersionID), + SourceVersionID: res.Header.Get(HeaderCopySourceVersionID), + PartNumber: input.PartNumber, + ETag: out.ETag, + LastModified: out.LastModified, + }, nil +} + +func copyRangeV2(start, end int64) string { + cr := "" + if start == 0 && end == 0 { + return cr + } + if start > end { + return cr + } + cr = fmt.Sprintf("bytes=%d-%d", start, end) + return cr +} + +// UploadPartCopyV2 copy a part of object as a part of a multipart upload operation +func (cli *ClientV2) UploadPartCopyV2( + ctx context.Context, + input *UploadPartCopyV2Input) (*UploadPartCopyV2Output, error) { + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + if err := isValidBucketName(input.SrcBucket, false); err != nil { + return nil, err + } + if err := isValidKey(input.SrcKey, input.Key); err != nil { + return nil, err + } + + req := cli.newBuilder(input.Bucket, input.Key). + WithParams(*input) + + if input.CopySourceRange != "" { + req = req.WithHeader(HeaderCopySourceRange, input.CopySourceRange) + } else if input.CopySourceRangeEnd != 0 { + req = req.WithHeader(HeaderCopySourceRange, copyRangeV2(input.CopySourceRangeStart, input.CopySourceRangeEnd)) + } + + res, err := req.WithCopySource(input.SrcBucket, input.SrcKey). + WithRetry(nil, ServerErrorClassifier{}). + Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + var out uploadPartCopyOutput + if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &out); err != nil { + return nil, err + } + + lastModified, _ := time.ParseInLocation(http.TimeFormat, res.Header.Get(HeaderLastModified), time.UTC) + if out.ETag == "" { + return nil, &TosServerError{ + TosError: TosError{out.Message}, + RequestInfo: res.RequestInfo(), + Code: out.Code, + HostID: out.HostID, + Resource: out.Resource, + } + } + return &UploadPartCopyV2Output{ + RequestInfo: res.RequestInfo(), + PartNumber: input.PartNumber, + ETag: out.ETag, + LastModified: lastModified, + CopySourceVersionID: res.Header.Get(HeaderCopySourceVersionID), + ServerSideEncryption: res.Header.Get(HeaderServerSideEncryption), + ServerSideEncryptionKeyID: res.Header.Get(HeaderServerSideEncryptionKmsKeyID), + SSECAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm), + SSECKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5), + }, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/cors.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/cors.go new file mode 100644 index 0000000000..2207d574b8 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/cors.go @@ -0,0 +1,81 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +// GetBucketCORS get the bucket's CORS settings. +func (cli *ClientV2) GetBucketCORS(ctx context.Context, input *GetBucketCORSInput) (*GetBucketCORSOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("cors", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := GetBucketCORSOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + + return &output, nil +} + +// PutBucketCORS upsert the bucket's CORS settings. +func (cli *ClientV2) PutBucketCORS(ctx context.Context, input *PutBucketCORSInput) (*PutBucketCORSOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + data, contentMD5, err := marshalInput("PutBucketCORSInput", input) + if err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("cors", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := PutBucketCORSOutput{RequestInfo: res.RequestInfo()} + return &output, nil + +} + +// DeleteBucketCORS delete the bucket's all CORS settings. +func (cli *ClientV2) DeleteBucketCORS(ctx context.Context, input *DeleteBucketCORSInput) (*DeleteBucketCORSOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("cors", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteBucketCORSOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/crc.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/crc.go new file mode 100644 index 0000000000..63f8726e73 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/crc.go @@ -0,0 +1,113 @@ +package tos + +import ( + "hash" + "hash/crc64" +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + crc uint64 + tab *crc64.Table +} + +// NewCRC is similar with crc64.New, but you can set the initial value. +// Following methods are copied from package crc64 to implement Hash interface. +func NewCRC(tab *crc64.Table, init uint64) hash.Hash64 { return &digest{init, tab} } + +func (d *digest) Size() int { return crc64.Size } + +func (d *digest) BlockSize() int { return 1 } + +func (d *digest) Reset() { d.crc = 0 } + +func (d *digest) Write(p []byte) (n int, err error) { + d.crc = crc64.Update(d.crc, d.tab, p) + return len(p), nil +} + +func (d *digest) Sum64() uint64 { return d.crc } + +func (d *digest) Sum(in []byte) []byte { + s := d.Sum64() + return append(in, byte(s>>56), byte(s>>48), byte(s>>40), byte(s>>32), byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) +} + +// gf2Dim dimension of GF(2) vectors (length of CRC) +const gf2Dim int = 64 + +func gf2MatrixTimes(mat []uint64, vec uint64) uint64 { + var sum uint64 + for i := 0; vec != 0; i++ { + if vec&1 != 0 { + sum ^= mat[i] + } + + vec >>= 1 + } + return sum +} + +func gf2MatrixSquare(square []uint64, mat []uint64) { + for n := 0; n < gf2Dim; n++ { + square[n] = gf2MatrixTimes(mat, mat[n]) + } +} + +// CRC64Combine combines CRC64 +func CRC64Combine(crc1 uint64, crc2 uint64, len2 uint64) uint64 { + var even [gf2Dim]uint64 // Even-power-of-two zeros operator + var odd [gf2Dim]uint64 // Odd-power-of-two zeros operator + + // Degenerate case + if len2 == 0 { + return crc1 + } + + // Put operator for one zero bit in odd + odd[0] = crc64.ECMA // CRC64 polynomial + var row uint64 = 1 + for n := 1; n < gf2Dim; n++ { + odd[n] = row + row <<= 1 + } + + // Put operator for two zero bits in even + gf2MatrixSquare(even[:], odd[:]) + + // Put operator for four zero bits in odd + gf2MatrixSquare(odd[:], even[:]) + + // Apply len2 zeros to crc1, first square will put the operator for one zero byte, eight zero bits, in even + for { + // Apply zeros operator for this bit of len2 + gf2MatrixSquare(even[:], odd[:]) + + if len2&1 != 0 { + crc1 = gf2MatrixTimes(even[:], crc1) + } + + len2 >>= 1 + + // If no more bits set, then done + if len2 == 0 { + break + } + + // Another iteration of the loop with odd and even swapped + gf2MatrixSquare(odd[:], even[:]) + if len2&1 != 0 { + crc1 = gf2MatrixTimes(odd[:], crc1) + } + len2 >>= 1 + + // If no more bits set, then done + if len2 == 0 { + break + } + } + + // Return combined CRC + crc1 ^= crc2 + return crc1 +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/credential.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/credential.go new file mode 100644 index 0000000000..f8592fb65c --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/credential.go @@ -0,0 +1,161 @@ +package tos + +import ( + "sync/atomic" + "time" + "unsafe" + + "golang.org/x/sync/singleflight" +) + +type Credential struct { + AccessKeyID string + AccessKeySecret string + SecurityToken string +} + +// Credentials provides Credential +type Credentials interface { + Credential() Credential +} + +// StaticCredentials Credentials with static access-key and secret-key +type StaticCredentials struct { + accessKey string + secretKey string + securityToken string +} + +// NewStaticCredentials Credentials with static access-key and secret-key +// use StaticCredentials.WithSecurityToken to set security-token +// +// you can use it as: +// client, err := tos.NewClient(endpoint, tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey))) +// // do something more +// +// And you can use tos.WithPerRequestSigner set the 'Signer' for each request. +// +func NewStaticCredentials(accessKeyID, accessKeySecret string) *StaticCredentials { + return &StaticCredentials{ + accessKey: accessKeyID, + secretKey: accessKeySecret, + } +} + +// WithSecurityToken set security-token +func (sc *StaticCredentials) WithSecurityToken(securityToken string) { + sc.securityToken = securityToken +} + +func (sc *StaticCredentials) Credential() Credential { + return Credential{ + AccessKeyID: sc.accessKey, + AccessKeySecret: sc.secretKey, + SecurityToken: sc.securityToken, + } +} + +// WithoutSecretKeyCredentials Credentials with static access-key and no secret-key +// +// If you don't want to use secret-key directly, but use signed-key, you can use it as: +// signer := tos.NewSignV4(tos.NewWithoutSecretKeyCredentials(accessKey), region) +// signer.WithSigningKey(func(*SigningKeyInfo) []byte { return signingKey}) +// client, err := tos.NewClient(endpoint, tos.WithSigner(signer)) +// // do something more +// +// And you can use tos.WithPerRequestSigner set the 'Signer' for each request. +// +type WithoutSecretKeyCredentials struct { + accessKey string + securityToken string +} + +func NewWithoutSecretKeyCredentials(accessKeyID string) *WithoutSecretKeyCredentials { + return &WithoutSecretKeyCredentials{ + accessKey: accessKeyID, + securityToken: "", + } +} + +// WithSecurityToken set security-token +func (sc *WithoutSecretKeyCredentials) WithSecurityToken(securityToken string) { + sc.securityToken = securityToken +} + +func (sc *WithoutSecretKeyCredentials) Credential() Credential { + return Credential{ + AccessKeyID: sc.accessKey, + SecurityToken: sc.securityToken, + } +} + +// FederationToken contains Credential and Credential's expiration time +type FederationToken struct { + Credential Credential + Expiration time.Time +} + +// FederationTokenProvider provides FederationToken +type FederationTokenProvider interface { + FederationToken() (*FederationToken, error) +} + +// FederationCredentials implements Credentials interfaces with flushing Credential periodically +type FederationCredentials struct { + cachedToken *FederationToken + refreshing uint32 + preFetch time.Duration + tokenProvider FederationTokenProvider + flight singleflight.Group +} + +// NewFederationCredentials FederationCredentials implements Credentials interfaces with flushing Credential periodically +// +// use WithPreFetch set prefetch time +func NewFederationCredentials(tokenProvider FederationTokenProvider) (*FederationCredentials, error) { + cred, err := tokenProvider.FederationToken() + if err != nil { + return nil, err + } + + return &FederationCredentials{ + cachedToken: cred, + preFetch: 5 * time.Minute, + tokenProvider: tokenProvider, + }, nil +} + +// WithPreFetch set prefetch time +func (fc *FederationCredentials) WithPreFetch(preFetch time.Duration) { + fc.preFetch = preFetch +} + +func (fc *FederationCredentials) token() *FederationToken { + return (*FederationToken)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&fc.cachedToken)))) +} + +// Credential for Credentials interface +func (fc *FederationCredentials) Credential() Credential { + now := time.Now() + if token := fc.token(); now.After(token.Expiration) { // 已经过期 + _, _, _ = fc.flight.Do("flushing", func() (interface{}, error) { + flushed, err := fc.tokenProvider.FederationToken() + if err != nil { + return nil, err + } + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&fc.cachedToken)), unsafe.Pointer(flushed)) + return flushed, nil + }) + } else if now.Add(fc.preFetch).After(token.Expiration) && + atomic.LoadUint32(&fc.refreshing) == 0 { + // 将要过期, prefetch token + if atomic.CompareAndSwapUint32(&fc.refreshing, 0, 1) { + defer atomic.StoreUint32(&fc.refreshing, 0) + if newToken, err := fc.tokenProvider.FederationToken(); err == nil { + atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&fc.cachedToken)), unsafe.Pointer(newToken)) + } + } + } + + return fc.token().Credential +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/custom_domain.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/custom_domain.go new file mode 100644 index 0000000000..33b929db73 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/custom_domain.go @@ -0,0 +1,76 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +func (cli *ClientV2) PutBucketCustomDomain(ctx context.Context, input *PutBucketCustomDomainInput) (*PutBucketCustomDomainOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + body := putBucketCustomDomainInput{ + Rule: input.Rule, + } + data, contentMD5, err := marshalInput("PutBucketCustomDomainInput", body) + if err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("customdomain", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketCustomDomainOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) ListBucketCustomDomain(ctx context.Context, input *ListBucketCustomDomainInput) (*ListBucketCustomDomainOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("customdomain", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := ListBucketCustomDomainOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) DeleteBucketCustomDomain(ctx context.Context, input *DeleteBucketCustomDomainInput) (*DeleteBucketCustomDomainOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("customdomain", input.Domain). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteBucketCustomDomainOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/dns_cache.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/dns_cache.go new file mode 100644 index 0000000000..99581d692f --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/dns_cache.go @@ -0,0 +1,213 @@ +package tos + +import ( + "container/heap" + "context" + "net" + "strings" + "sync" + "time" +) + +const ( + DefaultCacheCap = 100 + VolceHostSuffix = "volces.com" + HostSplitSep = "." + HostSplitLength = 4 +) + +type cacheItem struct { + host string + ipList []string + expireAt time.Time + heapIndex int +} +type priorityQueue []*cacheItem + +func (p priorityQueue) Len() int { + return len(p) +} + +func (p priorityQueue) Peek() *cacheItem { + if p.Len() > 0 { + return p[0] + } + return nil +} + +func (p priorityQueue) Less(i, j int) bool { + return p[i].expireAt.Before(p[j].expireAt) +} + +func (p priorityQueue) Swap(i, j int) { + p[i], p[j] = p[j], p[i] + p[i].heapIndex = i + p[j].heapIndex = j +} + +func (p *priorityQueue) Push(x interface{}) { + n := len(*p) + item := x.(*cacheItem) + item.heapIndex = n + *p = append(*p, item) +} + +func (p *priorityQueue) Pop() interface{} { + old := *p + n := len(old) + item := old[n-1] + old[n-1] = nil + item.heapIndex = -1 + *p = old[0 : n-1] + return item +} + +type cache struct { + lock sync.RWMutex + heap *priorityQueue + cleanTime time.Time + data map[string]cacheItem + expiration time.Duration +} + +func (c *cache) Remove(key string, removeIp string) { + c.lock.Lock() + defer c.lock.Unlock() + data, ok := c.data[key] + if !ok { + return + } + value := make([]string, 0, len(data.ipList)) + for _, ip := range data.ipList { + if ip == removeIp { + continue + } + value = append(value, ip) + } + + // 没有有效的 IP 将缓存删除 + if len(value) == 0 { + delete(c.data, key) + return + } + + data.ipList = value + c.data[key] = data + +} + +func (c *cache) Get(key string) ([]string, bool) { + c.lock.RLock() + data, ok := c.data[key] + c.lock.RUnlock() + if !ok { + return nil, false + } + if data.expireAt.Before(time.Now()) { + return nil, false + } + return data.ipList, true +} + +func (c *cache) cleanCache() { + c.cleanTime = time.Now().Add(c.expiration) + maxCleanCount := 5 + for i := 0; i < maxCleanCount; i++ { + item := c.heap.Peek() + if item == nil { + return + } + + if item.expireAt.Before(time.Now()) { + heap.Pop(c.heap) + data, ok := c.data[item.host] + if ok && data.expireAt == item.expireAt { + delete(c.data, item.host) + } + } else { + return + } + } +} + +func (c *cache) Put(key string, ipList []string) { + c.lock.Lock() + defer c.lock.Unlock() + item := cacheItem{ + ipList: ipList, + expireAt: time.Now().Add(c.expiration), + host: key, + } + c.data[key] = item + heap.Push(c.heap, &item) + + // 大于 Cap + if c.heap.Len() > DefaultCacheCap { + item := heap.Pop(c.heap).(*cacheItem) + if item == nil { + return + } + data, ok := c.data[item.host] + if ok && data.expireAt == item.expireAt { + delete(c.data, item.host) + } + } + + if time.Now().After(c.cleanTime) { + c.cleanCache() + } +} + +type resolver struct { + cache *cache +} + +func newResolver(expiration time.Duration) *resolver { + pq := make(priorityQueue, 0) + return &resolver{cache: &cache{ + heap: &pq, + cleanTime: time.Now().Add(expiration), + data: make(map[string]cacheItem), + expiration: expiration, + }} +} + +func ipToStringList(ips []net.IP) []string { + res := make([]string, len(ips)) + for i, ip := range ips { + res[i] = ip.String() + } + return res +} + +func wrappedHost(host string) string { + if !strings.HasSuffix(host, VolceHostSuffix) { + return host + } + hostSplit := strings.Split(host, HostSplitSep) + if len(hostSplit) != HostSplitLength { + return host + } + return strings.Join(hostSplit[1:], HostSplitSep) + +} + +func (r *resolver) GetIpList(ctx context.Context, host string) ([]string, error) { + + ipList, ok := r.cache.Get(wrappedHost(host)) + if ok { + return ipList, nil + } + ips, err := net.LookupIP(host) + if err != nil { + return nil, err + } + ipsStr := ipToStringList(ips) + r.cache.Put(wrappedHost(host), ipsStr) + + return ipsStr, nil +} + +func (r *resolver) Remove(host string, ip string) { + r.cache.Remove(wrappedHost(host), ip) +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/download_file.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/download_file.go new file mode 100644 index 0000000000..994e76f8ed --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/download_file.go @@ -0,0 +1,388 @@ +package tos + +import ( + "context" + "crypto/md5" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" +) + +func getDownloadCheckpoint(input *DownloadFileInput, init func(input *HeadObjectV2Output) (*downloadCheckpoint, error), output *HeadObjectV2Output) (checkpoint *downloadCheckpoint, err error) { + enabled := input.EnableCheckpoint + checkpointPath := input.CheckpointFile + if !enabled { + return init(output) + } + + checkpoint = &downloadCheckpoint{} + loadCheckPoint(checkpointPath, checkpoint) + if checkpoint.Valid(input, output) { + return + } + + parentDir := filepath.Dir(checkpointPath) + stat, err := os.Stat(parentDir) + if err != nil { + err = os.MkdirAll(parentDir, os.ModePerm) + if err != nil { + return nil, newTosClientError(err.Error(), err) + } + } else if !stat.IsDir() { + return nil, newTosClientError("Fail to create folder due to a same file exists.", nil) + } + + file, err := os.Create(checkpointPath) + if err != nil { + return nil, newTosClientError(err.Error(), err) + } + _ = file.Close() + + checkpoint, err = init(output) + if err != nil { + return nil, err + } + + err = checkpoint.WriteToFile() + if err != nil { + return nil, err + } + + return +} + +func (cli *ClientV2) DownloadFile(ctx context.Context, input *DownloadFileInput) (*DownloadFileOutput, error) { + err := validateDownloadInput(input, cli.isCustomDomain) + if err != nil { + return nil, err + } + headOutput, err := cli.HeadObjectV2(ctx, &input.HeadObjectV2Input) + if err != nil { + return nil, err + } + + needDownload, err := parseDownloadFilePath(input) + if err != nil { + return nil, err + } + if !needDownload { + return &DownloadFileOutput{*headOutput}, nil + } + + event := downloadEvent{input: input} + init := func(output *HeadObjectV2Output) (*downloadCheckpoint, error) { + err := createDownloadTempFile(input, event) + if err != nil { + return nil, err + } + return initDownloadCheckpoint(input, headOutput) + } + checkpoint, err := getDownloadCheckpoint(input, init, headOutput) + if err != nil { + return nil, err + } + cleaner := func() { + _ = os.Remove(input.CheckpointFile) + _ = os.Remove(input.tempFile) + } + bindCancelHookWithCleaner(input.CancelHook, cleaner) + return cli.downloadFile(ctx, headOutput, checkpoint, input, event) +} + +// loadCheckPoint load UploadFile checkpoint or DownloadFile checkpoint. +// checkpoint must be a pointer +func loadCheckPoint(path string, checkpoint interface{}) { + contents, err := ioutil.ReadFile(path) + if err != nil && !os.IsNotExist(err) { + return + } + if len(contents) == 0 { + return + } + json.Unmarshal(contents, &checkpoint) +} +func isDir(filePath string) bool { + stat, err := os.Stat(filePath) + if err != nil { + _, fileName := filepath.Split(filePath) + return fileName == "" + } + return stat.IsDir() +} + +// if file is a directory, append suffix to it to make a file name +func withSuffixIfDir(filePath string, suffix string) string { + if isDir(filePath) { + return filepath.Clean(filepath.Join(filePath, suffix)) + } + return filePath +} + +func getDownloadCheckPointPath(checkpointPath, filePath, bucket, key, versionId string) string { + fileName := strings.Join([]string{filepath.Base(filePath), checkpointPathMd5(bucket, key, versionId), "download"}, ".") + if len(checkpointPath) == 0 { + dirName := filepath.Dir(filePath) + return filepath.Clean(filepath.Join(dirName, fileName)) + } + return withSuffixIfDir(checkpointPath, fileName) + +} + +func checkpointPathMd5(bucket string, key string, versionId string) string { + var data []byte + if versionId != "" { + data = []byte(strings.Join([]string{bucket, key, versionId}, ".")) + } else { + data = []byte(strings.Join([]string{bucket, key}, ".")) + + } + r := md5.Sum(data) + return base64.URLEncoding.EncodeToString(r[:]) +} + +func parseDownloadFilePath(input *DownloadFileInput) (needDownloadFile bool, err error) { + input.filePath = input.FilePath + inputFile := input.filePath + isDirRes := isDir(input.filePath) + if isDirRes { + input.filePath = filepath.Clean(filepath.Join(input.filePath, input.Key)) + } + input.tempFile = input.filePath + TempFileSuffix + + if input.EnableCheckpoint { + input.CheckpointFile = getDownloadCheckPointPath(input.CheckpointFile, input.filePath, input.Bucket, input.Key, input.VersionID) + } + + if isDirRes && strings.HasSuffix(input.Key, "/") { + err := os.MkdirAll(filepath.Join(inputFile, input.Key), os.ModePerm) + if err != nil { + return false, InvalidFilePath.withCause(err) + } + return false, nil + } + + return true, nil + +} + +func validateDownloadInput(input *DownloadFileInput, isCustomDomain bool) error { + if err := isValidNames(input.Bucket, input.Key, isCustomDomain); err != nil { + return err + } + + if input.PartSize == 0 { + input.PartSize = DefaultPartSize + } + if input.PartSize < MinPartSize || input.PartSize > MaxPartSize { + return newTosClientError("The input part size is invalid, please set it range from 5MB to 5GB", nil) + } + + if input.TaskNum < 1 { + input.TaskNum = 1 + } + if input.TaskNum > 1000 { + input.TaskNum = 1000 + } + return nil +} + +func initDownloadCheckpoint(input *DownloadFileInput, headOutput *HeadObjectV2Output) (*downloadCheckpoint, error) { + partsNum := headOutput.ContentLength / input.PartSize + remainder := headOutput.ContentLength % input.PartSize + if remainder != 0 { + partsNum++ + } + parts := make([]downloadPartInfo, partsNum) + for i := int64(0); i < partsNum; i++ { + parts[i] = downloadPartInfo{ + PartNumber: int(i + 1), + RangeStart: i * input.PartSize, + RangeEnd: (i+1)*input.PartSize - 1, + } + } + if remainder != 0 { + parts[partsNum-1].RangeEnd = (partsNum-1)*input.PartSize + remainder - 1 + } + if len(parts) > 10000 { + return nil, newTosClientError("tos: part count too many", nil) + } + return &downloadCheckpoint{ + checkpointPath: input.CheckpointFile, + Bucket: input.Bucket, + Key: input.Key, + VersionID: input.VersionID, + PartSize: input.PartSize, + IfMatch: input.IfMatch, + IfModifiedSince: input.IfModifiedSince, + IfNoneMatch: input.IfNoneMatch, + IfUnmodifiedSince: input.IfUnmodifiedSince, + SSECAlgorithm: input.SSECAlgorithm, + SSECKeyMD5: input.SSECKey, + ObjectInfo: objectInfo{ + Etag: headOutput.ETag, + HashCrc64ecma: headOutput.HashCrc64ecma, + LastModified: headOutput.LastModified, + ObjectSize: headOutput.ContentLength, + }, + FileInfo: downloadFileInfo{ + FilePath: input.filePath, + TempFilePath: input.tempFile, + }, + PartsInfo: parts, + }, nil +} + +func checkAndCreateDir(filePath string) error { + dir := filepath.Dir(filePath) + stat, err := os.Stat(dir) + if err != nil { + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + return err + } + } else if !stat.IsDir() { + return fmt.Errorf("dir name same as file name. ") + } + return nil +} + +func createDownloadTempFile(input *DownloadFileInput, event downloadEvent) error { + wrapErr := func(err error) error { + event.postDownloadEvent(&DownloadEvent{ + Type: enum.DownloadEventCreateTempFileFailed, + Bucket: input.Bucket, + Key: input.Key, + VersionID: input.VersionID, + FilePath: input.filePath, + TempFilePath: &input.tempFile, + CheckpointFile: &input.CheckpointFile, + }) + return newTosClientError("tos: create temp file failed.", err) + } + + err := checkAndCreateDir(input.tempFile) + if err != nil { + return wrapErr(err) + } + file, err := os.Create(input.tempFile) + if err != nil { + return wrapErr(err) + } + _ = file.Close() + event.postDownloadEvent(&DownloadEvent{ + Type: enum.DownloadEventCreateTempFileSucceed, + Bucket: input.Bucket, + Key: input.Key, + VersionID: input.VersionID, + FilePath: input.filePath, + TempFilePath: &input.tempFile, + CheckpointFile: &input.CheckpointFile, + }) + return nil +} + +func getDownloadTasks(cli *ClientV2, ctx context.Context, headOutput *HeadObjectV2Output, + checkpoint *downloadCheckpoint, input *DownloadFileInput) []task { + tasks := make([]task, 0) + consumed := int64(0) + subtotal := int64(0) + for _, part := range checkpoint.PartsInfo { + if !part.IsCompleted { + tasks = append(tasks, &downloadTask{ + cli: cli, + ctx: ctx, + input: input, + partNumber: part.PartNumber, + rangeStart: part.RangeStart, + rangeEnd: part.RangeEnd, + consumed: &consumed, + subtotal: &subtotal, + total: headOutput.ContentLength, + enableCRC64: cli.enableCRC, + }) + } else { + consumed += part.RangeEnd - part.RangeStart + 1 + } + } + return tasks +} + +func (d downloadEvent) newDownloadEvent() *DownloadEvent { + return &DownloadEvent{ + Bucket: d.input.Bucket, + Key: d.input.Key, + VersionID: d.input.VersionID, + FilePath: d.input.filePath, + CheckpointFile: &d.input.CheckpointFile, + TempFilePath: &d.input.tempFile, + } +} + +func (d downloadEvent) newDownloadPartSucceedEvent(part downloadPartInfo) *DownloadEvent { + event := d.newSucceedEvent(enum.DownloadEventDownloadPartSucceed) + event.DowloadPartInfo = &DownloadPartInfo{ + PartNumber: part.PartNumber, + RangeStart: part.RangeStart, + RangeEnd: part.RangeEnd, + } + return event +} + +func (d downloadEvent) newSucceedEvent(eventType enum.DownloadEventType) *DownloadEvent { + event := d.newDownloadEvent() + event.Type = eventType + return event +} + +func (d downloadEvent) newFailedEvent(err error, eventType enum.DownloadEventType) *DownloadEvent { + event := d.newDownloadEvent() + event.Type = eventType + event.Err = err + return event +} + +func (d downloadEvent) postDownloadEvent(event *DownloadEvent) { + if d.input.DownloadEventListener != nil { + d.input.DownloadEventListener.EventChange(event) + } +} + +func (cli *ClientV2) downloadFile(ctx context.Context, + headOutput *HeadObjectV2Output, checkpoint *downloadCheckpoint, input *DownloadFileInput, event downloadEvent) (*DownloadFileOutput, error) { + // prepare tasks + tasks := getDownloadTasks(cli, ctx, headOutput, checkpoint, input) + routinesNum := min(input.TaskNum, len(tasks)) + tg := newTaskGroup(getCancelHandle(input.CancelHook), routinesNum, checkpoint, event, input.EnableCheckpoint, tasks) + tg.RunWorker() + // start adding tasks + postDataTransferStatus(input.DataTransferListener, &DataTransferStatus{ + Type: enum.DataTransferStarted, + }) + tg.Scheduler() + success, err := tg.Wait() + if err != nil { + _ = os.Remove(input.tempFile) + } + + if success < len(tasks) { + return nil, newTosClientError("tos: some download task failed.", nil) + } + // Check CRC64 + if cli.enableCRC && headOutput.HashCrc64ecma != 0 && combineCRCInDownload(checkpoint.PartsInfo) != headOutput.HashCrc64ecma { + return nil, newTosClientError("tos: crc of entire file mismatch.", nil) + } + err = os.Rename(input.tempFile, input.filePath) + if err != nil { + event.postDownloadEvent(event.newFailedEvent(err, enum.DownloadEventRenameTempFileFailed)) + return nil, err + } + event.postDownloadEvent(event.newSucceedEvent(enum.DownloadEventRenameTempFileSucceed)) + _ = os.Remove(checkpoint.checkpointPath) + return &DownloadFileOutput{*headOutput}, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum/enum.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum/enum.go new file mode 100644 index 0000000000..1a33151156 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum/enum.go @@ -0,0 +1,181 @@ +package enum + +type ACLType string + +const ( + ACLPrivate ACLType = "private" + ACLPublicRead ACLType = "public-read" + ACLPublicReadWrite ACLType = "public-read-write" + ACLAuthRead ACLType = "authenticated-read" + ACLBucketOwnerRead ACLType = "bucket-owner-read" + ACLBucketOwnerFullControl ACLType = "bucket-owner-full-control" + ACLLogDeliveryWrite ACLType = "log-delivery-write" + ACLBucketOwnerEntrusted ACLType = "bucket-owner-entrusted" +) + +type StorageClassType string + +const ( + StorageClassStandard StorageClassType = "STANDARD" + StorageClassIa StorageClassType = "IA" + StorageClassArchiveFr StorageClassType = "ARCHIVE_FR" + StorageClassIntelligentTiering StorageClassType = "INTELLIGENT_TIERING" + StorageClassColdArchive StorageClassType = "COLD_ARCHIVE" +) + +type MetadataDirectiveType string + +const ( + // MetadataDirectiveReplace replace source object metadata when calling CopyObject + MetadataDirectiveReplace MetadataDirectiveType = "REPLACE" + + // MetadataDirectiveCopy copy source object metadata when calling CopyObject + MetadataDirectiveCopy MetadataDirectiveType = "COPY" +) + +type AzRedundancyType string + +const ( + AzRedundancySingleAz AzRedundancyType = "single-az" + AzRedundancyMultiAz AzRedundancyType = "multi-az" +) + +type PermissionType string + +const ( + PermissionRead PermissionType = "READ" + PermissionWrite PermissionType = "WRITE" + PermissionReadAcp PermissionType = "READ_ACP" + PermissionWriteAcp PermissionType = "WRITE_ACP" + PermissionFullControl PermissionType = "FULL_CONTROL" +) + +type GranteeType string + +const ( + GranteeGroup GranteeType = "Group" + GranteeUser GranteeType = "CanonicalUser" +) + +type CannedType string + +const ( + CannedAllUsers CannedType = "AllUsers" + CannedAuthenticatedUsers CannedType = "AuthenticatedUsers" +) + +type DataTransferType int + +const ( + DataTransferStarted DataTransferType = 1 + DataTransferRW DataTransferType = 2 + DataTransferSucceed DataTransferType = 3 + DataTransferFailed DataTransferType = 4 +) + +type HttpMethodType string + +const ( + HttpMethodGet HttpMethodType = "GET" + HttpMethodPut HttpMethodType = "PUT" + HttpMethodPost HttpMethodType = "POST" + HttpMethodDelete HttpMethodType = "DELETE" + HttpMethodHead HttpMethodType = "HEAD" +) + +type UploadEventType int + +const ( + UploadEventCreateMultipartUploadSucceed UploadEventType = 1 + UploadEventCreateMultipartUploadFailed UploadEventType = 2 + UploadEventUploadPartSucceed UploadEventType = 3 + UploadEventUploadPartFailed UploadEventType = 4 + UploadEventUploadPartAborted UploadEventType = 5 // The task needs to be interrupted in case of 403, 404, 405 errors + UploadEventCompleteMultipartUploadSucceed UploadEventType = 6 + UploadEventCompleteMultipartUploadFailed UploadEventType = 7 +) + +type DownloadEventType int + +const ( + DownloadEventCreateTempFileSucceed DownloadEventType = 1 + DownloadEventCreateTempFileFailed DownloadEventType = 2 + DownloadEventDownloadPartSucceed DownloadEventType = 3 + DownloadEventDownloadPartFailed DownloadEventType = 4 + DownloadEventDownloadPartAborted DownloadEventType = 5 // The task needs to be interrupted in case of 403, 404, 405 errors + DownloadEventRenameTempFileSucceed DownloadEventType = 6 + DownloadEventRenameTempFileFailed DownloadEventType = 7 +) + +type CertStatusType string + +const ( + CertStatusBound CertStatusType = "CertBound" + CertStatusUnbound CertStatusType = "CertUnbound" + CertStatusExpired CertStatusType = "CertExpired" +) + +type StorageClassInheritDirectiveType string + +const ( + StorageClassIDDestinationBucket StorageClassInheritDirectiveType = "DESTINATION_BUCKET" + StorageClassIDSourceObject StorageClassInheritDirectiveType = "SOURCE_OBJECT" +) + +type StatusType string + +const ( + StatusEnabled StatusType = "Enabled" + StatusDisabled StatusType = "Disabled" +) + +const ( + LifecycleStatusEnabled StatusType = "Enabled" + LifecycleStatusDisabled StatusType = "Disabled" +) + +type RedirectType string + +const ( + RedirectTypeMirror RedirectType = "Mirror" + RedirectTypeAsync RedirectType = "Async" +) + +const ( + SSETosAlg = "AES256" + SSEKMS = "kms" +) + +type VersioningStatusType string + +const ( + VersioningStatusEnable VersioningStatusType = "Enabled" + VersioningStatusSuspended VersioningStatusType = "Suspended" +) + +type ProtocolType string + +const ( + ProtocolHttp ProtocolType = "http" + ProtocolHttps ProtocolType = "https" +) + +type CopyEventType int + +const ( + CopyEventCreateMultipartUploadSucceed CopyEventType = 1 + CopyEventCreateMultipartUploadFailed CopyEventType = 2 + CopyEventUploadPartCopySuccess CopyEventType = 3 + CopyEventUploadPartCopyFailed CopyEventType = 4 + CopyEventUploadPartCopyAborted CopyEventType = 5 + CopyEventCompleteMultipartUploadSucceed CopyEventType = 6 + CopyEventCompleteMultipartUploadFailed CopyEventType = 7 +) + +type TierType string + +const ( + TierStandard TierType = "Standard" + TierExpedited TierType = "Expedited" + TierBulk TierType = "Bulk" +) diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/error.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/error.go new file mode 100644 index 0000000000..a30a09b740 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/error.go @@ -0,0 +1,323 @@ +package tos + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" +) + +var ( + InputIsNilClientError = newTosClientError("input is nil. ", nil) + InputInvalidClientError = newTosClientError("input data is invalid. ", nil) + InvalidPartNumber = newTosClientError("input part number is invalid. ", nil) + InvalidUploadID = newTosClientError("input upload id is invalid. ", nil) + InvalidBucketNameLength = newTosClientError("invalid bucket name, the length must be [3, 63]", nil) + InvalidBucketNameCharacter = newTosClientError("invalid bucket name, the character set is illegal", nil) + InvalidBucketNameStartingOrEnding = newTosClientError("invalid bucket name, the bucket name can be neither starting with '-' nor ending with '-'", nil) + InvalidObjectNameLength = newTosClientError("invalid object name, the length must be [1, 696]", nil) + InvalidObjectNameStartingOrEnding = newTosClientError("invalid object name, the object name can not start with '\\'", nil) + InvalidObjectNameCharacterSet = newTosClientError("invalid object name, the character set is illegal", nil) + InvalidACL = newTosClientError("invalid acl type", nil) + InvalidStorageClass = newTosClientError("invalid storage class", nil) + InvalidGrantee = newTosClientError("invalid grantee type", nil) + InvalidCanned = newTosClientError("invalid canned type", nil) + InvalidAzRedundancy = newTosClientError("invalid az redundancy type", nil) + InvalidMetadataDirective = newTosClientError("invalid metadata directive type", nil) + InvalidPermission = newTosClientError("invalid permission type", nil) + InvalidSSECAlgorithm = newTosClientError("invalid encryption-decryption algorithm", nil) + InvalidPartSize = newTosClientError("invalid part size, the size must be [5242880, 5368709120]", nil) + InvalidSrcFilePath = newTosClientError("invalid file path, the file does not exist", nil) + InvalidFilePartNum = newTosClientError("unsupported part number, the maximum is 10000", nil) + InvalidMarshal = newTosClientError("unable to do serialization/deserialization", nil) + InvalidPreSignedURLExpires = newTosClientError("invalid pre signed url expires, the time must be less 604800 seconds.", nil) + InvalidPreSignedConditions = newTosClientError("invalid pre signed url conditions.", nil) + InvalidFilePath = newTosClientError("invalid file path", nil) + InvalidCheckpointFilePath = newTosClientError("invalid checkpoint file path", nil) + CrcCheckFail = newTosClientError("crc check not equal", nil) + InvalidS3Endpoint = newTosClientError("do not support s3 endpoint, please use tos endpoint", nil) + NotSupportSeek = newTosClientError("reader not support seek", nil) + ProxyNotSupportHttps = newTosClientError("proxy not support https", nil) + ProxyUrlInvalid = newTosClientError("proxy url invalid", nil) + NotificationConfigurationsInvalid = newTosClientError("Notification Configurations invalid", nil) + InvalidCompleteAllPartsLength = newTosClientError("Should not specify both complete all and Parts", nil) + InvalidPartsLength = newTosClientError("You must specify at least one part", nil) + InvlidDeleteMultiObjectsLength = newTosClientError("You must specify at least one object", nil) +) + +type TosError struct { + Message string +} + +func (e *TosError) Error() string { + return e.Message +} + +// for simplify code +func newTosClientError(message string, cause error) *TosClientError { + return &TosClientError{ + TosError: TosError{ + Message: message, + }, + Cause: cause, + } +} + +type TosClientError struct { + TosError + Cause error +} + +func (t *TosClientError) withCause(err error) error { + t.Cause = err + return t +} + +// try to unmarshal server error from response +func newTosServerError(res *Response) error { + data, err := ioutil.ReadAll(io.LimitReader(res.Body, 64<<10)) // avoid too large + if err != nil && len(data) <= 0 { + return &TosServerError{ + TosError: TosError{"tos: server returned an empty body"}, + RequestInfo: res.RequestInfo(), + } + } + se := Error{StatusCode: res.StatusCode} + if err = json.Unmarshal(data, &se); err != nil { + return &TosServerError{ + TosError: TosError{"tos: server returned an invalid body"}, + RequestInfo: res.RequestInfo(), + } + } + return &TosServerError{ + TosError: TosError{se.Message}, + RequestInfo: res.RequestInfo(), + Code: se.Code, + HostID: se.HostID, + Resource: se.Resource, + } +} + +// 服务端错误定义参考：https://www.volcengine.com/docs/6349/74874 +type TosServerError struct { + TosError `json:"TosError"` + RequestInfo `json:"RequestInfo"` + Code string `json:"Code,omitempty"` + HostID string `json:"HostID,omitempty"` + Resource string `json:"Resource,omitempty"` +} + +type Error struct { + StatusCode int `json:"-"` + Code string `json:"Code,omitempty"` + Message string `json:"Message,omitempty"` + RequestID string `json:"RequestId,omitempty"` + HostID string `json:"HostId,omitempty"` + Resource string `json:"Resource,omitempty"` +} + +func (e *Error) Error() string { + return fmt.Sprintf("tos: request error: StatusCode=%d, Code=%s, Message=%q, RequestID=%s, HostID=%s", + e.StatusCode, e.Code, e.Message, e.RequestID, e.HostID) +} + +// Code return error code saved in TosServerError +func Code(err error) string { + if er, ok := err.(*TosServerError); ok { + return er.Code + } + return "" +} + +// StatueCode return status code saved in TosServerError or UnexpectedStatusCodeError +// +// Deprecated: use StatusCode instead +func StatueCode(err error) int { + return StatusCode(err) +} + +// StatusCode return status code saved in TosServerError or UnexpectedStatusCodeError +func StatusCode(err error) int { + if er, ok := err.(*TosServerError); ok { + return er.StatusCode + } + if er, ok := err.(*UnexpectedStatusCodeError); ok { + return er.StatusCode + } + return 0 +} + +func RequestID(err error) string { + switch ev := err.(type) { + case *TosServerError: + return ev.RequestID + case *UnexpectedStatusCodeError: + return ev.RequestID + case *ChecksumError: + return ev.RequestID + case *SerializeError: + return ev.RequestID + } + return "" +} + +type UnexpectedStatusCodeError struct { + StatusCode int `json:"StatusCode,omitempty"` + ExpectedCodes []int `json:"ExpectedCodes,omitempty"` + RequestID string `json:"RequestId,omitempty"` + expectedCodes [2]int + responseMsg string + err Error +} + +func NewUnexpectedStatusCodeError(statusCode int, expectedCode int, expectedCodes ...int) *UnexpectedStatusCodeError { + err := UnexpectedStatusCodeError{ + StatusCode: statusCode, + } + err.ExpectedCodes = err.expectedCodes[:0] + err.ExpectedCodes = append(err.ExpectedCodes, expectedCode) + err.ExpectedCodes = append(err.ExpectedCodes, expectedCodes...) + return &err +} + +func (us *UnexpectedStatusCodeError) WithRequestBody(res *Response) *UnexpectedStatusCodeError { + data, err := ioutil.ReadAll(io.LimitReader(res.Body, 64<<10)) + if err != nil || len(data) <= 0 { + return us + } + us.responseMsg = string(data) + se := Error{StatusCode: res.StatusCode} + err = json.Unmarshal(data, &se) + if err != nil { + return us + } + us.err = se + return us +} + +func (us *UnexpectedStatusCodeError) WithRequestID(requestID string) *UnexpectedStatusCodeError { + us.RequestID = requestID + return us +} + +func (us *UnexpectedStatusCodeError) GoString() string { + if us.responseMsg != "" { + return fmt.Sprintf("tos.UnexpectedStatusCodeError{StatusCode:%d, ExpectedCodes:%v, RequestID:%s, ResponseErr:%s}", + us.StatusCode, us.ExpectedCodes, us.RequestID, us.responseMsg) + } + return fmt.Sprintf("tos.UnexpectedStatusCodeError{StatusCode:%d, ExpectedCodes:%v, RequestID:%s}", + us.StatusCode, us.ExpectedCodes, us.RequestID) +} + +func (us *UnexpectedStatusCodeError) Error() string { + if us.responseMsg != "" { + return fmt.Sprintf("tos: unexpected status code error: StatusCode=%d, ExpectedCodes=%v, RequestID=%s, ResponseErr:%s", + us.StatusCode, us.ExpectedCodes, us.RequestID, us.responseMsg) + } + return fmt.Sprintf("tos: unexpected status code error: StatusCode=%d, ExpectedCodes=%v, RequestID=%s", + us.StatusCode, us.ExpectedCodes, us.RequestID) +} + +type ChecksumError struct { + RequestID string `json:"RequestId,omitempty"` + ExpectedChecksum string `json:"ExpectedChecksum,omitempty"` + ActualChecksum string `json:"ActualChecksum,omitempty"` +} + +func (ce *ChecksumError) Error() string { + return fmt.Sprintf("tos: checksum error: RequestID=%s, ExpectedChecksum=%s, ActualChecksum=%s", + ce.RequestID, ce.ExpectedChecksum, ce.ActualChecksum) +} + +type SerializeError struct { + RequestID string `json:"RequestId,omitempty"` + Message string `json:"Message,omitempty"` +} + +func (se *SerializeError) Error() string { + return fmt.Sprintf("tos: serialize error: RequestID=%s, Message=%q", se.RequestID, se.Message) +} + +func checkError(res *Response, readBody bool, okCode int, okCodes ...int) error { + if res.StatusCode == okCode { + return nil + } + for _, code := range okCodes { + if res.StatusCode == code { + return nil + } + } + defer res.Close() + if readBody && res.StatusCode >= http.StatusBadRequest && res.Body != nil { + return newTosServerError(res) + // fall through + } + unexpected := NewUnexpectedStatusCodeError(res.StatusCode, okCode, okCodes...). + WithRequestID(res.RequestInfo().RequestID) + if readBody && res.Body != nil { + unexpected = unexpected.WithRequestBody(res) + } + return &TosServerError{ + TosError: TosError{unexpected.Error()}, + RequestInfo: res.RequestInfo(), + Code: unexpected.err.Code, + HostID: unexpected.err.HostID, + Resource: unexpected.err.Resource, + } +} + +// StatusCodeClassifier classifies Errors. +// If the error is nil, it returns NoRetry; +// if the error is TimeoutException or can be interpreted as TosServerError, and the StatusCode is 5xx or 429, it returns Retry; +// otherwise, it returns NoRetry. +type StatusCodeClassifier struct{} + +// Classify implements the classifier interface. +func (classifier StatusCodeClassifier) Classify(err error) retryAction { + if err == nil { + return NoRetry + } + e, ok := err.(*TosServerError) + if ok { + if e.StatusCode >= 500 || e.StatusCode == 429 { + return Retry + } + } + cErr, ok := err.(*TosClientError) + if ok { + _, ok = cErr.Cause.(interface{ Timeout() bool }) + if ok { + return Retry + } + } + + return NoRetry +} + +// ServerErrorClassifier classify errors returned by POST method. +// If the error is nil, it returns NoRetry; +// if the error can be interpreted as TosServerError and its StatusCode is 5xx or 429, it returns Retry; +// otherwise, it returns NoRetry. +type ServerErrorClassifier struct{} + +// Classify implements the classifier interface. +func (classifier ServerErrorClassifier) Classify(err error) retryAction { + if err == nil { + return NoRetry + } + e, ok := err.(*TosServerError) + if ok { + if e.StatusCode >= 500 || e.StatusCode == 429 { + return Retry + } + } + return NoRetry +} + +type NoRetryClassifier struct{} + +// Classify implements the classifier interface. +func (classifier NoRetryClassifier) Classify(_ error) retryAction { + return NoRetry +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/fetch.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/fetch.go new file mode 100644 index 0000000000..a70d86bdc9 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/fetch.go @@ -0,0 +1,149 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +const ( + FetchTaskStateFailed = "Failed" + FetchTaskStateSucceed = "Succeed" + FetchTaskStateExpired = "Expired" + FetchTaskStateRunning = "Running" +) + +type FetchObjectInput struct { + URL string `json:"URL,omitempty"` // required + Key string `json:"Key,omitempty"` // required + IgnoreSameKey bool `json:"IgnoreSameKey,omitempty"` // optional, default value is false + ContentMD5 string `json:"ContentMD5,omitempty"` // hex-encoded md5, optional +} + +type FetchObjectOutput struct { + RequestInfo `json:"-"` + VersionID string `json:"VersionId,omitempty"` // may be empty + ETag string `json:"ETag,omitempty"` +} + +type fetchObjectInput struct { + URL string `json:"URL,omitempty"` // required + IgnoreSameKey bool `json:"IgnoreSameKey,omitempty"` // optional, default value is false + ContentMD5 string `json:"ContentMD5,omitempty"` // base64-encoded md5, optional +} + +// FetchObject fetch an object from specified URL +// options: +// WithMeta set meta header(s) +// WithServerSideEncryptionCustomer set server side encryption options +// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl +// Calling FetchObject will be blocked util fetch operation is finished +func (bkt *Bucket) FetchObject(ctx context.Context, input *FetchObjectInput, options ...Option) (*FetchObjectOutput, error) { + if err := isValidKey(input.Key); err != nil { + return nil, err + } + + data, contentMD5, err := marshalInput("FetchObjectInput", &fetchObjectInput{ + URL: input.URL, + IgnoreSameKey: input.IgnoreSameKey, + ContentMD5: input.ContentMD5, + }) + if err != nil { + return nil, err + } + + res, err := bkt.client.newBuilder(bkt.name, input.Key, options...). + WithQuery("fetch", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, ServerErrorClassifier{}). + Request(ctx, http.MethodPost, bytes.NewReader(data), bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + out := FetchObjectOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(out.RequestID, res.Body, &out); err != nil { + return nil, err + } + + out.VersionID = res.Header.Get(HeaderVersionID) + return &out, nil +} + +type PutFetchTaskInput struct { + URL string `json:"URL,omitempty"` // required + Object string `json:"Object,omitempty"` // object key, required + IgnoreSameKey bool `json:"IgnoreSameKey,omitempty"` // optional, default value is false + ContentMD5 string `json:"ContentMD5,omitempty"` // hex-encoded md5, optional +} + +type PutFetchTaskOutput struct { + RequestInfo `json:"-"` + TaskID string `json:"TaskId,omitempty"` +} + +// PutFetchTask put a fetch task to a bucket +// options: +// WithMeta set meta header(s) +// WithServerSideEncryptionCustomer set server side encryption options +// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl +// Calling PutFetchTask will return immediately after the task created. +func (bkt *Bucket) PutFetchTask(ctx context.Context, input *PutFetchTaskInput, options ...Option) (*PutFetchTaskOutput, error) { + if err := isValidKey(input.Object); err != nil { + return nil, err + } + + data, contentMD5, err := marshalInput("PutFetchTaskInput", input) + if err != nil { + return nil, err + } + + res, err := bkt.client.newBuilder(bkt.name, "", options...). + WithQuery("fetchTask", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, ServerErrorClassifier{}). + Request(ctx, http.MethodPost, bytes.NewReader(data), bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + out := PutFetchTaskOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(out.RequestID, res.Body, &out); err != nil { + return nil, err + } + return &out, nil +} + +type GetFetchTaskInput struct { + TaskID string `json:"taskID,omitempty"` +} + +type GetFetchTaskOutput struct { + RequestInfo `json:"-"` + State string `json:"State,omitempty"` + // Cause string `json:"Cause,omitempty"` +} + +// GetFetchTask query the task state by the TaskID +// Task state: +// FetchTaskStateFailed = "Failed" +// FetchTaskStateSucceed = "Succeed" +// FetchTaskStateExpired = "Expired" +// FetchTaskStateRunning = "Running" +func (bkt *Bucket) GetFetchTask(ctx context.Context, input *GetFetchTaskInput, options ...Option) (*GetFetchTaskOutput, error) { + res, err := bkt.client.newBuilder(bkt.name, "", options...). + WithQuery("fetchTask", ""). + WithQuery("taskId", input.TaskID). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + out := GetFetchTaskOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(out.RequestID, res.Body, &out); err != nil { + return nil, err + } + return &out, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/http_trace.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/http_trace.go new file mode 100644 index 0000000000..b07f374410 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/http_trace.go @@ -0,0 +1,106 @@ +package tos + +import ( + "crypto/tls" + "fmt" + "net/http" + "net/http/httptrace" + "sync/atomic" +) + +type accessLogRequest struct { + clientDnsCost int64 + clientDialCost int64 + clientTlsHandShakeCost int64 + clientSendHeadersAndBodyCost int64 + clientWaitResponseCost int64 + clientSendRequestCost int64 + actionStartMs int64 +} + +func newAccessLogRequest(actionStartMs int64) *accessLogRequest { + return &accessLogRequest{ + clientDnsCost: -1, + clientDialCost: -1, + clientTlsHandShakeCost: -1, + clientSendHeadersAndBodyCost: -1, + clientWaitResponseCost: -1, + clientSendRequestCost: -1, + actionStartMs: actionStartMs, + } +} + +func (r *accessLogRequest) PrintAccessLog(logger Logger, req *http.Request, response *http.Response) { + if logger == nil { + return + } + atomic.CompareAndSwapInt64(&r.clientSendRequestCost, -1, GetUnixTimeMs()-r.actionStartMs) + var requestId *string + if response != nil { + requestId = StringPtr(response.Header.Get(HeaderRequestID)) + } + prefix := buildPrefix(requestId) + if req != nil { + logger.Debug(fmt.Sprintf("%s, method: %s, host: %s, request uri: %s, dns cost: %d ms, dial cost: %d ms, tls handshake cost: %d ms, send headers and body cost: %d ms, wait response cost: %d ms, request cost: %d ms", + prefix, req.Method, req.URL.Host, req.URL.EscapedPath(), r.clientDnsCost, r.clientDialCost, r.clientTlsHandShakeCost, + r.clientSendHeadersAndBodyCost, r.clientWaitResponseCost, r.clientSendRequestCost)) + } else { + logger.Debug(fmt.Sprintf("%s, dns cost: %d ms, dial cost: %d ms, tls handshake cost: %d ms, send headers and body cost: %d ms, wait response cost: %d ms, request cost: %d ms", + prefix, r.clientDnsCost, r.clientDialCost, r.clientTlsHandShakeCost, r.clientSendHeadersAndBodyCost, r.clientWaitResponseCost, r.clientSendRequestCost)) + } +} + +func buildPrefix(requestId *string) string { + prefix := "" + + if requestId != nil { + prefix = fmt.Sprintf("[requestId: %s] %s", *requestId, prefix) + } + return prefix +} + +func getClientTrace(actionStartMs int64) (*httptrace.ClientTrace, *accessLogRequest) { + var dnsStart int64 + var dialStart int64 + var tlsHandShakeStart int64 + var sendHeadersAndBodyStart int64 + var waitResponseStart int64 + r := newAccessLogRequest(actionStartMs) + trace := &httptrace.ClientTrace{ + GotFirstResponseByte: func() { + r.clientWaitResponseCost = GetUnixTimeMs() - waitResponseStart + }, + DNSStart: func(info httptrace.DNSStartInfo) { + dnsStart = GetUnixTimeMs() + }, + DNSDone: func(info httptrace.DNSDoneInfo) { + r.clientDnsCost = GetUnixTimeMs() - dnsStart + }, + ConnectStart: func(network, addr string) { + dialStart = GetUnixTimeMs() + }, + ConnectDone: func(network, addr string, err error) { + now := GetUnixTimeMs() + sendHeadersAndBodyStart = now + r.clientDialCost = now - dialStart + }, + TLSHandshakeStart: func() { + tlsHandShakeStart = GetUnixTimeMs() + }, + TLSHandshakeDone: func(state tls.ConnectionState, err error) { + now := GetUnixTimeMs() + sendHeadersAndBodyStart = now + r.clientTlsHandShakeCost = now - tlsHandShakeStart + }, + + GotConn: func(httptrace.GotConnInfo) { + sendHeadersAndBodyStart = GetUnixTimeMs() + }, + + WroteRequest: func(info httptrace.WroteRequestInfo) { + waitResponseStart = GetUnixTimeMs() + r.clientSendHeadersAndBodyCost = waitResponseStart - sendHeadersAndBodyStart + }, + } + return trace, r +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/lifecycle.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/lifecycle.go new file mode 100644 index 0000000000..96bac7011a --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/lifecycle.go @@ -0,0 +1,116 @@ +package tos + +import ( + "bytes" + "context" + "net/http" + "time" +) + +func (cli *ClientV2) parseLifecycleInput(input *PutBucketLifecycleInput) putBucketLifecycleInput { + lifecycleInput := make([]lifecycleRule, 0, len(input.Rules)) + for _, lifecycle := range input.Rules { + var exp *expiration + if lifecycle.Expiration != nil { + exp = &expiration{ + Days: lifecycle.Expiration.Days, + } + if !lifecycle.Expiration.Date.IsZero() { + exp.Date = lifecycle.Expiration.Date.Format(time.RFC3339) + } + } + + transitionList := make([]transition, 0, len(lifecycle.Transitions)) + + for _, trans := range lifecycle.Transitions { + t := transition{ + Days: trans.Days, + StorageClass: trans.StorageClass, + } + if !trans.Date.IsZero() { + t.Date = trans.Date.Format(time.RFC3339) + } + transitionList = append(transitionList, t) + } + + lifecycleInput = append(lifecycleInput, lifecycleRule{ + ID: lifecycle.ID, + Prefix: lifecycle.Prefix, + Status: lifecycle.Status, + Transitions: transitionList, + Expiration: exp, + NonCurrentVersionTransition: lifecycle.NonCurrentVersionTransition, + NoCurrentVersionExpiration: lifecycle.NoCurrentVersionExpiration, + Tag: lifecycle.Tag, + AbortInCompleteMultipartUpload: lifecycle.AbortInCompleteMultipartUpload, + }) + + } + return putBucketLifecycleInput{Rules: lifecycleInput} +} +func (cli *ClientV2) PutBucketLifecycle(ctx context.Context, input *PutBucketLifecycleInput) (*PutLifecycleOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + data, contentMD5, err := marshalInput("PutBucketLifecycleInput", cli.parseLifecycleInput(input)) + if err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("lifecycle", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutLifecycleOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketLifecycle(ctx context.Context, input *GetBucketLifecycleInput) (*GetBucketLifecycleOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("lifecycle", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketLifecycleOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) DeleteBucketLifecycle(ctx context.Context, input *DeleteBucketLifecycleInput) (*DeleteBucketLifecycleOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("lifecycle", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteBucketLifecycleOutput{RequestInfo: res.RequestInfo()} + return &output, nil + +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/logger.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/logger.go new file mode 100644 index 0000000000..011d77ab92 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/logger.go @@ -0,0 +1,9 @@ +package tos + +type Logger interface { + Debug(args ...interface{}) + Info(args ...interface{}) + Warn(args ...interface{}) + Error(args ...interface{}) + Fatal(args ...interface{}) +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/metadata.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/metadata.go new file mode 100644 index 0000000000..c3801e8c64 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/metadata.go @@ -0,0 +1,149 @@ +package tos + +import ( + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" +) + +// ObjectMeta object metadata +type ObjectMeta struct { + ContentLength int64 `json:"ContentLength,omitempty"` + ContentType string `json:"ContentType,omitempty"` + ContentMD5 string `json:"ContentMD5,omitempty"` + ContentLanguage string `json:"ContentLanguage,omitempty"` + ContentEncoding string `json:"ContentEncoding,omitempty"` + ContentDisposition string `json:"ContentDisposition,omitempty"` + LastModified string `json:"LastModified,omitempty"` + CacheControl string `json:"CacheControl,omitempty"` + Expires string `json:"Expires,omitempty"` + ETag string `json:"ETag,omitempty"` + VersionID string `json:"VersionId,omitempty"` + DeleteMarker bool `json:"DeleteMarker,omitempty"` + ObjectType string `json:"ObjectType,omitempty"` // "" or "Appendable" + StorageClass string `json:"StorageClass,omitempty"` + Restore string `json:"Restore,omitempty"` + Metadata map[string]string `json:"Metadata,omitempty"` + Tag string `json:"Tag,omitempty"` + SSECustomerAlgorithm string `json:"SSECustomerAlgorithm,omitempty"` + SSECustomerKeyMD5 string `json:"SSECustomerKeyMD5,omitempty"` + CSType string `json:"CSType,omitempty"` +} + +type ObjectMetaV2 struct { + ETag string + LastModified time.Time + DeleteMarker bool + SSECAlgorithm string + SSECKeyMD5 string + VersionID string + WebsiteRedirectLocation string + ObjectType string + HashCrc64ecma uint64 + StorageClass enum.StorageClassType + Meta Metadata + ContentLength int64 + ContentType string + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + Expires time.Time + ServerSideEncryption string + ServerSideEncryptionKeyID string +} + +func (om *ObjectMeta) fromResponse(res *Response) { + om.ETag = res.Header.Get(HeaderETag) + om.LastModified = res.Header.Get(HeaderLastModified) + om.DeleteMarker, _ = strconv.ParseBool(res.Header.Get(HeaderDeleteMarker)) + om.SSECustomerAlgorithm = res.Header.Get(HeaderSSECustomerAlgorithm) + om.SSECustomerKeyMD5 = res.Header.Get(HeaderSSECustomerKeyMD5) + om.VersionID = res.Header.Get(HeaderVersionID) + + om.ObjectType = res.Header.Get(HeaderObjectType) + om.StorageClass = res.Header.Get(HeaderStorageClass) + om.Metadata = userMetadata(res.Header) + + om.ContentLength = res.ContentLength + om.ContentType = res.Header.Get(HeaderContentType) + om.CacheControl = res.Header.Get(HeaderCacheControl) + om.ContentDisposition = res.Header.Get(HeaderContentDisposition) + om.ContentEncoding = res.Header.Get(HeaderContentEncoding) + om.ContentLanguage = res.Header.Get(HeaderContentLanguage) + om.Expires = res.Header.Get(HeaderExpires) + + om.ContentMD5 = res.Header.Get(HeaderContentMD5) + om.Restore = res.Header.Get(HeaderRestore) + om.Tag = res.Header.Get(HeaderTag) + om.CSType = res.Header.Get(HeaderCSType) +} + +func (om *ObjectMetaV2) fromResponseV2(res *Response) { + lastModified, _ := time.ParseInLocation(http.TimeFormat, res.Header.Get(HeaderLastModified), time.UTC) + deleteMarker, _ := strconv.ParseBool(res.Header.Get(HeaderDeleteMarker)) + // If s is empty or contains invalid digits, err.Err = ErrSyntax and the returned value is 0; + crc64, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64) + length, _ := strconv.ParseInt(res.Header.Get(HeaderContentLength), 10, 64) + expires, _ := time.ParseInLocation(http.TimeFormat, res.Header.Get(HeaderExpires), time.UTC) + om.ETag = res.Header.Get(HeaderETag) + om.LastModified = lastModified + om.DeleteMarker = deleteMarker + om.SSECAlgorithm = res.Header.Get(HeaderSSECustomerAlgorithm) + om.SSECKeyMD5 = res.Header.Get(HeaderContentMD5) + om.VersionID = res.Header.Get(HeaderVersionID) + om.WebsiteRedirectLocation = res.Header.Get(HeaderWebsiteRedirectLocation) + om.ObjectType = res.Header.Get(HeaderObjectType) + om.HashCrc64ecma = crc64 + om.StorageClass = enum.StorageClassType(res.Header.Get(HeaderStorageClass)) + om.Meta = &CustomMeta{m: userMetadata(res.Header)} + om.ContentLength = length + om.ContentType = res.Header.Get(HeaderContentType) + om.CacheControl = res.Header.Get(HeaderCacheControl) + om.ContentDisposition, _ = url.QueryUnescape(res.Header.Get(HeaderContentDisposition)) + om.ContentEncoding = res.Header.Get(HeaderContentEncoding) + om.ContentLanguage = res.Header.Get(HeaderContentLanguage) + om.Expires = expires + om.ServerSideEncryption = res.Header.Get(HeaderServerSideEncryption) + om.ServerSideEncryptionKeyID = res.Header.Get(HeaderServerSideEncryptionKmsKeyID) +} + +func userMetadata(header http.Header) map[string]string { + + meta := make(map[string]string) + for key := range header { + if strings.HasPrefix(key, HeaderMetaPrefix) { + kk, err := url.QueryUnescape(key[len(HeaderMetaPrefix):]) + if err != nil { + kk = key[len(HeaderMetaPrefix):] + } + meta[strings.ToLower(kk)], err = url.QueryUnescape(header.Get(key)) + if err != nil { + meta[strings.ToLower(kk)] = header.Get(key) + } + } + } + return meta +} + +func parseUserMetaData(userMeta []userMeta) Metadata { + if len(userMeta) == 0 { + return nil + } + metas := make(map[string]string, len(userMeta)) + for _, meta := range userMeta { + kk, err := url.QueryUnescape(meta.Key) + if err != nil { + kk = meta.Key + } + metas[strings.ToLower(kk)], err = url.QueryUnescape(meta.Value) + if err != nil { + metas[strings.ToLower(kk)] = meta.Value + } + } + return &CustomMeta{metas} +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mime.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mime.go new file mode 100644 index 0000000000..310b943c12 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mime.go @@ -0,0 +1,572 @@ +package tos + +import "path" + +var mime = map[string]string{ + "3gp": "video/3gpp", + "7z": "application/x-7z-compressed", + "abw": "application/x-abiword", + "ai": "application/postscript", + "aif": "audio/x-aiff", + "aifc": "audio/x-aiff", + "aiff": "audio/x-aiff", + "alc": "chemical/x-alchemy", + "amr": "audio/amr", + "anx": "application/annodex", + "apk": "application/vnd.android.package-archive", + "appcache": "text/cache-manifest", + "art": "image/x-jg", + "asc": "text/plain", + "asf": "video/x-ms-asf", + "aso": "chemical/x-ncbi-asn1-binary", + "asx": "video/x-ms-asf", + "atom": "application/atom+xml", + "atomcat": "application/atomcat+xml", + "atomsrv": "application/atomserv+xml", + "au": "audio/basic", + "avi": "video/x-msvideo", + "awb": "audio/amr-wb", + "axa": "audio/annodex", + "axv": "video/annodex", + "b": "chemical/x-molconn-Z", + "bak": "application/x-trash", + "bat": "application/x-msdos-program", + "bcpio": "application/x-bcpio", + "bib": "text/x-bibtex", + "bin": "application/octet-stream", + "bmp": "image/x-ms-bmp", + "boo": "text/x-boo", + "book": "application/x-maker", + "brf": "text/plain", + "bsd": "chemical/x-crossfire", + "c": "text/x-csrc", + "c++": "text/x-c++src", + "c3d": "chemical/x-chem3d", + "cab": "application/x-cab", + "cac": "chemical/x-cache", + "cache": "chemical/x-cache", + "cap": "application/vnd.tcpdump.pcap", + "cascii": "chemical/x-cactvs-binary", + "cat": "application/vnd.ms-pki.seccat", + "cbin": "chemical/x-cactvs-binary", + "cbr": "application/x-cbr", + "cbz": "application/x-cbz", + "cc": "text/x-c++src", + "cda": "application/x-cdf", + "cdf": "application/x-cdf", + "cdr": "image/x-coreldraw", + "cdt": "image/x-coreldrawtemplate", + "cdx": "chemical/x-cdx", + "cdy": "application/vnd.cinderella", + "cef": "chemical/x-cxf", + "cer": "chemical/x-cerius", + "chm": "chemical/x-chemdraw", + "chrt": "application/x-kchart", + "cif": "chemical/x-cif", + "class": "application/java-vm", + "cls": "text/x-tex", + "cmdf": "chemical/x-cmdf", + "cml": "chemical/x-cml", + "cod": "application/vnd.rim.cod", + "com": "application/x-msdos-program", + "cpa": "chemical/x-compass", + "cpio": "application/x-cpio", + "cpp": "text/x-c++src", + "cpt": "application/mac-compactpro", + "cr2": "image/x-canon-cr2", + "crl": "application/x-pkcs7-crl", + "crt": "application/x-x509-ca-cert", + "crw": "image/x-canon-crw", + "csd": "audio/csound", + "csf": "chemical/x-cache-csf", + "csh": "application/x-csh", + "csm": "chemical/x-csml", + "csml": "chemical/x-csml", + "css": "text/css", + "csv": "text/csv", + "ctab": "chemical/x-cactvs-binary", + "ctx": "chemical/x-ctx", + "cu": "application/cu-seeme", + "cub": "chemical/x-gaussian-cube", + "cxf": "chemical/x-cxf", + "cxx": "text/x-c++src", + "d": "text/x-dsrc", + "davmount": "application/davmount+xml", + "dcm": "application/dicom", + "dcr": "application/x-director", + "ddeb": "application/vnd.debian.binary-package", + "dif": "video/dv", + "diff": "text/x-diff", + "dir": "application/x-director", + "djv": "image/vnd.djvu", + "djvu": "image/vnd.djvu", + "dl": "video/dl", + "dll": "application/x-msdos-program", + "dmg": "application/x-apple-diskimage", + "dms": "application/x-dms", + "doc": "application/msword", + "docm": "application/vnd.ms-word.document.macroEnabled.12", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dot": "application/msword", + "dotm": "application/vnd.ms-word.template.macroEnabled.12", + "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "dv": "video/dv", + "dvi": "application/x-dvi", + "dx": "chemical/x-jcamp-dx", + "dxr": "application/x-director", + "emb": "chemical/x-embl-dl-nucleotide", + "embl": "chemical/x-embl-dl-nucleotide", + "eml": "message/rfc822", + "eot": "application/vnd.ms-fontobject", + "eps": "application/postscript", + "eps2": "application/postscript", + "eps3": "application/postscript", + "epsf": "application/postscript", + "epsi": "application/postscript", + "erf": "image/x-epson-erf", + "es": "application/ecmascript", + "etx": "text/x-setext", + "exe": "application/x-msdos-program", + "ez": "application/andrew-inset", + "fb": "application/x-maker", + "fbdoc": "application/x-maker", + "fch": "chemical/x-gaussian-checkpoint", + "fchk": "chemical/x-gaussian-checkpoint", + "fig": "application/x-xfig", + "flac": "audio/flac", + "fli": "video/fli", + "flv": "video/x-flv", + "fm": "application/x-maker", + "frame": "application/x-maker", + "frm": "application/x-maker", + "gal": "chemical/x-gaussian-log", + "gam": "chemical/x-gamess-input", + "gamin": "chemical/x-gamess-input", + "gan": "application/x-ganttproject", + "gau": "chemical/x-gaussian-input", + "gcd": "text/x-pcs-gcd", + "gcf": "application/x-graphing-calculator", + "gcg": "chemical/x-gcg8-sequence", + "gen": "chemical/x-genbank", + "gf": "application/x-tex-gf", + "gif": "image/gif", + "gjc": "chemical/x-gaussian-input", + "gjf": "chemical/x-gaussian-input", + "gl": "video/gl", + "gnumeric": "application/x-gnumeric", + "gpt": "chemical/x-mopac-graph", + "gsf": "application/x-font", + "gsm": "audio/x-gsm", + "gtar": "application/x-gtar", + "gz": "application/gzip", + "h": "text/x-chdr", + "h++": "text/x-c++hdr", + "hdf": "application/x-hdf", + "hh": "text/x-c++hdr", + "hin": "chemical/x-hin", + "hpp": "text/x-c++hdr", + "hqx": "application/mac-binhex40", + "hs": "text/x-haskell", + "hta": "application/hta", + "htc": "text/x-component", + "htm": "text/html", + "html": "text/html", + "hwp": "application/x-hwp", + "hxx": "text/x-c++hdr", + "ica": "application/x-ica", + "ice": "x-conference/x-cooltalk", + "ico": "image/vnd.microsoft.icon", + "ics": "text/calendar", + "icz": "text/calendar", + "ief": "image/ief", + "iges": "model/iges", + "igs": "model/iges", + "iii": "application/x-iphone", + "info": "application/x-info", + "inp": "chemical/x-gamess-input", + "ins": "application/x-internet-signup", + "iso": "application/x-iso9660-image", + "isp": "application/x-internet-signup", + "ist": "chemical/x-isostar", + "istr": "chemical/x-isostar", + "jad": "text/vnd.sun.j2me.app-descriptor", + "jam": "application/x-jam", + "jar": "application/java-archive", + "java": "text/x-java", + "jdx": "chemical/x-jcamp-dx", + "jmz": "application/x-jmol", + "jng": "image/x-jng", + "jnlp": "application/x-java-jnlp-file", + "jp2": "image/jp2", + "jpe": "image/jpeg", + "jpeg": "image/jpeg", + "jpf": "image/jpx", + "jpg": "image/jpeg", + "jpg2": "image/jp2", + "jpm": "image/jpm", + "jpx": "image/jpx", + "js": "application/javascript", + "json": "application/json", + "kar": "audio/midi", + "key": "application/pgp-keys", + "kil": "application/x-killustrator", + "kin": "chemical/x-kinemage", + "kml": "application/vnd.google-earth.kml+xml", + "kmz": "application/vnd.google-earth.kmz", + "kpr": "application/x-kpresenter", + "kpt": "application/x-kpresenter", + "ksp": "application/x-kspread", + "kwd": "application/x-kword", + "kwt": "application/x-kword", + "latex": "application/x-latex", + "lha": "application/x-lha", + "lhs": "text/x-literate-haskell", + "lin": "application/bbolin", + "lsf": "video/x-la-asf", + "lsx": "video/x-la-asf", + "ltx": "text/x-tex", + "ly": "text/x-lilypond", + "lyx": "application/x-lyx", + "lzh": "application/x-lzh", + "lzx": "application/x-lzx", + "m3g": "application/m3g", + "m3u": "audio/x-mpegurl", + "m3u8": "application/x-mpegURL", + "m4a": "audio/mpeg", + "maker": "application/x-maker", + "man": "application/x-troff-man", + "mbox": "application/mbox", + "mcif": "chemical/x-mmcif", + "mcm": "chemical/x-macmolecule", + "mdb": "application/msaccess", + "me": "application/x-troff-me", + "mesh": "model/mesh", + "mid": "audio/midi", + "midi": "audio/midi", + "mif": "application/x-mif", + "mkv": "video/x-matroska", + "mm": "application/x-freemind", + "mmd": "chemical/x-macromodel-input", + "mmf": "application/vnd.smaf", + "mml": "text/mathml", + "mmod": "chemical/x-macromodel-input", + "mng": "video/x-mng", + "moc": "text/x-moc", + "mol": "chemical/x-mdl-molfile", + "mol2": "chemical/x-mol2", + "moo": "chemical/x-mopac-out", + "mop": "chemical/x-mopac-input", + "mopcrt": "chemical/x-mopac-input", + "mov": "video/quicktime", + "movie": "video/x-sgi-movie", + "mp2": "audio/mpeg", + "mp3": "audio/mpeg", + "mp4": "video/mp4", + "mpc": "chemical/x-mopac-input", + "mpe": "video/mpeg", + "mpeg": "video/mpeg", + "mpega": "audio/mpeg", + "mpg": "video/mpeg", + "mpga": "audio/mpeg", + "mph": "application/x-comsol", + "mpv": "video/x-matroska", + "ms": "application/x-troff-ms", + "msh": "model/mesh", + "msi": "application/x-msi", + "mvb": "chemical/x-mopac-vib", + "mxf": "application/mxf", + "mxu": "video/vnd.mpegurl", + "nb": "application/mathematica", + "nbp": "application/mathematica", + "nc": "application/x-netcdf", + "nef": "image/x-nikon-nef", + "nwc": "application/x-nwc", + "o": "application/x-object", + "oda": "application/oda", + "odb": "application/vnd.oasis.opendocument.database", + "odc": "application/vnd.oasis.opendocument.chart", + "odf": "application/vnd.oasis.opendocument.formula", + "odg": "application/vnd.oasis.opendocument.graphics", + "odi": "application/vnd.oasis.opendocument.image", + "odm": "application/vnd.oasis.opendocument.text-master", + "odp": "application/vnd.oasis.opendocument.presentation", + "ods": "application/vnd.oasis.opendocument.spreadsheet", + "odt": "application/vnd.oasis.opendocument.text", + "oga": "audio/ogg", + "ogg": "audio/ogg", + "ogv": "video/ogg", + "ogx": "application/ogg", + "old": "application/x-trash", + "one": "application/onenote", + "onepkg": "application/onenote", + "onetmp": "application/onenote", + "onetoc2": "application/onenote", + "opf": "application/oebps-package+xml", + "opus": "audio/ogg", + "orc": "audio/csound", + "orf": "image/x-olympus-orf", + "otf": "application/font-sfnt", + "otg": "application/vnd.oasis.opendocument.graphics-template", + "oth": "application/vnd.oasis.opendocument.text-web", + "otp": "application/vnd.oasis.opendocument.presentation-template", + "ots": "application/vnd.oasis.opendocument.spreadsheet-template", + "ott": "application/vnd.oasis.opendocument.text-template", + "oza": "application/x-oz-application", + "p": "text/x-pascal", + "p7r": "application/x-pkcs7-certreqresp", + "pac": "application/x-ns-proxy-autoconfig", + "pas": "text/x-pascal", + "pat": "image/x-coreldrawpattern", + "patch": "text/x-diff", + "pbm": "image/x-portable-bitmap", + "pcap": "application/vnd.tcpdump.pcap", + "pcf": "application/x-font-pcf", + "pcf.Z": "application/x-font-pcf", + "pcx": "image/pcx", + "pdb": "chemical/x-pdb", + "pdf": "application/pdf", + "pfa": "application/x-font", + "pfb": "application/x-font", + "pfr": "application/font-tdpfr", + "pgm": "image/x-portable-graymap", + "pgn": "application/x-chess-pgn", + "pgp": "application/pgp-encrypted", + "php": "#application/x-httpd-php", + "php3": "#application/x-httpd-php3", + "php3p": "#application/x-httpd-php3-preprocessed", + "php4": "#application/x-httpd-php4", + "php5": "#application/x-httpd-php5", + "phps": "#application/x-httpd-php-source", + "pht": "#application/x-httpd-php", + "phtml": "#application/x-httpd-php", + "pk": "application/x-tex-pk", + "pl": "text/x-perl", + "pls": "audio/x-scpls", + "pm": "text/x-perl", + "png": "image/png", + "pnm": "image/x-portable-anymap", + "pot": "text/plain", + "potm": "application/vnd.ms-powerpoint.template.macroEnabled.12", + "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", + "ppam": "application/vnd.ms-powerpoint.addin.macroEnabled.12", + "ppm": "image/x-portable-pixmap", + "pps": "application/vnd.ms-powerpoint", + "ppsm": "application/vnd.ms-powerpoint.slideshow.macroEnabled.12", + "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "ppt": "application/vnd.ms-powerpoint", + "pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "prf": "application/pics-rules", + "prt": "chemical/x-ncbi-asn1-ascii", + "ps": "application/postscript", + "psd": "image/x-photoshop", + "py": "text/x-python", + "pyc": "application/x-python-code", + "pyo": "application/x-python-code", + "qgs": "application/x-qgis", + "qt": "video/quicktime", + "qtl": "application/x-quicktimeplayer", + "ra": "audio/x-pn-realaudio", + "ram": "audio/x-pn-realaudio", + "rar": "application/rar", + "ras": "image/x-cmu-raster", + "rb": "application/x-ruby", + "rd": "chemical/x-mdl-rdfile", + "rdf": "application/rdf+xml", + "rdp": "application/x-rdp", + "rgb": "image/x-rgb", + "rhtml": "#application/x-httpd-eruby", + "rm": "audio/x-pn-realaudio", + "roff": "application/x-troff", + "ros": "chemical/x-rosdal", + "rpm": "application/x-redhat-package-manager", + "rss": "application/x-rss+xml", + "rtf": "application/rtf", + "rtx": "text/richtext", + "rxn": "chemical/x-mdl-rxnfile", + "scala": "text/x-scala", + "sce": "application/x-scilab", + "sci": "application/x-scilab", + "sco": "audio/csound", + "scr": "application/x-silverlight", + "sct": "text/scriptlet", + "sd": "chemical/x-mdl-sdfile", + "sd2": "audio/x-sd2", + "sda": "application/vnd.stardivision.draw", + "sdc": "application/vnd.stardivision.calc", + "sdd": "application/vnd.stardivision.impress", + "sds": "application/vnd.stardivision.chart", + "sdw": "application/vnd.stardivision.writer", + "ser": "application/java-serialized-object", + "sfd": "application/vnd.font-fontforge-sfd", + "sfv": "text/x-sfv", + "sgf": "application/x-go-sgf", + "sgl": "application/vnd.stardivision.writer-global", + "sh": "application/x-sh", + "shar": "application/x-shar", + "shp": "application/x-qgis", + "shtml": "text/html", + "shx": "application/x-qgis", + "sid": "audio/prs.sid", + "sig": "application/pgp-signature", + "sik": "application/x-trash", + "silo": "model/mesh", + "sis": "application/vnd.symbian.install", + "sisx": "x-epoc/x-sisx-app", + "sit": "application/x-stuffit", + "sitx": "application/x-stuffit", + "skd": "application/x-koan", + "skm": "application/x-koan", + "skp": "application/x-koan", + "skt": "application/x-koan", + "sldm": "application/vnd.ms-powerpoint.slide.macroEnabled.12", + "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", + "smi": "application/smil+xml", + "smil": "application/smil+xml", + "snd": "audio/basic", + "spc": "chemical/x-galactic-spc", + "spl": "application/x-futuresplash", + "spx": "audio/ogg", + "sql": "application/x-sql", + "src": "application/x-wais-source", + "srt": "text/plain", + "stc": "application/vnd.sun.xml.calc.template", + "std": "application/vnd.sun.xml.draw.template", + "sti": "application/vnd.sun.xml.impress.template", + "stw": "application/vnd.sun.xml.writer.template", + "sty": "text/x-tex", + "sv4cpio": "application/x-sv4cpio", + "sv4crc": "application/x-sv4crc", + "svg": "image/svg+xml", + "svgz": "image/svg+xml", + "sw": "chemical/x-swissprot", + "swf": "application/x-shockwave-flash", + "swfl": "application/x-shockwave-flash", + "sxc": "application/vnd.sun.xml.calc", + "sxd": "application/vnd.sun.xml.draw", + "sxg": "application/vnd.sun.xml.writer.global", + "sxi": "application/vnd.sun.xml.impress", + "sxm": "application/vnd.sun.xml.math", + "sxw": "application/vnd.sun.xml.writer", + "t": "application/x-troff", + "tar": "application/x-tar", + "taz": "application/x-gtar-compressed", + "tcl": "application/x-tcl", + "tex": "text/x-tex", + "texi": "application/x-texinfo", + "texinfo": "application/x-texinfo", + "text": "text/plain", + "tgf": "chemical/x-mdl-tgf", + "tgz": "application/x-gtar-compressed", + "thmx": "application/vnd.ms-officetheme", + "tif": "image/tiff", + "tiff": "image/tiff", + "tk": "text/x-tcl", + "tm": "text/texmacs", + "torrent": "application/x-bittorrent", + "tr": "application/x-troff", + "ts": "video/MP2T", + "tsp": "application/dsptype", + "tsv": "text/tab-separated-values", + "ttf": "application/font-sfnt", + "ttl": "text/turtle", + "txt": "text/plain", + "uls": "text/iuls", + "ustar": "application/x-ustar", + "val": "chemical/x-ncbi-asn1-binary", + "vcard": "text/vcard", + "vcd": "application/x-cdlink", + "vcf": "text/vcard", + "vcs": "text/x-vcalendar", + "vmd": "chemical/x-vmd", + "vms": "chemical/x-vamas-iso14976", + "vrm": "x-world/x-vrml", + "vrml": "model/vrml", + "vsd": "application/vnd.visio", + "vss": "application/vnd.visio", + "vst": "application/vnd.visio", + "vsw": "application/vnd.visio", + "wad": "application/x-doom", + "wasm": "application/wasm", + "wav": "audio/x-wav", + "wax": "audio/x-ms-wax", + "wbmp": "image/vnd.wap.wbmp", + "wbxml": "application/vnd.wap.wbxml", + "webm": "video/webm", + "wk": "application/x-123", + "wm": "video/x-ms-wm", + "wma": "audio/x-ms-wma", + "wmd": "application/x-ms-wmd", + "wml": "text/vnd.wap.wml", + "wmlc": "application/vnd.wap.wmlc", + "wmls": "text/vnd.wap.wmlscript", + "wmlsc": "application/vnd.wap.wmlscriptc", + "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wmz": "application/x-ms-wmz", + "woff": "application/font-woff", + "wp5": "application/vnd.wordperfect5.1", + "wpd": "application/vnd.wordperfect", + "wrl": "model/vrml", + "wsc": "text/scriptlet", + "wvx": "video/x-ms-wvx", + "wz": "application/x-wingz", + "x3d": "model/x3d+xml", + "x3db": "model/x3d+binary", + "x3dv": "model/x3d+vrml", + "xbm": "image/x-xbitmap", + "xcf": "application/x-xcf", + "xcos": "application/x-scilab-xcos", + "xht": "application/xhtml+xml", + "xhtml": "application/xhtml+xml", + "xlam": "application/vnd.ms-excel.addin.macroEnabled.12", + "xlb": "application/vnd.ms-excel", + "xls": "application/vnd.ms-excel", + "xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xlt": "application/vnd.ms-excel", + "xltm": "application/vnd.ms-excel.template.macroEnabled.12", + "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "xml": "application/xml", + "xpi": "application/x-xpinstall", + "xpm": "image/x-xpixmap", + "xsd": "application/xml", + "xsl": "application/xslt+xml", + "xslt": "application/xslt+xml", + "xspf": "application/xspf+xml", + "xtel": "chemical/x-xtel", + "xul": "application/vnd.mozilla.xul+xml", + "xwd": "image/x-xwindowdump", + "xyz": "chemical/x-xyz", + "xz": "application/x-xz", + "zip": "application/zip", +} + +type ContentTypeRecognizer interface { + ContentType(objectKey string) string +} + +type ExtensionBasedContentTypeRecognizer struct{} + +func (er ExtensionBasedContentTypeRecognizer) ContentType(objectKey string) string { + if len(objectKey) == 0 { + return "" + } + extName := path.Ext(objectKey) + if len(extName) > 0 && extName[0] == '.' { + extName = extName[1:] + } + contentType, ok := mime[extName] + if ok { + return contentType + } + return "binary/octet-stream" +} + +type EmptyContentTypeRecognizer struct{} + +func (er EmptyContentTypeRecognizer) ContentType(objectKey string) string { + _ = objectKey + return "" +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mirror_back.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mirror_back.go new file mode 100644 index 0000000000..fb3e76bcfb --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mirror_back.go @@ -0,0 +1,76 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +func (cli *ClientV2) PutBucketMirrorBack(ctx context.Context, input *PutBucketMirrorBackInput) (*PutBucketMirrorBackOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + data, contentMD5, err := marshalInput("PutBucketMirrorBackInput", putBucketMirrorBackInput{ + Rules: input.Rules, + }) + if err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("mirror", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketMirrorBackOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketMirrorBack(ctx context.Context, input *GetBucketMirrorBackInput) (*GetBucketMirrorBackOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("mirror", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketMirrorBackOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) DeleteBucketMirrorBack(ctx context.Context, input *DeleteBucketMirrorBackInput) (*DeleteBucketMirrorBackOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("mirror", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteBucketMirrorBackOutput{RequestInfo: res.RequestInfo()} + return &output, nil + +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/multipart.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/multipart.go new file mode 100644 index 0000000000..2898fee734 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/multipart.go @@ -0,0 +1,513 @@ +package tos + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "hash" + "io" + "io/ioutil" + "net/http" + "os" + "sort" + "strconv" +) + +// CreateMultipartUpload create a multipart upload operation +// objectKey: the name of object +// options: WithContentType set Content-Type, +// WithContentDisposition set Content-Disposition, +// WithContentLanguage set Content-Language, +// WithContentEncoding set Content-Encoding, +// WithCacheControl set Cache-Control, +// WithExpires set Expires, +// WithMeta set meta header(s), +// WithContentSHA256 set Content-Sha256, +// WithContentMD5 set Content-MD5 +// WithExpires set Expires, +// WithServerSideEncryptionCustomer set server side encryption options +// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl +// +// Deprecated: use CreateMultipartUpload of ClientV2 instead +func (bkt *Bucket) CreateMultipartUpload(ctx context.Context, objectKey string, options ...Option) (*CreateMultipartUploadOutput, error) { + if err := isValidKey(objectKey); err != nil { + return nil, err + } + + res, err := bkt.client.newBuilder(bkt.name, objectKey, options...). + WithQuery("uploads", ""). + WithRetry(nil, ServerErrorClassifier{}). + Request(ctx, http.MethodPost, nil, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + var upload multipartUpload + if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &upload); err != nil { + return nil, err + } + + return &CreateMultipartUploadOutput{ + RequestInfo: res.RequestInfo(), + Bucket: upload.Bucket, + Key: upload.Key, + UploadID: upload.UploadID, + SSECustomerAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm), + SSECustomerKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5), + }, nil +} + +// CreateMultipartUploadV2 create a multipart upload operation +func (cli *ClientV2) CreateMultipartUploadV2( + ctx context.Context, + input *CreateMultipartUploadV2Input) (*CreateMultipartUploadV2Output, error) { + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + if err := isValidKey(input.Key); err != nil { + return nil, err + } + if err := isValidSSECAlgorithm(input.SSECAlgorithm); len(input.SSECAlgorithm) != 0 && err != nil { + return nil, err + } + if err := isValidACL(input.ACL); len(input.ACL) != 0 && err != nil { + return nil, err + } + if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) != 0 && err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, input.Key). + WithQuery("uploads", ""). + WithParams(*input). + WithRetry(nil, ServerErrorClassifier{}). + Request(ctx, http.MethodPost, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + var upload multipartUpload + if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &upload); err != nil { + return nil, err + } + + return &CreateMultipartUploadV2Output{ + RequestInfo: res.RequestInfo(), + Bucket: upload.Bucket, + Key: upload.Key, + UploadID: upload.UploadID, + SSECAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm), + SSECKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5), + EncodingType: res.Header.Get(HeaderContentEncoding), + ServerSideEncryption: res.Header.Get(HeaderServerSideEncryption), + ServerSideEncryptionKeyID: res.Header.Get(HeaderServerSideEncryptionKmsKeyID), + }, nil +} + +// UploadPart upload a part for a multipart upload operation +// input: the parameters, some fields is required, e.g. Key, UploadID, PartNumber and PartNumber +// +// If uploading 'Content' with known Content-Length, please add option tos.WithContentLength +// +// Deprecated: use UploadPart of ClientV2 instead +func (bkt *Bucket) UploadPart(ctx context.Context, input *UploadPartInput, options ...Option) (*UploadPartOutput, error) { + if err := isValidKey(input.Key); err != nil { + return nil, err + } + var ( + onRetry func(req *Request) error = nil + cf classifier + content = input.Content + ) + cf = NoRetryClassifier{} + if seeker, ok := content.(io.Seeker); ok { + start, err := seeker.Seek(0, io.SeekCurrent) + if err == nil { + onRetry = func(req *Request) error { + // PutObject/UploadPartV2 can be treated as an idempotent semantics if the request message body + // supports a reset operation. e.g. the request message body is a string, + // a local file handle, binary data in memory + if seeker, ok := req.Content.(io.Seeker); ok { + _, err := seeker.Seek(start, io.SeekStart) + if err != nil { + return err + } + } else { + return newTosClientError("Io Reader not support retry", nil) + } + return nil + } + cf = StatusCodeClassifier{} + } + } + res, err := bkt.client.newBuilder(bkt.name, input.Key, options...). + WithQuery("uploadId", input.UploadID). + WithQuery("partNumber", strconv.Itoa(input.PartNumber)). + WithRetry(onRetry, cf). + Request(ctx, http.MethodPut, input.Content, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + return &UploadPartOutput{ + RequestInfo: res.RequestInfo(), + PartNumber: input.PartNumber, + ETag: res.Header.Get(HeaderETag), + SSECustomerAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm), + SSECustomerKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5), + }, nil +} + +// UploadPartV2 upload a part for a multipart upload operation +func (cli *ClientV2) UploadPartV2(ctx context.Context, input *UploadPartV2Input) (*UploadPartV2Output, error) { + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + if err := isValidSSECAlgorithm(input.SSECAlgorithm); len(input.SSECAlgorithm) != 0 && err != nil { + return nil, err + } + + var ( + checker hash.Hash64 + content = input.Content + contentLength = input.ContentLength + ) + + if input == nil { + return nil, InputInvalidClientError + } + + if input.PartNumber == 0 { + return nil, InvalidPartNumber + } + + if input.UploadID == "" { + return nil, InvalidUploadID + } + + if contentLength == 0 { + contentLength = tryResolveLength(content) + } + + if cli.enableCRC { + checker = NewCRC(DefaultCrcTable(), 0) + } + + var ( + onRetry func(req *Request) error = nil + cf classifier + ) + + if content != nil { + content = wrapReader(content, contentLength, input.DataTransferListener, input.RateLimiter, &crcChecker{checker: checker}) + } + + cf = NoRetryClassifier{} + if seeker, ok := content.(io.Seeker); ok { + start, err := seeker.Seek(0, io.SeekCurrent) + if err == nil { + onRetry = func(req *Request) error { + // PutObject/UploadPartV2 can be treated as an idempotent semantics if the request message body + // supports a reset operation. e.g. the request message body is a string, + // a local file handle, binary data in memory + if seeker, ok := req.Content.(io.Seeker); ok { + _, err := seeker.Seek(start, io.SeekStart) + if err != nil { + return err + } + } else { + return newTosClientError("Io Reader not support retry", nil) + } + return nil + } + cf = StatusCodeClassifier{} + } + } + + res, err := cli.newBuilder(input.Bucket, input.Key). + WithParams(*input). + WithContentLength(input.ContentLength). + WithRetry(onRetry, cf). + Request(ctx, http.MethodPut, content, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + if err = checkCrc64(res, checker); err != nil { + return nil, err + } + checksum, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64) + return &UploadPartV2Output{ + RequestInfo: res.RequestInfo(), + PartNumber: input.PartNumber, + ETag: res.Header.Get(HeaderETag), + SSECAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm), + SSECKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5), + HashCrc64ecma: checksum, + ServerSideEncryptionKeyID: res.Header.Get(HeaderServerSideEncryptionKmsKeyID), + ServerSideEncryption: res.Header.Get(HeaderServerSideEncryption), + }, nil +} + +// UploadPartFromFile upload a part for a multipart upload operation from file +func (cli *ClientV2) UploadPartFromFile(ctx context.Context, input *UploadPartFromFileInput) (*UploadPartFromFileOutput, error) { + file, err := os.Open(input.FilePath) + if err != nil { + return nil, err + } + _, err = file.Seek(int64(input.Offset), io.SeekStart) + + if err != nil { + return nil, err + } + output, err := cli.UploadPartV2(ctx, &UploadPartV2Input{ + UploadPartBasicInput: input.UploadPartBasicInput, + Content: file, + ContentLength: input.PartSize, + }) + if err != nil { + return nil, err + } + return &UploadPartFromFileOutput{*output}, nil +} + +// CompleteMultipartUpload complete a multipart upload operation +// input: input.Key the object name, +// input.UploadID the uploadID got from CreateMultipartUpload +// input.UploadedParts upload part output got from UploadPart or UploadPartCopy +// +// Deprecated: use CompleteMultipartUpload of ClientV2 instead +func (bkt *Bucket) CompleteMultipartUpload(ctx context.Context, input *CompleteMultipartUploadInput, options ...Option) (*CompleteMultipartUploadOutput, error) { + if err := isValidKey(input.Key); err != nil { + return nil, err + } + multipart := partsToComplete{Parts: make(uploadedParts, 0, len(input.UploadedParts))} + for _, p := range input.UploadedParts { + multipart.Parts = append(multipart.Parts, p.uploadedPart()) + } + + sort.Sort(multipart.Parts) + data, err := json.Marshal(&multipart) + if err != nil { + return nil, InvalidMarshal + } + + res, err := bkt.client.newBuilder(bkt.name, input.Key, options...). + WithQuery("uploadId", input.UploadID). + WithRetry(OnRetryFromStart, ServerErrorClassifier{}). + Request(ctx, http.MethodPost, bytes.NewReader(data), bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + return &CompleteMultipartUploadOutput{ + RequestInfo: res.RequestInfo(), + VersionID: res.Header.Get(HeaderVersionID), + }, nil +} + +// CompleteMultipartUploadV2 complete a multipart upload operation +func (cli *ClientV2) CompleteMultipartUploadV2( + ctx context.Context, input *CompleteMultipartUploadV2Input) (*CompleteMultipartUploadV2Output, error) { + + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + reqBuilder := cli.newBuilder(input.Bucket, input.Key). + WithParams(*input) + var err error + var res *Response + if input.CompleteAll { + if len(input.Parts) != 0 { + return nil, InvalidCompleteAllPartsLength + } + reqBuilder.WithHeader("x-tos-complete-all", "yes") + res, err = reqBuilder.WithRetry(nil, ServerErrorClassifier{}).Request(ctx, http.MethodPost, nil, cli.roundTripper(http.StatusOK)) + } else { + if len(input.Parts) == 0 { + return nil, InvalidPartsLength + } + multipart := partsToComplete{Parts: make(uploadedParts, 0, len(input.Parts))} + for _, p := range input.Parts { + multipart.Parts = append(multipart.Parts, p.uploadedPart()) + } + + sort.Sort(multipart.Parts) + data, marshalErr := json.Marshal(&multipart) + if marshalErr != nil { + return nil, InvalidMarshal + } + res, err = reqBuilder.WithRetry(OnRetryFromStart, ServerErrorClassifier{}).Request(ctx, http.MethodPost, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + } + if err != nil { + return nil, err + } + defer res.Close() + crc64, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64) + output := &CompleteMultipartUploadV2Output{ + RequestInfo: res.RequestInfo(), + VersionID: res.Header.Get(HeaderVersionID), + HashCrc64ecma: crc64, + } + var callbackResult string + if input.Callback == "" { + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + } else { + callbackRes, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, &TosServerError{ + TosError: TosError{Message: fmt.Sprintf("tos: read callback result err:%s", err.Error())}, + RequestInfo: res.RequestInfo(), + } + } + if len(callbackRes) > 0 { + callbackResult = string(callbackRes) + } + output.ETag = res.Header.Get(HeaderETag) + output.Location = res.Header.Get(HeaderLocation) + } + output.CallbackResult = callbackResult + output.ServerSideEncryption = res.Header.Get(HeaderServerSideEncryption) + output.ServerSideEncryptionKeyID = res.Header.Get(HeaderServerSideEncryptionKmsKeyID) + return output, nil +} + +// AbortMultipartUpload abort a multipart upload operation +// +// Deprecated: use AbortMultipartUpload of ClientV2 instead +func (bkt *Bucket) AbortMultipartUpload(ctx context.Context, input *AbortMultipartUploadInput, options ...Option) (*AbortMultipartUploadOutput, error) { + if err := isValidKey(input.Key); err != nil { + return nil, err + } + res, err := bkt.client.newBuilder(bkt.name, input.Key, options...). + WithQuery("uploadId", input.UploadID). + WithRetry(nil, ServerErrorClassifier{}). + Request(ctx, http.MethodDelete, nil, bkt.client.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + return &AbortMultipartUploadOutput{RequestInfo: res.RequestInfo()}, nil +} + +// AbortMultipartUpload abort a multipart upload operation +func (cli *ClientV2) AbortMultipartUpload(ctx context.Context, input *AbortMultipartUploadInput) (*AbortMultipartUploadOutput, error) { + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, input.Key). + WithParams(*input). + WithRetry(nil, ServerErrorClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + return &AbortMultipartUploadOutput{RequestInfo: res.RequestInfo()}, nil +} + +// ListUploadedParts List Uploaded Parts +// objectKey: the object name +// input: key, uploadID and other parameters +// +// Deprecated: use ListParts of ClientV2 instead +func (bkt *Bucket) ListUploadedParts(ctx context.Context, input *ListUploadedPartsInput, options ...Option) (*ListUploadedPartsOutput, error) { + if err := isValidKey(input.Key); err != nil { + return nil, err + } + + res, err := bkt.client.newBuilder(bkt.name, input.Key, options...). + WithQuery("uploadId", input.UploadID). + WithQuery("max-parts", strconv.Itoa(input.MaxParts)). + WithQuery("part-number-marker", strconv.Itoa(input.PartNumberMarker)). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := ListUploadedPartsOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +// ListParts List Uploaded Parts +func (cli *ClientV2) ListParts(ctx context.Context, input *ListPartsInput) (*ListPartsOutput, error) { + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, input.Key). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := ListPartsOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +// ListMultipartUploads list multipart uploads +// +// Deprecated: use ListMultipartUploads of ClientV2 instead +func (bkt *Bucket) ListMultipartUploads(ctx context.Context, input *ListMultipartUploadsInput, options ...Option) (*ListMultipartUploadsOutput, error) { + res, err := bkt.client.newBuilder(bkt.name, "", options...). + WithQuery("uploads", ""). + WithQuery("prefix", input.Prefix). + WithQuery("delimiter", input.Delimiter). + WithQuery("key-marker", input.KeyMarker). + WithQuery("upload-id-marker", input.UploadIDMarker). + WithQuery("max-uploads", strconv.Itoa(input.MaxUploads)). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := ListMultipartUploadsOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +// ListMultipartUploadsV2 list multipart uploads +func (cli *ClientV2) ListMultipartUploadsV2( + ctx context.Context, + input *ListMultipartUploadsV2Input) (*ListMultipartUploadsV2Output, error) { + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("uploads", ""). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := ListMultipartUploadsV2Output{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/notification.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/notification.go new file mode 100644 index 0000000000..97ab273aaa --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/notification.go @@ -0,0 +1,54 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +func (cli *ClientV2) PutBucketNotification(ctx context.Context, input *PutBucketNotificationInput) (*PutBucketNotificationOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + data, contentMD5, err := marshalInput("PutBucketNotification", putBucketNotificationInput{CloudFunctionConfigurations: input.CloudFunctionConfigurations, RocketMQConfigurations: input.RocketMQConfigurations}) + if err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("notification", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketNotificationOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketNotification(ctx context.Context, input *GetBucketNotificationInput) (*GetBucketNotificationOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("notification", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketNotificationOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/object.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/object.go new file mode 100644 index 0000000000..22ff195fa3 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/object.go @@ -0,0 +1,1121 @@ +package tos + +import ( + "bytes" + "context" + "errors" + "fmt" + "hash" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strconv" +) + +type Bucket struct { + name string + client *Client + baseClient *baseClient +} + +// GetObject get data and metadata of an object +// objectKey: the name of object +// options: WithVersionID which version of this object +// WithRange the range of content, +// WithIfModifiedSince return if the object modified after the given date, otherwise return status code 304 +// WithIfUnmodifiedSince, WithIfMatch, WithIfNoneMatch set If-Unmodified-Since, If-Match and If-None-Match +// +// Deprecated: use GetObject of ClientV2 instead +func (bkt *Bucket) GetObject(ctx context.Context, objectKey string, options ...Option) (*GetObjectOutput, error) { + if err := isValidKey(objectKey); err != nil { + return nil, err + } + rb := bkt.client.newBuilder(bkt.name, objectKey, options...) + res, err := rb.WithRetry(nil, StatusCodeClassifier{}).Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(expectedCode(rb))) + if err != nil { + return nil, err + } + output := GetObjectOutput{ + RequestInfo: res.RequestInfo(), + ContentRange: rb.Header.Get(HeaderContentRange), + Content: res.Body, + } + output.ObjectMeta.fromResponse(res) + return &output, nil +} + +func (cli *ClientV2) copyToFile(fileName string, reader io.Reader) error { + fd, err := os.OpenFile(filepath.Clean(fileName), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, DefaultFilePerm) + if err != nil { + return err + } + defer fd.Close() + _, err = io.Copy(fd, reader) + if err != nil { + return err + } + return nil +} + +// GetObjectToFile get object and write it to file +func (cli *ClientV2) GetObjectToFile(ctx context.Context, input *GetObjectToFileInput) (*GetObjectToFileOutput, error) { + + err := checkAndCreateDir(input.FilePath) + if err != nil { + return nil, InvalidFilePath.withCause(err) + } + + tempFilePath := input.FilePath + TempFileSuffix + + get, err := cli.GetObjectV2(ctx, &input.GetObjectV2Input) + if err != nil { + return nil, err + } + defer get.Content.Close() + err = cli.copyToFile(tempFilePath, get.Content) + if err != nil { + return nil, newTosClientError("GetObject to File error", err) + } + + err = os.Rename(tempFilePath, input.FilePath) + if err != nil { + return nil, err + } + return &GetObjectToFileOutput{get.GetObjectBasicOutput}, nil +} + +// GetObjectV2 get data and metadata of an object +func (cli *ClientV2) GetObjectV2(ctx context.Context, input *GetObjectV2Input) (*GetObjectV2Output, error) { + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + rb := cli.newBuilder(input.Bucket, input.Key). + WithParams(*input).WithRetry(nil, StatusCodeClassifier{}) + if input.Range != "" { + rb.WithHeader(HeaderRange, input.Range) + } else if input.RangeEnd != 0 || input.RangeStart != 0 { + if input.RangeEnd < input.RangeStart { + return nil, errors.New("tos: invalid range") + } + // set rb.Range will change expected code + rb.Range = &Range{Start: input.RangeStart, End: input.RangeEnd} + rb.WithHeader(HeaderRange, rb.Range.String()) + } + res, err := rb.Request(ctx, http.MethodGet, nil, cli.roundTripper(expectedCode(rb))) + if err != nil { + return nil, err + } + basic := GetObjectBasicOutput{ + RequestInfo: res.RequestInfo(), + ContentRange: res.Header.Get(HeaderContentRange), + } + basic.ObjectMetaV2.fromResponseV2(res) + var serverCrc uint64 + var checker hash.Hash64 + // 200 为完整请求 + if res.StatusCode == http.StatusOK && cli.enableCRC { + serverCrc = basic.HashCrc64ecma + checker = NewCRC(DefaultCrcTable(), 0) + + } + output := GetObjectV2Output{ + GetObjectBasicOutput: basic, + Content: wrapReader(res.Body, res.ContentLength, input.DataTransferListener, input.RateLimiter, &crcChecker{checker: checker, serverCrc: serverCrc}), + } + return &output, nil +} + +// HeadObject get metadata of an object +// objectKey: the name of object +// options: WithVersionID which version of this object +// WithRange the range of content, +// WithIfModifiedSince return if the object modified after the given date, otherwise return status code 304 +// WithIfUnmodifiedSince, WithIfMatch, WithIfNoneMatch set If-Unmodified-Since, If-Match and If-None-Match +// +// Deprecated: use HeadObject of ClientV2 instead +func (bkt *Bucket) HeadObject(ctx context.Context, objectKey string, options ...Option) (*HeadObjectOutput, error) { + if err := isValidKey(objectKey); err != nil { + return nil, err + } + + rb := bkt.client.newBuilder(bkt.name, objectKey, options...) + + res, err := rb.WithRetry(nil, StatusCodeClassifier{}).Request(ctx, http.MethodHead, nil, bkt.client.roundTripper(expectedCode(rb))) + if err != nil { + return nil, err + } + defer res.Close() + + output := HeadObjectOutput{ + RequestInfo: res.RequestInfo(), + ContentRange: rb.Header.Get(HeaderContentRange), + } + output.ObjectMeta.fromResponse(res) + return &output, nil +} + +// HeadObjectV2 get metadata of an object +func (cli *ClientV2) HeadObjectV2(ctx context.Context, input *HeadObjectV2Input) (*HeadObjectV2Output, error) { + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + + rb := cli.newBuilder(input.Bucket, input.Key). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}) + res, err := rb.Request(ctx, http.MethodHead, nil, cli.roundTripper(expectedCode(rb))) + if err != nil { + return nil, err + } + defer res.Close() + + output := HeadObjectV2Output{ + RequestInfo: res.RequestInfo(), + } + output.ObjectMetaV2.fromResponseV2(res) + return &output, nil +} + +func expectedCode(rb *requestBuilder) int { + okCode := http.StatusOK + if rb.Header.Get(HeaderRange) != "" || rb.Query.Get(QueryPartNumber) != "" { + okCode = http.StatusPartialContent + } + return okCode +} + +// DeleteObject delete an object +// objectKey: the name of object +// options: WithVersionID which version of this object will be deleted +// +// Deprecated: use DeleteObject of ClientV2 instead +func (bkt *Bucket) DeleteObject(ctx context.Context, objectKey string, options ...Option) (*DeleteObjectOutput, error) { + if err := isValidKey(objectKey); err != nil { + return nil, err + } + + res, err := bkt.client.newBuilder(bkt.name, objectKey, options...).WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, bkt.client.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + deleteMarker, _ := strconv.ParseBool(res.Header.Get(HeaderDeleteMarker)) + return &DeleteObjectOutput{ + RequestInfo: res.RequestInfo(), + DeleteMarker: deleteMarker, + VersionID: res.Header.Get(HeaderVersionID), + }, nil +} + +// DeleteObjectV2 delete an object +func (cli *ClientV2) DeleteObjectV2(ctx context.Context, input *DeleteObjectV2Input) (*DeleteObjectV2Output, error) { + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, input.Key). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + deleteMarker, _ := strconv.ParseBool(res.Header.Get(HeaderDeleteMarker)) + return &DeleteObjectV2Output{ + DeleteObjectOutput{ + RequestInfo: res.RequestInfo(), + DeleteMarker: deleteMarker, + VersionID: res.Header.Get(HeaderVersionID)}}, nil +} + +// DeleteMultiObjects delete multi-objects +// input: the objects will be deleted +// +// Deprecated: use DeleteMultiObjects of ClientV2 instead +func (bkt *Bucket) DeleteMultiObjects(ctx context.Context, input *DeleteMultiObjectsInput, options ...Option) (*DeleteMultiObjectsOutput, error) { + for _, object := range input.Objects { + if err := isValidKey(object.Key); err != nil { + return nil, err + } + } + + in, contentMD5, err := marshalInput("DeleteMultiObjectsInput", deleteMultiObjectsInput{ + Objects: input.Objects, + Quiet: input.Quiet, + }) + if err != nil { + return nil, err + } + res, err := bkt.client.newBuilder(bkt.name, "", options...). + WithHeader(HeaderContentMD5, contentMD5). + WithQuery("delete", ""). + WithRetry(OnRetryFromStart, ServerErrorClassifier{}). + Request(ctx, http.MethodPost, bytes.NewReader(in), bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := DeleteMultiObjectsOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +// DeleteMultiObjects delete multi-objects +func (cli *ClientV2) DeleteMultiObjects(ctx context.Context, input *DeleteMultiObjectsInput) (*DeleteMultiObjectsOutput, error) { + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + if len(input.Objects) == 0 { + return nil, InvlidDeleteMultiObjectsLength + } + + for _, object := range input.Objects { + if err := isValidKey(object.Key); err != nil { + return nil, err + } + } + in, contentMD5, err := marshalInput("DeleteMultiObjectsInput", deleteMultiObjectsInput{ + Objects: input.Objects, + Quiet: input.Quiet, + }) + if err != nil { + return nil, err + } + // POST method, don't retry + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("delete", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, ServerErrorClassifier{}). + Request(ctx, http.MethodPost, bytes.NewReader(in), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteMultiObjectsOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +// PutObject put an object +// objectKey: the name of object +// content: the content of object +// options: WithContentType set Content-Type, +// WithContentDisposition set Content-Disposition, +// WithContentLanguage set Content-Language, +// WithContentEncoding set Content-Encoding, +// WithCacheControl set Cache-Control, +// WithExpires set Expires, +// WithMeta set meta header(s), +// WithContentSHA256 set Content-Sha256, +// WithContentMD5 set Content-MD5 +// WithExpires set Expires, +// WithServerSideEncryptionCustomer set server side encryption options +// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl +// +// NOTICE: only content with a known length is supported now, +// e.g, bytes.Buffer, bytes.Reader, strings.Reader, os.File, io.LimitedReader, net.Buffers. +// if the parameter content(an io.Reader) is not one of these, +// please use io.LimitReader(reader, length) to wrap this reader or use the WithContentLength option. +// +// Deprecated: use PutObjectV2 of ClientV2 instead +func (bkt *Bucket) PutObject(ctx context.Context, objectKey string, content io.Reader, options ...Option) (*PutObjectOutput, error) { + if err := isValidKey(objectKey); err != nil { + return nil, err + } + var ( + onRetry func(req *Request) error = nil + classifier classifier + ) + classifier = NoRetryClassifier{} + if seeker, ok := content.(io.Seeker); ok { + start, err := seeker.Seek(0, io.SeekCurrent) + if err == nil { + onRetry = func(req *Request) error { + // PutObject/UploadPart can be treated as an idempotent semantics if the request message body + // supports a reset operation. e.g. the request message body is a string, + // a local file handle, binary data in memory + if seeker, ok := req.Content.(io.Seeker); ok { + _, err := seeker.Seek(start, io.SeekStart) + if err != nil { + return err + } + } else { + return newTosClientError("Io Reader not support retry", nil) + } + return nil + } + classifier = StatusCodeClassifier{} + } + } + res, err := bkt.client.newBuilder(bkt.name, objectKey, options...). + WithRetry(onRetry, classifier). + Request(ctx, http.MethodPut, content, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + return &PutObjectOutput{ + RequestInfo: res.RequestInfo(), + ETag: res.Header.Get(HeaderETag), + VersionID: res.Header.Get(HeaderVersionID), + SSECustomerAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm), + SSECustomerKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5), + }, nil +} + +func skipEscape(i byte) bool { + return (i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || (i >= '0' && i <= '9') || + i == '-' || + i == '.' || + i == '_' || + i == '~' +} + +func escapeHeader(s string) string { + var buf bytes.Buffer + for i := 0; i < len(s); i++ { + c := s[i] + if skipEscape(c) { + buf.WriteByte(c) + } else { + fmt.Fprintf(&buf, "%%%02X", c) + } + } + return buf.String() +} + +func existChinese(s string) bool { + r := []rune(s) + + for i := 0; i < len(r); i++ { + if r[i] >= 0x4E00 && r[i] <= 0x9FA5 { + return true + } + } + return false +} + +// url-encode Chinese characters only +func headerEncode(s string) string { + return escapeHeader(s) +} + +func checkCrc64(res *Response, checker hash.Hash64) error { + if res.Header.Get(HeaderHashCrc64ecma) == "" || checker == nil { + return nil + } + crc64, err := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64) + if err != nil { + return &TosServerError{ + TosError: TosError{"tos: server returned invalid crc"}, + RequestInfo: res.RequestInfo(), + } + } + if checker.Sum64() != crc64 { + return &TosServerError{ + TosError: TosError{Message: fmt.Sprintf("tos: crc64 check failed, expected:%d, in fact:%d", crc64, checker.Sum64())}, + RequestInfo: res.RequestInfo(), + } + } + return nil +} + +type crcChecker struct { + checker hash.Hash64 + serverCrc uint64 +} + +type nopCloser struct { + base io.Reader +} + +func wrapCloser(reader io.Reader) io.ReadCloser { + return &nopCloser{base: reader} +} + +func (n2 nopCloser) Seek(offset int64, whence int) (int64, error) { + seeker, ok := n2.base.(io.Seeker) + if !ok { + return 0, NotSupportSeek + } + return seeker.Seek(offset, whence) +} + +func (n2 nopCloser) Read(p []byte) (n int, err error) { + return n2.base.Read(p) +} + +func (n2 nopCloser) Close() error { + return nil +} + +// wrapReader wrap reader with some extension function. +// If reader can be interpreted as io.ReadCloser, use itself as base ReadCloser, else wrap it a NopCloser. +func wrapReader(reader io.Reader, totalBytes int64, listener DataTransferListener, limiter RateLimiter, crcChecker *crcChecker) io.ReadCloser { + var wrapped io.ReadCloser + // get base ReadCloser + if rc, ok := reader.(io.ReadCloser); ok { + wrapped = rc + } else { + wrapped = wrapCloser(reader) + } + // wrap with listener + if listener != nil { + wrapped = &readCloserWithListener{ + listener: listener, + base: wrapped, + consumed: 0, + total: totalBytes, + } + } + // wrap with limiter + if limiter != nil { + wrapped = &ReadCloserWithLimiter{ + limiter: limiter, + base: wrapped, + } + } + // wrap with crc64 checker + if crcChecker != nil && crcChecker.checker != nil { + wrapped = &readCloserWithCRC{ + serverCrc: crcChecker.serverCrc, + checker: crcChecker.checker, + base: wrapped, + } + } + return wrapped +} + +// PutObjectV2 put an object +func (cli *ClientV2) PutObjectV2(ctx context.Context, input *PutObjectV2Input) (*PutObjectV2Output, error) { + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + if err := isValidSSECAlgorithm(input.SSECAlgorithm); len(input.SSECAlgorithm) != 0 && err != nil { + return nil, err + } + + if err := isValidACL(input.ACL); len(input.ACL) != 0 && err != nil { + return nil, err + } + + if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) != 0 && err != nil { + return nil, err + } + + var ( + checker hash.Hash64 + content = input.Content + contentLength = input.ContentLength + ) + if cli.enableCRC { + checker = NewCRC(DefaultCrcTable(), 0) + } + if contentLength <= 0 { + contentLength = tryResolveLength(content) + } + + var ( + onRetry func(req *Request) error = nil + classifier classifier + ) + if content != nil { + content = wrapReader(content, contentLength, input.DataTransferListener, input.RateLimiter, &crcChecker{checker: checker}) + } + classifier = NoRetryClassifier{} + if seeker, ok := content.(io.Seeker); ok { + start, err := seeker.Seek(0, io.SeekCurrent) + if err == nil { + onRetry = func(req *Request) error { + // PutObject/UploadPart can be treated as an idempotent semantics if the request message body + // supports a reset operation. e.g. the request message body is a string, + // a local file handle, binary data in memory + if seeker, ok := req.Content.(io.Seeker); ok { + _, err := seeker.Seek(start, io.SeekStart) + if err != nil { + return err + } + } else { + return newTosClientError("Io Reader not support retry", nil) + } + return nil + } + classifier = StatusCodeClassifier{} + } + } + + rb := cli.newBuilder(input.Bucket, input.Key). + WithContentLength(contentLength). + WithParams(*input). + WithRetry(onRetry, classifier) + res, err := rb.Request(ctx, http.MethodPut, content, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + if err = checkCrc64(res, checker); err != nil { + return nil, err + } + crc64, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64) + callbackResult := "" + if input.Callback != "" && res.Body != nil { + callbackRes, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, &TosServerError{ + TosError: TosError{Message: fmt.Sprintf("tos: read callback result err:%s", err.Error())}, + RequestInfo: res.RequestInfo(), + } + } + if len(callbackRes) > 0 { + callbackResult = string(callbackRes) + } + } + + return &PutObjectV2Output{ + RequestInfo: res.RequestInfo(), + ETag: res.Header.Get(HeaderETag), + SSECAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm), + SSECKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5), + VersionID: res.Header.Get(HeaderVersionID), + ServerSideEncryption: res.Header.Get(HeaderServerSideEncryption), + ServerSideEncryptionKeyID: res.Header.Get(HeaderServerSideEncryptionKmsKeyID), + CallbackResult: callbackResult, + HashCrc64ecma: crc64, + }, nil +} + +// PutObjectFromFile put an object from file +func (cli *ClientV2) PutObjectFromFile(ctx context.Context, input *PutObjectFromFileInput) (*PutObjectFromFileOutput, error) { + file, err := os.Open(input.FilePath) + if err != nil { + return nil, err + } + defer file.Close() + putOutput, err := cli.PutObjectV2(ctx, &PutObjectV2Input{ + PutObjectBasicInput: input.PutObjectBasicInput, + Content: file, + }) + if err != nil { + return nil, err + } + return &PutObjectFromFileOutput{*putOutput}, err +} + +// AppendObject append content at the tail of an appendable object +// objectKey: the name of object +// content: the content of object +// offset: append position, equals to the current object-size +// options: WithContentType set Content-Type, +// WithContentDisposition set Content-Disposition, +// WithContentLanguage set Content-Language, +// WithContentEncoding set Content-Encoding, +// WithCacheControl set Cache-Control, +// WithExpires set Expires, +// WithMeta set meta header(s), +// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl +// above options only take effect when offset parameter is 0. +// WithContentSHA256 set Content-Sha256, +// WithContentMD5 set Content-MD5. +// +// NOTICE: only content with a known length is supported now, +// e.g, bytes.Buffer, bytes.Reader, strings.Reader, os.File, io.LimitedReader, net.Buffers. +// if the parameter content(an io.Reader) is not one of these, +// please use io.LimitReader(reader, length) to wrap this reader or use the WithContentLength option. +// +// Deprecated: use AppendObject of ClientV2 instead +func (bkt *Bucket) AppendObject(ctx context.Context, objectKey string, content io.Reader, offset int64, options ...Option) (*AppendObjectOutput, error) { + if err := isValidKey(objectKey); err != nil { + return nil, err + } + + res, err := bkt.client.newBuilder(bkt.name, objectKey, options...). + WithQuery("append", ""). + WithQuery("offset", strconv.FormatInt(offset, 10)). + WithRetry(nil, NoRetryClassifier{}). + Request(ctx, http.MethodPost, content, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + nextOffset := res.Header.Get(HeaderNextAppendOffset) + appendOffset, err := strconv.ParseInt(nextOffset, 10, 64) + if err != nil { + return nil, fmt.Errorf("tos: server return unexpected Next-Append-Offset header %q", nextOffset) + } + return &AppendObjectOutput{ + RequestInfo: res.RequestInfo(), + ETag: res.Header.Get(HeaderETag), + NextAppendOffset: appendOffset, + }, nil +} + +func (bkt *Bucket) PutObjectTagging(ctx context.Context, input *PutObjectTaggingInput, option ...Option) (*PutObjectTaggingOutput, error) { + return bkt.baseClient.PutObjectTagging(ctx, input, option...) +} + +func (bkt *Bucket) GetObjectTagging(ctx context.Context, input *GetObjectTaggingInput, option ...Option) (*GetObjectTaggingOutput, error) { + return bkt.baseClient.GetObjectTagging(ctx, input, option...) +} + +func (bkt *Bucket) DeleteObjectTagging(ctx context.Context, input *DeleteObjectTaggingInput, option ...Option) (*DeleteObjectTaggingOutput, error) { + return bkt.baseClient.DeleteObjectTagging(ctx, input, option...) +} + +func (bkt *Bucket) RestoreObject(ctx context.Context, input *RestoreObjectInput, option ...Option) (*RestoreObjectOutput, error) { + return bkt.baseClient.RestoreObject(ctx, input, option...) +} + +// AppendObjectV2 append content at the tail of an appendable object +func (cli *ClientV2) AppendObjectV2(ctx context.Context, input *AppendObjectV2Input) (*AppendObjectV2Output, error) { + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + var ( + checker hash.Hash64 + content = input.Content + contentLength = input.ContentLength + ) + if contentLength <= 0 { + contentLength = tryResolveLength(content) + } + if cli.enableCRC { + checker = NewCRC(DefaultCrcTable(), input.PreHashCrc64ecma) + } + if content != nil { + content = wrapReader(content, contentLength, input.DataTransferListener, input.RateLimiter, &crcChecker{checker: checker}) + } + res, err := cli.newBuilder(input.Bucket, input.Key). + WithQuery("append", ""). + WithParams(*input). + WithContentLength(contentLength). + WithRetry(nil, NoRetryClassifier{}). + Request(ctx, http.MethodPost, content, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + nextOffset := res.Header.Get(HeaderNextAppendOffset) + appendOffset, err := strconv.ParseInt(nextOffset, 10, 64) + if err != nil { + return nil, &TosServerError{ + TosError: TosError{fmt.Sprintf("tos: server return unexpected Next-Append-Offset header %q", nextOffset)}, + RequestInfo: res.RequestInfo(), + } + } + if err = checkCrc64(res, checker); err != nil { + return nil, err + } + crc64, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64) + return &AppendObjectV2Output{ + RequestInfo: res.RequestInfo(), + VersionID: res.Header.Get(HeaderVersionID), + NextAppendOffset: appendOffset, + HashCrc64ecma: crc64, + }, nil +} + +// SetObjectMeta overwrites metadata of the object +// objectKey: the name of object +// options: WithContentType set Content-Type, +// WithContentDisposition set Content-Disposition, +// WithContentLanguage set Content-Language, +// WithContentEncoding set Content-Encoding, +// WithCacheControl set Cache-Control, +// WithExpires set Expires, +// WithMeta set meta header(s), +// WithVersionID which version of this object will be set +// +// NOTICE: SetObjectMeta always overwrites all previous metadata +// +// Deprecated: use SetObjectMeta of ClientV2 instead +func (bkt *Bucket) SetObjectMeta(ctx context.Context, objectKey string, options ...Option) (*SetObjectMetaOutput, error) { + if err := isValidKey(objectKey); err != nil { + return nil, err + } + + res, err := bkt.client.newBuilder(bkt.name, objectKey, options...). + WithQuery("metadata", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodPost, nil, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + return &SetObjectMetaOutput{RequestInfo: res.RequestInfo()}, nil +} + +// SetObjectMeta overwrites metadata of the object +func (cli *ClientV2) SetObjectMeta(ctx context.Context, input *SetObjectMetaInput) (*SetObjectMetaOutput, error) { + if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, input.Key). + WithQuery("metadata", ""). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodPost, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + return &SetObjectMetaOutput{RequestInfo: res.RequestInfo()}, nil +} + +// ListObjects list objects of a bucket +// +// Deprecated: use ListObjectsV2 of ClientV2 instead +func (bkt *Bucket) ListObjects(ctx context.Context, input *ListObjectsInput, options ...Option) (*ListObjectsOutput, error) { + res, err := bkt.client.newBuilder(bkt.name, "", options...). + WithQuery("prefix", input.Prefix). + WithQuery("delimiter", input.Delimiter). + WithQuery("marker", input.Marker). + WithQuery("max-keys", strconv.Itoa(input.MaxKeys)). + WithQuery("encoding-type", input.EncodingType). + WithQuery("fetch-meta", strconv.FormatBool(input.FetchMeta)). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + internalOutput := &listObjectsOutput{} + if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &internalOutput); err != nil { + return nil, err + } + output := ListObjectsOutput{ + RequestInfo: res.RequestInfo(), + Name: internalOutput.Name, + Prefix: internalOutput.Prefix, + Marker: internalOutput.Marker, + MaxKeys: internalOutput.MaxKeys, + NextMarker: internalOutput.NextMarker, + Delimiter: internalOutput.Delimiter, + IsTruncated: internalOutput.IsTruncated, + EncodingType: internalOutput.EncodingType, + CommonPrefixes: internalOutput.CommonPrefixes, + Contents: nil, + } + contents := make([]ListedObject, 0, len(internalOutput.Contents)) + for _, content := range internalOutput.Contents { + contents = append(contents, ListedObject{ + Key: content.Key, + LastModified: content.LastModified, + ETag: content.ETag, + Size: content.Size, + Owner: content.Owner, + StorageClass: content.StorageClass, + Type: content.Type, + Meta: parseUserMetaData(content.Meta), + }) + } + output.Contents = contents + return &output, nil +} + +// ListObjectsV2 list objects of a bucket +// Deprecated: use ListObjectsType2 of ClientV2 instead +func (cli *ClientV2) ListObjectsV2(ctx context.Context, input *ListObjectsV2Input) (*ListObjectsV2Output, error) { + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + temp := listObjectsV2Output{ + RequestInfo: res.RequestInfo(), + } + if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil { + return nil, err + } + contents := make([]ListedObjectV2, 0, len(temp.Contents)) + for _, object := range temp.Contents { + var hashCrc uint64 + if len(object.HashCrc64ecma) == 0 { + hashCrc = 0 + } else { + hashCrc, err = strconv.ParseUint(object.HashCrc64ecma, 10, 64) + if err != nil { + return nil, &TosServerError{ + TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"}, + RequestInfo: RequestInfo{RequestID: temp.RequestID}, + } + } + } + + contents = append(contents, ListedObjectV2{ + Key: object.Key, + LastModified: object.LastModified, + ETag: object.ETag, + Size: object.Size, + Owner: object.Owner, + StorageClass: object.StorageClass, + HashCrc64ecma: uint64(hashCrc), + Meta: parseUserMetaData(object.Meta), + }) + } + output := ListObjectsV2Output{ + RequestInfo: temp.RequestInfo, + Name: temp.Name, + Prefix: temp.Prefix, + Marker: temp.Marker, + MaxKeys: temp.MaxKeys, + NextMarker: temp.NextMarker, + Delimiter: temp.Delimiter, + IsTruncated: temp.IsTruncated, + EncodingType: temp.EncodingType, + CommonPrefixes: temp.CommonPrefixes, + Contents: contents, + } + return &output, nil +} + +func (cli *ClientV2) listObjectsType2(ctx context.Context, input *ListObjectsType2Input) (*ListObjectsType2Output, error) { + res, err := cli.newBuilder(input.Bucket, ""). + WithParams(*input). + WithQuery("list-type", "2"). + WithQuery("fetch-owner", "true"). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + temp := listObjectsType2Output{ + RequestInfo: res.RequestInfo(), + } + if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil { + return nil, err + } + contents := make([]ListedObjectV2, 0, len(temp.Contents)) + for _, object := range temp.Contents { + var hashCrc uint64 + if len(object.HashCrc64ecma) == 0 { + hashCrc = 0 + } else { + hashCrc, err = strconv.ParseUint(object.HashCrc64ecma, 10, 64) + if err != nil { + return nil, &TosServerError{ + TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"}, + RequestInfo: RequestInfo{RequestID: temp.RequestID}, + } + } + } + contents = append(contents, ListedObjectV2{ + Key: object.Key, + LastModified: object.LastModified, + ETag: object.ETag, + Size: object.Size, + Owner: object.Owner, + StorageClass: object.StorageClass, + HashCrc64ecma: hashCrc, + Meta: parseUserMetaData(object.Meta), + }) + } + output := ListObjectsType2Output{ + RequestInfo: temp.RequestInfo, + Name: temp.Name, + ContinuationToken: temp.ContinuationToken, + Prefix: temp.Prefix, + MaxKeys: temp.MaxKeys, + KeyCount: temp.KeyCount, + Delimiter: temp.Delimiter, + IsTruncated: temp.IsTruncated, + EncodingType: temp.EncodingType, + CommonPrefixes: temp.CommonPrefixes, + NextContinuationToken: temp.NextContinuationToken, + Contents: contents, + } + return &output, nil +} + +func (cli *ClientV2) ListObjectsType2(ctx context.Context, input *ListObjectsType2Input) (*ListObjectsType2Output, error) { + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + copyInput := *input + input = ©Input + if input.MaxKeys == 0 { + input.MaxKeys = DefaultListMaxKeys + } + if input.ListOnlyOnce { + return cli.listObjectsType2(ctx, input) + } + var output *ListObjectsType2Output + for { + res, err := cli.listObjectsType2(ctx, input) + if err != nil { + return nil, err + } + if output == nil { + output = res + } else { + output.KeyCount += res.KeyCount + output.IsTruncated = res.IsTruncated + output.NextContinuationToken = res.NextContinuationToken + output.Contents = append(output.Contents, res.Contents...) + output.CommonPrefixes = append(output.CommonPrefixes, res.CommonPrefixes...) + } + if !res.IsTruncated || len(res.Contents) >= input.MaxKeys { + break + } + input.ContinuationToken = res.NextContinuationToken + input.MaxKeys = input.MaxKeys - res.KeyCount + } + + return output, nil +} + +// ListObjectVersions list multi-version objects of a bucket +// +// Deprecated: use ListObjectV2Versions of ClientV2 instead +func (bkt *Bucket) ListObjectVersions(ctx context.Context, input *ListObjectVersionsInput, options ...Option) (*ListObjectVersionsOutput, error) { + res, err := bkt.client.newBuilder(bkt.name, "", options...). + WithQuery("prefix", input.Prefix). + WithQuery("delimiter", input.Delimiter). + WithQuery("key-marker", input.KeyMarker). + WithQuery("max-keys", strconv.Itoa(input.MaxKeys)). + WithQuery("encoding-type", input.EncodingType). + WithQuery("fetch-meta", strconv.FormatBool(input.FetchMeta)). + WithQuery("versions", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + interOutput := listObjectVersionsOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(interOutput.RequestID, res.Body, &interOutput); err != nil { + return nil, err + } + output := ListObjectVersionsOutput{ + RequestInfo: interOutput.RequestInfo, + Name: interOutput.Name, + Prefix: interOutput.Prefix, + KeyMarker: interOutput.KeyMarker, + VersionIDMarker: interOutput.VersionIDMarker, + Delimiter: interOutput.Delimiter, + EncodingType: interOutput.EncodingType, + MaxKeys: interOutput.MaxKeys, + NextKeyMarker: interOutput.NextKeyMarker, + NextVersionIDMarker: interOutput.NextVersionIDMarker, + IsTruncated: interOutput.IsTruncated, + CommonPrefixes: interOutput.CommonPrefixes, + DeleteMarkers: interOutput.DeleteMarkers, + } + + contents := make([]ListedObjectVersion, 0, len(interOutput.Versions)) + for _, content := range interOutput.Versions { + contents = append(contents, ListedObjectVersion{ + Key: content.Key, + IsLatest: content.IsLatest, + LastModified: content.LastModified, + ETag: content.ETag, + Size: content.Size, + Owner: content.Owner, + StorageClass: content.StorageClass, + Type: content.Type, + VersionID: content.VersionID, + Meta: parseUserMetaData(content.Meta), + }) + } + output.Versions = contents + + return &output, nil +} + +// ListObjectVersionsV2 list multi-version objects of a bucket +func (cli *ClientV2) ListObjectVersionsV2( + ctx context.Context, + input *ListObjectVersionsV2Input) (*ListObjectVersionsV2Output, error) { + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithParams(*input). + WithQuery("versions", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + temp := listObjectVersionsV2Output{RequestInfo: res.RequestInfo()} + + if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil { + return nil, err + } + versions := make([]ListedObjectVersionV2, 0, len(temp.Versions)) + for _, version := range temp.Versions { + var hashCrc uint64 + if len(version.HashCrc64ecma) == 0 { + hashCrc = 0 + } else { + hashCrc, err = strconv.ParseUint(version.HashCrc64ecma, 10, 64) + if err != nil { + return nil, &TosServerError{ + TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"}, + RequestInfo: RequestInfo{RequestID: temp.RequestID}, + } + } + } + versions = append(versions, ListedObjectVersionV2{ + Key: version.Key, + LastModified: version.LastModified, + ETag: version.ETag, + IsLatest: version.IsLatest, + Size: version.Size, + Owner: version.Owner, + StorageClass: version.StorageClass, + VersionID: version.VersionID, + HashCrc64ecma: hashCrc, + Meta: parseUserMetaData(version.Meta), + }) + } + output := ListObjectVersionsV2Output{ + RequestInfo: temp.RequestInfo, + Name: temp.Name, + Prefix: temp.Prefix, + KeyMarker: temp.KeyMarker, + VersionIDMarker: temp.VersionIDMarker, + Delimiter: temp.Delimiter, + EncodingType: temp.EncodingType, + MaxKeys: temp.MaxKeys, + NextKeyMarker: temp.NextKeyMarker, + NextVersionIDMarker: temp.NextVersionIDMarker, + IsTruncated: temp.IsTruncated, + CommonPrefixes: temp.CommonPrefixes, + DeleteMarkers: temp.DeleteMarkers, + Versions: versions, + } + + return &output, nil +} + +func (cli *ClientV2) RestoreObject(ctx context.Context, input *RestoreObjectInput) (*RestoreObjectOutput, error) { + return cli.baseClient.RestoreObject(ctx, input) +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/options.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/options.go new file mode 100644 index 0000000000..048e7371d3 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/options.go @@ -0,0 +1,274 @@ +package tos + +import ( + "net/http" + "time" +) + +type Option func(*requestBuilder) + +// WithContentType set Content-Type header +// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta +func WithContentType(contentType string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderContentType, contentType) + } +} + +// WithContentLength set Content-Length header +// used in Bucket.PutObject Bucket.AppendObject Bucket.UploadPart +// +// If the length of the content is known, it is better to add this option when Put, Append or Upload. +func WithContentLength(length int64) Option { + return func(rb *requestBuilder) { + rb.WithContentLength(length) + } +} + +// WithCacheControl set Cache-Control header +// used in Bucket.PutObject Bucket.AppendObject +// Bucket.CreateMultipartUpload Bucket.SetObjectMeta +func WithCacheControl(cacheControl string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderCacheControl, cacheControl) + } +} + +// WithContentDisposition set Content-Disposition header +// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta +func WithContentDisposition(contentDisposition string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderContentDisposition, contentDisposition) + } +} + +// WithContentEncoding set Content-Encoding header +// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta +func WithContentEncoding(contentEncoding string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderContentEncoding, contentEncoding) + } +} + +// WithContentLanguage set Content-Language header +// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta +func WithContentLanguage(contentLanguage string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderContentLanguage, contentLanguage) + } +} + +// WithContentMD5 set Content-MD5 header +func WithContentMD5(contentMD5 string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderContentMD5, contentMD5) + } +} + +// WithContentSHA256 set X-Tos-Content-Sha256 header +func WithContentSHA256(contentSHA256 string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderContentSha256, contentSHA256) + } +} + +// WithExpires set Expires header +// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta +func WithExpires(expires time.Time) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderExpires, expires.Format(http.TimeFormat)) + } +} + +// WithServerSideEncryptionCustomer set server-side-encryption parameters +// used in Bucket.PutObject Bucket.CreateMultipartUpload +func WithServerSideEncryptionCustomer(ssecAlgorithm, ssecKey, ssecKeyMD5 string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderSSECustomerAlgorithm, ssecAlgorithm) + rb.Header.Set(HeaderSSECustomerKey, ssecKey) + rb.Header.Set(HeaderSSECustomerKeyMD5, ssecKeyMD5) + } +} + +// WithIfModifiedSince set If-Modified-Since header +// used in Bucket.GetObject Bucket.HeadObject +func WithIfModifiedSince(since time.Time) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderIfModifiedSince, since.Format(http.TimeFormat)) + } +} + +// WithIfUnmodifiedSince set If-Unmodified-Since header +// used in Bucket.GetObject Bucket.HeadObject +func WithIfUnmodifiedSince(since time.Time) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderIfUnmodifiedSince, since.Format(http.TimeFormat)) + } +} + +// WithIfMatch set If-Match header +func WithIfMatch(ifMatch string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderIfMatch, ifMatch) + } +} + +// WithIfNoneMatch set If-None-Match header +func WithIfNoneMatch(ifNoneMatch string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderIfNoneMatch, ifNoneMatch) + } +} + +// WithCopySourceIfMatch set X-Tos-Copy-Source-If-Match header +// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom Bucket.UploadPartCopy +func WithCopySourceIfMatch(ifMatch string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderCopySourceIfMatch, ifMatch) + } +} + +// WithCopySourceIfNoneMatch set X-Tos-Copy-Source-If-None-Match +// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom Bucket.UploadPartCopy +func WithCopySourceIfNoneMatch(ifNoneMatch string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderCopySourceIfNoneMatch, ifNoneMatch) + } +} + +// WithCopySourceIfModifiedSince set X-Tos-Copy-Source-If-Modified-Since header +// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom Bucket.UploadPartCopy +func WithCopySourceIfModifiedSince(ifModifiedSince string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderCopySourceIfModifiedSince, ifModifiedSince) + } +} + +// WithCopySourceIfUnmodifiedSince set X-Tos-Copy-Source-If-Unmodified-Since header +// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom Bucket.UploadPartCopy +func WithCopySourceIfUnmodifiedSince(ifUnmodifiedSince string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderCopySourceIfUnmodifiedSince, ifUnmodifiedSince) + } +} + +// WithMeta set meta header +// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject Bucket.SetObjectMeta +func WithMeta(key, value string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderMetaPrefix+key, value) + } +} + +// WithRange set Range header +// used in Bucket.GetObject Bucket.HeadObject +func WithRange(start, end int64) Option { + return func(rb *requestBuilder) { + rb.Range = &Range{Start: start, End: end} + rb.Header.Set(HeaderRange, rb.Range.String()) + } +} + +// WithVersionID set version parameter +// used in Bucket.GetObject Bucket.HeadObject Bucket.DeleteObject +// Bucket.GetObjectAcl Bucket.SetObjectMeta +// Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom +// Client.PreSignedURL +func WithVersionID(versionID string) Option { + return func(rb *requestBuilder) { + rb.Query.Add("versionId", versionID) + } +} + +// WithMetadataDirective set X-Tos-Metadata-Directive header +// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom +func WithMetadataDirective(directive string) Option { + return func(rb *requestBuilder) { + rb.Header.Add(HeaderMetadataDirective, directive) + } +} + +// WithACL set X-Tos-Acl header +// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject +func WithACL(acl string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderACL, acl) + } +} + +// WithACLGrantFullControl X-Tos-Grant-Full-Control header +// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject +func WithACLGrantFullControl(grantFullControl string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderGrantFullControl, grantFullControl) + } +} + +// WithACLGrantRead set X-Tos-Grant-Read header +// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject +func WithACLGrantRead(grantRead string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderGrantRead, grantRead) + } +} + +// WithACLGrantReadAcp set X-Tos-Grant-Read-Acp header +// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject +func WithACLGrantReadAcp(grantReadAcp string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderGrantReadAcp, grantReadAcp) + } +} + +// WithACLGrantWrite set X-Tos-Grant-Write header +// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject +func WithACLGrantWrite(grantWrite string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderGrantWrite, grantWrite) + } +} + +// WithACLGrantWriteAcp set X-Tos-Grant-Write-Acp header +// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject +func WithACLGrantWriteAcp(grantWriteAcp string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderGrantWriteAcp, grantWriteAcp) + } +} + +// WithWebsiteRedirectLocation set X-Tos-Website-Redirect-Location header +func WithWebsiteRedirectLocation(redirectLocation string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(HeaderWebsiteRedirectLocation, redirectLocation) + } +} + +// WithPerRequestSigner set Signer for a request +// +// use this option when you need set request-level signature parameter(s). +// for example, use different ak and sk for each request. +// +// if 'signer' set to nil, the request will not be signed. +func WithPerRequestSigner(signer Signer) Option { + return func(rb *requestBuilder) { + rb.Signer = signer + } +} + +// WithHeader add request http header. +// +// NOTICE: use it carefully. +func WithHeader(key, value string) Option { + return func(rb *requestBuilder) { + rb.Header.Set(key, value) + } +} + +// WithQuery add request query parameter +// +// NOTICE: use it carefully. +func WithQuery(key, value string) Option { + return func(rb *requestBuilder) { + rb.Query.Set(key, value) + } +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/parse_output.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/parse_output.go new file mode 100644 index 0000000000..7e3cf2ab09 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/parse_output.go @@ -0,0 +1,249 @@ +package tos + +import ( + "net/http" + "strconv" +) + +// ParseListObjectsType2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *ListObjectsType2Output type. +func ParseListObjectsType2Output(httpRes *http.Response) (*ListObjectsType2Output, error) { + + res := &Response{ + StatusCode: httpRes.StatusCode, + ContentLength: httpRes.ContentLength, + Header: httpRes.Header, + Body: httpRes.Body, + } + + err := checkError(res, true, 200) + if err != nil { + return nil, err + } + defer res.Close() + + temp := listObjectsType2Output{ + RequestInfo: res.RequestInfo(), + } + if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil { + return nil, err + } + contents := make([]ListedObjectV2, 0, len(temp.Contents)) + for _, object := range temp.Contents { + var hashCrc uint64 + if len(object.HashCrc64ecma) == 0 { + hashCrc = 0 + } else { + hashCrc, err = strconv.ParseUint(object.HashCrc64ecma, 10, 64) + if err != nil { + return nil, &TosServerError{ + TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"}, + RequestInfo: RequestInfo{RequestID: temp.RequestID}, + } + } + } + contents = append(contents, ListedObjectV2{ + Key: object.Key, + LastModified: object.LastModified, + ETag: object.ETag, + Size: object.Size, + Owner: object.Owner, + StorageClass: object.StorageClass, + HashCrc64ecma: hashCrc, + }) + } + output := ListObjectsType2Output{ + RequestInfo: temp.RequestInfo, + Name: temp.Name, + ContinuationToken: temp.ContinuationToken, + Prefix: temp.Prefix, + MaxKeys: temp.MaxKeys, + KeyCount: temp.KeyCount, + Delimiter: temp.Delimiter, + IsTruncated: temp.IsTruncated, + EncodingType: temp.EncodingType, + CommonPrefixes: temp.CommonPrefixes, + NextContinuationToken: temp.NextContinuationToken, + Contents: contents, + } + return &output, nil +} + +// ParseListObjectsV2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *ListObjectsV2Output type. +func ParseListObjectsV2Output(httpRes *http.Response) (*ListObjectsV2Output, error) { + + res := &Response{ + StatusCode: httpRes.StatusCode, + ContentLength: httpRes.ContentLength, + Header: httpRes.Header, + Body: httpRes.Body, + } + + err := checkError(res, true, 200) + if err != nil { + return nil, err + } + defer res.Close() + + temp := listObjectsV2Output{ + RequestInfo: res.RequestInfo(), + } + if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil { + return nil, err + } + contents := make([]ListedObjectV2, 0, len(temp.Contents)) + for _, object := range temp.Contents { + var hashCrc uint64 + if len(object.HashCrc64ecma) == 0 { + hashCrc = 0 + } else { + hashCrc, err = strconv.ParseUint(object.HashCrc64ecma, 10, 64) + if err != nil { + return nil, &TosServerError{ + TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"}, + RequestInfo: RequestInfo{RequestID: temp.RequestID}, + } + } + } + contents = append(contents, ListedObjectV2{ + Key: object.Key, + LastModified: object.LastModified, + ETag: object.ETag, + Size: object.Size, + Owner: object.Owner, + StorageClass: object.StorageClass, + HashCrc64ecma: uint64(hashCrc), + }) + } + output := ListObjectsV2Output{ + RequestInfo: temp.RequestInfo, + Name: temp.Name, + Prefix: temp.Prefix, + Marker: temp.Marker, + MaxKeys: temp.MaxKeys, + NextMarker: temp.NextMarker, + Delimiter: temp.Delimiter, + IsTruncated: temp.IsTruncated, + EncodingType: temp.EncodingType, + CommonPrefixes: temp.CommonPrefixes, + Contents: contents, + } + return &output, nil +} + +// ParseListObjectVersionsV2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *ListObjectVersionsV2Output type. +func ParseListObjectVersionsV2Output(httpRes *http.Response) (*ListObjectVersionsV2Output, error) { + + res := &Response{ + StatusCode: httpRes.StatusCode, + ContentLength: httpRes.ContentLength, + Header: httpRes.Header, + Body: httpRes.Body, + } + + err := checkError(res, true, 200) + if err != nil { + return nil, err + } + defer res.Close() + + temp := listObjectVersionsV2Output{RequestInfo: res.RequestInfo()} + + if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil { + return nil, err + } + versions := make([]ListedObjectVersionV2, 0, len(temp.Versions)) + for _, version := range temp.Versions { + var hashCrc uint64 + if len(version.HashCrc64ecma) == 0 { + hashCrc = 0 + } else { + hashCrc, err = strconv.ParseUint(version.HashCrc64ecma, 10, 64) + if err != nil { + return nil, &TosServerError{ + TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"}, + RequestInfo: RequestInfo{RequestID: temp.RequestID}, + } + } + } + versions = append(versions, ListedObjectVersionV2{ + Key: version.Key, + LastModified: version.LastModified, + ETag: version.ETag, + IsLatest: version.IsLatest, + Size: version.Size, + Owner: version.Owner, + StorageClass: version.StorageClass, + VersionID: version.VersionID, + HashCrc64ecma: hashCrc, + }) + } + output := ListObjectVersionsV2Output{ + RequestInfo: temp.RequestInfo, + Name: temp.Name, + Prefix: temp.Prefix, + KeyMarker: temp.KeyMarker, + VersionIDMarker: temp.VersionIDMarker, + Delimiter: temp.Delimiter, + EncodingType: temp.EncodingType, + MaxKeys: temp.MaxKeys, + NextKeyMarker: temp.NextKeyMarker, + NextVersionIDMarker: temp.NextVersionIDMarker, + IsTruncated: temp.IsTruncated, + CommonPrefixes: temp.CommonPrefixes, + DeleteMarkers: temp.DeleteMarkers, + Versions: versions, + } + + return &output, nil +} + +// ParseHeadObjectV2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *HeadObjectV2Output type. +func ParseHeadObjectV2Output(httpRes *http.Response) (*HeadObjectV2Output, error) { + + res := &Response{ + StatusCode: httpRes.StatusCode, + ContentLength: httpRes.ContentLength, + Header: httpRes.Header, + Body: httpRes.Body, + } + + err := checkError(res, false, 200) + if err != nil { + return nil, err + } + defer res.Close() + + output := HeadObjectV2Output{ + RequestInfo: res.RequestInfo(), + } + output.ObjectMetaV2.fromResponseV2(res) + return &output, nil +} + +// ParseGetObjectV2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *GetObjectV2Output type. +func ParseGetObjectV2Output(httpRes *http.Response, expectedCode int) (*GetObjectV2Output, error) { + + res := &Response{ + StatusCode: httpRes.StatusCode, + ContentLength: httpRes.ContentLength, + Header: httpRes.Header, + Body: httpRes.Body, + } + + err := checkError(res, true, expectedCode) + if err != nil { + return nil, err + } + + basic := GetObjectBasicOutput{ + RequestInfo: res.RequestInfo(), + ContentRange: res.Header.Get(HeaderContentRange), + } + basic.ObjectMetaV2.fromResponseV2(res) + + output := GetObjectV2Output{ + GetObjectBasicOutput: basic, + Content: wrapReader(res.Body, res.ContentLength, nil, nil, nil), + } + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/policy.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/policy.go new file mode 100644 index 0000000000..c8589679c1 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/policy.go @@ -0,0 +1,180 @@ +package tos + +import ( + "bytes" + "context" + "io/ioutil" + "net/http" + "strings" +) + +type BucketPolicy struct { + Policy string `json:"Policy,omitempty"` +} + +type GetBucketPolicyOutput struct { + RequestInfo `json:"-"` + Policy string `json:"Policy,omitempty"` +} + +type PutBucketPolicyOutput struct { + RequestInfo `json:"-"` +} + +type DeleteBucketPolicyOutput struct { + RequestInfo `json:"-"` +} + +type GetBucketPolicyV2Input struct { + Bucket string `json:"-"` +} + +type GetBucketPolicyV2Output struct { + RequestInfo `json:"-"` + Policy string `json:"Policy,omitempty"` +} + +type putBucketPolicyV2Input struct { + Policy string `json:"Policy,omitempty"` +} + +type PutBucketPolicyV2Input struct { + Bucket string `json:"-"` + Policy string `json:"Policy,omitempty"` +} + +type PutBucketPolicyV2Output struct { + RequestInfo `json:"-"` +} + +type DeleteBucketPolicyV2Input struct { + Bucket string `json:"-"` +} +type DeleteBucketPolicyV2Output struct { + RequestInfo +} + +// GetBucketPolicy get bucket access policy +func (cli *Client) GetBucketPolicy(ctx context.Context, bucket string) (*GetBucketPolicyOutput, error) { + if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + res, err := cli.newBuilder(bucket, ""). + WithQuery("policy", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + return &GetBucketPolicyOutput{ + RequestInfo: res.RequestInfo(), + Policy: string(data), + }, nil +} + +// PutBucketPolicy set bucket access policy +func (cli *Client) PutBucketPolicy(ctx context.Context, bucket string, policy *BucketPolicy) (*PutBucketPolicyOutput, error) { + if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(bucket, ""). + WithQuery("policy", ""). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, strings.NewReader(policy.Policy), cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + return &PutBucketPolicyOutput{RequestInfo: res.RequestInfo()}, nil +} + +// DeleteBucketPolicy delete bucket access policy +func (cli *Client) DeleteBucketPolicy(ctx context.Context, bucket string) (*DeleteBucketPolicyOutput, error) { + if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + res, err := cli.newBuilder(bucket, ""). + WithQuery("policy", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + return &DeleteBucketPolicyOutput{RequestInfo: res.RequestInfo()}, nil +} + +func (cli *ClientV2) PutBucketPolicyV2(ctx context.Context, input *PutBucketPolicyV2Input) (*PutBucketPolicyV2Output, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("policy", ""). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader([]byte(input.Policy)), cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketPolicyV2Output{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketPolicyV2(ctx context.Context, input *GetBucketPolicyV2Input) (*GetBucketPolicyV2Output, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("policy", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketPolicyV2Output{RequestInfo: res.RequestInfo()} + data, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + output.Policy = string(data) + return &output, nil +} + +func (cli *ClientV2) DeleteBucketPolicyV2(ctx context.Context, input *DeleteBucketPolicyV2Input) (*DeleteBucketPolicyV2Output, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("policy", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteBucketPolicyV2Output{RequestInfo: res.RequestInfo()} + return &output, nil + +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/proxy.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/proxy.go new file mode 100644 index 0000000000..63317919ab --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/proxy.go @@ -0,0 +1,55 @@ +package tos + +import ( + "net/url" + "strconv" +) + +type Proxy struct { + proxyHost string + proxyUserName string + proxyPassword string + proxyPort int +} + +func (p *Proxy) getRawUrl() string { + return p.proxyHost + ":" + strconv.Itoa(p.proxyPort) +} + +func (p *Proxy) Url() *url.URL { + proxyURL, _ := url.Parse(p.getRawUrl()) + if p.proxyUserName != "" && p.proxyPassword != "" { + proxyURL.User = url.UserPassword(p.proxyUserName, p.proxyPassword) + } else if p.proxyUserName != "" { + proxyURL.User = url.User(p.proxyUserName) + } + return proxyURL +} + +func NewProxy(proxyHost string, proxyPort int) (*Proxy, error) { + proxyUrl, err := url.Parse(proxyHost) + if err != nil { + return nil, ProxyUrlInvalid.withCause(err) + } + if proxyUrl.Scheme == "" { + proxyHost = "http://" + proxyHost + } + if proxyUrl.Scheme == "https" { + return nil, ProxyNotSupportHttps + } + + if _, err := url.Parse(proxyHost + ":" + strconv.Itoa(proxyPort)); err != nil { + if err != nil { + return nil, err + } + } + return &Proxy{ + proxyHost: proxyHost, + proxyPort: proxyPort, + }, nil +} + +func (p *Proxy) WithAuth(username string, password string) { + p.proxyUserName = username + p.proxyPassword = password +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rate_limiter.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rate_limiter.go new file mode 100644 index 0000000000..db8f1ef11e --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rate_limiter.go @@ -0,0 +1,58 @@ +package tos + +import ( + "sync" + "time" +) + +const ( + minRate = 1024 + minCapacity = 10 * 1024 +) + +type defaultRateLimit struct { + rate int64 + capacity int64 + + currentAmount int64 + sync.Mutex + lastConsumeTime time.Time +} + +func NewDefaultRateLimit(rate int64, capacity int64) RateLimiter { + + if rate < minRate { + rate = minRate + } + if capacity < minCapacity { + capacity = minCapacity + } + return &defaultRateLimit{ + rate: rate, + capacity: capacity, + lastConsumeTime: time.Now(), + currentAmount: capacity, + Mutex: sync.Mutex{}, + } +} + +func (d *defaultRateLimit) Acquire(want int64) (ok bool, timeToWait time.Duration) { + d.Lock() + defer d.Unlock() + if want > d.capacity { + want = d.capacity + } + increment := int64(time.Since(d.lastConsumeTime).Seconds() * float64(d.rate)) + if increment+d.currentAmount > d.capacity { + d.currentAmount = d.capacity + } else { + d.currentAmount += increment + } + if want > d.currentAmount { + timeToWaitSec := float64(want-d.currentAmount) / float64(d.rate) + return false, time.Duration(timeToWaitSec * float64(time.Second)) + } + d.lastConsumeTime = time.Now() + d.currentAmount -= want + return true, 0 +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/realtime_log.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/realtime_log.go new file mode 100644 index 0000000000..a12b065a49 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/realtime_log.go @@ -0,0 +1,76 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +func (cli *ClientV2) PutBucketRealTimeLog(ctx context.Context, input *PutBucketRealTimeLogInput) (*PutBucketRealTimeLogOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + body := putBucketRealTimeLogInput{ + Configuration: input.Configuration, + } + data, contentMD5, err := marshalInput("PutBucketRealTimeLogInput", body) + if err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("realtimeLog", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketRealTimeLogOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketRealTimeLog(ctx context.Context, input *GetBucketRealTimeLogInput) (*GetBucketRealTimeLogOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("realtimeLog", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketRealTimeLogOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) DeleteBucketRealTimeLog(ctx context.Context, input *DeleteBucketRealTimeLogInput) (*DeleteBucketRealTimeLogOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("realtimeLog", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteBucketRealTimeLogOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rename.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rename.go new file mode 100644 index 0000000000..93cc1ba162 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rename.go @@ -0,0 +1,129 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +type PutBucketRenameInput struct { + Bucket string `json:"-"` + RenameEnable bool `json:"RenameEnable"` +} + +type PutBucketRenameOutput struct { + RequestInfo +} + +type GetBucketRenameInput struct { + Bucket string +} + +type GetBucketRenameOutput struct { + RequestInfo + RenameEnable bool +} + +type DeleteBucketRenameInput struct { + Bucket string +} + +type DeleteBucketRenameOutput struct { + RequestInfo +} + +func (cli *ClientV2) PutBucketRename(ctx context.Context, input *PutBucketRenameInput) (*PutBucketRenameOutput, error) { + + if input == nil { + return nil, InputIsNilClientError + } + + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + data, contentMD5, err := marshalInput("PutBucketRename", input) + if err != nil { + return nil, err + } + + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("rename", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketRenameOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketRename(ctx context.Context, input *GetBucketRenameInput) (*GetBucketRenameOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + req := cli.newBuilder(input.Bucket, ""). + WithQuery("rename", ""). + WithRetry(nil, StatusCodeClassifier{}) + + res, err := req.Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketRenameOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) DeleteBucketRename(ctx context.Context, input *DeleteBucketRenameInput) (*DeleteBucketRenameOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("rename", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteBucketRenameOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) RenameObject(ctx context.Context, input *RenameObjectInput) (*RenameObjectOutput, error) { + + if input == nil { + return nil, InputIsNilClientError + } + + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, input.Key). + WithQuery("rename", ""). + WithParams(*input). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := RenameObjectOutput{RequestInfo: res.RequestInfo()} + + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/replication.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/replication.go new file mode 100644 index 0000000000..4c740966f7 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/replication.go @@ -0,0 +1,79 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +func (cli *ClientV2) PutBucketReplication(ctx context.Context, input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + data, contentMD5, err := marshalInput("PutBucketReplication", putBucketReplicationInput{Role: input.Role, Rules: input.Rules}) + if err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("replication", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketReplicationOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketReplication(ctx context.Context, input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + req := cli.newBuilder(input.Bucket, ""). + WithQuery("replication", ""). + WithQuery("progress", ""). + WithRetry(nil, StatusCodeClassifier{}) + + if input.RuleID != "" { + req.WithQuery("rule-id", input.RuleID) + } + res, err := req.Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketReplicationOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) DeleteBucketReplication(ctx context.Context, input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("replication", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteBucketReplicationOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/request.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/request.go new file mode 100644 index 0000000000..93d630adf7 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/request.go @@ -0,0 +1,425 @@ +package tos + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "reflect" + "strconv" + "strings" + "time" +) + +type urlMode int + +const ( + // urlModePath url pattern is http(s)://{bucket}.domain/{object} + urlModeDefault = 0 + // urlModePath url pattern is http(s)://domain/{bucket}/{object} + urlModePath = 1 +) + +type Request struct { + Scheme string + Method string + Host string + Path string + ContentLength *int64 + Content io.Reader + Query url.Values + Header http.Header +} + +func (req *Request) URL() string { + u := url.URL{ + Scheme: req.Scheme, + Host: req.Host, + Path: req.Path, + RawQuery: req.Query.Encode(), + } + return u.String() +} + +func OnRetryFromStart(req *Request) error { + if seek, ok := req.Content.(io.Seeker); ok { + _, err := seek.Seek(0, io.SeekStart) + return err + } + return nil +} + +// Range represents a range of an object +type Range struct { + Start int64 + End int64 +} + +// HTTP Range header +func (hr *Range) String() string { + return fmt.Sprintf("bytes=%d-%d", hr.Start, hr.End) +} + +type CopySource struct { + srcBucket string + srcObjectKey string +} + +type requestBuilder struct { + Signer Signer + Scheme string + Host string + Bucket string + Object string + URLMode urlMode + ContentLength *int64 + Range *Range + Query url.Values + Header http.Header + Retry *retryer + OnRetry func(req *Request) error + Classifier classifier + CopySource *CopySource + IsCustomDomain bool + // CheckETag bool + // CheckCRC32 bool +} + +func (rb *requestBuilder) WithRetry(onRetry func(req *Request) error, classifier classifier) *requestBuilder { + if onRetry == nil { + rb.OnRetry = func(req *Request) error { return nil } + } else { + rb.OnRetry = onRetry + } + if classifier == nil { + rb.Classifier = NoRetryClassifier{} + } else { + rb.Classifier = classifier + } + return rb +} + +func (rb *requestBuilder) WithCopySource(srcBucket, srcObjectKey string) *requestBuilder { + rb.CopySource = &CopySource{ + srcBucket: srcBucket, + srcObjectKey: srcObjectKey, + } + return rb +} + +func (rb *requestBuilder) WithQuery(key, value string) *requestBuilder { + rb.Query.Add(key, value) + return rb +} + +func (rb *requestBuilder) WithHeader(key, value string) *requestBuilder { + if len(value) > 0 { + rb.Header.Set(key, value) + } + return rb +} + +func convertToString(iface interface{}, tag *reflect.StructTag) string { + // return empty string if value is zero except filed with "default" tag + var result string + switch v := iface.(type) { + case string: + result = v + case int: + if v != 0 { + result = strconv.Itoa(v) + } else { + result = tag.Get("default") + } + case int64: + if v != 0 { + result = strconv.Itoa(int(v)) + } else { + result = tag.Get("default") + } + case time.Time: + if !v.IsZero() { + result = v.Format(http.TimeFormat) + } + case bool: + result = strconv.FormatBool(v) + default: + if reflect.TypeOf(iface).Kind() == reflect.String { + result = reflect.ValueOf(iface).String() + } + } + return result +} + +// WithParams will set filed with tag "header" in input to rb.Header. +func (rb *requestBuilder) WithParams(input interface{}) *requestBuilder { + + t := reflect.TypeOf(input) + v := reflect.ValueOf(input) + for i := 0; i < v.NumField(); i++ { + filed := t.Field(i) + if filed.Type.Kind() == reflect.Struct { + rb.WithParams(v.Field(i).Interface()) + } + location := filed.Tag.Get("location") + switch location { + case "header": + value := convertToString(v.Field(i).Interface(), &filed.Tag) + if filed.Tag.Get("encodeChinese") == "true" { + value = headerEncode(value) + } + rb.WithHeader(filed.Tag.Get("locationName"), value) + case "headers": + if headers, ok := v.Field(i).Interface().(map[string]string); ok { + for k, v := range headers { + rb.Header.Set(HeaderMetaPrefix+headerEncode(k), headerEncode(v)) + } + return rb + } + case "query": + v := convertToString(v.Field(i).Interface(), &filed.Tag) + if len(v) > 0 { + rb.WithQuery(filed.Tag.Get("locationName"), v) + } + } + } + return rb +} + +func (rb *requestBuilder) WithContentLength(length int64) *requestBuilder { + rb.ContentLength = &length + return rb +} + +func (rb *requestBuilder) hostPath() (string, string) { + + if rb.IsCustomDomain { + if len(rb.Object) > 0 { + return rb.Host, "/" + rb.Object + } + return rb.Host, "/" + } + + if rb.URLMode == urlModePath { + if len(rb.Object) > 0 { + return rb.Host, "/" + rb.Bucket + "/" + rb.Object + } + return rb.Host, "/" + rb.Bucket // rb.Bucket may be empty "" + } + // URLModeDefault + if len(rb.Bucket) == 0 { + return rb.Host, "/" + } + return rb.Bucket + "." + rb.Host, "/" + rb.Object +} + +func (rb *requestBuilder) build(method string, content io.Reader) *Request { + host, path := rb.hostPath() + req := &Request{ + Scheme: rb.Scheme, + Method: method, + Host: host, + Path: path, + Content: content, + Query: rb.Query, + Header: rb.Header, + } + + if content != nil { + if rb.ContentLength != nil { + req.ContentLength = rb.ContentLength + } else if length := tryResolveLength(content); length >= 0 { + req.ContentLength = &length + } + } + return req +} + +func (rb *requestBuilder) Build(method string, content io.Reader) *Request { + req := rb.build(method, content) + if rb.CopySource != nil { + versionID := req.Query.Get("versionId") + req.Query.Del("versionId") + req.Header.Add(HeaderCopySource, copySource(rb.CopySource.srcBucket, rb.CopySource.srcObjectKey, versionID)) + } + if rb.Signer != nil { + signed := rb.Signer.SignHeader(req) + for key, values := range signed { + req.Header[key] = values + } + } + return req +} + +type roundTripper func(ctx context.Context, req *Request) (*Response, error) + +func (rb *requestBuilder) Request(ctx context.Context, method string, + content io.Reader, roundTripper roundTripper) (*Response, error) { + + var ( + req *Request + res *Response + err error + ) + + req = rb.Build(method, content) + + if rb.Retry != nil { + work := func() (err error) { + err = rb.OnRetry(req) + if err != nil { + return err + } + res, err = roundTripper(ctx, req) + return err + } + err = rb.Retry.Run(ctx, work, rb.Classifier) + if err != nil { + return nil, err + } + return res, err + } + res, err = roundTripper(ctx, req) + + return res, err +} + +func (rb *requestBuilder) PreSignedURL(method string, ttl time.Duration) (string, error) { + req := rb.build(method, nil) + if rb.Signer == nil { + return "", errors.New("tos: credentials is not set when the tos.Client was created") + } + + query := rb.Signer.SignQuery(req, ttl) + for k, v := range query { + req.Query[k] = v + } + return req.URL(), nil +} + +type RequestInfo struct { + RequestID string + ID2 string + StatusCode int + Header http.Header +} + +type Response struct { + StatusCode int + ContentLength int64 + Header http.Header + Body io.ReadCloser +} + +func (r *Response) RequestInfo() RequestInfo { + return RequestInfo{ + RequestID: r.Header.Get(HeaderRequestID), + ID2: r.Header.Get(HeaderID2), + StatusCode: r.StatusCode, + Header: r.Header, + } +} + +func (r *Response) Close() error { + if r.Body != nil { + return r.Body.Close() + } + return nil +} + +func marshalOutput(requestID string, reader io.Reader, output interface{}) error { + // Although status code is ok, we need to check if response body is valid. + // If response body is invalid, TosServerError should be raised. But we can't + // unmarshal error from response body now. + data, err := ioutil.ReadAll(reader) + if err != nil { + return &TosServerError{ + TosError: TosError{Message: "tos: unmarshal response body failed."}, + RequestInfo: RequestInfo{RequestID: requestID}, + } + } + data = bytes.TrimSpace(data) + if len(data) == 0 { + return &TosServerError{ + TosError: TosError{Message: "server returns empty result"}, + RequestInfo: RequestInfo{RequestID: requestID}, + } + } + if err = json.Unmarshal(data, output); err != nil { + return &TosServerError{ + TosError: TosError{Message: err.Error()}, + RequestInfo: RequestInfo{RequestID: requestID}, + } + } + return nil +} + +func marshalInput(name string, input interface{}) ([]byte, string, error) { + data, err := json.Marshal(input) + if err != nil { + return nil, "", InvalidMarshal + } + + sum := md5.Sum(data) + return data, base64.StdEncoding.EncodeToString(sum[:]), nil +} + +func fileUnreadLength(file *os.File) (int64, error) { + offset, err := file.Seek(0, io.SeekCurrent) + if err != nil { + return 0, err + } + + stat, err := file.Stat() + if err != nil { + return 0, err + } + + size := stat.Size() + if offset > size || offset < 0 { + return 0, newTosClientError("tos: unexpected file size and(or) offset", nil) + } + + return size - offset, nil +} + +func tryResolveLength(reader io.Reader) int64 { + switch v := reader.(type) { + case *bytes.Buffer: + return int64(v.Len()) + case *bytes.Reader: + return int64(v.Len()) + case *strings.Reader: + return int64(v.Len()) + case *os.File: + length, err := fileUnreadLength(v) + if err != nil { + return -1 + } + return length + case *io.LimitedReader: + return v.N + case *net.Buffers: + if v != nil { + length := int64(0) + for _, p := range *v { + length += int64(len(p)) + } + return length + } + return 0 + default: + return -1 + } +} + +func Int64(value int64) *int64 { return &value } diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/resumable_copy.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/resumable_copy.go new file mode 100644 index 0000000000..31f3e579e7 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/resumable_copy.go @@ -0,0 +1,289 @@ +package tos + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" +) + +func parseResumableCopyObjectPath(input *ResumableCopyObjectInput) { + isDirRes := isDir(input.CheckpointFile) + if isDirRes || input.CheckpointFile == "" { + input.CheckpointFile = filepath.Clean(filepath.Join(input.CheckpointFile, fmt.Sprintf("%s.%s.%s.%s.%s", input.SrcBucket, input.SrcKey, input.SrcVersionID, input.Bucket, input.Key))) + } +} + +func loadExistCopyCheckPoint(ctx context.Context, cli *ClientV2, input *ResumableCopyObjectInput, headOutput *HeadObjectV2Output) (*copyObjectCheckpoint, bool) { + checkpoint := ©ObjectCheckpoint{} + var err error + loadCheckPoint(input.CheckpointFile, checkpoint) + if checkpoint.Valid(input, headOutput) { + return checkpoint, true + } else if checkpoint.Bucket != "" && checkpoint.Key != "" && checkpoint.UploadID != "" { + // 尝试去 abort + _, err = cli.AbortMultipartUpload(ctx, + &AbortMultipartUploadInput{ + Bucket: checkpoint.Bucket, + Key: checkpoint.Key, + UploadID: checkpoint.UploadID}) + if err != nil && cli.logger != nil { + cli.logger.Debug("fail to abort upload task: %s, err:%s", checkpoint.UploadID, err.Error()) + } + } + return nil, false +} + +func getResumableCopyObjectCheckpoint(ctx context.Context, cli *ClientV2, input *ResumableCopyObjectInput, headOutput *HeadObjectV2Output, init func() (*copyObjectCheckpoint, error)) (checkpoint *copyObjectCheckpoint, err error) { + if !input.EnableCheckpoint { + return init() + } + + parseResumableCopyObjectPath(input) + + checkpoint, exist := loadExistCopyCheckPoint(ctx, cli, input, headOutput) + + if exist { + return checkpoint, nil + } + err = checkAndCreateDir(input.CheckpointFile) + if err != nil { + return nil, InvalidCheckpointFilePath.withCause(err) + } + + file, err := os.Create(input.CheckpointFile) + if err != nil { + return nil, newTosClientError("tos: create checkpoint file failed", err) + } + _ = file.Close() + checkpoint, err = init() + if err != nil { + return nil, err + } + + err = checkpoint.WriteToFile() + if err != nil { + return nil, err + } + + return +} + +func initCopyPartsInfo(headOutput *HeadObjectV2Output, partSize int64) ([]copyPartInfo, error) { + if headOutput.ContentLength == 0 { + return []copyPartInfo{{ + PartNumber: 1, + IsZeroSize: true, + }}, nil + } + partCount := headOutput.ContentLength / partSize + remainder := headOutput.ContentLength % partSize + if remainder != 0 { + partCount++ + } + if partCount > 10000 { + return nil, InvalidFilePartNum + } + parts := make([]copyPartInfo, 0, partCount) + for i := int64(0); i < partCount; i++ { + part := copyPartInfo{ + PartNumber: i + 1, + CopySourceRangeStart: i * partSize, + CopySourceRangeEnd: (i+1)*partSize - 1, + CopySourceRange: fmt.Sprintf("bytes=%d-%d", i*partSize, (i+1)*partSize-1), + } + parts = append(parts, part) + } + if remainder != 0 { + parts[partCount-1].CopySourceRangeEnd = (partCount-1)*partSize + remainder - 1 + parts[partCount-1].CopySourceRange = fmt.Sprintf("bytes=%d-%d", (partCount-1)*partSize, (partCount-1)*partSize+remainder-1) + } + return parts, nil +} + +func initCopyCheckpoint(input *ResumableCopyObjectInput, headOutput *HeadObjectV2Output) (*copyObjectCheckpoint, error) { + parts, err := initCopyPartsInfo(headOutput, input.PartSize) + if err != nil { + return nil, err + } + cp := ©ObjectCheckpoint{ + Bucket: input.Bucket, + Key: input.Key, + SrcBucket: input.SrcBucket, + SrcVersionID: input.SrcVersionID, + PartSize: input.PartSize, + UploadID: "", + CopySourceIfMatch: input.CopySourceIfMatch, + CopySourceIfModifiedSince: input.CopySourceIfModifiedSince, + CopySourceIfNoneMatch: input.CopySourceIfNoneMatch, + CopySourceIfUnmodifiedSince: input.CopySourceIfUnmodifiedSince, + CopySourceSSECAlgorithm: input.CopySourceSSECAlgorithm, + CopySourceSSECKeyMD5: input.CopySourceSSECKeyMD5, + SSECAlgorithm: input.SSECAlgorithm, + SSECKeyMD5: input.SSECKeyMD5, + EncodingType: input.EncodingType, + CopySourceObjectInfo: objectInfo{ + Etag: headOutput.ETag, + HashCrc64ecma: headOutput.HashCrc64ecma, + LastModified: headOutput.LastModified, + ObjectSize: headOutput.ContentLength, + }, + PartsInfo: parts, + CheckpointPath: input.CheckpointFile, + } + return cp, nil +} +func prepareCopyTasks(cli *ClientV2, ctx context.Context, checkpoint *copyObjectCheckpoint, input *ResumableCopyObjectInput) []task { + tasks := make([]task, 0) + for _, part := range checkpoint.PartsInfo { + if !part.IsCompleted { + tasks = append(tasks, ©Task{ + cli: cli, + ctx: ctx, + input: input, + UploadID: checkpoint.UploadID, + PartNumber: part.PartNumber, + PartInfo: part, + }) + } + } + return tasks +} + +func (cli *ClientV2) copyPart(ctx context.Context, cp *copyObjectCheckpoint, input *ResumableCopyObjectInput, event *copyEvent) (*ResumableCopyObjectOutput, error) { + tasks := prepareCopyTasks(cli, ctx, cp, input) + routinesNum := min(input.TaskNum, len(tasks)) + cancelHandle := getCancelHandle(input.CancelHook) + tg := newTaskGroup(cancelHandle, routinesNum, cp, event, input.EnableCheckpoint, tasks) + abort := func() error { + _, err := cli.AbortMultipartUpload(ctx, + &AbortMultipartUploadInput{ + Bucket: input.Bucket, + Key: input.Key, + UploadID: cp.UploadID}) + return err + } + bindCancelHookWithAborter(input.CancelHook, abort) + tg.RunWorker() + tg.Scheduler() + success, taskErr := tg.Wait() + if taskErr != nil { + if err := abort(); err != nil { + return nil, err + } + return nil, taskErr + } + // handle results + if success < len(tasks) { + return nil, newTosClientError("tos: some tasks copy failed.", nil) + } + complete, err := cli.CompleteMultipartUploadV2(ctx, &CompleteMultipartUploadV2Input{ + Bucket: input.Bucket, + Key: input.Key, + UploadID: cp.UploadID, + Parts: cp.GetParts(), + }) + if err != nil { + event.postCopyEvent(&CopyEvent{ + Type: enum.CopyEventCompleteMultipartUploadFailed, + Err: err, + Bucket: input.Bucket, + Key: input.Key, + UploadID: &cp.UploadID, + SrcBucket: input.SrcBucket, + SrcKey: input.SrcKey, + SrcVersionID: input.SrcVersionID, + CheckpointFile: &input.CheckpointFile, + }) + return nil, err + } + event.postCopyEvent(&CopyEvent{ + Type: enum.CopyEventCompleteMultipartUploadSucceed, + Err: err, + Bucket: input.Bucket, + Key: input.Key, + UploadID: &cp.UploadID, + SrcBucket: input.SrcBucket, + SrcKey: input.SrcKey, + SrcVersionID: input.SrcVersionID, + CheckpointFile: &input.CheckpointFile, + }) + _ = os.Remove(input.CheckpointFile) + + return &ResumableCopyObjectOutput{ + RequestInfo: complete.RequestInfo, + Bucket: complete.Bucket, + Key: complete.Key, + UploadID: cp.UploadID, + Etag: complete.ETag, + Location: complete.Location, + VersionID: complete.VersionID, + HashCrc64ecma: complete.HashCrc64ecma, + SSECAlgorithm: cp.SSECAlgorithm, + SSECKeyMD5: cp.SSECKeyMD5, + EncodingType: cp.EncodingType, + }, nil +} + +func (cli *ClientV2) ResumableCopyObject(ctx context.Context, input *ResumableCopyObjectInput) (*ResumableCopyObjectOutput, error) { + rawInput := *input + copyInput := &rawInput + headOutput, err := cli.HeadObjectV2(ctx, &HeadObjectV2Input{ + Bucket: copyInput.SrcBucket, + Key: copyInput.SrcKey, + VersionID: copyInput.SrcVersionID, + SSECAlgorithm: copyInput.CopySourceSSECAlgorithm, + SSECKey: copyInput.CopySourceSSECKey, + SSECKeyMD5: copyInput.CopySourceSSECKeyMD5, + IfModifiedSince: copyInput.CopySourceIfModifiedSince, + IfNoneMatch: copyInput.CopySourceIfNoneMatch, + IfUnmodifiedSince: copyInput.CopySourceIfUnmodifiedSince, + IfMatch: copyInput.CopySourceIfMatch, + }) + if err != nil { + return nil, err + } + event := ©Event{input: copyInput} + init := func() (*copyObjectCheckpoint, error) { + return initCopyCheckpoint(copyInput, headOutput) + } + cp, err := getResumableCopyObjectCheckpoint(ctx, cli, copyInput, headOutput, init) + if err != nil { + return nil, err + } + if cp.UploadID == "" { + created, err := cli.CreateMultipartUploadV2(ctx, ©Input.CreateMultipartUploadV2Input) + if err != nil { + event.postCopyEvent(&CopyEvent{ + Type: enum.CopyEventCreateMultipartUploadFailed, + Err: err, + Bucket: copyInput.Bucket, + Key: copyInput.Key, + SrcBucket: copyInput.SrcBucket, + SrcKey: copyInput.SrcKey, + SrcVersionID: copyInput.SrcVersionID, + CheckpointFile: ©Input.CheckpointFile, + }) + return nil, err + } + event.uploadID = created.UploadID + event.postCopyEvent(&CopyEvent{ + Type: enum.CopyEventCreateMultipartUploadSucceed, + Bucket: copyInput.Bucket, + Key: copyInput.Key, + SrcBucket: copyInput.SrcBucket, + SrcKey: copyInput.SrcKey, + SrcVersionID: copyInput.SrcVersionID, + CheckpointFile: ©Input.CheckpointFile, + }) + cp.UploadID = created.UploadID + } + cleaner := func() { + _ = os.Remove(copyInput.CheckpointFile) + } + bindCancelHookWithCleaner(copyInput.CancelHook, cleaner) + + return cli.copyPart(ctx, cp, copyInput, event) +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/sign_v4.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/sign_v4.go new file mode 100644 index 0000000000..2adcfdeef8 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/sign_v4.go @@ -0,0 +1,414 @@ +package tos + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +const ( + emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + unsignedPayload = "UNSIGNED-PAYLOAD" + signPrefix = "TOS4-HMAC-SHA256" + iso8601Layout = "20060102T150405Z" + yyMMdd = "20060102" + serverTimeFormat = "2006-01-02T15:04:05Z" + authorization = "Authorization" + + v4Algorithm = "X-Tos-Algorithm" + v4Credential = "X-Tos-Credential" + v4Date = "X-Tos-Date" + v4Expires = "X-Tos-Expires" + v4SignedHeaders = "X-Tos-SignedHeaders" + v4Signature = "X-Tos-Signature" + v4SignatureLower = "x-tos-signature" + v4ContentSHA256 = "X-Tos-Content-Sha256" + v4SecurityToken = "X-Tos-Security-Token" + + v4Prefix = "x-tos" +) + +func defaultSigningQueryV4(key string) bool { + return key != v4SignatureLower +} + +func defaultSigningHeaderV4(key string, isSigningQuery bool) bool { + return (key == "content-type" && !isSigningQuery) || strings.HasPrefix(key, v4Prefix) +} + +func UTCNow() time.Time { + return time.Now().UTC() +} + +type Signer interface { + SignHeader(req *Request) http.Header + + SignQuery(req *Request, ttl time.Duration) url.Values +} + +type SigningKeyInfo struct { + Date string + Region string + Credential *Credential +} + +type SignV4 struct { + credentials Credentials + region string + signingHeader func(key string, isSigningQuery bool) bool + signingQuery func(key string) bool + now func() time.Time + signingKey func(*SigningKeyInfo) []byte + logger Logger +} + +type signedRes struct { + CanonicalString string + StringToSign string + Sign string +} + +type signedHeader struct { + CanonicalString string + StringToSign string + Header http.Header +} + +type signedQuery struct { + CanonicalString string + StringToSign string + Query url.Values +} + +type SignV4Option func(*SignV4) + +func (sv *SignV4) WithSignLogger(logger Logger) { + sv.logger = logger +} + +// NewSignV4 create SignV4 +// use WithSignKey to set self-defined sign-key generator +// use WithSignLogger to set logger +func NewSignV4(credentials Credentials, region string) *SignV4 { + signV4 := &SignV4{ + credentials: credentials, + region: region, + signingHeader: defaultSigningHeaderV4, + signingQuery: defaultSigningQueryV4, + now: UTCNow, + signingKey: SigningKey, + } + return signV4 +} + +// WithSigningKey for self-defined sign-key generator +func (sv *SignV4) WithSigningKey(signingKey func(*SigningKeyInfo) []byte) { + sv.signingKey = signingKey +} + +func (sv *SignV4) signedHeader(header http.Header, isSignedQuery bool) KVs { + var signed = make(KVs, 0, 10) + for key, values := range header { + kk := strings.ToLower(key) + if sv.signingHeader(kk, isSignedQuery) { + vv := make([]string, 0, len(values)) + for _, value := range values { + vv = append(vv, strings.Join(strings.Fields(value), " ")) + } + signed = append(signed, KV{Key: kk, Values: vv}) + } + } + return signed +} + +func (sv *SignV4) signedQuery(query url.Values, extra url.Values) KVs { + var signed = make(KVs, 0, len(query)+len(extra)) + for key, values := range query { + if sv.signingQuery(strings.ToLower(key)) { + signed = append(signed, KV{Key: key, Values: values}) + } + } + for key, values := range extra { + if sv.signingQuery(strings.ToLower(key)) { + signed = append(signed, KV{Key: key, Values: values}) + } + } + return signed +} + +func (sv *SignV4) canonicalRequest(method, path, contentSha256 string, header, query KVs) string { + const split = byte('\n') + var buf bytes.Buffer + buf.Grow(512) + + // Method + buf.WriteString(method) + buf.WriteByte(split) + + // URI + buf.Write(encodePath(path)) + buf.WriteByte(split) + + // query + buf.Write(encodeQuery(query)) + buf.WriteByte(split) + + // canonical headers + keys := make([]string, 0, len(header)) + for _, kv := range header { + keys = append(keys, kv.Key) + buf.WriteString(kv.Key) + buf.WriteByte(':') + buf.WriteString(strings.Join(kv.Values, ",")) + buf.WriteByte('\n') + } + buf.WriteByte(split) + + // signed headers + buf.WriteString(strings.Join(keys, ";")) + buf.WriteByte(split) + + if len(contentSha256) > 0 { + buf.WriteString(contentSha256) + } else { + buf.WriteString(emptySHA256) + } + return buf.String() +} + +func SigningKey(info *SigningKeyInfo) []byte { + date := hmacSHA256([]byte(info.Credential.AccessKeySecret), []byte(info.Date)) + region := hmacSHA256(date, []byte(info.Region)) + service := hmacSHA256(region, []byte("tos")) + return hmacSHA256(service, []byte("request")) +} + +func (sv *SignV4) doSign(method, path, contentSha256 string, header, query KVs, now time.Time, cred *Credential) signedRes { + const split = byte('\n') + + canonicalStr := sv.canonicalRequest(method, path, contentSha256, header, query) + + var buf bytes.Buffer + buf.Grow(len(signPrefix) + 128) + + buf.WriteString(signPrefix) + buf.WriteByte(split) + + buf.WriteString(now.Format(iso8601Layout)) + buf.WriteByte(split) + + date := now.Format(yyMMdd) + buf.WriteString(date) // yyMMdd + '/' + region + '/' + service + '/' + request + buf.WriteByte('/') + buf.WriteString(sv.region) + + buf.WriteString("/tos/request") + buf.WriteByte(split) + + sum := sha256.Sum256([]byte(canonicalStr)) + buf.WriteString(hex.EncodeToString(sum[:])) + + signK := sv.signingKey(&SigningKeyInfo{Date: date, Region: sv.region, Credential: cred}) + sign := hmacSHA256(signK, buf.Bytes()) + return signedRes{ + CanonicalString: canonicalStr, + StringToSign: buf.String(), + Sign: hex.EncodeToString(sign), + } + +} + +func (sv *SignV4) SignHeader(req *Request) http.Header { + signed := make(http.Header, 4) + now := sv.now() + date := now.Format(iso8601Layout) + contentSha256 := req.Header.Get(v4ContentSHA256) + + signedHeader := sv.signedHeader(req.Header, false) + signedHeader = append(signedHeader, KV{Key: strings.ToLower(v4Date), Values: []string{date}}) + signedHeader = append(signedHeader, KV{Key: "date", Values: []string{date}}) + signedHeader = append(signedHeader, KV{Key: "host", Values: []string{req.Host}}) + // if len(contentSha256) == 0 { + // signedHeader = append(signedHeader, KV{Key: strings.ToLower(v4ContentSHA256), Values: []string{unsignedPayload}}) + // signed.Set(v4ContentSHA256, unsignedPayload) + // } + + cred := sv.credentials.Credential() + if sts := cred.SecurityToken; len(sts) > 0 { + signedHeader = append(signedHeader, KV{Key: strings.ToLower(v4SecurityToken), Values: []string{sts}}) + signed.Set(v4SecurityToken, sts) + } + + sort.Sort(signedHeader) + signedQuery := sv.signedQuery(req.Query, nil) + + signRes := sv.doSign(req.Method, req.Path, contentSha256, signedHeader, signedQuery, now, &cred) + credential := fmt.Sprintf("%s/%s/%s/tos/request", cred.AccessKeyID, now.Format(yyMMdd), sv.region) + auth := fmt.Sprintf("TOS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", credential, joinKeys(signedHeader), signRes.Sign) + + signed.Set(authorization, auth) + signed.Set(v4Date, date) + signed.Set("Date", date) + if sv.logger != nil { + sv.logger.Debug("[tos] CanonicalString:" + "\n" + signRes.CanonicalString + "\n") + sv.logger.Debug("[tos] StringToSign:" + "\n" + signRes.StringToSign + "\n") + } + return signed +} + +func (sv *SignV4) SignQuery(req *Request, ttl time.Duration) url.Values { + now := sv.now() + date := now.Format(iso8601Layout) + query := req.Query + extra := make(url.Values) + + cred := sv.credentials.Credential() + credential := fmt.Sprintf("%s/%s/%s/tos/request", cred.AccessKeyID, now.Format(yyMMdd), sv.region) + extra.Add(v4Algorithm, signPrefix) + extra.Add(v4Credential, credential) + + extra.Add(v4Date, date) + extra.Add(v4Expires, strconv.FormatInt(ttl.Milliseconds()/1000, 10)) + if sts := cred.SecurityToken; len(sts) > 0 { + extra.Add(v4SecurityToken, sts) + } + + signedHeader := sv.signedHeader(req.Header, true) + signedHeader = append(signedHeader, KV{Key: "host", Values: []string{req.Host}}) + sort.Sort(signedHeader) + + extra.Add(v4SignedHeaders, joinKeys(signedHeader)) + signedQuery := sv.signedQuery(query, extra) + + signRes := sv.doSign(req.Method, req.Path, unsignedPayload, signedHeader, signedQuery, now, &cred) + extra.Add(v4Signature, signRes.Sign) + if sv.logger != nil { + sv.logger.Debug("[tos] CanonicalString:" + "\n" + signRes.CanonicalString + "\n") + sv.logger.Debug("[tos] StringToSign:" + "\n" + signRes.StringToSign + "\n") + } + return extra +} + +type KV struct { + Key string + Values []string +} + +type KVs []KV + +func (kvs KVs) Len() int { return len(kvs) } +func (kvs KVs) Swap(i, j int) { kvs[i], kvs[j] = kvs[j], kvs[i] } +func (kvs KVs) Less(i, j int) bool { return kvs[i].Key < kvs[j].Key } + +func joinKeys(kvs KVs) string { + keys := make([]string, 0, len(kvs)) + for i := range kvs { + keys = append(keys, kvs[i].Key) + } + sort.Strings(keys) + return strings.Join(keys, ";") +} + +func hmacSHA256(key []byte, value []byte) []byte { + h := hmac.New(sha256.New, key) + h.Write(value) + return h.Sum(nil) +} + +var ( + nonEscape [256]bool +) + +// ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' || ch == '~' || ch == '.') +func init() { + for i := 'a'; i <= 'z'; i++ { + nonEscape[i] = true + } + for i := 'A'; i <= 'Z'; i++ { + nonEscape[i] = true + } + for i := '0'; i <= '9'; i++ { + nonEscape[i] = true + } + nonEscape['-'] = true + nonEscape['_'] = true + nonEscape['.'] = true + nonEscape['~'] = true +} + +func encodePath(path string) []byte { + if len(path) == 0 { + return []byte{'/'} + } + + return URIEncode(path, false) +} + +func encodeQuery(query KVs) []byte { + if len(query) == 0 { + return make([]byte, 0) + } + + var buf bytes.Buffer + buf.Grow(512) + + sort.Sort(query) + for _, kv := range query { + keyEscaped := URIEncode(kv.Key, true) + for _, v := range kv.Values { + if buf.Len() > 0 { + buf.WriteByte('&') + } + buf.Write(keyEscaped) + buf.WriteByte('=') + buf.Write(URIEncode(v, true)) + } + } + return buf.Bytes() +} + +func URIEncode(in string, encodeSlash bool) []byte { + hexCount := 0 + for i := 0; i < len(in); i++ { + c := uint8(in[i]) + if c == '/' { + if encodeSlash { + hexCount++ + } + } else if !nonEscape[c] { + hexCount++ + } + } + encoded := make([]byte, len(in)+2*hexCount) + for i, j := 0, 0; i < len(in); i++ { + c := uint8(in[i]) + if c == '/' { + if encodeSlash { + encoded[j] = '%' + encoded[j+1] = '2' + encoded[j+2] = 'F' + j += 3 + } else { + encoded[j] = c + j++ + } + } else if !nonEscape[c] { + encoded[j] = '%' + encoded[j+1] = "0123456789ABCDEF"[c>>4] + encoded[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + } else { + encoded[j] = c + j++ + } + } + return encoded +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/tagging.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/tagging.go new file mode 100644 index 0000000000..4dfacf8085 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/tagging.go @@ -0,0 +1,17 @@ +package tos + +import ( + "context" +) + +func (cli *ClientV2) PutObjectTagging(ctx context.Context, input *PutObjectTaggingInput) (*PutObjectTaggingOutput, error) { + return cli.baseClient.PutObjectTagging(ctx, input) +} + +func (cli *ClientV2) GetObjectTagging(ctx context.Context, input *GetObjectTaggingInput) (*GetObjectTaggingOutput, error) { + return cli.baseClient.GetObjectTagging(ctx, input) +} + +func (cli *ClientV2) DeleteObjectTagging(ctx context.Context, input *DeleteObjectTaggingInput) (*DeleteObjectTaggingOutput, error) { + return cli.baseClient.DeleteObjectTagging(ctx, input) +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/transport.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/transport.go new file mode 100644 index 0000000000..6e1a89dd5e --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/transport.go @@ -0,0 +1,254 @@ +package tos + +import ( + "context" + "crypto/tls" + "math/rand" + "net" + "net/http" + "net/http/httptrace" + "time" +) + +type TransportConfig struct { + + // MaxIdleConns same as http.Transport MaxIdleConns. Default is 1024. + MaxIdleConns int + + // MaxIdleConnsPerHost same as http.Transport MaxIdleConnsPerHost. Default is 1024. + MaxIdleConnsPerHost int + + // MaxConnsPerHost same as http.Transport MaxConnsPerHost. Default is no limit. + MaxConnsPerHost int + + // RequestTimeout same as http.Client Timeout + // Deprecated: use ReadTimeout or WriteTimeout instead + RequestTimeout time.Duration + + // DialTimeout same as net.Dialer Timeout + DialTimeout time.Duration + + // KeepAlive same as net.Dialer KeepAlive + KeepAlive time.Duration + + // IdleConnTimeout same as http.Transport IdleConnTimeout + IdleConnTimeout time.Duration + + // TLSHandshakeTimeout same as http.Transport TLSHandshakeTimeout + TLSHandshakeTimeout time.Duration + + // ResponseHeaderTimeout same as http.Transport ResponseHeaderTimeout + ResponseHeaderTimeout time.Duration + + // ExpectContinueTimeout same as http.Transport ExpectContinueTimeout + ExpectContinueTimeout time.Duration + + // ReadTimeout see net.Conn SetReadDeadline + ReadTimeout time.Duration + + // WriteTimeout set net.Conn SetWriteDeadline + WriteTimeout time.Duration + + // InsecureSkipVerify set tls.Config InsecureSkipVerify + InsecureSkipVerify bool + + // DNSCacheTime Set Dns Cache Time. + DNSCacheTime time.Duration + + // Proxy Set http proxy for http client. + Proxy *Proxy +} + +type Transport interface { + RoundTrip(context.Context, *Request) (*Response, error) +} + +type DefaultTransport struct { + client http.Client + logger Logger +} + +func (d *DefaultTransport) WithDefaultTransportLogger(logger Logger) { + d.logger = logger +} + +// NewDefaultTransport create a DefaultTransport with config +func NewDefaultTransport(config *TransportConfig) *DefaultTransport { + var r *resolver + + if config.DNSCacheTime >= time.Minute { + r = newResolver(config.DNSCacheTime) + } + + transport := &http.Transport{ + DialContext: (&TimeoutDialer{ + Dialer: net.Dialer{ + Timeout: config.DialTimeout, + KeepAlive: config.KeepAlive, + }, + resolver: r, + ReadTimeout: config.ReadTimeout, + WriteTimeout: config.WriteTimeout, + }).DialContext, + MaxIdleConns: config.MaxIdleConns, + MaxIdleConnsPerHost: config.MaxIdleConnsPerHost, + MaxConnsPerHost: config.MaxConnsPerHost, + IdleConnTimeout: config.IdleConnTimeout, + TLSHandshakeTimeout: config.TLSHandshakeTimeout, + ResponseHeaderTimeout: config.ResponseHeaderTimeout, + ExpectContinueTimeout: config.ExpectContinueTimeout, + DisableCompression: true, + // #nosec G402 + TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}, + } + + if config.Proxy != nil && config.Proxy.proxyHost != "" { + transport.Proxy = http.ProxyURL(config.Proxy.Url()) + } + + return &DefaultTransport{ + client: http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Transport: transport, + }, + } +} + +// newDefaultTranposrtWithHTTPTransport +func newDefaultTranposrtWithHTTPTransport(transport http.RoundTripper) *DefaultTransport { + return &DefaultTransport{ + client: http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + Transport: transport, + }, + } +} + +// NewDefaultTransportWithClient crate a DefaultTransport with a http.Client +func NewDefaultTransportWithClient(client http.Client) *DefaultTransport { + return &DefaultTransport{client: client} +} + +func (dt *DefaultTransport) RoundTrip(ctx context.Context, req *Request) (*Response, error) { + hr, err := http.NewRequestWithContext(ctx, req.Method, req.URL(), req.Content) + if err != nil { + return nil, newTosClientError(err.Error(), err) + } + + if req.ContentLength != nil { + hr.ContentLength = *req.ContentLength + } + + for key, values := range req.Header { + hr.Header[key] = values + } + + var accessLog *accessLogRequest + if dt.logger != nil { + var trace *httptrace.ClientTrace + trace, accessLog = getClientTrace(GetUnixTimeMs()) + ctx = httptrace.WithClientTrace(ctx, trace) + hr = hr.WithContext(ctx) + } + res, err := dt.client.Do(hr) + + if accessLog != nil { + accessLog.PrintAccessLog(dt.logger, hr, res) + } + + if err != nil { + return nil, newTosClientError(err.Error(), err) + } + + return &Response{ + StatusCode: res.StatusCode, + ContentLength: res.ContentLength, + Header: res.Header, + Body: res.Body, + }, nil +} + +type TimeoutConn struct { + net.Conn + readTimeout time.Duration + writeTimeout time.Duration + zero time.Time +} + +func NewTimeoutConn(conn net.Conn, readTimeout, writeTimeout time.Duration) *TimeoutConn { + return &TimeoutConn{ + Conn: conn, + readTimeout: readTimeout, + writeTimeout: writeTimeout, + } +} + +func (tc *TimeoutConn) Read(b []byte) (n int, err error) { + timeout := tc.readTimeout > 0 + if timeout { + _ = tc.SetReadDeadline(time.Now().Add(tc.readTimeout)) + } + + n, err = tc.Conn.Read(b) + if timeout { + _ = tc.SetReadDeadline(time.Now().Add(tc.readTimeout * 5)) + } + return n, err +} + +func (tc *TimeoutConn) Write(b []byte) (n int, err error) { + timeout := tc.writeTimeout > 0 + if timeout { + _ = tc.SetWriteDeadline(time.Now().Add(tc.writeTimeout)) + } + + n, err = tc.Conn.Write(b) + if tc.readTimeout > 0 { + _ = tc.SetReadDeadline(time.Now().Add(tc.readTimeout * 5)) + } + return n, err +} + +type TimeoutDialer struct { + net.Dialer + resolver *resolver + ReadTimeout time.Duration + WriteTimeout time.Duration +} + +func (d *TimeoutDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + if d.resolver != nil { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + ipList, err := d.resolver.GetIpList(ctx, host) + if err != nil { + return nil, err + } + + // 随机打乱 IP List + rand.Shuffle(len(ipList), func(i, j int) { + ipList[i], ipList[j] = ipList[j], ipList[i] + }) + + for _, ip := range ipList { + conn, err := d.Dialer.DialContext(ctx, network, ip+":"+port) + if err == nil { + return NewTimeoutConn(conn, d.ReadTimeout, d.WriteTimeout), nil + } else { + d.resolver.Remove(address, ip) + } + } + } + + conn, err := d.Dialer.DialContext(ctx, network, address) + if err != nil { + return nil, err + } + return NewTimeoutConn(conn, d.ReadTimeout, d.WriteTimeout), nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type.go new file mode 100644 index 0000000000..d6f5cd0ec7 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type.go @@ -0,0 +1,2221 @@ +package tos + +import ( + "fmt" + "io" + "net/url" + "time" + + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" +) + +type Grantee struct { + ID string `json:"ID,omitempty"` + DisplayName string `json:"DisplayName,omitempty"` + Type string `json:"Type,omitempty"` + URI string `json:"Canned,omitempty"` +} + +type GranteeV2 struct { + ID string `json:"ID,omitempty"` + DisplayName string `json:"DisplayName,omitempty"` + Type enum.GranteeType `json:"Type,omitempty"` + Canned enum.CannedType `json:"Canned,omitempty"` +} + +type GrantV2 struct { + GranteeV2 GranteeV2 `json:"Grantee,omitempty"` + Permission enum.PermissionType `json:"Permission,omitempty"` +} + +type Grant struct { + Grantee Grantee `json:"Grantee,omitempty"` + Permission enum.PermissionType `json:"Permission,omitempty"` +} + +type ObjectAclGrant struct { + ACL string `json:"ACL,omitempty"` + GrantFullControl string `json:"GrantFullControl,omitempty"` + GrantRead string `json:"GrantRead,omitempty"` + GrantReadAcp string `json:"GrantReadAcp,omitempty"` + // Deprecated: GrantWrite will be ignored + GrantWrite string `json:"GrantWrite,omitempty"` + GrantWriteAcp string `json:"GrantWriteAcp,omitempty"` +} + +type ObjectAclRules struct { + Owner Owner `json:"Owner,omitempty"` + Grants []Grant `json:"Grants,omitempty"` +} + +type GetObjectAclOutput struct { + RequestInfo `json:"-"` + VersionID string `json:"VersionId,omitempty"` + Owner Owner `json:"Owner,omitempty"` + Grants []Grant `json:"Grants,omitempty"` +} + +type bucketACL struct { + Owner Owner `json:"Owner,omitempty"` + GrantList []GrantV2 `json:"Grants,omitempty"` +} + +type PutBucketACLInput struct { + Bucket string + ACLType enum.ACLType `location:"header" locationName:"X-Tos-Acl"` // optional + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` // optional + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` // optional + GrantReadAcp string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` // optional + GrantWrite string `location:"header" locationName:"X-Tos-Grant-Write"` // optional + GrantWriteAcp string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` // optional + + Owner Owner `json:"Owner,omitempty"` + Grants []GrantV2 `json:"Grants,omitempty"` +} + +type PutBucketACLOutput struct { + RequestInfo +} + +type GetBucketACLInput struct { + Bucket string +} + +type GetBucketACLOutput struct { + RequestInfo + Owner Owner `json:"Owner,omitempty"` + Grants []GrantV2 `json:"Grants,omitempty"` +} + +type GetObjectACLInput struct { + Bucket string + Key string + VersionID string `location:"query" locationName:"versionId"` +} + +type GetObjectACLOutput struct { + RequestInfo `json:"-"` + VersionID string `json:"VersionID,omitempty"` + Owner Owner `json:"Owner,omitempty"` + Grants []GrantV2 `json:"Grants,omitempty"` + BucketOwnerEntrusted bool `json:"BucketOwnerEntrusted"` +} + +// PutObjectAclInput AclGrant, AclRules can not set both. +type PutObjectAclInput struct { + Key string `json:"Key,omitempty"` // the object, required + VersionID string `json:"VersionId,omitempty"` // the version id of the object, optional + AclGrant *ObjectAclGrant `json:"AclGrant,omitempty"` // set acl by header + AclRules *ObjectAclRules `json:"AclRules,omitempty"` // set acl by rules +} + +type PutObjectACLInput struct { + Bucket string + Key string // the object, required + VersionID string `location:"query" locationName:"versionId"` // optional + ACL enum.ACLType `location:"header" locationName:"X-Tos-Acl"` // optional + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` // optional + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` // optional + GrantReadAcp string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` // optional + + // Deprecated + GrantWrite string `location:"header" locationName:"X-Tos-Grant-Write"` // optional + GrantWriteAcp string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` // optional + Owner Owner + Grants []GrantV2 + BucketOwnerEntrusted bool +} + +type PutObjectAclOutput struct { + RequestInfo `json:"-"` +} + +type PutObjectACLOutput struct { + PutObjectAclOutput +} + +type putFetchTaskV2Input struct { + URL string `json:"URL,omitempty"` + IgnoreSameKey bool `json:"IgnoreSameKey,omitempty"` + HexMD5 string `json:"ContentMD5,omitempty"` + Object string `json:"Object,omitempty"` +} + +type PutFetchTaskInputV2 struct { + Bucket string + Key string + + ACL enum.ACLType `location:"header" locationName:"X-Tos-Acl"` + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` + GrantReadACP string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` + GrantWriteACP string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` + StorageClass enum.StorageClassType `location:"header" locationName:"X-Tos-Storage-Class"` + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` + Meta map[string]string `location:"headers"` + + URL string `json:"URL,omitempty"` + IgnoreSameKey bool `json:"IgnoreSameKey,omitempty"` + HexMD5 string `json:"ContentMD5,omitempty"` +} + +type PutFetchTaskOutputV2 struct { + RequestInfo + TaskID string +} + +type FetchObjectInputV2 struct { + Bucket string + Key string + ACL enum.ACLType `location:"header" locationName:"X-Tos-Acl"` + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` + GrantReadACP string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` + GrantWriteACP string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` + StorageClass enum.StorageClassType `location:"header" locationName:"X-Tos-Storage-Class"` + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` + Meta map[string]string `location:"headers"` + + URL string `json:"URL,omitempty"` + IgnoreSameKey bool `json:"IgnoreSameKey,omitempty"` + HexMD5 string `json:"ContentMD5,omitempty"` +} + +type FetchObjectOutputV2 struct { + RequestInfo + VersionID string `json:"VersionId,omitempty"` + Etag string `json:"Etag,omitempty"` + SSECAlgorithm string `json:"SSECAlgorithm,omitempty"` + SSECKeyMD5 string `json:"SSECKeyMD5,omitempty"` +} + +type PreSingedPostSignatureInput struct { + Bucket string + Key string + Expires int64 + Conditions []PostSignatureCondition + ContentLengthRange *ContentLengthRange +} + +type PreSingedPostSignatureOutput struct { + OriginPolicy string + Policy string + Algorithm string + Credential string + Date string + Signature string +} + +type ContentLengthRange struct { + RangeStart int64 + RangeEnd int64 +} + +type PostSignatureCondition struct { + Key string + Value string + Operator *string +} + +type PreSingedPolicyURLInput struct { + Bucket string + Expires int64 + Conditions []PolicySignatureCondition + AlternativeEndpoint string + IsCustomDomain bool +} + +type PreSingedPolicyURLOutput struct { + PreSignedPolicyURLGenerator + SignatureQuery string + bucket string + host string + scheme string + isCustomDomain bool +} + +type PolicySignatureCondition struct { + Key string + Value string + Operator *string +} + +type PreSignedPolicyURLGenerator interface { + GetSignedURLForList(bucket string, additionalQuery map[string]string) string + GetSignedURLForGetOrHead(bucket, key string, additionalQuery map[string]string) string +} + +func (output *PreSingedPolicyURLOutput) GetSignedURLForList(additionalQuery map[string]string) string { + query := make(url.Values) + for k, v := range additionalQuery { + query.Add(k, v) + } + queryStr := query.Encode() + if queryStr != "" { + queryStr = "&" + queryStr + } + var domain string + if output.isCustomDomain { + domain = output.host + } else { + domain = fmt.Sprintf("%s.%s", output.bucket, output.host) + } + str := fmt.Sprintf("%s://%s/?%s%s", output.scheme, domain, output.SignatureQuery, queryStr) + return str +} +func (output *PreSingedPolicyURLOutput) GetSignedURLForGetOrHead(key string, additionalQuery map[string]string) string { + query := make(url.Values) + for k, v := range additionalQuery { + query.Add(k, v) + } + queryStr := query.Encode() + if queryStr != "" { + queryStr = "&" + queryStr + } + var domain string + if output.isCustomDomain { + domain = output.host + } else { + domain = fmt.Sprintf("%s.%s", output.bucket, output.host) + } + str := fmt.Sprintf("%s://%s/%s?%s%s", output.scheme, domain, key, output.SignatureQuery, queryStr) + return str +} + +type PreSignedURLInput struct { + HTTPMethod enum.HttpMethodType + Bucket string + Key string + Expires int64 // Expiration time in seconds, default 3600 seconds, max 7 days, range [1, 604800] + Header map[string]string + Query map[string]string + AlternativeEndpoint string + IsCustomDomain *bool +} + +type PreSignedURLOutput struct { + SignedUrl string // Pre-signed URL + SignedHeader map[string]string // The actual header fields contained in the pre-signature +} + +type CreateBucketInput struct { + Bucket string `json:"Bucket,omitempty"` // required + ACL string `json:"ACL,omitempty"` // optional + GrantFullControl string `json:"GrantFullControl,omitempty"` // optional + GrantRead string `json:"GrantRead,omitempty"` // optional + GrantReadAcp string `json:"GrantReadAcp,omitempty"` // optional + GrantWrite string `json:"GrantWrite,omitempty"` // optional + GrantWriteAcp string `json:"GrantWriteAcp,omitempty"` // optional +} + +type CreateBucketV2Input struct { + Bucket string // required + ACL enum.ACLType `location:"header" locationName:"X-Tos-Acl"` // optional + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` // optional + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` // optional + GrantReadAcp string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` // optional + GrantWrite string `location:"header" locationName:"X-Tos-Grant-Write"` // optional + GrantWriteAcp string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` // optional + StorageClass enum.StorageClassType `location:"header" locationName:"X-Tos-Storage-Class"` // setting the default storage type for buckets + AzRedundancy enum.AzRedundancyType `location:"header" locationName:"X-Tos-Az-Redundancy"` // setting the AZ type for buckets +} + +type CreateBucketOutput struct { + RequestInfo `json:"-"` + Location string `json:"Location,omitempty"` +} + +type CreateBucketV2Output struct { + CreateBucketOutput +} + +type HeadBucketOutput struct { + RequestInfo `json:"-"` + Region string `json:"Region,omitempty"` + StorageClass enum.StorageClassType `json:"StorageClass,omitempty"` + AzRedundancy enum.AzRedundancyType `json:"AzRedundancy"` +} + +type GetBucketCORSInput struct { + Bucket string +} + +type CorsRule struct { + AllowedOrigin []string `json:"AllowedOrigins,omitempty"` + AllowedMethod []string `json:"AllowedMethods,omitempty"` + AllowedHeader []string `json:"AllowedHeaders,omitempty"` + ExposeHeader []string `json:"ExposeHeaders,omitempty"` + MaxAgeSeconds int `json:"MaxAgeSeconds,omitempty"` +} + +type GetBucketCORSOutput struct { + RequestInfo `json:"-"` + CORSRules []CorsRule `json:"CORSRules,omitempty"` +} + +type PutBucketCORSInput struct { + Bucket string `json:"-"` + CORSRules []CorsRule `json:"CORSRules,omitempty"` +} + +type PutBucketCORSOutput struct { + RequestInfo `json:"-"` +} + +type DeleteBucketCORSInput struct { + Bucket string +} + +type DeleteBucketCORSOutput struct { + RequestInfo `json:"-"` +} + +type HeadBucketInput struct { + Bucket string +} + +type DeleteBucketInput struct { + Bucket string +} + +type DeleteBucketOutput struct { + RequestInfo `json:"-"` +} + +type ListedOwner struct { + ID string `json:"ID,omitempty"` +} + +type ListBucketsOutput struct { + RequestInfo `json:"-"` + Buckets []ListedBucket `json:"Buckets,omitempty"` + Owner ListedOwner `json:"Owner,omitempty"` +} + +type Owner struct { + ID string `json:"ID,omitempty"` + DisplayName string `json:"DisplayName,omitempty"` +} + +type ListedBucket struct { + CreationDate string `json:"CreationDate,omitempty"` + Name string `json:"Name,omitempty"` + Location string `json:"Location,omitempty"` + ExtranetEndpoint string `json:"ExtranetEndpoint,omitempty"` + IntranetEndpoint string `json:"IntranetEndpoint,omitempty"` +} + +type ListBucketsInput struct{} + +type PutObjectBasicInput struct { + Bucket string + Key string + ContentLength int64 `location:"header" locationName:"Content-Length"` + ContentMD5 string `location:"header" locationName:"Content-MD5"` + ContentSHA256 string `location:"header" locationName:"X-Tos-Content-Sha256"` + CacheControl string `location:"header" locationName:"Cache-Control"` + ContentDisposition string `location:"header" locationName:"Content-Disposition" encodeChinese:"true"` + ContentEncoding string `location:"header" locationName:"Content-Encoding"` + ContentLanguage string `location:"header" locationName:"Content-Language"` + ContentType string `location:"header" locationName:"Content-Type"` + Expires time.Time `location:"header" locationName:"Expires"` + ACL enum.ACLType `location:"header" locationName:"X-Tos-Acl"` + + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` // optional + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` // optional + GrantReadAcp string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` // optional + GrantWriteAcp string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` // optional + + Callback string `location:"header" locationName:"X-Tos-Callback"` + CallbackVar string `location:"header" locationName:"X-Tos-Callback-Var"` + WebsiteRedirectLocation string `location:"header" locationName:"X-Tos-Website-Redirect-Location"` + StorageClass enum.StorageClassType `location:"header" locationName:"X-Tos-Storage-Class"` + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` + ServerSideEncryption string `location:"header" locationName:"X-Tos-Server-Side-Encryption"` + ServerSideEncryptionKeyID string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Kms-Key-Id"` + TrafficLimit int64 `location:"header" locationName:"X-Tos-Traffic-Limit"` + ForbidOverwrite bool `location:"header" locationName:"X-Tos-Forbid-Overwrite"` + IfMatch string `location:"header" locationName:"X-Tos-If-Match"` + Meta map[string]string `location:"headers"` + DataTransferListener DataTransferListener + RateLimiter RateLimiter +} + +type PutObjectV2Input struct { + PutObjectBasicInput + Content io.Reader +} + +type PutObjectV2Output struct { + RequestInfo + ETag string + SSECAlgorithm string + SSECKeyMD5 string + VersionID string + CallbackResult string + HashCrc64ecma uint64 + ServerSideEncryption string + ServerSideEncryptionKeyID string +} + +type PutObjectOutput struct { + RequestInfo `json:"-"` + ETag string `json:"ETag,omitempty"` + VersionID string `json:"VersionId,omitempty"` + SSECustomerAlgorithm string `json:"SSECustomerAlgorithm,omitempty"` + SSECustomerKeyMD5 string `json:"SSECustomerKeyMD5,omitempty"` +} + +type PutObjectFromFileInput struct { + PutObjectBasicInput + FilePath string +} + +type PutObjectFromFileOutput struct { + PutObjectV2Output +} + +type CommonHeaders struct { + ContentLength int64 `location:"header" locationName:"Content-Length"` + ContentMD5 string `location:"header" locationName:"Content-MD5"` + ContentSHA256 string `location:"header" locationName:"X-Tos-Content-Sha256"` + CacheControl string `location:"header" locationName:"Cache-Control"` + ContentDisposition string `location:"header" locationName:"Content-Disposition" encodeChinese:"true"` + ContentEncoding string `location:"header" locationName:"Content-Encoding"` + ContentLanguage string `location:"header" locationName:"Content-Language"` + ContentType string `location:"header" locationName:"Content-Type"` + Expires time.Time `location:"header" locationName:"Expires"` + ACL enum.ACLType `location:"header" locationName:"X-Tos-Acl"` + + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` // optional + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` // optional + GrantReadAcp string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` // optional + GrantWriteAcp string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` // optional + + WebsiteRedirectLocation string `location:"header" locationName:"X-Tos-Website-Redirect-Location"` + StorageClass enum.StorageClassType `location:"header" locationName:"X-Tos-Storage-Class"` +} + +type SSEHeaders struct { + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` + ServerSideEncryption string `location:"header" locationName:"X-Tos-Server-Side-Encryption"` +} + +type AppendObjectV2Input struct { + Bucket string + Key string + Offset int64 `location:"query" locationName:"offset" default:"0"` + Content io.Reader + ContentLength int64 `location:"header" locationName:"Content-Length"` + ContentMD5 string `location:"header" locationName:"Content-MD5"` + ContentSHA256 string `location:"header" locationName:"X-Tos-Content-Sha256"` + CacheControl string `location:"header" locationName:"Cache-Control"` + ContentDisposition string `location:"header" locationName:"Content-Disposition" encodeChinese:"true"` + ContentEncoding string `location:"header" locationName:"Content-Encoding"` + ContentLanguage string `location:"header" locationName:"Content-Language"` + ContentType string `location:"header" locationName:"Content-Type"` + Expires time.Time `location:"header" locationName:"Expires"` + ACL enum.ACLType `location:"header" locationName:"X-Tos-Acl"` + + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` // optional + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` // optional + GrantReadAcp string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` // optional + GrantWriteAcp string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` // optional + + WebsiteRedirectLocation string `location:"header" locationName:"X-Tos-Website-Redirect-Location"` + StorageClass enum.StorageClassType `location:"header" locationName:"X-Tos-Storage-Class"` + TrafficLimit int64 `location:"header" locationName:"X-Tos-Traffic-Limit"` + IfMatch string `location:"header" locationName:"X-Tos-If-Match"` + + Meta map[string]string `location:"headers"` + DataTransferListener DataTransferListener + RateLimiter RateLimiter + PreHashCrc64ecma uint64 +} + +type AppendObjectOutput struct { + RequestInfo `json:"-"` + ETag string `json:"ETag,omitempty"` + NextAppendOffset int64 `json:"NextAppendOffset,omitempty"` +} + +type AppendObjectV2Output struct { + RequestInfo `json:"-"` + VersionID string `json:"VersionID,omitempty"` + NextAppendOffset int64 `json:"NextAppendOffset,omitempty"` + HashCrc64ecma uint64 `json:"HashCrc64Ecma,omitempty"` +} + +type SetObjectMetaInput struct { + Bucket string + Key string + VersionID string `location:"query" locationName:"versionId"` + + CacheControl string `location:"header" locationName:"Cache-Control"` + ContentDisposition string `location:"header" locationName:"Content-Disposition"` + ContentEncoding string `location:"header" locationName:"Content-Encoding"` + ContentLanguage string `location:"header" locationName:"Content-Language"` + ContentType string `location:"header" locationName:"Content-Type"` + Expires time.Time `location:"header" locationName:"Expires"` + + Meta map[string]string `location:"headers"` +} + +type SetObjectMetaOutput struct { + RequestInfo `json:"-"` +} + +type ListObjectsV2Input struct { + Bucket string + ListObjectsInput +} + +type ListObjectsType2Input struct { + Bucket string + Prefix string `location:"query" locationName:"prefix"` + Delimiter string `location:"query" locationName:"delimiter"` + StartAfter string `location:"query" locationName:"start-after"` + ContinuationToken string `location:"query" locationName:"continuation-token"` + MaxKeys int `location:"query" locationName:"max-keys"` + EncodingType string `location:"query" locationName:"encoding-type"` + FetchMeta bool `location:"query" locationName:"fetch-meta"` + ListOnlyOnce bool +} + +type ListObjectsType2Output struct { + RequestInfo + Name string `json:"Name,omitempty"` + Prefix string `json:"Prefix,omitempty"` + ContinuationToken string `json:"ContinuationToken,omitempty"` + KeyCount int `json:"KeyCount,omitempty"` + MaxKeys int `json:"MaxKeys,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + NextContinuationToken string `json:"NextContinuationToken,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Contents []ListedObjectV2 `json:"Contents,omitempty"` +} + +type ListObjectsInput struct { + Prefix string `location:"query" locationName:"prefix"` + Delimiter string `location:"query" locationName:"delimiter"` + Marker string `location:"query" locationName:"marker"` + MaxKeys int `location:"query" locationName:"max-keys"` + EncodingType string `location:"query" locationName:"encoding-type"` // "" or "url" + FetchMeta bool `location:"query" locationName:"fetch-meta"` + // Deprecated + Reverse bool +} + +type ListedObject struct { + Key string `json:"Key,omitempty"` + LastModified string `json:"LastModified,omitempty"` + ETag string `json:"ETag,omitempty"` + Size int64 `json:"Size,omitempty"` + Owner Owner `json:"Owner,omitempty"` + StorageClass string `json:"StorageClass,omitempty"` + Type string `json:"Type,omitempty"` + Meta Metadata `json:"UserMeta,omitempty"` +} + +type listedObject struct { + Key string `json:"Key,omitempty"` + LastModified string `json:"LastModified,omitempty"` + ETag string `json:"ETag,omitempty"` + Size int64 `json:"Size,omitempty"` + Owner Owner `json:"Owner,omitempty"` + StorageClass string `json:"StorageClass,omitempty"` + Type string `json:"Type,omitempty"` + Meta []userMeta `json:"UserMeta,omitempty"` +} + +type ListedObjectV2 struct { + Key string + LastModified time.Time + ETag string + Size int64 + Owner Owner + StorageClass enum.StorageClassType + HashCrc64ecma uint64 + Meta Metadata +} + +type userMeta struct { + Key string `json:"Key"` + Value string `json:"Value"` +} + +type listedObjectV2 struct { + Key string `json:"Key,omitempty"` + LastModified time.Time `json:"LastModified,omitempty"` + ETag string `json:"ETag,omitempty"` + Size int64 `json:"Size,omitempty"` + Owner Owner `json:"Owner,omitempty"` + StorageClass enum.StorageClassType `json:"StorageClass,omitempty"` + HashCrc64ecma string `json:"HashCrc64Ecma,omitempty"` + Meta []userMeta `json:"UserMeta,omitempty"` +} + +type ListedCommonPrefix struct { + Prefix string `json:"Prefix,omitempty"` +} + +type listObjectsOutput struct { + Name string `json:"Name,omitempty"` // bucket name + Prefix string `json:"Prefix,omitempty"` + Marker string `json:"Marker,omitempty"` + MaxKeys int64 `json:"MaxKeys,omitempty"` + NextMarker string `json:"NextMarker,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Contents []listedObject `json:"Contents,omitempty"` +} + +type ListObjectsOutput struct { + RequestInfo `json:"-"` + Name string `json:"Name,omitempty"` // bucket name + Prefix string `json:"Prefix,omitempty"` + Marker string `json:"Marker,omitempty"` + MaxKeys int64 `json:"MaxKeys,omitempty"` + NextMarker string `json:"NextMarker,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Contents []ListedObject `json:"Contents,omitempty"` +} + +type ListObjectsV2Output struct { + RequestInfo `json:"-"` + Name string `json:"Name,omitempty"` + Prefix string `json:"Prefix,omitempty"` + Marker string `json:"Marker,omitempty"` + MaxKeys int64 `json:"MaxKeys,omitempty"` + NextMarker string `json:"NextMarker,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Contents []ListedObjectV2 `json:"Contents,omitempty"` +} + +type listObjectsV2Output struct { + RequestInfo `json:"-"` + Name string `json:"Name,omitempty"` + Prefix string `json:"Prefix,omitempty"` + Marker string `json:"Marker,omitempty"` + MaxKeys int64 `json:"MaxKeys,omitempty"` + NextMarker string `json:"NextMarker,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Contents []listedObjectV2 `json:"Contents,omitempty"` +} + +type listObjectsType2Output struct { + RequestInfo `json:"-"` + Name string `json:"Name,omitempty"` + Prefix string `json:"Prefix,omitempty"` + ContinuationToken string `json:"ContinuationToken,omitempty"` + KeyCount int `json:"KeyCount,omitempty"` + MaxKeys int `json:"MaxKeys,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + NextContinuationToken string `json:"NextContinuationToken,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Contents []listedObjectV2 `json:"Contents,omitempty"` +} + +type ListObjectVersionsInput struct { + Prefix string `location:"query" locationName:"prefix"` + Delimiter string `location:"query" locationName:"delimiter"` + KeyMarker string `location:"query" locationName:"key-marker"` + VersionIDMarker string `location:"query" locationName:"version-id-marker"` + MaxKeys int `location:"query" locationName:"max-keys"` + EncodingType string `location:"query" locationName:"encoding-type"` // "" or "url" + FetchMeta bool `location:"query" locationName:"fetch-meta"` +} + +type ListObjectVersionsV2Input struct { + Bucket string `json:"Prefix,omitempty"` + ListObjectVersionsInput +} + +type ListedObjectVersion struct { + ETag string `json:"ETag,omitempty"` + IsLatest bool `json:"IsLatest,omitempty"` + Key string `json:"Key,omitempty"` + LastModified string `json:"LastModified,omitempty"` + Owner Owner `json:"Owner,omitempty"` + Size int64 `json:"Size,omitempty"` + StorageClass string `json:"StorageClass,omitempty"` + Type string `json:"Type,omitempty"` + VersionID string `json:"VersionId,omitempty"` + Meta Metadata `json:"UserMeta,omitempty"` +} + +type listedObjectVersionV2 struct { + Key string + LastModified time.Time + ETag string + IsLatest bool + Size int64 + Owner Owner + StorageClass enum.StorageClassType + VersionID string + HashCrc64ecma string + Meta []userMeta `json:"UserMeta,omitempty"` +} + +type ListedObjectVersionV2 struct { + Key string + LastModified time.Time + ETag string + IsLatest bool + Size int64 + Owner Owner + StorageClass enum.StorageClassType + VersionID string + HashCrc64ecma uint64 + Meta Metadata `json:"UserMeta,omitempty"` +} + +type ListedDeleteMarkerEntry struct { + IsLatest bool `json:"IsLatest,omitempty"` + Key string `json:"Key,omitempty"` + LastModified string `json:"LastModified,omitempty"` + Owner Owner `json:"Owner,omitempty"` + VersionID string `json:"VersionId,omitempty"` +} + +type ListedDeleteMarker struct { + Key string + LastModified time.Time + IsLatest bool + Owner Owner + VersionID string +} + +type listObjectVersionsV2Output struct { + RequestInfo `json:"-"` + Name string `json:"Name,omitempty"` // bucket name + Prefix string `json:"Prefix,omitempty"` + KeyMarker string `json:"KeyMarker,omitempty"` + VersionIDMarker string `json:"VersionIdMarker,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + MaxKeys int `json:"MaxKeys,omitempty"` + NextKeyMarker string `json:"NextKeyMarker,omitempty"` + NextVersionIDMarker string `json:"NextVersionIdMarker,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Versions []listedObjectVersionV2 `json:"Versions,omitempty"` + DeleteMarkers []ListedDeleteMarker `json:"DeleteMarkers,omitempty"` +} + +type ListObjectVersionsV2Output struct { + RequestInfo `json:"-"` + Name string `json:"Name,omitempty"` // bucket name + Prefix string `json:"Prefix,omitempty"` + KeyMarker string `json:"KeyMarker,omitempty"` + VersionIDMarker string `json:"VersionIdMarker,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + MaxKeys int `json:"MaxKeys,omitempty"` + NextKeyMarker string `json:"NextKeyMarker,omitempty"` + NextVersionIDMarker string `json:"NextVersionIdMarker,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Versions []ListedObjectVersionV2 `json:"Versions,omitempty"` + DeleteMarkers []ListedDeleteMarker `json:"DeleteMarkers,omitempty"` +} + +type listedObjectVersion struct { + ETag string `json:"ETag,omitempty"` + IsLatest bool `json:"IsLatest,omitempty"` + Key string `json:"Key,omitempty"` + LastModified string `json:"LastModified,omitempty"` + Owner Owner `json:"Owner,omitempty"` + Size int64 `json:"Size,omitempty"` + StorageClass string `json:"StorageClass,omitempty"` + Type string `json:"Type,omitempty"` + VersionID string `json:"VersionId,omitempty"` + Meta []userMeta `json:"UserMeta,omitempty"` +} + +type listObjectVersionsOutput struct { + RequestInfo `json:"-"` + Name string `json:"Name,omitempty"` // bucket name + Prefix string `json:"Prefix,omitempty"` + KeyMarker string `json:"KeyMarker,omitempty"` + VersionIDMarker string `json:"VersionIdMarker,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + MaxKeys int64 `json:"MaxKeys,omitempty"` + NextKeyMarker string `json:"NextKeyMarker,omitempty"` + NextVersionIDMarker string `json:"NextVersionIdMarker,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Versions []listedObjectVersion `json:"Versions,omitempty"` + DeleteMarkers []ListedDeleteMarkerEntry `json:"DeleteMarkers,omitempty"` +} + +type ListObjectVersionsOutput struct { + RequestInfo `json:"-"` + Name string `json:"Name,omitempty"` // bucket name + Prefix string `json:"Prefix,omitempty"` + KeyMarker string `json:"KeyMarker,omitempty"` + VersionIDMarker string `json:"VersionIdMarker,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + MaxKeys int64 `json:"MaxKeys,omitempty"` + NextKeyMarker string `json:"NextKeyMarker,omitempty"` + NextVersionIDMarker string `json:"NextVersionIdMarker,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + CommonPrefixes []ListedCommonPrefix `json:"CommonPrefixes,omitempty"` + Versions []ListedObjectVersion `json:"Versions,omitempty"` + DeleteMarkers []ListedDeleteMarkerEntry `json:"DeleteMarkers,omitempty"` +} + +type GetObjectOutput struct { + RequestInfo `json:"-"` + ContentRange string `json:"ContentRange,omitempty"` + Content io.ReadCloser `json:"-"` + ObjectMeta +} + +type GetObjectV2Input struct { + Bucket string + Key string + VersionID string `location:"query" locationName:"versionId"` + + IfMatch string `location:"header" locationName:"If-Match"` + IfModifiedSince time.Time `location:"header" locationName:"If-Modified-Since"` + IfNoneMatch string `location:"header" locationName:"If-None-Match"` + IfUnmodifiedSince time.Time `location:"header" locationName:"If-Unmodified-Since"` + + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` + TrafficLimit int64 `location:"header" locationName:"X-Tos-Traffic-Limit"` + + ResponseCacheControl string `location:"query" locationName:"response-cache-control"` + ResponseContentDisposition string `location:"query" locationName:"response-content-disposition"` + ResponseContentEncoding string `location:"query" locationName:"response-content-encoding"` + ResponseContentLanguage string `location:"query" locationName:"response-content-language"` + ResponseContentType string `location:"query" locationName:"response-content-type"` + ResponseExpires time.Time `location:"query" locationName:"response-expires"` + Process string `location:"query" locationName:"x-tos-process"` + + RangeStart int64 + RangeEnd int64 + Range string + + DataTransferListener DataTransferListener + RateLimiter RateLimiter + // Deprecated Not Use + PartNumber int +} + +type GetObjectBasicOutput struct { + RequestInfo + ContentRange string // don't move into ObjectMetaV2 + ObjectMetaV2 +} + +type GetObjectV2Output struct { + GetObjectBasicOutput + Content io.ReadCloser +} + +type GetObjectToFileInput struct { + GetObjectV2Input + FilePath string +} + +type GetObjectToFileOutput struct { + GetObjectBasicOutput +} + +type HeadObjectV2Input struct { + Bucket string + Key string + VersionID string `location:"query" locationName:"versionId"` + + IfMatch string `location:"header" locationName:"If-Match"` + IfModifiedSince time.Time `location:"header" locationName:"If-Modified-Since"` + IfNoneMatch string `location:"header" locationName:"If-None-Match"` + IfUnmodifiedSince time.Time `location:"header" locationName:"If-Unmodified-Since"` + + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` +} + +type HeadObjectOutput struct { + RequestInfo `json:"-"` + ContentRange string `json:"ContentRange,omitempty"` + ObjectMeta +} + +type HeadObjectV2Output struct { + RequestInfo `json:"-"` + ObjectMetaV2 +} + +type DeleteObjectV2Input struct { + Bucket string + Key string + VersionID string `location:"query" locationName:"versionId"` +} + +type DeleteObjectOutput struct { + RequestInfo `json:"-"` + DeleteMarker bool `json:"DeleteMarker,omitempty"` + VersionID string `json:"VersionId,omitempty"` +} + +type DeleteObjectV2Output struct { + DeleteObjectOutput +} + +type ObjectTobeDeleted struct { + Key string `json:"Key,omitempty"` + VersionID string `json:"VersionId,omitempty"` +} + +type DeleteMultiObjectsInput struct { + Bucket string + Objects []ObjectTobeDeleted `json:"Objects,omitempty"` + Quiet bool `json:"Quiet,omitempty"` +} + +type Deleted struct { + Key string `json:"Key,omitempty"` + VersionID string `json:"VersionId,omitempty"` + DeleteMarker *bool `json:"DeleteMarker,omitempty"` + DeleteMarkerVersionID string `json:"DeleteMarkerVersionId,omitempty"` +} + +type DeletedV2 struct { + Key string `json:"Key,omitempty"` + VersionID string `json:"VersionId,omitempty"` + DeleteMarker bool `json:"DeleteMarker,omitempty"` + DeleteMarkerVersionID string `json:"DeleteMarkerVersionId,omitempty"` +} + +type DeleteError struct { + Code string `json:"Code,omitempty"` + Message string `json:"Message,omitempty"` + Key string `json:"Key,omitempty"` + VersionID string `json:"VersionId,omitempty"` +} + +type DeleteMultiObjectsOutput struct { + RequestInfo `json:"-"` + Deleted []DeletedV2 `json:"Deleted,omitempty"` // 删除成功的Object列表 + Error []DeleteError `json:"Error,omitempty"` // 删除失败的Object列表 +} + +type CopyObjectInput struct { + Bucket string + Key string + SrcBucket string + SrcKey string + SrcVersionID string `location:"query" locationName:"versionId"` + CacheControl string `location:"header" locationName:"Cache-Control"` + ContentDisposition string `location:"header" locationName:"Content-Disposition" encodeChinese:"true"` + ContentEncoding string `location:"header" locationName:"Content-Encoding"` + ContentLanguage string `location:"header" locationName:"Content-Language"` + ContentType string `location:"header" locationName:"Content-Type"` + Expires time.Time `location:"header" locationName:"Expires"` + ACL enum.ACLType `location:"header" locationName:"X-Tos-Acl"` + + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` // optional + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` // optional + GrantReadAcp string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` // optional + GrantWriteAcp string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` // optional + + WebsiteRedirectLocation string `location:"header" locationName:"X-Tos-Website-Redirect-Location"` + StorageClass enum.StorageClassType `location:"header" locationName:"X-Tos-Storage-Class"` + + CopySourceIfMatch string `location:"header" locationName:"X-Tos-Copy-Source-If-Match"` + CopySourceIfModifiedSince time.Time `location:"header" locationName:"X-Tos-Copy-Source-If-Modified-Since"` + CopySourceIfNoneMatch string `location:"header" locationName:"X-Tos-Copy-Source-If-None-Match"` + CopySourceIfUnmodifiedSince time.Time `location:"header" locationName:"X-Tos-Copy-Source-If-Unmodified-Since"` + + CopySourceSSECAlgorithm string `location:"header" locationName:"X-Tos-Copy-Source-Server-Side-Encryption-Customer-Algorithm"` + CopySourceSSECKey string `location:"header" locationName:"X-Tos-Copy-Source-Server-Side-Encryption-Customer-Key"` + CopySourceSSECKeyMD5 string `location:"header" locationName:"X-Tos-Copy-Source-Server-Side-Encryption-Customer-Key-MD5"` + ServerSideEncryption string `location:"header" locationName:"X-Tos-Server-Side-Encryption"` + ServerSideEncryptionKeyID string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Kms-Key-Id"` + + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + TrafficLimit int64 `location:"header" locationName:"X-Tos-Traffic-Limit"` + ForbidOverwrite bool `location:"header" locationName:"X-Tos-Forbid-Overwrite"` + IfMatch string `location:"header" locationName:"X-Tos-If-Match"` + MetadataDirective enum.MetadataDirectiveType `location:"header" locationName:"X-Tos-Metadata-Directive"` + Meta map[string]string `location:"headers"` +} + +type copyObjectOutput struct { + ETag string `json:"ETag,omitempty"` // at body + LastModified string `json:"LastModified,omitempty"` // at body + Error +} + +type CopyObjectOutput struct { + RequestInfo `json:"-"` + VersionID string `json:"VersionId,omitempty"` + SourceVersionID string `json:"SourceVersionId,omitempty"` + ETag string `json:"ETag,omitempty"` // at body + LastModified string `json:"LastModified,omitempty"` // at body + SSECAlgorithm string `json:"SSECAlgorithm,omitempty"` + SSECKeyMD5 string `json:"SSECKeyMD5,omitempty"` + ServerSideEncryption string `json:"ServerSideEncryption,omitempty"` + ServerSideEncryptionKeyID string `json:"ServerSideEncryptionKmsKeyId,omitempty"` +} + +type UploadPartCopyInput struct { + UploadID string `json:"UploadId,omitempty"` + DestinationKey string `json:"DestinationKey,omitempty"` + SourceBucket string `json:"SourceBucket,omitempty"` + SourceKey string `json:"SourceKey,omitempty"` + SourceVersionID string `json:"SourceVersionId,omitempty"` // optional + StartOffset *int64 `json:"StartOffset,omitempty"` // optional + PartSize *int64 `json:"PartSize,omitempty"` // optional + PartNumber int `json:"PartNumber,omitempty"` +} + +type UploadPartCopyOutput struct { + RequestInfo `json:"-"` + VersionID string `json:"VersionId,omitempty"` + SourceVersionID string `json:"SourceVersionId,omitempty"` + PartNumber int `json:"PartNumber,omitempty"` + ETag string `json:"ETag,omitempty"` + LastModified string `json:"LastModified,omitempty"` +} + +type UploadPartCopyV2Input struct { + Bucket string + Key string + UploadID string `location:"query" locationName:"uploadId"` + PartNumber int `location:"query" locationName:"partNumber"` + + SrcBucket string + SrcKey string + SrcVersionID string `location:"query" locationName:"versionId"` + CopySourceRangeStart int64 + CopySourceRangeEnd int64 + CopySourceRange string + + CopySourceIfMatch string `location:"header" locationName:"X-Tos-Copy-Source-If-Match"` + CopySourceIfModifiedSince time.Time `location:"header" locationName:"X-Tos-Copy-Source-If-Modified-Since"` + CopySourceIfNoneMatch string `location:"header" locationName:"X-Tos-Copy-Source-If-None-Match"` + CopySourceIfUnmodifiedSince time.Time `location:"header" locationName:"X-Tos-Copy-Source-If-Unmodified-Since"` + + CopySourceSSECAlgorithm string `location:"header" locationName:"X-Tos-Copy-Source-Server-Side-Encryption-Customer-Algorithm"` + CopySourceSSECKey string `location:"header" locationName:"X-Tos-Copy-Source-Server-Side-Encryption-Customer-Key"` + CopySourceSSECKeyMD5 string `location:"header" locationName:"X-Tos-Copy-Source-Server-Side-Encryption-Customer-Key-MD5"` + + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + TrafficLimit int64 `location:"header" locationName:"X-Tos-Traffic-Limit"` +} + +type UploadPartCopyV2Output struct { + RequestInfo + PartNumber int + ETag string + LastModified time.Time + CopySourceVersionID string + ServerSideEncryption string + ServerSideEncryptionKeyID string + SSECAlgorithm string + SSECKeyMD5 string +} + +type CreateMultipartUploadV2Input struct { + Bucket string + Key string + EncodingType string `location:"query" locationName:"encoding-type"` // "" or "url" + CacheControl string `location:"header" locationName:"Cache-Control"` + ContentDisposition string `location:"header" locationName:"Content-Disposition" encodeChinese:"true"` + ContentEncoding string `location:"header" locationName:"Content-Encoding"` + ContentLanguage string `location:"header" locationName:"Content-Language"` + ContentType string `location:"header" locationName:"Content-Type"` + Expires time.Time `location:"header" locationName:"Expires"` + ACL enum.ACLType `location:"header" locationName:"X-Tos-Acl"` + + GrantFullControl string `location:"header" locationName:"X-Tos-Grant-Full-Control"` // optional + GrantRead string `location:"header" locationName:"X-Tos-Grant-Read"` // optional + GrantReadAcp string `location:"header" locationName:"X-Tos-Grant-Read-Acp"` // optional + GrantWriteAcp string `location:"header" locationName:"X-Tos-Grant-Write-Acp"` // optional + + WebsiteRedirectLocation string `location:"header" locationName:"X-Tos-Website-Redirect-Location"` + StorageClass enum.StorageClassType `location:"header" locationName:"X-Tos-Storage-Class"` + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` + ServerSideEncryption string `location:"header" locationName:"X-Tos-Server-Side-Encryption"` + ServerSideEncryptionKeyID string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Kms-Key-Id"` + ForbidOverwrite bool `location:"header" locationName:"X-Tos-Forbid-Overwrite"` + Meta map[string]string `location:"headers"` +} + +type RenameObjectInput struct { + Bucket string + Key string + NewKey string `location:"query" locationName:"name"` +} + +type RenameObjectOutput struct { + RequestInfo +} + +type CreateMultipartUploadOutput struct { + RequestInfo `json:"-"` + Bucket string `json:"Bucket,omitempty"` + Key string `json:"Key,omitempty"` + UploadID string `json:"UploadId,omitempty"` + SSECustomerAlgorithm string `json:"SSECustomerAlgorithm,omitempty"` + SSECustomerKeyMD5 string `json:"SSECustomerKeyMD5,omitempty"` +} + +type CreateMultipartUploadV2Output struct { + RequestInfo `json:"-"` + Bucket string `json:"Bucket,omitempty"` + Key string `json:"Key,omitempty"` + UploadID string `json:"UploadID,omitempty"` + SSECAlgorithm string `json:"SSECAlgorithm,omitempty"` + SSECKeyMD5 string `json:"SSECKeyMD5,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + ServerSideEncryption string `json:"ServerSideEncryption,omitempty"` + ServerSideEncryptionKeyID string `json:"ServerSideEncryptionKeyID,omitempty"` +} + +type UploadPartInput struct { + Key string `json:"Key,omitempty"` + UploadID string `json:"UploadId,omitempty"` + PartNumber int `json:"PartNumber,omitempty"` + Content io.Reader `json:"-"` +} + +type UploadPartOutput struct { + RequestInfo `json:"-"` + PartNumber int `json:"PartNumber,omitempty"` + ETag string `json:"ETag,omitempty"` + SSECustomerAlgorithm string `json:"SSECustomerAlgorithm,omitempty"` + SSECustomerKeyMD5 string `json:"SSECustomerKeyMD5,omitempty"` +} + +func (up *UploadPartOutput) uploadedPart() uploadedPart { + return uploadedPart{PartNumber: up.PartNumber, ETag: up.ETag} +} + +type UploadPartBasicInput struct { + Bucket string + Key string + UploadID string `location:"query" locationName:"uploadId"` + PartNumber int `location:"query" locationName:"partNumber"` + + ContentMD5 string `location:"header" locationName:"Content-MD5"` + + SSECAlgorithm string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Algorithm"` + SSECKey string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key"` + SSECKeyMD5 string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Customer-Key-MD5"` + ServerSideEncryption string `location:"header" locationName:"X-Tos-Server-Side-Encryption"` + ServerSideEncryptionKeyID string `location:"header" locationName:"X-Tos-Server-Side-Encryption-Kms-Key-Id"` + + TrafficLimit int64 `location:"header" locationName:"X-Tos-Traffic-Limit"` + + DataTransferListener DataTransferListener + RateLimiter RateLimiter +} + +type UploadPartV2Input struct { + UploadPartBasicInput + Content io.Reader + ContentLength int64 `location:"header" locationName:"Content-Length"` +} + +type UploadPartV2Output struct { + RequestInfo + PartNumber int + ETag string + SSECAlgorithm string + SSECKeyMD5 string + HashCrc64ecma uint64 + ServerSideEncryption string + ServerSideEncryptionKeyID string +} + +func (up *UploadPartV2Output) uploadedPart() uploadedPart { + return uploadedPart{PartNumber: up.PartNumber, ETag: up.ETag} +} + +type UploadPartFromFileInput struct { + UploadPartBasicInput + FilePath string + Offset uint64 // 当前分段在文件中的起始位置 + PartSize int64 // 当前分段长度，该字段等同于 Content-Length 头域 +} + +type UploadPartFromFileOutput struct { + UploadPartV2Output +} + +type UploadedPart struct { + PartNumber int32 `json:"PartNumber,omitempty"` // Part编号 + ETag string `json:"ETag,omitempty"` // ETag + LastModified string `json:"LastModified,omitempty"` // 最后一次修改时间 + Size int64 `json:"Size,omitempty"` // Part大小 +} + +func (part *UploadedPart) uploadedPart() uploadedPart { + return uploadedPart{ + PartNumber: int(part.PartNumber), + ETag: part.ETag, + } +} + +type UploadedPartV2 struct { + PartNumber int `json:"PartNumber,omitempty"` // Part编号 + ETag string `json:"ETag,omitempty"` // ETag + LastModified time.Time `json:"LastModified,omitempty"` // 最后一次修改时间 + Size int64 `json:"Size,omitempty"` // Part大小 +} + +func (part UploadedPartV2) uploadedPart() uploadedPart { + return uploadedPart{PartNumber: part.PartNumber, ETag: part.ETag} +} + +type MultipartUploadedPart interface { + uploadedPart() uploadedPart +} + +type CompleteMultipartUploadInput struct { + Key string `json:"Key,omitempty"` + UploadID string `json:"UploadId,omitempty"` + UploadedParts []MultipartUploadedPart `json:"UploadedParts,omitempty"` +} + +type CompleteMultipartUploadOutput struct { + RequestInfo `json:"-"` + VersionID string `json:"VersionId,omitempty"` +} + +type CompleteMultipartUploadV2Input struct { + Bucket string + Key string + CompleteAll bool + UploadID string `location:"query" locationName:"uploadId"` + Callback string `location:"header" locationName:"X-Tos-Callback"` + CallbackVar string `location:"header" locationName:"X-Tos-Callback-Var"` + ForbidOverwrite bool `location:"header" locationName:"X-Tos-Forbid-Overwrite"` + Parts []UploadedPartV2 +} + +type CompleteMultipartUploadV2Output struct { + RequestInfo + Bucket string + Key string + ETag string + Location string + CompletedParts []UploadedPartV2 + VersionID string + HashCrc64ecma uint64 + CallbackResult string + ServerSideEncryption string + ServerSideEncryptionKeyID string +} + +type AbortMultipartUploadInput struct { + // Bucket is needed in V2 api + Bucket string + Key string + UploadID string `location:"query" locationName:"uploadId"` +} + +type AbortMultipartUploadOutput struct { + RequestInfo `json:"-"` +} + +type UploadInfo struct { + Key string `json:"Key,omitempty"` + UploadId string `json:"UploadId,omitempty"` + Owner Owner `json:"Owner,omitempty"` + StorageClass string `json:"StorageClass,omitempty"` + Initiated string `json:"Initiated,omitempty"` +} + +type UploadCommonPrefix struct { + Prefix string `json:"Prefix"` +} + +type ListMultipartUploadsInput struct { + Prefix string `json:"Prefix,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + KeyMarker string `json:"KeyMarker,omitempty"` + UploadIDMarker string `json:"UploadIdMarker,omitempty"` + MaxUploads int `json:"MaxUploads,omitempty"` +} + +type ListMultipartUploadsOutput struct { + RequestInfo `json:"-"` + Bucket string `json:"Bucket,omitempty"` + KeyMarker string `json:"KeyMarker,omitempty"` + UploadIdMarker string `json:"UploadIdMarker,omitempty"` + NextKeyMarker string `json:"NextKeyMarker,omitempty"` + NextUploadIdMarker string `json:"NextUploadIdMarker,omitempty"` + Delimiter string `json:"Delimiter,omitempty"` + Prefix string `json:"Prefix,omitempty"` + MaxUploads int32 `json:"MaxUploads,omitempty"` + IsTruncated bool `json:"IsTruncated,omitempty"` + Upload []UploadInfo `json:"Uploads,omitempty"` + CommonPrefixes []UploadCommonPrefix `json:"CommonPrefixes,omitempty"` +} + +type ListMultipartUploadsV2Input struct { + Bucket string + Prefix string `location:"query" locationName:"uploads"` + Delimiter string `location:"query" locationName:"delimiter"` + KeyMarker string `location:"query" locationName:"key-marker"` + UploadIDMarker string `location:"query" locationName:"upload-id-marker"` + MaxUploads int `location:"query" locationName:"max-uploads"` + EncodingType string `location:"query" locationName:"encoding-type"` // "" or "url" +} + +type ListedUpload struct { + Key string + UploadID string + Owner Owner + StorageClass enum.StorageClassType + Initiated time.Time +} + +type ListMultipartUploadsV2Output struct { + RequestInfo + Bucket string + Prefix string + KeyMarker string + UploadIDMarker string + MaxUploads int + Delimiter string + IsTruncated bool + EncodingType string + NextKeyMarker string + NextUploadIDMarker string + CommonPrefixes []ListedCommonPrefix + Uploads []ListedUpload +} + +type ListUploadedPartsInput struct { + Key string `json:"Key,omitempty"` + UploadID string `json:"UploadId,omitempty"` + MaxParts int `json:"MaxParts,omitempty"` // 最大Part个数 + PartNumberMarker int `json:"NextPartNumberMarker,omitempty"` // 起始Part的位置 +} + +type ListUploadedPartsOutput struct { + RequestInfo `json:"-"` + Bucket string `json:"Bucket,omitempty"` // Bucket名称 + Key string `json:"Key,omitempty"` // Object名称 + UploadID string `json:"UploadId,omitempty"` // 上传ID + PartNumberMarker int `json:"PartNumberMarker,omitempty"` // 当前页起始位置 + NextPartNumberMarker int `json:"NextPartNumberMarker,omitempty"` // 下一个Part的位置 + MaxParts int `json:"MaxParts,omitempty"` // 最大Part个数 + IsTruncated bool `json:"IsTruncated,omitempty"` // 是否完全上传完成 + StorageClass string `json:"StorageClass,omitempty"` // 存储类型 + Owner Owner `json:"Owner,omitempty"` // 属主 + UploadedParts []UploadedPart `json:"Parts,omitempty"` // 已完成的Part +} + +type ListPartsInput struct { + Bucket string + Key string + UploadID string `location:"query" locationName:"uploadId"` + PartNumberMarker int `location:"query" locationName:"part-number-marker"` + MaxParts int `location:"query" locationName:"max-parts"` + EncodingType string `location:"query" locationName:"encoding-type"` // "" or "url" +} + +type ListPartsOutput struct { + RequestInfo + Bucket string + Key string + UploadID string + PartNumberMarker int + MaxParts int + IsTruncated bool + EncodingType string + + NextPartNumberMarker int + StorageClass enum.StorageClassType + Owner Owner + Parts []UploadedPartV2 +} + +type putBucketLifecycleInput struct { + Rules []lifecycleRule `json:"Rules,omitempty"` +} + +type lifecycleRule struct { + ID string `json:"ID,omitempty"` + Prefix string `json:"Prefix,omitempty"` + Status enum.StatusType `json:"Status,omitempty"` + Transitions []transition `json:"Transitions,omitempty"` + Expiration *expiration `json:"Expiration,omitempty"` + NonCurrentVersionTransition []NonCurrentVersionTransition `json:"NoncurrentVersionTransitions,omitempty"` + NoCurrentVersionExpiration *NoCurrentVersionExpiration `json:"NoncurrentVersionExpiration,omitempty"` + Tag []Tag `json:"Tags,omitempty"` + AbortInCompleteMultipartUpload *AbortInCompleteMultipartUpload `json:"AbortIncompleteMultipartUpload,omitempty"` +} + +type PutBucketLifecycleInput struct { + Bucket string + Rules []LifecycleRule `json:"Rules,omitempty"` +} + +type GetBucketLifecycleInput struct { + Bucket string +} + +type GetBucketLifecycleOutput struct { + RequestInfo + Rules []LifecycleRule `json:"Rules"` +} + +type DeleteBucketLifecycleInput struct { + Bucket string +} + +type DeleteBucketLifecycleOutput struct { + RequestInfo +} + +type LifecycleRule struct { + ID string `json:"ID,omitempty"` + Prefix string `json:"Prefix,omitempty"` + Status enum.StatusType `json:"Status,omitempty"` + Transitions []Transition `json:"Transitions,omitempty"` + Expiration *Expiration `json:"Expiration,omitempty"` + NonCurrentVersionTransition []NonCurrentVersionTransition `json:"NoncurrentVersionTransitions,omitempty"` + NoCurrentVersionExpiration *NoCurrentVersionExpiration `json:"NoncurrentVersionExpiration,omitempty"` + Tag []Tag `json:"Tags,omitempty"` + AbortInCompleteMultipartUpload *AbortInCompleteMultipartUpload `json:"AbortIncompleteMultipartUpload,omitempty"` +} + +type AbortInCompleteMultipartUpload struct { + DaysAfterInitiation int `json:"DaysAfterInitiation,omitempty"` +} + +type Tag struct { + Key string `json:"Key"` + Value string `json:"Value"` +} + +type NoCurrentVersionExpiration struct { + NoCurrentDays int `json:"NoncurrentDays,omitempty"` +} + +type NonCurrentVersionTransition struct { + NonCurrentDays int `json:"NoncurrentDays,omitempty"` + StorageClass enum.StorageClassType `json:"StorageClass,omitempty"` +} + +type transition struct { + Days int `json:"Days,omitempty"` + Date string `json:"Date,omitempty"` + StorageClass enum.StorageClassType `json:"StorageClass,omitempty"` +} + +type Transition struct { + Days int `json:"Days,omitempty"` + Date time.Time `json:"Date,omitempty"` + StorageClass enum.StorageClassType `json:"StorageClass,omitempty"` +} + +type Expiration struct { + Days int `json:"Days,omitempty"` + Date time.Time `json:"Date,omitempty"` +} + +type expiration struct { + Days int `json:"Days,omitempty"` + Date string `json:"Date,omitempty"` +} + +type PutLifecycleOutput struct { + RequestInfo +} + +type PutBucketMirrorBackOutput struct { + RequestInfo +} + +type putBucketMirrorBackInput struct { + Rules []MirrorBackRule `json:"Rules"` +} + +type PutBucketMirrorBackInput struct { + Bucket string + Rules []MirrorBackRule +} + +type MirrorBackRule struct { + ID string `json:"ID,omitempty"` + Condition Condition `json:"Condition,omitempty"` + Redirect Redirect `json:"Redirect,omitempty"` +} + +type Condition struct { + HttpCode int `json:"HttpCode,omitempty"` + KeyPrefix string `json:"KeyPrefix,omitempty"` + KeySuffix string `json:"KeySuffix,omitempty"` +} + +type Redirect struct { + RedirectType enum.RedirectType `json:"RedirectType,omitempty"` + FetchSourceOnRedirect bool `json:"FetchSourceOnRedirect,omitempty"` + PassQuery bool `json:"PassQuery,omitempty"` + FollowRedirect bool `json:"FollowRedirect,omitempty"` + MirrorHeader MirrorHeader `json:"MirrorHeader,omitempty"` + PublicSource PublicSource `json:"PublicSource,omitempty"` + Transform Transform `json:"Transform,omitempty"` +} + +type Transform struct { + WithKeyPrefix string `json:"WithKeyPrefix,omitempty"` + WithKeySuffix string `json:"WithKeySuffix,omitempty"` + ReplaceKeyPrefix ReplaceKeyPrefix `json:"ReplaceKeyPrefix,omitempty"` +} + +type ReplaceKeyPrefix struct { + KeyPrefix string `json:"KeyPrefix,omitempty"` + ReplaceWith string `json:"ReplaceWith,omitempty"` +} + +type PublicSource struct { + SourceEndpoint SourceEndpoint `json:"SourceEndpoint,omitempty"` + FixedEndpoint bool `json:"FixedEndpoint,omitempty"` +} + +type GetBucketMirrorBackInput struct { + Bucket string +} + +type GetBucketMirrorBackOutput struct { + RequestInfo + Rules []MirrorBackRule +} + +type DeleteObjectTaggingInput struct { + Bucket string + Key string + VersionID string `location:"query" locationName:"versionId"` +} + +type GetObjectTaggingInput struct { + Bucket string + Key string + VersionID string `location:"query" locationName:"versionId"` +} +type putObjectTaggingInput struct { + TagSet TagSet `json:"TagSet"` +} +type PutObjectTaggingInput struct { + Bucket string + Key string + VersionID string `location:"query" locationName:"versionId"` + TagSet TagSet `json:"TagSet"` +} + +type PutObjectTaggingOutput struct { + RequestInfo + VersionID string +} + +type GetObjectTaggingOutput struct { + RequestInfo + VersionID string + TagSet TagSet +} + +type TagSet struct { + Tags []Tag +} + +type DeleteObjectTaggingOutput struct { + RequestInfo + VersionID string +} + +type DeleteBucketMirrorBackInput struct { + Bucket string +} + +type DeleteBucketMirrorBackOutput struct { + RequestInfo +} + +type SourceEndpoint struct { + Primary []string `json:"Primary,omitempty"` + Follower []string `json:"Follower,omitempty"` +} +type MirrorHeader struct { + PassAll bool `json:"PassAll,omitempty"` + Pass []string `json:"Pass,omitempty"` + Remove []string `json:"Remove,omitempty"` +} + +type PutBucketStorageClassInput struct { + Bucket string + StorageClass enum.StorageClassType `location:"header" locationName:"X-Tos-Storage-Class"` +} + +type PutBucketStorageClassOutput struct { + RequestInfo +} + +type GetBucketLocationInput struct { + Bucket string +} + +type GetBucketLocationOutput struct { + RequestInfo `json:"-"` + Region string `json:"Region,omitempty"` + ExtranetEndpoint string `json:"ExtranetEndpoint,omitempty"` + IntranetEndpoint string `json:"IntranetEndpoint,omitempty"` +} + +type CancelHook interface { + // Cancel 取消断点上传\断点下载事, isAbort 为 true 时删除上下文信息和临时文件，为 false 时只是中断当前执行，该接口只能调用一次 + Cancel(isAbort bool) + // to make user unable to implement this interface + internal() +} + +type DownloadFileInput struct { + HeadObjectV2Input + FilePath string + filePath string + PartSize int64 + TaskNum int + EnableCheckpoint bool + CheckpointFile string + tempFile string + TrafficLimit int64 + DownloadEventListener DownloadEventListener + DataTransferListener DataTransferListener + RateLimiter RateLimiter + CancelHook CancelHook // user can not set this filed +} + +func (d *DownloadFileInput) withCancelHook(hook CancelHook) { + d.CancelHook = hook +} + +type DownloadFileOutput struct { + HeadObjectV2Output +} + +type DownloadEvent struct { + Type enum.DownloadEventType + Err error // not empty when it occurs when failed, aborted event occurs + Bucket string + Key string + VersionID string + FilePath string // path of the file to download to + CheckpointFile *string // path to checkpoint file + TempFilePath *string // path fo the temp file + // not empty when download part event occurs + DowloadPartInfo *DownloadPartInfo +} + +// DownloadPartInfo is returned when DownloadEvent occur +type DownloadPartInfo struct { + PartNumber int + RangeStart int64 + RangeEnd int64 +} + +type DownloadEventListener interface { + EventChange(event *DownloadEvent) +} + +type UploadFileInput struct { + CreateMultipartUploadV2Input + + FilePath string + PartSize int64 + TaskNum int + EnableCheckpoint bool + CheckpointFile string + TrafficLimit int64 `location:"header" locationName:"X-Tos-Traffic-Limit"` + + DataTransferListener DataTransferListener + UploadEventListener UploadEventListener + RateLimiter RateLimiter + // cancelHook 支持取消断点续传任务 + CancelHook CancelHook +} + +func NewCancelHook() CancelHook { + return &canceler{ + cancelHandle: make(chan struct{}), + } +} + +// UploadPartInfo is returned when UploadEvent occur +type UploadPartInfo struct { + PartNumber int + PartSize int64 + Offset int64 + // upload part succeed 事件发生时有值 + ETag *string + HashCrc64ecma *uint64 +} + +type UploadEvent struct { + Type enum.UploadEventType + Err error // failed, aborted 事件发生时不为空 + Bucket string + Key string + UploadID *string + CheckpointFile *string // 断点续传文件全路径 + // upload part 相关事件发生时有值 + UploadPartInfo *UploadPartInfo +} + +type UploadEventListener interface { + EventChange(event *UploadEvent) +} + +type UploadFileOutput struct { + RequestInfo + Bucket string + Key string + UploadID string + ETag string + Location string + VersionID string + HashCrc64ecma uint64 + SSECAlgorithm string + SSECKeyMD5 string + EncodingType string +} + +type DataTransferStatus struct { + TotalBytes int64 + ConsumedBytes int64 // bytes read/written + RWOnceBytes int64 // bytes read/written this time + Type enum.DataTransferType +} + +type putBucketNotificationInput struct { + CloudFunctionConfigurations []CloudFunctionConfiguration `json:"CloudFunctionConfigurations"` + RocketMQConfigurations []RocketMQConfiguration `json:"RocketMQConfigurations"` +} + +type RocketMQConf struct { + InstanceID string `json:"InstanceId"` + Topic string `json:"Topic"` + AccessKeyID string `json:"AccessKeyId"` +} + +type RocketMQConfiguration struct { + ID string `json:"RuleId"` + Role string `json:"Role"` + Events []string `json:"Events"` + Filter Filter `json:"Filter"` + RocketMQ RocketMQConf `json:"RocketMQ"` +} + +type PutBucketNotificationInput struct { + Bucket string `json:"-"` + CloudFunctionConfigurations []CloudFunctionConfiguration `json:"CloudFunctionConfigurations"` + RocketMQConfigurations []RocketMQConfiguration `json:"RocketMQConfigurations"` +} + +type PutBucketNotificationOutput struct { + RequestInfo +} + +type CloudFunctionConfiguration struct { + ID string `json:"RuleId"` + Events []string `json:"Events"` + Filter Filter `json:"Filter"` + CloudFunction string `json:"CloudFunction"` +} + +type Filter struct { + Key FilterKey `json:"TOSKey"` +} + +type FilterKey struct { + Rules []FilterRule `json:"FilterRules"` +} + +type FilterRule struct { + Name string `json:"Name"` + Value string `json:"Value"` +} + +type GetBucketNotificationInput struct { + Bucket string +} + +type GetBucketNotificationOutput struct { + RequestInfo + CloudFunctionConfigurations []CloudFunctionConfiguration `json:"CloudFunctionConfigurations"` + RocketMQConfigurations []RocketMQConfiguration `json:"RocketMQConfigurations"` +} + +type putBucketVersioningInput struct { + Status enum.VersioningStatusType `json:"Status"` +} + +type PutBucketVersioningInput struct { + Bucket string + Status enum.VersioningStatusType +} + +type PutBucketVersioningOutput struct { + RequestInfo +} + +type GetBucketVersioningInput struct { + Bucket string +} + +type GetBucketVersioningOutputV2 struct { + RequestInfo + Status enum.VersioningStatusType `json:"Status"` +} + +type putBucketWebsiteInput struct { + RedirectAllRequestsTo *RedirectAllRequestsTo `json:"RedirectAllRequestsTo,omitempty"` + IndexDocument *IndexDocument `json:"IndexDocument,omitempty"` + ErrorDocument *ErrorDocument `json:"ErrorDocument,omitempty"` + RoutingRules []RoutingRule `json:"RoutingRules,omitempty"` +} + +type PutBucketWebsiteInput struct { + Bucket string + RedirectAllRequestsTo *RedirectAllRequestsTo `json:"RedirectAllRequestsTo,omitempty"` + IndexDocument *IndexDocument `json:"IndexDocument,omitempty"` + ErrorDocument *ErrorDocument `json:"ErrorDocument,omitempty"` + RoutingRules *RoutingRules `json:"RoutingRules,omitempty"` +} + +type RedirectAllRequestsTo struct { + HostName string `json:"HostName"` + Protocol string `json:"Protocol,omitempty"` +} + +type IndexDocument struct { + Suffix string `json:"Suffix"` + ForbiddenSubDir bool `json:"ForbiddenSubDir,omitempty"` +} + +type ErrorDocument struct { + Key string `json:"Key"` +} + +type RoutingRules struct { + Rules []RoutingRule `json:"RoutingRules,omitempty"` +} + +type RoutingRule struct { + Condition RoutingRuleCondition `json:"Condition"` + Redirect RoutingRuleRedirect `json:"Redirect"` +} + +type RoutingRuleCondition struct { + KeyPrefixEquals string `json:"KeyPrefixEquals,omitempty"` + HttpErrorCodeReturnedEquals int `json:"HttpErrorCodeReturnedEquals,omitempty"` +} + +type RoutingRuleRedirect struct { + Protocol enum.ProtocolType `json:"Protocol,omitempty"` + HostName string `json:"HostName,omitempty"` + ReplaceKeyPrefixWith string `json:"ReplaceKeyPrefixWith,omitempty"` + ReplaceKeyWith string `json:"ReplaceKeyWith,omitempty"` + HttpRedirectCode int `json:"HttpRedirectCode,omitempty"` +} + +type PutBucketWebsiteOutput struct { + RequestInfo +} + +type GetBucketWebsiteInput struct { + Bucket string +} + +type GetBucketWebsiteOutput struct { + RequestInfo + RedirectAllRequestsTo *RedirectAllRequestsTo `json:"RedirectAllRequestsTo,omitempty"` + IndexDocument *IndexDocument `json:"IndexDocument,omitempty"` + ErrorDocument *ErrorDocument `json:"ErrorDocument,omitempty"` + RoutingRules []RoutingRule `json:"RoutingRules,omitempty"` +} + +type DeleteBucketWebsiteInput struct { + Bucket string +} + +type DeleteBucketWebsiteOutput struct { + RequestInfo +} + +type putBucketReplicationInput struct { + Role string `json:"Role"` + Rules []ReplicationRule `json:"Rules"` +} + +type PutBucketReplicationInput struct { + Bucket string + Role string + Rules []ReplicationRule +} + +type ReplicationRuleWithProgress struct { + ReplicationRule + Progress *Progress `json:"Progress,omitempty"` +} + +type ReplicationRule struct { + ID string `json:"ID"` + Status enum.StatusType `json:"Status"` + PrefixSet []string `json:"PrefixSet,omitempty"` + Destination Destination `json:"Destination"` + HistoricalObjectReplication enum.StatusType `json:"HistoricalObjectReplication"` +} + +type Destination struct { + Bucket string `json:"Bucket"` + Location string `json:"Location"` + StorageClass enum.StorageClassType `json:"StorageClass,omitempty"` + StorageClassInheritDirective enum.StorageClassInheritDirectiveType `json:"StorageClassInheritDirective,omitempty"` +} + +type Progress struct { + HistoricalObject float64 `json:"HistoricalObject"` + NewObject string `json:"NewObject"` +} + +type PutBucketReplicationOutput struct { + RequestInfo +} + +type GetBucketReplicationInput struct { + Bucket string + RuleID string +} + +type GetBucketReplicationOutput struct { + RequestInfo + Role string `json:"Role"` + Rules []ReplicationRuleWithProgress `json:"Rules"` +} +type DeleteBucketReplicationInput struct { + Bucket string +} + +type DeleteBucketReplicationOutput struct { + RequestInfo +} + +type putBucketRealTimeLogInput struct { + Configuration RealTimeLogConfiguration `json:"RealTimeLogConfiguration"` +} + +type PutBucketRealTimeLogInput struct { + Bucket string + Configuration RealTimeLogConfiguration +} + +type RealTimeLogConfiguration struct { + Role string `json:"Role"` + Configuration AccessLogConfiguration `json:"AccessLogConfiguration"` +} + +type AccessLogConfiguration struct { + UseServiceTopic bool `json:"UseServiceTopic"` + TLSProjectID string `json:"TLSProjectID"` + TLSTopicID string `json:"TLSTopicID"` +} + +type PutBucketRealTimeLogOutput struct { + RequestInfo +} + +type GetBucketRealTimeLogInput struct { + Bucket string +} + +type GetBucketRealTimeLogOutput struct { + RequestInfo + Configuration RealTimeLogConfiguration `json:"RealTimeLogConfiguration"` +} + +type DeleteBucketRealTimeLogInput struct { + Bucket string +} + +type DeleteBucketRealTimeLogOutput struct { + RequestInfo +} +type putBucketCustomDomainInput struct { + Rule CustomDomainRule `json:"CustomDomainRule,omitempty"` +} + +type PutBucketCustomDomainInput struct { + Bucket string + Rule CustomDomainRule +} + +type CustomDomainRule struct { + CertID string `json:"CertId"` + CertStatus enum.CertStatusType `json:"CertStatus"` + Domain string `json:"Domain"` + Forbidden bool `json:"Forbidden"` + ForbiddenReason string `json:"ForbiddenReason"` + Cname string `json:"Cname"` +} + +type PutBucketCustomDomainOutput struct { + RequestInfo +} + +type ListBucketCustomDomainInput struct { + Bucket string +} + +type ListBucketCustomDomainOutput struct { + RequestInfo + Rules []CustomDomainRule `json:"CustomDomainRules"` +} + +type DeleteBucketCustomDomainInput struct { + Bucket string + Domain string +} + +type DeleteBucketCustomDomainOutput struct { + RequestInfo +} + +type ResumableCopyObjectInput struct { + CreateMultipartUploadV2Input + + SrcBucket string + SrcKey string + SrcVersionID string + + CopySourceIfMatch string + CopySourceIfModifiedSince time.Time + CopySourceIfNoneMatch string + CopySourceIfUnmodifiedSince time.Time + + CopySourceSSECAlgorithm string + CopySourceSSECKey string + CopySourceSSECKeyMD5 string + + PartSize int64 + TaskNum int + EnableCheckpoint bool + CheckpointFile string + TrafficLimit int64 + + CopyEventListener CopyEventListener + CancelHook CancelHook +} + +type CopyPartInfo struct { + PartNumber int + CopySourceRangeStart int64 + CopySourceRangeEnd int64 + + // upload part copy succeed 时有值 + Etag *string +} + +type CopyEvent struct { + Type enum.CopyEventType + Err error + + Bucket string + Key string + UploadID *string + SrcBucket string + SrcKey string + SrcVersionID string + CheckpointFile *string + CopyPartInfo *copyPartInfo +} + +type CopyEventListener interface { + EventChange(event *CopyEvent) +} + +type ResumableCopyObjectOutput struct { + RequestInfo + Bucket string + Key string + UploadID string + Etag string + Location string + VersionID string + HashCrc64ecma uint64 + SSECAlgorithm string + SSECKeyMD5 string + EncodingType string +} + +type restoreObjectInput struct { + Days int `json:"Days"` + RestoreJobParameters *RestoreJobParameters `json:"RestoreJobParameters,omitempty"` +} +type RestoreObjectInput struct { + Bucket string + Key string + VersionID string `location:"query" locationName:"versionId"` + Days int + RestoreJobParameters *RestoreJobParameters +} + +type RestoreObjectOutput struct { + RequestInfo +} + +type RestoreJobParameters struct { + Tier enum.TierType `json:"Tier"` +} + +type DataTransferListener interface { + DataTransferStatusChange(status *DataTransferStatus) +} + +type RateLimiter interface { + // Acquire try to get a token. + // If ok, caller can read want bytes, else wait timeToWait and try again. + Acquire(want int64) (ok bool, timeToWait time.Duration) +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type_internal.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type_internal.go new file mode 100644 index 0000000000..74040374e8 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type_internal.go @@ -0,0 +1,1009 @@ +package tos + +import ( + "context" + "encoding/json" + "fmt" + "hash" + "hash/crc64" + "io" + "io/ioutil" + "os" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" +) + +const DefaultProgressCallbackSize = 512 * 1024 + +type Metadata interface { + AllKeys() []string + Get(key string) (string, bool) + Range(f func(key, value string) bool) +} + +type CustomMeta struct { + m map[string]string +} + +func (c *CustomMeta) AllKeys() []string { + keys := make([]string, 0, len(c.m)) + for k := range c.m { + keys = append(keys, k) + } + return keys +} + +func (c *CustomMeta) Get(key string) (val string, ok bool) { + val, ok = c.m[strings.ToLower(key)] + return +} + +func (c *CustomMeta) Range(f func(key, val string) bool) { + for k, v := range c.m { + if !f(k, v) { + break + } + } +} + +type multipartUpload struct { + Bucket string `json:"Bucket,omitempty"` + Key string `json:"Key,omitempty"` + UploadID string `json:"UploadId,omitempty"` +} + +type uploadedPart struct { + PartNumber int `json:"PartNumber"` + ETag string `json:"ETag"` +} + +type uploadedParts []uploadedPart + +func (p uploadedParts) Less(i, j int) bool { return p[i].PartNumber < p[j].PartNumber } +func (p uploadedParts) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p uploadedParts) Len() int { return len(p) } + +// only for marshal +type partsToComplete struct { + Parts uploadedParts `json:"Parts"` +} + +// only for Marshal +type deleteMultiObjectsInput struct { + Objects []ObjectTobeDeleted `json:"Objects,omitempty"` + Quiet bool `json:"Quiet,omitempty"` +} + +// only for Marshal +type accessControlList struct { + Owner Owner `json:"Owner,omitempty"` + Grants []GrantV2 `json:"Grants,omitempty"` + BucketOwnerEntrusted bool `json:"BucketOwnerEntrusted,omitempty"` +} + +type canceler struct { + called int32 + cancelHandle chan struct{} + // cleaner will clean all files need to be deleted + cleaner func() + // aborter will abort multi upload task + aborter func() error +} + +func (c *canceler) Cancel(isAbort bool) { + if c.cancelHandle == nil { + return + } + if atomic.CompareAndSwapInt32(&c.called, 0, 1) { + if isAbort { + if c.cleaner != nil { + c.cleaner() + } + if c.aborter != nil { + c.aborter() + } + } + close(c.cancelHandle) + } +} + +// do nothing +func (c *canceler) internal() {} + +type copyPartInfo struct { + PartNumber int64 `json:"part_number"` + CopySourceRange string `json:"copy_source_range"` + CopySourceRangeStart int64 `json:"copy_source_range_start"` + CopySourceRangeEnd int64 `json:"copy_source_range_end"` + Etag string `json:"etag"` + IsCompleted bool `json:"is_completed"` + IsZeroSize bool `json:"is_zero_size"` +} + +type copyObjectCheckpoint struct { + Bucket string `json:"bucket"` + Key string `json:"key"` + SrcBucket string `json:"src_bucket"` + SrcVersionID string `json:"src_version_id"` + PartSize int64 `json:"part_size"` + UploadID string `json:"upload_id"` + CopySourceIfMatch string `json:"copy_source_if_match"` + CopySourceIfModifiedSince time.Time `json:"copy_source_if_modified_since"` + CopySourceIfNoneMatch string `json:"copy_source_if_none_match"` + CopySourceIfUnmodifiedSince time.Time `json:"copy_source_if_unmodified_since"` + CopySourceSSECAlgorithm string `json:"copy_source_ssec_algorithm"` + CopySourceSSECKeyMD5 string `json:"copy_source_ssec_key_md5"` + SSECAlgorithm string `json:"ssec_algorithm"` + SSECKeyMD5 string `json:"ssec_key_md5"` + EncodingType string `json:"encoding_type"` + CopySourceObjectInfo objectInfo `json:"copy_source_object_info"` + PartsInfo []copyPartInfo `json:"parts_info"` + CheckpointPath string `json:"checkpoint_path"` +} + +func (c *copyObjectCheckpoint) Valid(input *ResumableCopyObjectInput, headOutput *HeadObjectV2Output) bool { + // 源对象发生改变 + if c.CopySourceObjectInfo.ObjectSize != headOutput.ContentLength || c.CopySourceObjectInfo.Etag != headOutput.ETag || + c.CopySourceObjectInfo.LastModified != headOutput.LastModified || c.CopySourceObjectInfo.HashCrc64ecma != headOutput.HashCrc64ecma { + return false + } + // 复制基本信息发生改变 + if c.Bucket != input.Bucket || input.Key != c.Key || + input.SrcBucket != c.SrcBucket || input.SrcVersionID != c.SrcVersionID || + input.PartSize != c.PartSize || input.EncodingType != c.EncodingType { + return false + } + + // 复制条件发生改变 + if c.CopySourceIfMatch != input.CopySourceIfMatch || c.CopySourceIfModifiedSince != input.CopySourceIfModifiedSince || + c.CopySourceIfUnmodifiedSince != input.CopySourceIfUnmodifiedSince || c.CopySourceIfNoneMatch != input.CopySourceIfNoneMatch { + return false + } + // 加密发生改变 + if c.SSECAlgorithm != input.SSECAlgorithm || c.SSECKeyMD5 != input.SSECKeyMD5 || + c.CopySourceSSECAlgorithm != input.CopySourceSSECAlgorithm || c.CopySourceSSECKeyMD5 != input.CopySourceSSECKeyMD5 { + return false + } + return true +} +func (c *copyObjectCheckpoint) WriteToFile() error { + + buffer, err := json.Marshal(c) + if err != nil { + return InvalidMarshal + } + err = ioutil.WriteFile(c.CheckpointPath, buffer, 0600) + if err != nil { + return newTosClientError(err.Error(), err) + } + return nil +} + +func (c *copyObjectCheckpoint) UpdatePartsInfo(result interface{}) { + part := result.(copyPartInfo) + c.PartsInfo[part.PartNumber-1] = part +} + +func (c *copyObjectCheckpoint) GetCheckPointFilePath() string { + return c.CheckpointPath +} + +func (c *copyObjectCheckpoint) GetParts() []UploadedPartV2 { + parts := make([]UploadedPartV2, 0, len(c.PartsInfo)) + for _, p := range c.PartsInfo { + parts = append(parts, UploadedPartV2{ + PartNumber: int(p.PartNumber), + ETag: p.Etag, + }) + } + return parts +} + +type objectInfo struct { + Etag string `json:"Etag,omitempty"` + HashCrc64ecma uint64 `json:"HashCrc64Ecma,omitempty"` + LastModified time.Time `json:"LastModified,omitempty"` + ObjectSize int64 `json:"ObjectSize,omitempty"` +} + +type downloadFileInfo struct { + FilePath string `json:"FilePath,omitempty"` + TempFilePath string `json:"TempFilePath,omitempty"` +} + +// downloadPartInfo is for checkpoint +type downloadPartInfo struct { + PartNumber int `json:"PartNumber,omitempty"` + RangeStart int64 `json:"RangeStart"` // not omit empty + RangeEnd int64 `json:"RangeEnd,omitempty"` + HashCrc64ecma uint64 `json:"HashCrc64Ecma,omitempty"` + IsCompleted bool `json:"IsCompleted"` // not omit empty +} + +type downloadCheckpoint struct { + checkpointPath string // this filed should not be marshaled + Bucket string `json:"Bucket,omitempty"` + Key string `json:"Key,omitempty"` + VersionID string `json:"VersionID,omitempty"` + PartSize int64 `json:"PartSize,omitempty"` + + IfMatch string `json:"IfMatch,omitempty"` + IfModifiedSince time.Time `json:"IfModifiedSince,omitempty"` + IfNoneMatch string `json:"IfNoneMatch,omitempty"` + IfUnmodifiedSince time.Time `json:"IfUnmodifiedSince,omitempty"` + + SSECAlgorithm string `json:"SSECAlgorithm,omitempty"` + SSECKeyMD5 string `json:"SSECKeyMD5,omitempty"` + ObjectInfo objectInfo `json:"ObjectInfo,omitempty"` + FileInfo downloadFileInfo `json:"FileInfo,omitempty"` + PartsInfo []downloadPartInfo `json:"PartsInfo,omitempty"` +} + +func (c *downloadCheckpoint) UpdatePartsInfo(result interface{}) { + part := result.(downloadPartInfo) + c.PartsInfo[part.PartNumber-1] = part + +} + +func (c *downloadCheckpoint) GetCheckPointFilePath() string { + return c.checkpointPath +} + +func (c *downloadCheckpoint) WriteToFile() error { + buffer, err := json.Marshal(c) + if err != nil { + return InvalidMarshal + } + err = ioutil.WriteFile(c.checkpointPath, buffer, 0600) + if err != nil { + return newTosClientError(err.Error(), err) + } + return nil +} + +func (c *downloadCheckpoint) Valid(input *DownloadFileInput, head *HeadObjectV2Output) bool { + if c.Bucket != input.Bucket || c.Key != input.Key || c.VersionID != input.VersionID || c.PartSize != input.PartSize || + c.IfMatch != input.IfMatch || c.IfModifiedSince != input.IfModifiedSince || c.IfNoneMatch != input.IfNoneMatch || + c.IfUnmodifiedSince != input.IfUnmodifiedSince || + c.SSECAlgorithm != input.SSECAlgorithm || c.SSECKeyMD5 != input.SSECKeyMD5 { + return false + } + + if c.ObjectInfo.Etag != head.ETag || c.ObjectInfo.HashCrc64ecma != head.HashCrc64ecma || + c.ObjectInfo.LastModified != head.LastModified || c.ObjectInfo.ObjectSize != head.ContentLength { + return false + } + if c.FileInfo.FilePath != input.filePath { + return false + } + return true +} + +type fileInfo struct { + LastModified int64 `json:"LastModified,omitempty"` + Size int64 `json:"Size"` +} + +// uploadPartInfo is for checkpoint +type uploadPartInfo struct { + uploadID *string // should not be marshaled + PartNumber int `json:"PartNumber"` + PartSize int64 `json:"PartSize"` + Offset uint64 `json:"Offset"` + ETag string `json:"ETag,omitempty"` + HashCrc64ecma uint64 `json:"HashCrc64Ecma,omitempty"` + IsCompleted bool `json:"IsCompleted"` +} + +type uploadCheckpoint struct { + checkpointPath string // this filed should not be marshaled + Bucket string `json:"Bucket,omitempty"` + Key string `json:"Key,omitempty"` + UploadID string `json:"UploadID,omitempty"` + PartSize int64 `json:"PartSize"` + SSECAlgorithm string `json:"SSECAlgorithm,omitempty"` + SSECKeyMD5 string `json:"SSECKeyMD5,omitempty"` + EncodingType string `json:"EncodingType,omitempty"` + FilePath string `json:"FilePath,omitempty"` + FileInfo fileInfo `json:"FileInfo"` + PartsInfo []uploadPartInfo `json:"PartsInfo,omitempty"` +} + +func (u *uploadCheckpoint) UpdatePartsInfo(result interface{}) { + part := result.(uploadPartInfo) + u.PartsInfo[part.PartNumber-1] = part + +} + +func (u *uploadCheckpoint) GetCheckPointFilePath() string { + return u.FilePath +} + +func (u *uploadCheckpoint) Valid(uploadFileStat os.FileInfo, bucketName, key, uploadFile string) bool { + if u.UploadID == "" || u.Bucket != bucketName || u.Key != key || u.FilePath != uploadFile { + return false + } + if u.FileInfo.Size != uploadFileStat.Size() || u.FileInfo.LastModified != uploadFileStat.ModTime().Unix() { + return false + } + return true +} + +func (u *uploadCheckpoint) GetParts() []UploadedPartV2 { + parts := make([]UploadedPartV2, 0, len(u.PartsInfo)) + for _, p := range u.PartsInfo { + parts = append(parts, UploadedPartV2{ + PartNumber: p.PartNumber, + ETag: p.ETag, + }) + } + return parts +} + +func (u *uploadCheckpoint) WriteToFile() error { + result, err := json.Marshal(u) + if err != nil { + return InvalidMarshal + } + err = ioutil.WriteFile(u.checkpointPath, result, 0600) + if err != nil { + return newTosClientError(err.Error(), err) + } + return nil +} + +type copyEvent struct { + input *ResumableCopyObjectInput + uploadID string +} + +func (c *copyEvent) postCopyEvent(event *CopyEvent) { + if c.input.CopyEventListener != nil { + c.input.CopyEventListener.EventChange(event) + } +} +func (c *copyEvent) PostEvent(eventType int, result interface{}, taskErr error) { + + event := &CopyEvent{ + Bucket: c.input.Bucket, + Key: c.input.Key, + UploadID: &c.uploadID, + SrcBucket: c.input.SrcBucket, + SrcKey: c.input.SrcKey, + SrcVersionID: c.input.SrcVersionID, + CheckpointFile: &c.input.CheckpointFile, + } + switch eventType { + case EventPartSucceed: + part, ok := result.(copyPartInfo) + if !ok { + return + } + event.CopyPartInfo = &part + event.Type = enum.CopyEventUploadPartCopySuccess + c.postCopyEvent(event) + case EventPartFailed: + event.Type = enum.CopyEventUploadPartCopyFailed + c.postCopyEvent(event) + case EventPartAborted: + event.Type = enum.CopyEventUploadPartCopyAborted + event.Err = taskErr + c.postCopyEvent(event) + default: + } +} + +type downloadEvent struct { + input *DownloadFileInput +} + +func (d downloadEvent) PostEvent(eventType int, result interface{}, taskErr error) { + switch eventType { + case EventPartSucceed: + part, ok := result.(downloadPartInfo) + if !ok { + return + } + d.postDownloadEvent(d.newDownloadPartSucceedEvent(part)) + case EventPartFailed: + d.postDownloadEvent(d.newFailedEvent(taskErr, enum.DownloadEventDownloadPartFailed)) + case EventPartAborted: + d.postDownloadEvent(d.newFailedEvent(taskErr, enum.DownloadEventDownloadPartAborted)) + default: + } +} + +type downloadTask struct { + cli *ClientV2 + ctx context.Context + input *DownloadFileInput + consumed *int64 + subtotal *int64 + total int64 + partNumber int + rangeStart int64 + rangeEnd int64 + enableCRC64 bool +} + +// Do the downloadTask, and return downloadPartInfo +func (t *downloadTask) do() (result interface{}, err error) { + input := t.getBaseInput().(GetObjectV2Input) + output, err := t.cli.GetObjectV2(t.ctx, &input) + if err != nil { + return nil, err + } + defer output.Content.Close() + file, err := os.OpenFile(t.input.tempFile, os.O_RDWR, DefaultFilePerm) + if err != nil { + return nil, err + } + defer func(file *os.File) { + _ = file.Close() + }(file) + var wrapped = output.Content + if t.input.DataTransferListener != nil { + wrapped = ¶llelReadCloserWithListener{ + listener: t.input.DataTransferListener, + base: wrapped, + consumed: t.consumed, + total: t.total, + subtotal: t.subtotal, + } + } + if t.input.RateLimiter != nil { + wrapped = &ReadCloserWithLimiter{ + limiter: t.input.RateLimiter, + base: wrapped, + } + } + var checker hash.Hash64 + if t.enableCRC64 { + checker = crc64.New(crc64.MakeTable(crc64.ECMA)) + wrapped = &readCloserWithCRC{ + checker: checker, + base: wrapped, + } + } + + _, err = file.Seek(t.rangeStart, io.SeekStart) + if err != nil { + return nil, err + } + written, err := io.Copy(file, wrapped) + if err != nil { + return nil, err + } + if written != (t.rangeEnd - t.rangeStart + 1) { + return nil, fmt.Errorf("io copy want length %d but get %d. ", t.rangeEnd-t.rangeStart+1, written) + } + part := downloadPartInfo{ + PartNumber: t.partNumber, + RangeStart: t.rangeStart, + RangeEnd: t.rangeEnd, + IsCompleted: true, + } + if t.enableCRC64 { + part.HashCrc64ecma = checker.Sum64() + } + return part, nil +} + +func (t *downloadTask) getBaseInput() interface{} { + return GetObjectV2Input{ + Bucket: t.input.Bucket, + Key: t.input.Key, + VersionID: t.input.VersionID, + IfMatch: t.input.IfMatch, + IfModifiedSince: t.input.IfModifiedSince, + IfNoneMatch: t.input.IfNoneMatch, + IfUnmodifiedSince: t.input.IfUnmodifiedSince, + SSECAlgorithm: t.input.SSECAlgorithm, + SSECKey: t.input.SSECKey, + SSECKeyMD5: t.input.SSECKeyMD5, + RangeStart: t.rangeStart, + RangeEnd: t.rangeEnd, + TrafficLimit: t.input.TrafficLimit, + // we want to Sent parallel Listener on output, so explicitly set listener of GetObjectV2Input nil here. + DataTransferListener: nil, + RateLimiter: nil, + } +} + +type uploadPostEvent struct { + input *UploadFileInput + checkPoint *uploadCheckpoint +} + +func (u *uploadPostEvent) PostEvent(eventType int, result interface{}, taskErr error) { + switch eventType { + case EventPartSucceed: + partInfo, ok := result.(uploadPartInfo) + if !ok { + return + } + u.postUploadEvent(u.newUploadPartSucceedEvent(u.input, partInfo)) + case EventPartFailed: + u.postUploadEvent(u.newUploadPartFailedEvent(u.input, u.checkPoint.UploadID, taskErr)) + case EventPartAborted: + u.postUploadEvent(u.newUploadPartAbortedEvent(u.input, u.checkPoint.UploadID, taskErr)) + + } +} + +type uploadTask struct { + cli *ClientV2 + input *UploadFileInput + consumed *int64 + subtotal *int64 + mutex *sync.Mutex + ctx context.Context + total int64 + UploadID string + ContentMD5 string + PartNumber int + Offset uint64 + PartSize int64 +} + +// Do the uploadTask, and return uploadPartInfo +func (t *uploadTask) do() (interface{}, error) { + file, err := os.Open(t.input.FilePath) + if err != nil { + return nil, newTosClientError(err.Error(), err) + } + _, err = file.Seek(int64(t.Offset), io.SeekStart) + if err != nil { + return nil, newTosClientError(err.Error(), err) + } + var wrapped = ioutil.NopCloser(io.LimitReader(file, t.input.PartSize)) + if t.input.DataTransferListener != nil { + wrapped = ¶llelReadCloserWithListener{ + listener: t.input.DataTransferListener, + base: wrapped, + total: t.total, + subtotal: t.subtotal, + consumed: t.consumed, + } + } + if t.input.RateLimiter != nil { + wrapped = &ReadCloserWithLimiter{ + limiter: t.input.RateLimiter, + base: wrapped, + } + } + input := t.getBaseInput().(UploadPartV2Input) + input.Content = wrapped + output, err := t.cli.UploadPartV2(t.ctx, &UploadPartV2Input{ + UploadPartBasicInput: input.UploadPartBasicInput, + Content: wrapped, + ContentLength: input.ContentLength, + }) + if err != nil { + return nil, err + } + return uploadPartInfo{ + uploadID: &t.UploadID, + PartNumber: output.PartNumber, + PartSize: t.PartSize, + Offset: t.Offset, + ETag: output.ETag, + HashCrc64ecma: output.HashCrc64ecma, + IsCompleted: true, + }, nil +} + +func (t *uploadTask) getBaseInput() interface{} { + return UploadPartV2Input{ + UploadPartBasicInput: UploadPartBasicInput{ + Bucket: t.input.Bucket, + Key: t.input.Key, + UploadID: t.UploadID, + PartNumber: t.PartNumber, + ContentMD5: t.ContentMD5, + SSECAlgorithm: t.input.SSECAlgorithm, + SSECKey: t.input.SSECKey, + SSECKeyMD5: t.input.SSECKeyMD5, + ServerSideEncryption: t.input.ServerSideEncryption, + TrafficLimit: t.input.TrafficLimit, + }, + ContentLength: t.PartSize, + } +} + +type retryAction int + +const ( + NoRetry retryAction = iota + Retry +) + +const ( + DefaultRetryBackoffBase = 100 * time.Millisecond + DefaultRetryTime = 3 +) + +type classifier interface { + Classify(error) retryAction +} + +func exponentialBackoff(n int, base time.Duration) []time.Duration { + backoffs := make([]time.Duration, n) + for i := 0; i < len(backoffs); i++ { + backoffs[i] = base + base *= 2 + } + return backoffs +} + +type retryer struct { + backoff []time.Duration + jitter float64 +} + +func (r *retryer) SetBackoff(backoff []time.Duration) { + r.backoff = backoff +} + +// newRetryer constructs a retryer with the given backoff pattern and classifier. The length of the backoff pattern +// indicates how many times an action will be retried, and the value at each index indicates the amount of time +// waited before each subsequent retry. The classifier is used to determine which errors should be retried and +// which should cause the retrier to fail fast. The DefaultClassifier is used if nil is passed. +func newRetryer(backoff []time.Duration) *retryer { + return &retryer{ + backoff: backoff, + } +} + +func worthToRetry(ctx context.Context, waitTime time.Duration) bool { + if ctx == nil { + return true + } + if ctx.Err() != nil { + return false + } + deadline, ok := ctx.Deadline() + if !ok { + return true + } + now := time.Now() + return now.UnixNano()+int64(waitTime) <= deadline.UnixNano() +} + +// Run executes the given work function, then classifies its return value based on the classifier. +// If the result is Succeed or Fail, the return value of the work function is +// returned to the caller. If the result is Retry, then Run sleeps according to its backoff policy +// before retrying. If the total number of retries is exceeded then the return value of the work function +// is returned to the caller regardless. +func (r *retryer) Run(ctx context.Context, work func() error, classifier classifier) error { + // run + ferr := work() + // try retry + for i := 0; i < len(r.backoff) && classifier.Classify(ferr) == Retry; i++ { + // 重试 + sleepTime := r.calcSleep(i) + if !worthToRetry(ctx, sleepTime) { + return ferr + } + time.Sleep(sleepTime) + ferr = work() + } + return ferr +} + +func (r *retryer) calcSleep(i int) time.Duration { + // take a random float in the range (-r.jitter, +r.jitter) and multiply it by the base amount + return r.backoff[i] +} + +// SetJitter sets the amount of jitter on each back-off to a factor between 0.0 and 1.0 (values outside this range +// are silently ignored). When a retry occurs, the back-off is adjusted by a random amount up to this value. +func (r *retryer) SetJitter(jit float64) { + if jit < 0 || jit > 1 { + return + } + r.jitter = jit +} + +// readCloserWithCRC warp io.ReadCloser with crc checker +type readCloserWithCRC struct { + serverCrc uint64 // Get Object 时对 content 进行校验 + checker hash.Hash64 + base io.ReadCloser +} + +func (r *readCloserWithCRC) Seek(offset int64, whence int) (int64, error) { + seeker, ok := r.base.(io.Seeker) + if !ok { + return 0, NotSupportSeek + } + + if whence != io.SeekCurrent { + r.checker.Reset() + } + + return seeker.Seek(offset, whence) +} + +func (r *readCloserWithCRC) Read(p []byte) (n int, err error) { + n, err = r.base.Read(p) + if n > 0 { + if n, err = r.checker.Write(p[:n]); err != nil { + return n, err + } + } + if err == io.EOF && r.serverCrc != 0 { + clientCRC := r.checker.Sum64() + if clientCRC != r.serverCrc { + return n, CrcCheckFail.withCause(fmt.Errorf("expect crc: %d , actual crc:%d", r.serverCrc, clientCRC)) + } + } + + return +} + +func (r *readCloserWithCRC) Close() error { + return r.base.Close() +} + +// parallelReadCloserWithListener warp multiple io.ReadCloser will be R/W in parallel with a same DataTransferListener +type parallelReadCloserWithListener struct { + listener DataTransferListener + base io.ReadCloser + consumed *int64 + subtotal *int64 + total int64 +} + +func (r *parallelReadCloserWithListener) Read(p []byte) (n int, err error) { + n, err = r.base.Read(p) + if err != nil && err != io.EOF { + postDataTransferStatus(r.listener, &DataTransferStatus{ + Type: enum.DataTransferFailed, + }) + return n, err + } + if n <= 0 { + return + } + subtotal := atomic.AddInt64(r.subtotal, int64(n)) + consumed := atomic.AddInt64(r.consumed, int64(n)) + + if subtotal >= DefaultProgressCallbackSize && atomic.CompareAndSwapInt64(r.subtotal, subtotal, 0) { + postDataTransferStatus(r.listener, &DataTransferStatus{ + Type: enum.DataTransferRW, + RWOnceBytes: subtotal, + ConsumedBytes: consumed, + TotalBytes: r.total, + }) + } + + if consumed == r.total { + for subtotal != 0 { + if atomic.CompareAndSwapInt64(r.subtotal, subtotal, 0) { + postDataTransferStatus(r.listener, &DataTransferStatus{ + Type: enum.DataTransferRW, + RWOnceBytes: subtotal, + ConsumedBytes: consumed, + TotalBytes: r.total, + }) + break + } else { + subtotal = atomic.LoadInt64(r.subtotal) + } + } + postDataTransferStatus(r.listener, &DataTransferStatus{ + Type: enum.DataTransferSucceed, + ConsumedBytes: consumed, + TotalBytes: r.total, + }) + } + return +} + +func (r *parallelReadCloserWithListener) Close() error { + return r.base.Close() +} + +// readCloserWithListener warp io.ReadCloser with DataTransferListener +type readCloserWithListener struct { + listener DataTransferListener + base io.ReadCloser + consumed int64 + subtotal int64 + total int64 + onceEof bool +} + +func (r *readCloserWithListener) Seek(offset int64, whence int) (int64, error) { + seeker, ok := r.base.(io.Seeker) + if !ok { + return 0, NotSupportSeek + } + if whence != io.SeekCurrent { + r.consumed = 0 + r.subtotal = 0 + r.onceEof = false + } + + return seeker.Seek(offset, whence) +} + +func (r *readCloserWithListener) Read(p []byte) (n int, err error) { + if r.consumed == 0 { + postDataTransferStatus(r.listener, &DataTransferStatus{ + Type: enum.DataTransferStarted, + }) + } + defer func() { + if err == io.EOF { + r.consumed += int64(n) + r.subtotal += int64(n) + if r.subtotal != 0 { + postDataTransferStatus(r.listener, &DataTransferStatus{ + Type: enum.DataTransferRW, + RWOnceBytes: r.subtotal, + ConsumedBytes: r.consumed, + TotalBytes: r.total, + }) + r.subtotal = 0 + } + + if !r.onceEof { + if r.total == -1 { + r.total = r.consumed + } + postDataTransferStatus(r.listener, &DataTransferStatus{ + Type: enum.DataTransferSucceed, + ConsumedBytes: r.consumed, + TotalBytes: r.total, + }) + r.onceEof = true + } + + } + }() + n, err = r.base.Read(p) + if err != nil && err != io.EOF { + postDataTransferStatus(r.listener, &DataTransferStatus{ + Type: enum.DataTransferFailed, + }) + return n, err + } + if n <= 0 || err == io.EOF { + return + } + r.consumed += int64(n) + r.subtotal += int64(n) + if r.subtotal >= DefaultProgressCallbackSize { + postDataTransferStatus(r.listener, &DataTransferStatus{ + Type: enum.DataTransferRW, + RWOnceBytes: r.subtotal, + ConsumedBytes: r.consumed, + TotalBytes: r.total, + }) + r.subtotal = 0 + } + return +} + +func (r *readCloserWithListener) Close() error { + return r.base.Close() +} + +// ReadCloserWithLimiter warp io.ReadCloser with DataTransferListener +type ReadCloserWithLimiter struct { + limiter RateLimiter + acquireN int + base io.ReadCloser +} + +func (r *ReadCloserWithLimiter) Seek(offset int64, whence int) (int64, error) { + seeker, ok := r.base.(io.Seeker) + if !ok { + return 0, NotSupportSeek + } + r.acquireN = 0 + return seeker.Seek(offset, whence) +} + +func (r *ReadCloserWithLimiter) Read(p []byte) (n int, err error) { + want := len(p) + if want > r.acquireN { + // 需要申请的配额 + want = want - r.acquireN + for { + ok, timeToWait := r.limiter.Acquire(int64(want)) + if ok { + break + } + time.Sleep(timeToWait) + } + r.acquireN += want + } + n, err = r.base.Read(p) + // 实际消耗的配额 + r.acquireN = r.acquireN - n + return n, err + +} + +func (r *ReadCloserWithLimiter) Close() error { + return r.base.Close() +} + +type copyTask struct { + cli *ClientV2 + input *ResumableCopyObjectInput + ctx context.Context + UploadID string + ContentMD5 string + PartNumber int64 + Offset uint64 + PartSize int64 + PartInfo copyPartInfo +} + +func (c *copyTask) do() (interface{}, error) { + uploadInput, ok := c.getBaseInput().(UploadPartV2Input) + if ok { + part, err := c.cli.UploadPartV2(c.ctx, &uploadInput) + if err != nil { + return nil, err + } + return copyPartInfo{PartNumber: int64(part.PartNumber), Etag: part.ETag, IsCompleted: true}, nil + } + input := c.getBaseInput().(UploadPartCopyV2Input) + output, err := c.cli.UploadPartCopyV2(c.ctx, &input) + if err != nil { + return nil, err + } + return copyPartInfo{ + PartNumber: int64(output.PartNumber), + CopySourceRange: input.CopySourceRange, + Etag: output.ETag, + IsCompleted: true, + }, nil + +} + +func (c *copyTask) getBaseInput() interface{} { + if c.PartInfo.IsZeroSize { + return UploadPartV2Input{UploadPartBasicInput: UploadPartBasicInput{ + Bucket: c.input.Bucket, + Key: c.input.Key, + UploadID: c.UploadID, + PartNumber: 1, + SSECAlgorithm: c.input.SSECAlgorithm, + SSECKey: c.input.SSECKey, + SSECKeyMD5: c.input.SSECKeyMD5, + }} + } + return UploadPartCopyV2Input{ + Bucket: c.input.Bucket, + Key: c.input.Key, + UploadID: c.UploadID, + PartNumber: int(c.PartNumber), + SrcBucket: c.input.SrcBucket, + SrcKey: c.input.SrcKey, + SrcVersionID: c.input.SrcVersionID, + CopySourceRangeStart: c.PartInfo.CopySourceRangeStart, + CopySourceRangeEnd: c.PartInfo.CopySourceRangeEnd, + CopySourceRange: c.PartInfo.CopySourceRange, + CopySourceIfMatch: c.input.CopySourceIfMatch, + CopySourceIfModifiedSince: c.input.CopySourceIfModifiedSince, + CopySourceIfNoneMatch: c.input.CopySourceIfNoneMatch, + CopySourceIfUnmodifiedSince: c.input.CopySourceIfUnmodifiedSince, + CopySourceSSECAlgorithm: c.input.CopySourceSSECAlgorithm, + CopySourceSSECKey: c.input.CopySourceSSECKey, + CopySourceSSECKeyMD5: c.input.CopySourceSSECKeyMD5, + SSECKey: c.input.SSECKey, + SSECKeyMD5: c.input.SSECKeyMD5, + SSECAlgorithm: c.input.SSECAlgorithm, + TrafficLimit: c.input.TrafficLimit, + } +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/upload_file.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/upload_file.go new file mode 100644 index 0000000000..664082e91f --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/upload_file.go @@ -0,0 +1,428 @@ +package tos + +import ( + "context" + "os" + "path/filepath" + "strings" + + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" +) + +// initUploadPartsInfo initialize parts info from file stat,return TosClientError if failed +func initUploadPartsInfo(uploadFileStat os.FileInfo, partSize int64) ([]uploadPartInfo, error) { + partCount := uploadFileStat.Size() / partSize + lastPartSize := uploadFileStat.Size() % partSize + if lastPartSize != 0 { + partCount++ + } + if partCount > 10000 { + return nil, InvalidFilePartNum + } + parts := make([]uploadPartInfo, 0, partCount) + for i := int64(0); i < partCount; i++ { + part := uploadPartInfo{ + PartNumber: int(i + 1), + PartSize: partSize, + Offset: uint64(i * partSize), + } + parts = append(parts, part) + } + if lastPartSize != 0 { + parts[partCount-1].PartSize = lastPartSize + } + if uploadFileStat.Size() == 0 { + parts = append(parts, uploadPartInfo{PartNumber: 1, PartSize: 0, Offset: 0}) + } + return parts, nil +} + +// initUploadCheckpoint initialize checkpoint file, return TosClientError if failed +func initUploadCheckpoint(input *UploadFileInput, stat os.FileInfo) (*uploadCheckpoint, error) { + parts, err := initUploadPartsInfo(stat, input.PartSize) + if err != nil { + return nil, err + } + checkPoint := &uploadCheckpoint{ + checkpointPath: input.CheckpointFile, + PartsInfo: parts, + Bucket: input.Bucket, + Key: input.Key, + PartSize: input.PartSize, + SSECAlgorithm: input.SSECAlgorithm, + SSECKeyMD5: input.SSECKeyMD5, + EncodingType: input.ContentEncoding, + FilePath: input.FilePath, + FileInfo: fileInfo{ + Size: stat.Size(), + LastModified: stat.ModTime().Unix(), + }, + } + return checkPoint, nil +} + +func getUploadCheckpointFilePath(checkpointPath, filePath string, bucket, key string) string { + fileName := strings.Join([]string{filepath.Base(filePath), checkpointPathMd5(bucket, key, ""), "upload"}, ".") + if len(checkpointPath) == 0 { + dirName := filepath.Dir(filePath) + return filepath.Join(dirName, fileName) + } + + return withSuffixIfDir(checkpointPath, fileName) +} + +// validateUploadInput validate upload input, return TosClientError failed +func validateUploadInput(input *UploadFileInput, stat os.FileInfo, isCustomDomain bool) error { + if err := isValidNames(input.Bucket, input.Key, isCustomDomain); err != nil { + return err + } + if input.PartSize == 0 { + input.PartSize = DefaultPartSize + } + if input.PartSize < MinPartSize || input.PartSize > MaxPartSize { + return InvalidPartSize + } + + if stat.IsDir() { + return newTosClientError("tos: does not support directory, please specific your file path.", nil) + } + if input.EnableCheckpoint { + // get correct checkpoint path + input.CheckpointFile = getUploadCheckpointFilePath(input.CheckpointFile, input.FilePath, input.Bucket, input.Key) + } + if input.TaskNum < 1 { + input.TaskNum = 1 + } + if input.TaskNum > 1000 { + input.TaskNum = 1000 + } + return nil +} + +func (u *uploadPostEvent) postUploadEvent(event *UploadEvent) { + if u.input.UploadEventListener != nil { + u.input.UploadEventListener.EventChange(event) + } +} + +func loadExistUploadCheckPoint(ctx context.Context, cli *ClientV2, input *UploadFileInput, srcFile os.FileInfo) (*uploadCheckpoint, bool) { + checkpoint := &uploadCheckpoint{} + var err error + loadCheckPoint(input.CheckpointFile, checkpoint) + if checkpoint.Valid(srcFile, input.Bucket, input.Key, input.FilePath) { + return checkpoint, true + } else if checkpoint.Bucket != "" && checkpoint.Key != "" && checkpoint.UploadID != "" { + // 尝试去 abort + _, err = cli.AbortMultipartUpload(ctx, + &AbortMultipartUploadInput{ + Bucket: checkpoint.Bucket, + Key: checkpoint.Key, + UploadID: checkpoint.UploadID}) + if err != nil && cli.logger != nil { + cli.logger.Debug("fail to abort upload task: %s, err:%s", checkpoint.UploadID, err.Error()) + } + } + return nil, false +} + +// getUploadCheckpoint get struct checkpoint from checkpoint file if checkpointPath is valid, +// or initialize from scratch with function init +func getUploadCheckpoint(ctx context.Context, cli *ClientV2, input *UploadFileInput, srcFile os.FileInfo, init func() (*uploadCheckpoint, error)) (checkpoint *uploadCheckpoint, err error) { + + if !input.EnableCheckpoint { + return init() + } + + checkpoint, exist := loadExistUploadCheckPoint(ctx, cli, input, srcFile) + + if exist { + return checkpoint, nil + } + err = checkAndCreateDir(input.CheckpointFile) + if err != nil { + return nil, InvalidCheckpointFilePath.withCause(err) + } + + file, err := os.Create(input.CheckpointFile) + if err != nil { + return nil, newTosClientError("tos: create checkpoint file failed", err) + } + _ = file.Close() + + checkpoint, err = init() + if err != nil { + return nil, err + } + + err = checkpoint.WriteToFile() + if err != nil { + return nil, err + } + + return +} + +func bindCancelHookWithAborter(hook CancelHook, aborter func() error) { + if hook == nil { + return + } + cancel := hook.(*canceler) + cancel.aborter = aborter +} + +func bindCancelHookWithCleaner(hook CancelHook, cleaner func()) { + if hook == nil { + return + } + cancel := hook.(*canceler) + cancel.cleaner = cleaner +} + +func (cli *ClientV2) UploadFile(ctx context.Context, input *UploadFileInput) (output *UploadFileOutput, err error) { + // avoid modifying on origin pointer + input = &(*input) + + stat, err := os.Stat(input.FilePath) + if err != nil { + return nil, InvalidSrcFilePath + } + + if err = validateUploadInput(input, stat, cli.isCustomDomain); err != nil { + return nil, err + } + + init := func() (*uploadCheckpoint, error) { + return initUploadCheckpoint(input, stat) + } + + // if the checkpoint file not exist, here we will create it + checkpoint, err := getUploadCheckpoint(ctx, cli, input, stat, init) + if err != nil { + return nil, err + } + + event := &uploadPostEvent{ + input: input, + checkPoint: checkpoint, + } + if checkpoint.UploadID == "" { + // create multipart upload task + created, err := cli.CreateMultipartUploadV2(ctx, &input.CreateMultipartUploadV2Input) + if err != nil { + event.postUploadEvent(&UploadEvent{ + Type: enum.UploadEventCreateMultipartUploadFailed, + Err: err, + Bucket: input.Bucket, + Key: input.Key, + CheckpointFile: &input.CheckpointFile, + }) + return nil, err + } + event.postUploadEvent(&UploadEvent{ + Type: enum.UploadEventCreateMultipartUploadSucceed, + Bucket: input.Bucket, + Key: input.Key, + UploadID: &created.UploadID, + CheckpointFile: &input.CheckpointFile, + }) + checkpoint.UploadID = created.UploadID + } + + cleaner := func() { + _ = os.Remove(input.CheckpointFile) + } + event.checkPoint = checkpoint + bindCancelHookWithCleaner(input.CancelHook, cleaner) + return cli.uploadPart(ctx, checkpoint, input, event) +} + +func prepareUploadTasks(cli *ClientV2, ctx context.Context, checkpoint *uploadCheckpoint, input *UploadFileInput) []task { + tasks := make([]task, 0) + consumed := int64(0) + subtotal := int64(0) + for _, part := range checkpoint.PartsInfo { + if !part.IsCompleted { + tasks = append(tasks, &uploadTask{ + cli: cli, + ctx: ctx, + input: input, + total: checkpoint.FileInfo.Size, + UploadID: checkpoint.UploadID, + PartNumber: part.PartNumber, + subtotal: &subtotal, + consumed: &consumed, + Offset: part.Offset, + PartSize: part.PartSize, + }) + } else { + consumed += part.PartSize + } + } + return tasks +} + +func (u *uploadPostEvent) newUploadPartSucceedEvent(input *UploadFileInput, part uploadPartInfo) *UploadEvent { + return &UploadEvent{ + Type: enum.UploadEventUploadPartSucceed, + Bucket: input.Bucket, + Key: input.Key, + UploadID: part.uploadID, + CheckpointFile: &input.CheckpointFile, + UploadPartInfo: &UploadPartInfo{ + PartNumber: part.PartNumber, + PartSize: part.PartSize, + Offset: int64(part.Offset), + ETag: &part.ETag, + HashCrc64ecma: &part.HashCrc64ecma, + }, + } +} + +func (u *uploadPostEvent) newUploadPartAbortedEvent(input *UploadFileInput, uploadID string, err error) *UploadEvent { + return &UploadEvent{ + Type: enum.UploadEventUploadPartAborted, + Err: err, + Bucket: input.Bucket, + Key: input.Key, + UploadID: &uploadID, + CheckpointFile: &input.CheckpointFile, + } +} + +func (u *uploadPostEvent) newUploadPartFailedEvent(input *UploadFileInput, uploadID string, err error) *UploadEvent { + return &UploadEvent{ + Type: enum.UploadEventUploadPartFailed, + Err: err, + Bucket: input.Bucket, + Key: input.Key, + UploadID: &uploadID, + CheckpointFile: &input.CheckpointFile, + } +} + +func (u *uploadPostEvent) newCompleteMultipartUploadFailedEvent(input *UploadFileInput, uploadID string, err error) *UploadEvent { + return &UploadEvent{ + Type: enum.UploadEventCompleteMultipartUploadFailed, + Err: err, + Bucket: input.Bucket, + Key: input.Key, + UploadID: &uploadID, + CheckpointFile: &input.CheckpointFile, + } +} + +func newCompleteMultipartUploadSucceedEvent(input *UploadFileInput, uploadID string) *UploadEvent { + return &UploadEvent{ + Type: enum.UploadEventCompleteMultipartUploadSucceed, + Bucket: input.Bucket, + Key: input.Key, + UploadID: &uploadID, + CheckpointFile: &input.CheckpointFile, + } +} + +func postDataTransferStatus(listener DataTransferListener, status *DataTransferStatus) { + if listener != nil { + listener.DataTransferStatusChange(status) + } +} + +func getCancelHandle(hook CancelHook) chan struct{} { + if c, ok := hook.(*canceler); ok { + return c.cancelHandle + } + return make(chan struct{}) +} + +func combineCRCInDownload(parts []downloadPartInfo) uint64 { + if len(parts) == 0 { + return 0 + } + crc := parts[0].HashCrc64ecma + for i := 1; i < len(parts); i++ { + crc = CRC64Combine(crc, parts[i].HashCrc64ecma, uint64(parts[i].RangeEnd-parts[i].RangeStart+1)) + } + return crc +} + +// combineCRCInParts calculates the total CRC of continuous parts +func combineCRCInParts(parts []uploadPartInfo) uint64 { + if parts == nil || len(parts) == 0 { + return 0 + } + crc := parts[0].HashCrc64ecma + for i := 1; i < len(parts); i++ { + crc = CRC64Combine(crc, parts[i].HashCrc64ecma, uint64(parts[i].PartSize)) + } + return crc +} + +func (cli *ClientV2) uploadPart(ctx context.Context, checkpoint *uploadCheckpoint, input *UploadFileInput, event *uploadPostEvent) (*UploadFileOutput, error) { + // prepare tasks + // if amount of tasks >= 10000, err "tos: part count too many" will be raised. + tasks := prepareUploadTasks(cli, ctx, checkpoint, input) + routinesNum := min(input.TaskNum, len(tasks)) + cancelHandle := getCancelHandle(input.CancelHook) + tg := newTaskGroup(cancelHandle, routinesNum, checkpoint, event, input.EnableCheckpoint, tasks) + abort := func() error { + _, err := cli.AbortMultipartUpload(ctx, + &AbortMultipartUploadInput{ + Bucket: input.Bucket, + Key: input.Key, + UploadID: checkpoint.UploadID}) + _ = os.Remove(input.CheckpointFile) + return err + } + bindCancelHookWithAborter(input.CancelHook, abort) + + tg.RunWorker() + // start adding tasks + postDataTransferStatus(input.DataTransferListener, &DataTransferStatus{ + TotalBytes: checkpoint.FileInfo.Size, + Type: enum.DataTransferStarted, + }) + + tg.Scheduler() + success, taskErr := tg.Wait() + if taskErr != nil { + if err := abort(); err != nil { + return nil, err + } + return nil, taskErr + } + // handle results + if success < len(tasks) { + return nil, newTosClientError("tos: some upload tasks failed.", nil) + } + complete, err := cli.CompleteMultipartUploadV2(ctx, &CompleteMultipartUploadV2Input{ + Bucket: input.Bucket, + Key: input.Key, + UploadID: checkpoint.UploadID, + Parts: checkpoint.GetParts(), + }) + if err != nil { + event.postUploadEvent(event.newCompleteMultipartUploadFailedEvent(input, checkpoint.UploadID, err)) + return nil, err + } + event.postUploadEvent(newCompleteMultipartUploadSucceedEvent(input, checkpoint.UploadID)) + + if cli.enableCRC && complete.HashCrc64ecma != 0 && combineCRCInParts(checkpoint.PartsInfo) != complete.HashCrc64ecma { + return nil, newTosClientError("tos: crc of entire file mismatch.", nil) + + } + _ = os.Remove(input.CheckpointFile) + + return &UploadFileOutput{ + RequestInfo: complete.RequestInfo, + Bucket: complete.Bucket, + Key: complete.Key, + UploadID: checkpoint.UploadID, + ETag: complete.ETag, + Location: complete.Location, + VersionID: complete.VersionID, + HashCrc64ecma: complete.HashCrc64ecma, + SSECAlgorithm: checkpoint.SSECAlgorithm, + SSECKeyMD5: checkpoint.SSECKeyMD5, + EncodingType: checkpoint.EncodingType, + }, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/util.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/util.go new file mode 100644 index 0000000000..14e6a4be56 --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/util.go @@ -0,0 +1,165 @@ +package tos + +import ( + "fmt" + "os" + "time" +) + +func min(a int, b int) int { + if a < b { + return a + } + return b +} + +const ( + EventPartSucceed = 3 + EventPartFailed = 4 + EventPartAborted = 5 // The task needs to be interrupted in case of 403, 404, 405 errors +) + +type task interface { + do() (interface{}, error) + getBaseInput() interface{} +} + +type checkPoint interface { + WriteToFile() error + UpdatePartsInfo(result interface{}) + GetCheckPointFilePath() string +} + +type taskGroup interface { + // Wait 等待执行结果, success 是此次成功的 task 数量 + Wait() (success int, err error) + // RunWorker 启动worker + RunWorker() + // Scheduler 分发任务 + Scheduler() +} + +type postEvent interface { + PostEvent(eventType int, result interface{}, taskErr error) +} + +type taskGroupImpl struct { + cancelHandle chan struct{} + abortHandle chan struct{} + errCh chan error + resultsCh chan interface{} + tasksCh chan task + routinesNum int + tasks []task + checkPoint checkPoint + enableCheckPoint bool + postEvent postEvent +} + +func (t *taskGroupImpl) Wait() (int, error) { + successNum := 0 + failNum := 0 +Loop: + for successNum+failNum < len(t.tasks) { + select { + case <-t.abortHandle: + break Loop + case <-t.cancelHandle: + break Loop + case part := <-t.resultsCh: + successNum++ + t.checkPoint.UpdatePartsInfo(part) + if t.enableCheckPoint { + t.checkPoint.WriteToFile() + } + t.postEvent.PostEvent(EventPartSucceed, part, nil) + case taskErr := <-t.errCh: + if StatusCode(taskErr) == 403 || StatusCode(taskErr) == 404 || StatusCode(taskErr) == 405 { + close(t.abortHandle) + _ = os.Remove(t.checkPoint.GetCheckPointFilePath()) + t.postEvent.PostEvent(EventPartAborted, nil, taskErr) + + return successNum, fmt.Errorf("status code not service error, err:%s. ", taskErr.Error()) + } + t.postEvent.PostEvent(EventPartFailed, nil, taskErr) + failNum++ + } + } + return successNum, nil +} + +func newTaskGroup(cancelHandle chan struct{}, routinesNum int, checkPoint checkPoint, postEvent postEvent, enableCheckPoint bool, tasks []task) taskGroup { + taskBufferSize := min(routinesNum, DefaultTaskBufferSize) + tasksCh := make(chan task, taskBufferSize) + return &taskGroupImpl{ + cancelHandle: cancelHandle, + abortHandle: make(chan struct{}), + errCh: make(chan error), + resultsCh: make(chan interface{}), + tasksCh: tasksCh, + routinesNum: routinesNum, + tasks: tasks, + checkPoint: checkPoint, + enableCheckPoint: enableCheckPoint, + postEvent: postEvent, + } +} + +func (t *taskGroupImpl) RunWorker() { + for i := 0; i < t.routinesNum; i++ { + go t.worker() + } + +} + +func (t *taskGroupImpl) Scheduler() { + go func() { + for _, task := range t.tasks { + select { + case <-t.cancelHandle: + return + case <-t.abortHandle: + return + default: + t.tasksCh <- task + } + } + + close(t.tasksCh) + }() + +} + +func (t *taskGroupImpl) worker() { + for { + select { + case <-t.cancelHandle: + return + case <-t.abortHandle: + return + case task, ok := <-t.tasksCh: + if !ok { + return + } + result, err := task.do() + if err != nil { + t.errCh <- err + } + if result != nil { + t.resultsCh <- result + } + } + } +} + +func GetUnixTimeMs() int64 { + return ToMillis(time.Now()) +} + +func ToMillis(t time.Time) int64 { + return t.UnixNano() / int64(time.Millisecond) +} + +func StringPtr(input string) *string { + return &input +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/versioning.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/versioning.go new file mode 100644 index 0000000000..4774a8589a --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/versioning.go @@ -0,0 +1,38 @@ +package tos + +import ( + "context" + "net/http" +) + +const ( + BucketVersioningEnable = "Enabled" + BucketVersioningSuspended = "Suspended" +) + +type GetBucketVersioningOutput struct { + RequestInfo `json:"-"` + Status string `json:"Status"` +} + +// GetBucketVersioning get the multi-version status of a bucket +func (cli *Client) GetBucketVersioning(ctx context.Context, bucket string) (*GetBucketVersioningOutput, error) { + if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil { + return nil, err + } + + res, err := cli.newBuilder(bucket, ""). + WithQuery("versioning", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + + output := GetBucketVersioningOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} diff --git a/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/website.go b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/website.go new file mode 100644 index 0000000000..8ddc60818e --- /dev/null +++ b/vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/website.go @@ -0,0 +1,82 @@ +package tos + +import ( + "bytes" + "context" + "net/http" +) + +func (cli *ClientV2) PutBucketWebsite(ctx context.Context, input *PutBucketWebsiteInput) (*PutBucketWebsiteOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + body := putBucketWebsiteInput{ + RedirectAllRequestsTo: input.RedirectAllRequestsTo, + IndexDocument: input.IndexDocument, + ErrorDocument: input.ErrorDocument, + } + if input.RoutingRules != nil { + body.RoutingRules = input.RoutingRules.Rules + } + data, contentMD5, err := marshalInput("PutBucketWebsiteInput", body) + if err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("website", ""). + WithHeader(HeaderContentMD5, contentMD5). + WithRetry(OnRetryFromStart, StatusCodeClassifier{}). + Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := PutBucketWebsiteOutput{RequestInfo: res.RequestInfo()} + return &output, nil +} + +func (cli *ClientV2) GetBucketWebsite(ctx context.Context, input *GetBucketWebsiteInput) (*GetBucketWebsiteOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("website", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK)) + if err != nil { + return nil, err + } + defer res.Close() + output := GetBucketWebsiteOutput{RequestInfo: res.RequestInfo()} + if err = marshalOutput(output.RequestID, res.Body, &output); err != nil { + return nil, err + } + return &output, nil +} + +func (cli *ClientV2) DeleteBucketWebsite(ctx context.Context, input *DeleteBucketWebsiteInput) (*DeleteBucketWebsiteOutput, error) { + if input == nil { + return nil, InputIsNilClientError + } + if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil { + return nil, err + } + res, err := cli.newBuilder(input.Bucket, ""). + WithQuery("website", ""). + WithRetry(nil, StatusCodeClassifier{}). + Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent)) + if err != nil { + return nil, err + } + defer res.Close() + + output := DeleteBucketWebsiteOutput{RequestInfo: res.RequestInfo()} + return &output, nil + +} diff --git a/vendor/github.com/volcengine/volc-sdk-golang/base/aes.go b/vendor/github.com/volcengine/volc-sdk-golang/base/aes.go new file mode 100644 index 0000000000..60c8399651 --- /dev/null +++ b/vendor/github.com/volcengine/volc-sdk-golang/base/aes.go @@ -0,0 +1,51 @@ +package base + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "errors" + "fmt" +) + +// AES CBC 加密 +func aesEncryptCBC(origData, key []byte) (crypted []byte, err error) { + defer func() { + if r := recover(); r != nil { + crypted = nil + err = errors.New(fmt.Sprintf("%v", r)) + } + }() + block, err := aes.NewCipher(key) + if err != nil { + return + } + + blockSize := block.BlockSize() + origData = zeroPadding(origData, blockSize) + blockMode := cipher.NewCBCEncrypter(block, key[:blockSize]) + crypted = make([]byte, len(origData)) + blockMode.CryptBlocks(crypted, origData) + return +} + +// AES CBC 加密后做一次Base64加密 +func aesEncryptCBCWithBase64(origData, key []byte) (string, error) { + cbc, err := aesEncryptCBC(origData, key) + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(cbc), nil +} + +func zeroPadding(ciphertext []byte, blockSize int) []byte { + padding := blockSize - len(ciphertext)%blockSize + if padding == 0 { + return ciphertext + } + + padtext := bytes.Repeat([]byte{byte(0)}, padding) + return append(ciphertext, padtext...) +} diff --git a/vendor/github.com/volcengine/volc-sdk-golang/base/client.go b/vendor/github.com/volcengine/volc-sdk-golang/base/client.go new file mode 100644 index 0000000000..e10abdf1b8 --- /dev/null +++ b/vendor/github.com/volcengine/volc-sdk-golang/base/client.go @@ -0,0 +1,290 @@ +package base + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +const ( + accessKey = "VOLC_ACCESSKEY" + secretKey = "VOLC_SECRETKEY" + + defaultScheme = "http" +) + +var _GlobalClient *http.Client + +func init() { + _GlobalClient = &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: 1000, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 10 * time.Second, + }, + } +} + +// Client 基础客户端 +type Client struct { + Client *http.Client + SdkVersion string + ServiceInfo *ServiceInfo + ApiInfoList map[string]*ApiInfo +} + +// NewClient 生成一个客户端 +func NewClient(info *ServiceInfo, apiInfoList map[string]*ApiInfo) *Client { + client := &Client{Client: _GlobalClient, ServiceInfo: info.Clone(), ApiInfoList: apiInfoList} + + if client.ServiceInfo.Scheme == "" { + client.ServiceInfo.Scheme = defaultScheme + } + + if os.Getenv(accessKey) != "" && os.Getenv(secretKey) != "" { + client.ServiceInfo.Credentials.AccessKeyID = os.Getenv(accessKey) + client.ServiceInfo.Credentials.SecretAccessKey = os.Getenv(secretKey) + } else if _, err := os.Stat(os.Getenv("HOME") + "/.volc/config"); err == nil { + if content, err := ioutil.ReadFile(os.Getenv("HOME") + "/.volc/config"); err == nil { + m := make(map[string]string) + json.Unmarshal(content, &m) + if accessKey, ok := m["ak"]; ok { + client.ServiceInfo.Credentials.AccessKeyID = accessKey + } + if secretKey, ok := m["sk"]; ok { + client.ServiceInfo.Credentials.SecretAccessKey = secretKey + } + } + } + + content, err := ioutil.ReadFile("VERSION") + if err == nil { + client.SdkVersion = strings.TrimSpace(string(content)) + client.ServiceInfo.Header.Set("User-Agent", strings.Join([]string{"volc-sdk-golang", client.SdkVersion}, "/")) + } + + return client +} + +func (serviceInfo *ServiceInfo) Clone() *ServiceInfo { + ret := new(ServiceInfo) + //base info + ret.Timeout = serviceInfo.Timeout + ret.Host = serviceInfo.Host + ret.Scheme = serviceInfo.Scheme + + //credential + ret.Credentials = serviceInfo.Credentials.Clone() + + // header + ret.Header = serviceInfo.Header.Clone() + return ret +} + +func (cred Credentials) Clone() Credentials { + return Credentials{ + Service: cred.Service, + Region: cred.Region, + SecretAccessKey: cred.SecretAccessKey, + AccessKeyID: cred.AccessKeyID, + SessionToken: cred.SessionToken, + } +} + +// SetAccessKey 设置AK +func (client *Client) SetAccessKey(ak string) { + if ak != "" { + client.ServiceInfo.Credentials.AccessKeyID = ak + } +} + +// SetSecretKey 设置SK +func (client *Client) SetSecretKey(sk string) { + if sk != "" { + client.ServiceInfo.Credentials.SecretAccessKey = sk + } +} + +// SetSessionToken +func (client *Client) SetSessionToken(token string) { + if token != "" { + client.ServiceInfo.Credentials.SessionToken = token + } +} + +// SetHost 设置Host +func (client *Client) SetHost(host string) { + if host != "" { + client.ServiceInfo.Host = host + } +} + +func (client *Client) SetScheme(scheme string) { + if scheme != "" { + client.ServiceInfo.Scheme = scheme + } +} + +// SetCredential 设置Credentials +func (client *Client) SetCredential(c Credentials) { + if c.AccessKeyID != "" { + client.ServiceInfo.Credentials.AccessKeyID = c.AccessKeyID + } + + if c.SecretAccessKey != "" { + client.ServiceInfo.Credentials.SecretAccessKey = c.SecretAccessKey + } + + if c.Region != "" { + client.ServiceInfo.Credentials.Region = c.Region + } + + if c.SessionToken != "" { + client.ServiceInfo.Credentials.SessionToken = c.SessionToken + } +} + +func (client *Client) SetTimeout(timeout time.Duration) { + if timeout > 0 { + client.ServiceInfo.Timeout = timeout + } +} + +// GetSignUrl 获取签名字符串 +func (client *Client) GetSignUrl(api string, query url.Values) (string, error) { + apiInfo := client.ApiInfoList[api] + + if apiInfo == nil { + return "", errors.New("相关api不存在") + } + + query = mergeQuery(query, apiInfo.Query) + + u := url.URL{ + Scheme: client.ServiceInfo.Scheme, + Host: client.ServiceInfo.Host, + Path: apiInfo.Path, + RawQuery: query.Encode(), + } + req, err := http.NewRequest(strings.ToUpper(apiInfo.Method), u.String(), nil) + + if err != nil { + return "", errors.New("构建request失败") + } + + return client.ServiceInfo.Credentials.SignUrl(req), nil +} + +// SignSts2 生成sts信息 +func (client *Client) SignSts2(inlinePolicy *Policy, expire time.Duration) (*SecurityToken2, error) { + var err error + sts := new(SecurityToken2) + if sts.AccessKeyID, sts.SecretAccessKey, err = createTempAKSK(); err != nil { + return nil, err + } + + if expire < time.Minute { + expire = time.Minute + } + + now := time.Now() + expireTime := now.Add(expire) + sts.CurrentTime = now.Format(time.RFC3339) + sts.ExpiredTime = expireTime.Format(time.RFC3339) + + innerToken, err := createInnerToken(client.ServiceInfo.Credentials, sts, inlinePolicy, expireTime.Unix()) + if err != nil { + return nil, err + } + + b, _ := json.Marshal(innerToken) + sts.SessionToken = "STS2" + base64.StdEncoding.EncodeToString(b) + return sts, nil +} + +// Query 发起Get的query请求 +func (client *Client) Query(api string, query url.Values) ([]byte, int, error) { + return client.requestWithContentType(api, query, "", "") +} + +// Json 发起Json的post请求 +func (client *Client) Json(api string, query url.Values, body string) ([]byte, int, error) { + return client.requestWithContentType(api, query, body, "application/json") +} + +// PostWithContentType 发起自定义 Content-Type 的 post 请求，Content-Type 不可以为空 +func (client *Client) PostWithContentType(api string, query url.Values, body string, ct string) ([]byte, int, error) { + return client.requestWithContentType(api, query, body, ct) +} + +func (client *Client) requestWithContentType(api string, query url.Values, body string, ct string) ([]byte, int, error) { + apiInfo := client.ApiInfoList[api] + + if apiInfo == nil { + return []byte(""), 500, errors.New("相关api不存在") + } + timeout := getTimeout(client.ServiceInfo.Timeout, apiInfo.Timeout) + header := mergeHeader(client.ServiceInfo.Header, apiInfo.Header) + query = mergeQuery(query, apiInfo.Query) + + u := url.URL{ + Scheme: client.ServiceInfo.Scheme, + Host: client.ServiceInfo.Host, + Path: apiInfo.Path, + RawQuery: query.Encode(), + } + var requestBody io.Reader + if body != "" { + requestBody = strings.NewReader(body) + } + req, err := http.NewRequest(strings.ToUpper(apiInfo.Method), u.String(), requestBody) + if err != nil { + return []byte(""), 500, errors.New("构建request失败") + } + req.Header = header + if ct != "" { + req.Header.Set("Content-Type", ct) + } + return client.makeRequest(api, req, timeout) +} + +// Post 发起Post请求 +func (client *Client) Post(api string, query url.Values, form url.Values) ([]byte, int, error) { + apiInfo := client.ApiInfoList[api] + form = mergeQuery(form, apiInfo.Form) + return client.requestWithContentType(api, query, form.Encode(), "application/x-www-form-urlencoded") +} + +func (client *Client) makeRequest(api string, req *http.Request, timeout time.Duration) ([]byte, int, error) { + req = client.ServiceInfo.Credentials.Sign(req) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + req = req.WithContext(ctx) + + resp, err := client.Client.Do(req) + if err != nil { + return []byte(""), 500, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return []byte(""), resp.StatusCode, err + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return body, resp.StatusCode, fmt.Errorf("api %s http code %d body %s", api, resp.StatusCode, string(body)) + } + + return body, resp.StatusCode, nil +} diff --git a/vendor/github.com/volcengine/volc-sdk-golang/base/model.go b/vendor/github.com/volcengine/volc-sdk-golang/base/model.go new file mode 100644 index 0000000000..caf1c8bd81 --- /dev/null +++ b/vendor/github.com/volcengine/volc-sdk-golang/base/model.go @@ -0,0 +1,109 @@ +package base + +import ( + "net/http" + "net/url" + "time" +) + +const ( + RegionCnNorth1 = "cn-north-1" + RegionUsEast1 = "us-east-1" + RegionApSingapore = "ap-singapore-1" + + timeFormatV4 = "20060102T150405Z" +) + +type ServiceInfo struct { + Timeout time.Duration + Scheme string + Host string + Header http.Header + Credentials Credentials +} + +type ApiInfo struct { + Method string + Path string + Query url.Values + Form url.Values + Timeout time.Duration + Header http.Header +} + +type Credentials struct { + AccessKeyID string + SecretAccessKey string + Service string + Region string + SessionToken string +} + +type metadata struct { + algorithm string + credentialScope string + signedHeaders string + date string + region string + service string +} + +// 统一的JSON返回结果 +type CommonResponse struct { + ResponseMetadata ResponseMetadata + Result interface{} `json:"Result,omitempty"` +} + +type BaseResp struct { + Status string + CreatedTime int64 + UpdatedTime int64 +} + +type ErrorObj struct { + CodeN int + Code string + Message string +} + +type ResponseMetadata struct { + RequestId string + Service string `json:",omitempty"` + Region string `json:",omitempty"` + Action string `json:",omitempty"` + Version string `json:",omitempty"` + Error *ErrorObj `json:",omitempty"` +} + +type Policy struct { + Statement []*Statement +} + +const ( + StatementEffectAllow = "Allow" + StatementEffectDeny = "Deny" +) + +type Statement struct { + Effect string + Action []string + Resource []string + Condition string `json:",omitempty"` +} + +type SecurityToken2 struct { + AccessKeyID string + SecretAccessKey string + SessionToken string + ExpiredTime string + CurrentTime string +} + +type InnerToken struct { + LTAccessKeyId string + AccessKeyId string + SignedSecretAccessKey string + ExpiredTime int64 + PolicyString string + Signature string +} diff --git a/vendor/github.com/volcengine/volc-sdk-golang/base/sign.go b/vendor/github.com/volcengine/volc-sdk-golang/base/sign.go new file mode 100644 index 0000000000..8342dbbb9c --- /dev/null +++ b/vendor/github.com/volcengine/volc-sdk-golang/base/sign.go @@ -0,0 +1,288 @@ +package base + +import ( + "bytes" + "crypto/hmac" + "crypto/md5" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "sort" + "strings" + "time" +) + +func (c Credentials) Sign(request *http.Request) *http.Request { + query := request.URL.Query() + + request.URL.RawQuery = query.Encode() + return Sign4(request, c) +} + +func (c Credentials) SignUrl(request *http.Request) string { + query := request.URL.Query() + ldt := timestampV4() + sdt := ldt[:8] + meta := new(metadata) + meta.date, meta.service, meta.region, meta.signedHeaders, meta.algorithm = sdt, c.Service, c.Region, "", "HMAC-SHA256" + meta.credentialScope = concat("/", meta.date, meta.region, meta.service, "request") + + query.Set("X-Date", ldt) + query.Set("X-NotSignBody", "") + query.Set("X-Credential", c.AccessKeyID+"/"+meta.credentialScope) + query.Set("X-Algorithm", meta.algorithm) + query.Set("X-SignedHeaders", meta.signedHeaders) + query.Set("X-SignedQueries", "") + keys := make([]string, 0, len(query)) + for k := range query { + keys = append(keys, k) + } + sort.Strings(keys) + query.Set("X-SignedQueries", strings.Join(keys, ";")) + + if c.SessionToken != "" { + query.Set("X-Security-Token", c.SessionToken) + } + + // Task 1 + hashedCanonReq := hashedSimpleCanonicalRequestV4(request, query, meta) + + // Task 2 + stringToSign := concat("\n", meta.algorithm, ldt, meta.credentialScope, hashedCanonReq) + + // Task 3 + signingKey := signingKeyV4(c.SecretAccessKey, meta.date, meta.region, meta.service) + signature := signatureV4(signingKey, stringToSign) + + query.Set("X-Signature", signature) + return query.Encode() +} + +// Sign4 signs a request with Signed Signature Version 4. +func Sign4(request *http.Request, credential Credentials) *http.Request { + keys := credential + + prepareRequestV4(request) + meta := new(metadata) + meta.service, meta.region = keys.Service, keys.Region + + // Task 0 设置SessionToken的header + if credential.SessionToken != "" { + request.Header.Set("X-Security-Token", credential.SessionToken) + } + + // Task 1 + hashedCanonReq := hashedCanonicalRequestV4(request, meta) + + // Task 2 + stringToSign := stringToSignV4(request, hashedCanonReq, meta) + + // Task 3 + signingKey := signingKeyV4(keys.SecretAccessKey, meta.date, meta.region, meta.service) + signature := signatureV4(signingKey, stringToSign) + + request.Header.Set("Authorization", buildAuthHeaderV4(signature, meta, keys)) + + return request +} + +func hashedSimpleCanonicalRequestV4(request *http.Request, query url.Values, meta *metadata) string { + payloadHash := hashSHA256([]byte("")) + + if request.URL.Path == "" { + request.URL.Path = "/" + } + + canonicalRequest := concat("\n", request.Method, normuri(request.URL.Path), normquery(query), "\n", meta.signedHeaders, payloadHash) + + return hashSHA256([]byte(canonicalRequest)) +} + +func hashedCanonicalRequestV4(request *http.Request, meta *metadata) string { + payload := readAndReplaceBody(request) + payloadHash := hashSHA256(payload) + request.Header.Set("X-Content-Sha256", payloadHash) + + request.Header.Set("Host", request.Host) + + var sortedHeaderKeys []string + for key := range request.Header { + switch key { + case "Content-Type", "Content-Md5", "Host", "X-Security-Token": + default: + if !strings.HasPrefix(key, "X-") { + continue + } + } + sortedHeaderKeys = append(sortedHeaderKeys, strings.ToLower(key)) + } + sort.Strings(sortedHeaderKeys) + + var headersToSign string + for _, key := range sortedHeaderKeys { + value := strings.TrimSpace(request.Header.Get(key)) + if key == "host" { + if strings.Contains(value, ":") { + split := strings.Split(value, ":") + port := split[1] + if port == "80" || port == "443" { + value = split[0] + } + } + } + headersToSign += key + ":" + value + "\n" + } + meta.signedHeaders = concat(";", sortedHeaderKeys...) + canonicalRequest := concat("\n", request.Method, normuri(request.URL.Path), normquery(request.URL.Query()), headersToSign, meta.signedHeaders, payloadHash) + + return hashSHA256([]byte(canonicalRequest)) +} + +func stringToSignV4(request *http.Request, hashedCanonReq string, meta *metadata) string { + requestTs := request.Header.Get("X-Date") + + meta.algorithm = "HMAC-SHA256" + meta.date = tsDateV4(requestTs) + meta.credentialScope = concat("/", meta.date, meta.region, meta.service, "request") + + return concat("\n", meta.algorithm, requestTs, meta.credentialScope, hashedCanonReq) +} + +func signatureV4(signingKey []byte, stringToSign string) string { + return hex.EncodeToString(hmacSHA256(signingKey, stringToSign)) +} + +func prepareRequestV4(request *http.Request) *http.Request { + necessaryDefaults := map[string]string{ + "Content-Type": "application/x-www-form-urlencoded; charset=utf-8", + "X-Date": timestampV4(), + } + + for header, value := range necessaryDefaults { + if request.Header.Get(header) == "" { + request.Header.Set(header, value) + } + } + + if request.URL.Path == "" { + request.URL.Path += "/" + } + + return request +} + +func signingKeyV4(secretKey, date, region, service string) []byte { + kDate := hmacSHA256([]byte(secretKey), date) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + kSigning := hmacSHA256(kService, "request") + return kSigning +} + +func buildAuthHeaderV4(signature string, meta *metadata, keys Credentials) string { + credential := keys.AccessKeyID + "/" + meta.credentialScope + + return meta.algorithm + + " Credential=" + credential + + ", SignedHeaders=" + meta.signedHeaders + + ", Signature=" + signature +} + +func timestampV4() string { + return now().Format(timeFormatV4) +} + +func tsDateV4(timestamp string) string { + return timestamp[:8] +} + +func hmacSHA256(key []byte, content string) []byte { + mac := hmac.New(sha256.New, key) + mac.Write([]byte(content)) + return mac.Sum(nil) +} + +func hashSHA256(content []byte) string { + h := sha256.New() + h.Write(content) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func hashMD5(content []byte) string { + h := md5.New() + h.Write(content) + return base64.StdEncoding.EncodeToString(h.Sum(nil)) +} + +func readAndReplaceBody(request *http.Request) []byte { + if request.Body == nil { + return []byte{} + } + payload, _ := ioutil.ReadAll(request.Body) + request.Body = ioutil.NopCloser(bytes.NewReader(payload)) + return payload +} + +func concat(delim string, str ...string) string { + return strings.Join(str, delim) +} + +var now = func() time.Time { + return time.Now().UTC() +} + +func normuri(uri string) string { + parts := strings.Split(uri, "/") + for i := range parts { + parts[i] = encodePathFrag(parts[i]) + } + return strings.Join(parts, "/") +} + +func encodePathFrag(s string) string { + hexCount := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c) { + hexCount++ + } + } + t := make([]byte, len(s)+2*hexCount) + j := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c) { + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + } else { + t[j] = c + j++ + } + } + return string(t) +} + +func shouldEscape(c byte) bool { + if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' { + return false + } + if '0' <= c && c <= '9' { + return false + } + if c == '-' || c == '_' || c == '.' || c == '~' { + return false + } + return true +} + +func normquery(v url.Values) string { + queryString := v.Encode() + + return strings.Replace(queryString, "+", "%20", -1) +} diff --git a/vendor/github.com/volcengine/volc-sdk-golang/base/utils.go b/vendor/github.com/volcengine/volc-sdk-golang/base/utils.go new file mode 100644 index 0000000000..1480e60030 --- /dev/null +++ b/vendor/github.com/volcengine/volc-sdk-golang/base/utils.go @@ -0,0 +1,180 @@ +package base + +import ( + "crypto/md5" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "math/rand" + "net/http" + "net/url" + "reflect" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func init() { + rand.Seed(time.Now().Unix()) +} + +func createTempAKSK() (accessKeyId string, plainSk string, err error) { + if accessKeyId, err = generateAccessKeyId("AKTP"); err != nil { + return + } + + plainSk, err = generateSecretKey() + if err != nil { + return + } + return +} + +func generateAccessKeyId(prefix string) (string, error) { + uuid := uuid.New() + + uidBase64 := base64.StdEncoding.EncodeToString([]byte(strings.Replace(uuid.String(), "-", "", -1))) + + s := strings.Replace(uidBase64, "=", "", -1) + s = strings.Replace(s, "/", "", -1) + s = strings.Replace(s, "+", "", -1) + s = strings.Replace(s, "-", "", -1) + return prefix + s, nil +} + +func randStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} + +func generateSecretKey() (string, error) { + randString32 := randStringRunes(32) + return aesEncryptCBCWithBase64([]byte(randString32), []byte("bytedance-isgood")) +} + +func createInnerToken(credentials Credentials, sts *SecurityToken2, inlinePolicy *Policy, t int64) (*InnerToken, error) { + var err error + innerToken := new(InnerToken) + + innerToken.LTAccessKeyId = credentials.AccessKeyID + innerToken.AccessKeyId = sts.AccessKeyID + innerToken.ExpiredTime = t + + key := md5.Sum([]byte(credentials.SecretAccessKey)) + innerToken.SignedSecretAccessKey, err = aesEncryptCBCWithBase64([]byte(sts.SecretAccessKey), key[:]) + if err != nil { + return nil, err + } + + if inlinePolicy != nil { + b, _ := json.Marshal(inlinePolicy) + innerToken.PolicyString = string(b) + } + + signStr := fmt.Sprintf("%s|%s|%d|%s|%s", innerToken.LTAccessKeyId, innerToken.AccessKeyId, innerToken.ExpiredTime, innerToken.SignedSecretAccessKey, innerToken.PolicyString) + + innerToken.Signature = hex.EncodeToString(hmacSHA256(key[:], signStr)) + return innerToken, nil +} + +func getTimeout(serviceTimeout, apiTimeout time.Duration) time.Duration { + timeout := time.Second + if serviceTimeout != time.Duration(0) { + timeout = serviceTimeout + } + if apiTimeout != time.Duration(0) { + timeout = apiTimeout + } + return timeout +} + +func mergeQuery(query1, query2 url.Values) (query url.Values) { + query = url.Values{} + if query1 != nil { + for k, vv := range query1 { + for _, v := range vv { + query.Add(k, v) + } + } + } + + if query2 != nil { + for k, vv := range query2 { + for _, v := range vv { + query.Add(k, v) + } + } + } + return +} + +func mergeHeader(header1, header2 http.Header) (header http.Header) { + header = http.Header{} + if header1 != nil { + for k, v := range header1 { + header.Set(k, strings.Join(v, ";")) + } + } + if header2 != nil { + for k, v := range header2 { + header.Set(k, strings.Join(v, ";")) + } + } + + return +} + +func NewAllowStatement(actions, resources []string) *Statement { + sts := new(Statement) + sts.Effect = "Allow" + sts.Action = actions + sts.Resource = resources + + return sts +} + +func NewDenyStatement(actions, resources []string) *Statement { + sts := new(Statement) + sts.Effect = "Deny" + sts.Action = actions + sts.Resource = resources + + return sts +} + +func ToUrlValues(i interface{}) (values url.Values) { + values = url.Values{} + iVal := reflect.ValueOf(i).Elem() + typ := iVal.Type() + for i := 0; i < iVal.NumField(); i++ { + f := iVal.Field(i) + // You ca use tags here... + // tag := typ.Field(i).Tag.Get("tagname") + // Convert each type into a string for the url.Values string map + var v string + switch f.Interface().(type) { + case int, int8, int16, int32, int64: + v = strconv.FormatInt(f.Int(), 10) + case uint, uint8, uint16, uint32, uint64: + v = strconv.FormatUint(f.Uint(), 10) + case float32: + v = strconv.FormatFloat(f.Float(), 'f', 4, 32) + case float64: + v = strconv.FormatFloat(f.Float(), 'f', 4, 64) + case []byte: + v = string(f.Bytes()) + case string: + v = f.String() + } + values.Set(typ.Field(i).Name, v) + } + return +} diff --git a/vendor/golang.org/x/sync/singleflight/singleflight.go b/vendor/golang.org/x/sync/singleflight/singleflight.go new file mode 100644 index 0000000000..8473fb7922 --- /dev/null +++ b/vendor/golang.org/x/sync/singleflight/singleflight.go @@ -0,0 +1,205 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package singleflight provides a duplicate function call suppression +// mechanism. +package singleflight // import "golang.org/x/sync/singleflight" + +import ( + "bytes" + "errors" + "fmt" + "runtime" + "runtime/debug" + "sync" +) + +// errGoexit indicates the runtime.Goexit was called in +// the user given function. +var errGoexit = errors.New("runtime.Goexit was called") + +// A panicError is an arbitrary value recovered from a panic +// with the stack trace during the execution of given function. +type panicError struct { + value interface{} + stack []byte +} + +// Error implements error interface. +func (p *panicError) Error() string { + return fmt.Sprintf("%v\n\n%s", p.value, p.stack) +} + +func newPanicError(v interface{}) error { + stack := debug.Stack() + + // The first line of the stack trace is of the form "goroutine N [status]:" + // but by the time the panic reaches Do the goroutine may no longer exist + // and its status will have changed. Trim out the misleading line. + if line := bytes.IndexByte(stack[:], '\n'); line >= 0 { + stack = stack[line+1:] + } + return &panicError{value: v, stack: stack} +} + +// call is an in-flight or completed singleflight.Do call +type call struct { + wg sync.WaitGroup + + // These fields are written once before the WaitGroup is done + // and are only read after the WaitGroup is done. + val interface{} + err error + + // These fields are read and written with the singleflight + // mutex held before the WaitGroup is done, and are read but + // not written after the WaitGroup is done. + dups int + chans []chan<- Result +} + +// Group represents a class of work and forms a namespace in +// which units of work can be executed with duplicate suppression. +type Group struct { + mu sync.Mutex // protects m + m map[string]*call // lazily initialized +} + +// Result holds the results of Do, so they can be passed +// on a channel. +type Result struct { + Val interface{} + Err error + Shared bool +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +// The return value shared indicates whether v was given to multiple callers. +func (g *Group) Do(key string, fn func() (interface{}, error)) (v interface{}, err error, shared bool) { + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + g.mu.Unlock() + c.wg.Wait() + + if e, ok := c.err.(*panicError); ok { + panic(e) + } else if c.err == errGoexit { + runtime.Goexit() + } + return c.val, c.err, true + } + c := new(call) + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + g.doCall(c, key, fn) + return c.val, c.err, c.dups > 0 +} + +// DoChan is like Do but returns a channel that will receive the +// results when they are ready. +// +// The returned channel will not be closed. +func (g *Group) DoChan(key string, fn func() (interface{}, error)) <-chan Result { + ch := make(chan Result, 1) + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + c.chans = append(c.chans, ch) + g.mu.Unlock() + return ch + } + c := &call{chans: []chan<- Result{ch}} + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + go g.doCall(c, key, fn) + + return ch +} + +// doCall handles the single call for a key. +func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) { + normalReturn := false + recovered := false + + // use double-defer to distinguish panic from runtime.Goexit, + // more details see https://golang.org/cl/134395 + defer func() { + // the given function invoked runtime.Goexit + if !normalReturn && !recovered { + c.err = errGoexit + } + + g.mu.Lock() + defer g.mu.Unlock() + c.wg.Done() + if g.m[key] == c { + delete(g.m, key) + } + + if e, ok := c.err.(*panicError); ok { + // In order to prevent the waiting channels from being blocked forever, + // needs to ensure that this panic cannot be recovered. + if len(c.chans) > 0 { + go panic(e) + select {} // Keep this goroutine around so that it will appear in the crash dump. + } else { + panic(e) + } + } else if c.err == errGoexit { + // Already in the process of goexit, no need to call again + } else { + // Normal return + for _, ch := range c.chans { + ch <- Result{c.val, c.err, c.dups > 0} + } + } + }() + + func() { + defer func() { + if !normalReturn { + // Ideally, we would wait to take a stack trace until we've determined + // whether this is a panic or a runtime.Goexit. + // + // Unfortunately, the only way we can distinguish the two is to see + // whether the recover stopped the goroutine from terminating, and by + // the time we know that, the part of the stack trace relevant to the + // panic has been discarded. + if r := recover(); r != nil { + c.err = newPanicError(r) + } + } + }() + + c.val, c.err = fn() + normalReturn = true + }() + + if !normalReturn { + recovered = true + } +} + +// Forget tells the singleflight to forget about a key. Future calls +// to Do for this key will call the function rather than waiting for +// an earlier call to complete. +func (g *Group) Forget(key string) { + g.mu.Lock() + delete(g.m, key) + g.mu.Unlock() +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ca380f9b0f..98eeb7e921 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -915,6 +915,13 @@ github.com/vmware/govmomi/vim25/progress github.com/vmware/govmomi/vim25/soap github.com/vmware/govmomi/vim25/types github.com/vmware/govmomi/vim25/xml +# github.com/volcengine/ve-tos-golang-sdk/v2 v2.6.2 +## explicit; go 1.13 +github.com/volcengine/ve-tos-golang-sdk/v2/tos +github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum +# github.com/volcengine/volc-sdk-golang v1.0.23 +## explicit; go 1.14 +github.com/volcengine/volc-sdk-golang/base # github.com/willf/bitset v1.1.9 ## explicit github.com/willf/bitset @@ -1051,6 +1058,7 @@ golang.org/x/oauth2/jwt # golang.org/x/sync v0.1.0 ## explicit golang.org/x/sync/errgroup +golang.org/x/sync/singleflight # golang.org/x/sys v0.9.0 ## explicit; go 1.17 golang.org/x/sys/cpu @@ -1243,8 +1251,6 @@ gopkg.in/alexcesaro/quotedprintable.v3 # gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d ## explicit gopkg.in/asn1-ber.v1 -# gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c -## explicit; go 1.11 # gopkg.in/fatih/set.v0 v0.2.1 ## explicit gopkg.in/fatih/set.v0 @@ -1432,7 +1438,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231012115531-f16047235f44 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231012115531-f16047235f44 => /root/projects/cloudmux ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing @@ -1512,6 +1518,8 @@ yunion.io/x/cloudmux/pkg/multicloud/remotefile yunion.io/x/cloudmux/pkg/multicloud/remotefile/provider yunion.io/x/cloudmux/pkg/multicloud/ucloud yunion.io/x/cloudmux/pkg/multicloud/ucloud/provider +yunion.io/x/cloudmux/pkg/multicloud/volcengine +yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider yunion.io/x/cloudmux/pkg/multicloud/zstack yunion.io/x/cloudmux/pkg/multicloud/zstack/provider # yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 @@ -1596,3 +1604,4 @@ yunion.io/x/sqlchemy/backends/sqlite # yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c ## explicit; go 1.12 yunion.io/x/structarg +# yunion.io/x/cloudmux => /root/projects/cloudmux diff --git a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/cloudaccount_const.go b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/cloudaccount_const.go index d877411f78..b0f1fb011b 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/cloudaccount_const.go +++ b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/cloudaccount_const.go @@ -44,6 +44,7 @@ const ( CLOUD_PROVIDER_BAIDU = "Baidu" CLOUD_PROVIDER_CUCLOUD = "ChinaUnion" CLOUD_PROVIDER_QINGCLOUD = "QingCloud" + CLOUD_PROVIDER_VOLCENGINE = "VolcEngine" CLOUD_PROVIDER_GENERICS3 = "S3" CLOUD_PROVIDER_CEPH = "Ceph" @@ -73,4 +74,5 @@ const ( CLOUD_ACCESS_ENV_CTYUN_CHINA = CLOUD_PROVIDER_CTYUN CLOUD_ACCESS_ENV_ECLOUD_CHINA = CLOUD_PROVIDER_ECLOUD CLOUD_ACCESS_ENV_JDCLOUD_CHINA = CLOUD_PROVIDER_JDCLOUD + CLOUD_ACCESS_ENV_VOLCENGINE_CHINA = CLOUD_PROVIDER_VOLCENGINE ) diff --git a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/guest_const.go b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/guest_const.go index f879358703..44d8570074 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/guest_const.go +++ b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/guest_const.go @@ -77,6 +77,7 @@ const ( HYPERVISOR_BAIDU = "baidu" HYPERVISOR_CUCLOUD = "cucloud" HYPERVISOR_QINGCLOUD = "qingcloud" + HYPERVISOR_VOLCENGINE = "volcengine" ) const ( diff --git a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/host_const.go b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/host_const.go index d5037ca7b9..2b56b8e38d 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/host_const.go +++ b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/host_const.go @@ -46,6 +46,7 @@ const ( HOST_TYPE_BAIDU = "baidu" HOST_TYPE_CUCLOUD = "cucloud" HOST_TYPE_QINGCLOUD = "qingcloud" + HOST_TYPE_VOLCENGINE = "volcengine" // # possible status HOST_ONLINE = "online" diff --git a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go index 018e23810b..697036bbb6 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go +++ b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go @@ -105,6 +105,11 @@ const ( STORAGE_ECLOUD_SSD = "ssd" // 高性能盘 STORAGE_ECLOUD_SSDEBS = "ssdebs" // 性能优化盘 STORAGE_ECLOUD_SYSTEM = "system" // 系统盘 + + // volcengine storage type + STORAGE_VOLCENGINE_FlexPL = "ESSD_FlexPL" // 极速型SSD(单盘最大IOPS 5万) + STORAGE_VOLCENGINE_PL0 = "ESSD_PL0" // 极速型SSD(单盘最大IOPS 1万) + STORAGE_VOLCENGINE_PTSSD = "PTSSD" // 性能型SSD(上一代产品) ) const ( diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/loader/loader.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/loader/loader.go index 6bead69634..86d0cfbab5 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/loader/loader.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/loader/loader.go @@ -43,6 +43,7 @@ import ( _ "yunion.io/x/cloudmux/pkg/multicloud/remotefile/provider" // private clouds _ "yunion.io/x/cloudmux/pkg/multicloud/ucloud/provider" // object storages _ "yunion.io/x/cloudmux/pkg/multicloud/zstack/provider" // private clouds + _ "yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider" ) func init() { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/bucket.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/bucket.go new file mode 100644 index 0000000000..a6de036d0a --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/bucket.go @@ -0,0 +1,416 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "time" + + tos "github.com/volcengine/ve-tos-golang-sdk/v2/tos" + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/fileutils" +) + +type SBucket struct { + multicloud.SBaseBucket + VolcEngineTags + region *SRegion + + Name string + Location string + CreationDate time.Time + StorageClass string +} + +func (b *SBucket) GetProjectId() string { + return "" +} + +func (b *SBucket) GetGlobalId() string { + return b.Name +} + +func (b *SBucket) GetName() string { + return b.Name +} + +func (b *SBucket) GetLocation() string { + return b.Location +} + +func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion { + return b.region +} + +func (b *SBucket) GetCreatedAt() time.Time { + return b.CreationDate +} + +func (b *SBucket) GetStorageClass() string { + return b.StorageClass +} + +func (b *SBucket) GetStats() cloudprovider.SBucketStats { + stats, _ := cloudprovider.GetIBucketStats(b) + return stats +} + +func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { + ret := []cloudprovider.SBucketAccessUrl{ + { + Url: fmt.Sprintf("%s.%s", b.Name, b.region.getTOSExternalDomain()), + Description: "ExtranetEndpoint", + Primary: true, + }, + { + Url: fmt.Sprintf("%s.%s", b.Name, b.region.getTOSInternalDomain()), + Description: "IntranetEndpoint", + }, + } + return ret +} + +func grantToCannedAcl(acls []tos.GrantV2) cloudprovider.TBucketACLType { + switch { + case len(acls) == 1: + if acls[0].Permission == enum.PermissionFullControl { + return cloudprovider.ACLPrivate + } + case len(acls) == 2: + for _, g := range acls { + if g.GranteeV2.Type == enum.GranteeGroup && g.GranteeV2.Canned == enum.CannedAuthenticatedUsers && g.Permission == enum.PermissionRead { + return cloudprovider.ACLAuthRead + } + if g.GranteeV2.Type == enum.GranteeGroup && g.GranteeV2.Canned == enum.CannedAllUsers && g.Permission == enum.PermissionRead { + return cloudprovider.ACLPublicRead + } + } + case len(acls) == 3: + for _, g := range acls { + if g.GranteeV2.Type == enum.GranteeGroup && g.GranteeV2.Canned == enum.CannedAllUsers && g.Permission == enum.PermissionRead { + return cloudprovider.ACLPublicReadWrite + } + } + } + return cloudprovider.ACLUnknown +} + +func (b *SBucket) GetAcl() cloudprovider.TBucketACLType { + acl := cloudprovider.ACLPrivate + toscli, err := b.region.GetTosClient() + if err != nil { + log.Errorf("GetTosClient fail %s", err) + return acl + } + input := tos.GetBucketACLInput{} + input.Bucket = b.Name + output, err := toscli.GetBucketACL(context.Background(), &input) + if err != nil { + log.Errorf("toscli.GetBucketAcl fail %s", err) + return acl + } + return grantToCannedAcl(output.Grants) +} + +func (b *SBucket) SetAcl(aclStr cloudprovider.TBucketACLType) error { + toscli, err := b.region.GetTosClient() + if err != nil { + return errors.Wrapf(err, "Get TosClient") + } + input := tos.PutBucketACLInput{} + input.Bucket = b.Name + input.ACLType = enum.ACLType(aclStr) + _, err = toscli.PutBucketACL(context.Background(), &input) + if err != nil { + return errors.Wrapf(err, "PutBucketAcl") + } + return nil +} + +func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) (string, error) { + return "", errors.ErrNotImplemented +} + +func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { + toscli, err := b.region.GetTosClient() + if err != nil { + return errors.Wrapf(err, "GetTosClient") + } + _, err = toscli.AbortMultipartUpload(ctx, &tos.AbortMultipartUploadInput{Bucket: b.Name, Key: key, UploadID: uploadId}) + if err != nil { + return errors.Wrapf(err, "AbortMultipartUploadWithContext") + } + return nil +} + +func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { + toscli, err := b.region.GetTosClient() + if err != nil { + return errors.Wrap(err, "GetTosClient") + } + parts := make([]tos.UploadedPartV2, len(partEtags)) + for i := range partEtags { + parts[i].PartNumber = int(i + 1) + parts[i].ETag = partEtags[i] + } + _, err = toscli.CompleteMultipartUploadV2(ctx, &tos.CompleteMultipartUploadV2Input{Bucket: b.Name, Key: key, UploadID: uploadId, Parts: parts}) + if err != nil { + return errors.Wrapf(err, "CompleteMultipartUploadV2") + } + return nil +} + +func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { + toscli, err := b.region.GetTosClient() + if err != nil { + return errors.Wrap(err, "GetTosClient") + } + if len(cannedAcl) == 0 { + cannedAcl = b.GetAcl() + } + var metaDir string + metaHdr := make(map[string]string) + cacheControl := "" + if meta != nil { + for k, v := range meta { + if len(v) == 0 || len(v[0]) == 0 { + continue + } + switch http.CanonicalHeaderKey(k) { + case cloudprovider.META_HEADER_CACHE_CONTROL: + cacheControl = v[0] + case cloudprovider.META_HEADER_CONTENT_TYPE: + cacheControl = v[0] + case cloudprovider.META_HEADER_CONTENT_LANGUAGE: + cacheControl = v[0] + case cloudprovider.META_HEADER_CONTENT_ENCODING: + cacheControl = v[0] + case cloudprovider.META_HEADER_CONTENT_DISPOSITION: + cacheControl = v[0] + default: + metaHdr[k] = v[0] + } + } + metaDir = "REPLACE" + } else { + metaDir = "COPY" + } + input := tos.CopyObjectInput{Bucket: b.Name, Key: destKey, SrcKey: fmt.Sprintf("%s/%s", srcBucket, url.PathEscape(srcKey)), StorageClass: enum.StorageClassType(storageClassStr), ACL: enum.ACLType(cannedAcl), MetadataDirective: enum.MetadataDirectiveType(metaDir)} + if len(cacheControl) > 0 { + input.CacheControl = cacheControl + } + if len(metaHdr) > 0 { + input.Meta = metaHdr + } + _, err = toscli.CopyObject(ctx, &input) + if err != nil { + return errors.Wrapf(err, "CopyObject") + } + return nil +} + +func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + toscli, err := b.region.GetTosClient() + if err != nil { + return "", errors.Wrap(err, "GetTosClient") + } + input := tos.UploadPartCopyV2Input{} + input.Bucket = b.Name + input.Key = key + input.UploadID = uploadId + input.PartNumber = partNumber + input.SrcBucket = srcBucket + input.SrcKey = srcKey + if srcLength > 0 { + input.CopySourceRange = fmt.Sprintf("bytes=%d-%d", srcOffset, srcOffset+srcLength-1) + } + output, err := toscli.UploadPartCopyV2(ctx, &input) + if err != nil { + return "", errors.Wrapf(err, "CopyPart") + } + return output.ETag, nil +} + +func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, part io.Reader, partSize int64, offset, totalSize int64) (string, error) { + toscli, err := b.region.GetTosClient() + if err != nil { + return "", errors.Wrap(err, "GetTosClient") + } + input := tos.UploadPartV2Input{} + input.Bucket = b.Name + input.Key = key + input.UploadID = uploadId + input.PartNumber = int(partIndex) + seeker, err := fileutils.NewReadSeeker(part, partSize) + if err != nil { + return "", errors.Wrap(err, "newFakeSeeker") + } + defer seeker.Close() + input.Content = seeker + input.ContentLength = partSize + output, err := toscli.UploadPartV2(ctx, &input) + if err != nil { + return "", errors.Wrapf(err, "UploadPart") + } + return output.ETag, nil +} + +func (b *SBucket) DeleteObject(ctx context.Context, key string) error { + toscli, err := b.region.GetTosClient() + if err != nil { + return errors.Wrap(err, "GetTosClient") + } + input := tos.DeleteObjectV2Input{} + _, err = toscli.DeleteObjectV2(ctx, &input) + if err != nil { + return errors.Wrap(err, "DeleteObject") + } + return nil +} + +func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + toscli, err := b.region.GetTosClient() + if err != nil { + return nil, errors.Wrap(err, "GetTosClient") + } + input := tos.GetObjectV2Input{} + output, err := toscli.GetObjectV2(ctx, &input) + if err != nil { + return nil, errors.Wrap(err, "DeleteObject") + } + return output.Content, nil +} + +func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + result := cloudprovider.SListObjectResult{} + toscli, err := b.region.GetTosClient() + if err != nil { + return result, errors.Wrap(err, "GetTosClient") + } + input := tos.ListObjectsV2Input{} + input.Bucket = b.Name + if len(prefix) > 0 { + input.Prefix = prefix + } + if len(marker) > 0 { + input.Marker = marker + } + if len(delimiter) > 0 { + input.Delimiter = delimiter + } + if maxCount > 0 { + input.MaxKeys = maxCount + } + output, err := toscli.ListObjectsV2(context.Background(), &input) + if err != nil { + return result, errors.Wrap(err, "ListObjects") + } + for _, object := range output.Contents { + obj := &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + StorageClass: string(object.StorageClass), + Key: object.Key, + SizeBytes: object.Size, + ETag: object.ETag, + LastModified: object.LastModified, + }, + } + result.Objects = append(result.Objects, obj) + } + if output.CommonPrefixes != nil { + result.CommonPrefixes = make([]cloudprovider.ICloudObject, len(output.CommonPrefixes)) + for i, commonPrefix := range output.CommonPrefixes { + result.CommonPrefixes[i] = &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{Key: commonPrefix.Prefix}, + } + } + } + result.IsTruncated = output.IsTruncated + result.NextMarker = output.NextMarker + return result, nil +} + +func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + return "", errors.ErrNotImplemented +} + +func (b *SBucket) PutObject(ctx context.Context, key string, body io.Reader, sizeBytes int64, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { + if sizeBytes < 0 { + return errors.Error("context length expected") + } + toscli, err := b.region.GetTosClient() + if err != nil { + return errors.Wrapf(err, "GetTosClient") + } + input := tos.PutObjectV2Input{} + input.Bucket = b.Name + input.Key = key + seeker, err := fileutils.NewReadSeeker(body, sizeBytes) + if err != nil { + return errors.Wrap(err, "newFakeSeeker") + } + defer seeker.Close() + input.Content = body + input.ContentLength = sizeBytes + + if meta != nil { + metaHdr := make(map[string]string) + for k, v := range meta { + if len(v) == 0 || len(v[0]) == 0 { + continue + } + switch http.CanonicalHeaderKey(k) { + case cloudprovider.META_HEADER_CACHE_CONTROL: + input.CacheControl = v[0] + case cloudprovider.META_HEADER_CONTENT_TYPE: + input.ContentType = v[0] + case cloudprovider.META_HEADER_CONTENT_MD5: + input.ContentMD5 = v[0] + case cloudprovider.META_HEADER_CONTENT_LANGUAGE: + input.ContentEncoding = v[0] + case cloudprovider.META_HEADER_CONTENT_ENCODING: + input.ContentDisposition = v[0] + default: + metaHdr[k] = v[0] + } + } + if len(metaHdr) > 0 { + input.Meta = metaHdr + } + } + + if len(cannedAcl) > 0 { + cannedAcl = b.GetAcl() + } + input.ACL = enum.ACLType(cannedAcl) + if len(storageClassStr) > 0 { + input.StorageClass = enum.StorageClassType(storageClassStr) + } + _, err = toscli.PutObjectV2(ctx, &input) + if err != nil { + return errors.Wrapf(err, "PutObject") + } + return nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/charge.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/charge.go new file mode 100644 index 0000000000..e4ab58987e --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/charge.go @@ -0,0 +1,48 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "time" + + api "yunion.io/x/cloudmux/pkg/apis/billing" +) + +const ( + PrePaidInstanceChargeType TChargeType = "PrePaid" + PostPaidInstanceChargeType TChargeType = "PostPaid" + DefaultInstanceChargeType = PostPaidInstanceChargeType +) + +func convertChargeType(ct TChargeType) string { + switch ct { + case PrePaidInstanceChargeType: + return api.BILLING_TYPE_PREPAID + case PostPaidInstanceChargeType: + return api.BILLING_TYPE_POSTPAID + default: + return "" + } +} + +func convertExpiredAt(expired time.Time) time.Time { + if !expired.IsZero() { + now := time.Now() + if expired.Sub(now) < time.Hour*24*365*6 { + return expired + } + } + return time.Time{} +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/disk.go new file mode 100644 index 0000000000..1e1aebedf1 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/disk.go @@ -0,0 +1,334 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "context" + "fmt" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" +) + +type SDisk struct { + storage *SStorage + multicloud.SDisk + VolcEngineTags + + ZoneId string + VolumeId string + VolumeName string + VolumeType string + Description string + InstanceId string + ImageId string + Size int + Status string + Kind string + CreatedAt time.Time + UpdatedAt time.Time + BillingType TChargeType + PayType string + TradeStatus int + ExpiredTime time.Time + DeleteWithInstance bool +} + +func (disk *SDisk) GetId() string { + return disk.VolumeId +} + +func (disk *SDisk) Delete(ctx context.Context) error { + _, err := disk.storage.zone.region.getDisk(disk.VolumeId) + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotFound { + return nil + } + return errors.Wrapf(err, "Failed to find disk %s when delete", disk.VolumeId) + } + + for { + err := disk.storage.zone.region.DeleteDisk(disk.VolumeId) + if err != nil { + if isError(err, "IncorrectDiskStatus") { + log.Infof("The disk is initializing, try later ...") + time.Sleep(10 * time.Second) + } else { + return errors.Wrapf(err, "DeleteDisk fail") + } + } else { + break + } + } + return cloudprovider.WaitDeleted(disk, 10*time.Second, 300*time.Second) // 5minutes +} + +func (disk *SDisk) Resize(ctx context.Context, sizeMb int64) error { + return disk.storage.zone.region.ResizeDisk(disk.VolumeId, sizeMb/1024) +} + +func (disk *SDisk) GetName() string { + if len(disk.VolumeName) > 0 { + return disk.VolumeName + } + return disk.VolumeId +} + +func (disk *SDisk) GetGlobalId() string { + return disk.VolumeId +} + +func (disk *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + return disk.storage, nil +} + +func (disk *SDisk) GetStatus() string { + switch disk.Status { + case "creating": + return api.DISK_ALLOCATING + default: + return api.DISK_READY + } +} + +func (disk *SDisk) Refresh() error { + new, err := disk.storage.zone.region.getDisk(disk.VolumeId) + if err != nil { + return err + } + return jsonutils.Update(disk, new) +} + +func (disk *SDisk) ResizeDisk(newSize int64) error { + // newSize 单位为 GB. 只能扩容，不能缩减。范围参考下面链接。 + // https://www.volcengine.com/docs/6396/76561 + return disk.storage.zone.region.ResizeDisk(disk.VolumeId, newSize) +} + +func (disk *SDisk) GetDiskFormat() string { + return "vhd" +} + +func (disk *SDisk) GetDiskSizeMB() int { + return disk.Size * 1024 +} + +func (disk *SDisk) GetIsAutoDelete() bool { + return disk.DeleteWithInstance +} + +func (disk *SDisk) GetTemplateId() string { + return disk.ImageId +} + +func (disk *SDisk) GetDiskType() string { + switch disk.Kind { + case "system": + return api.DISK_TYPE_SYS + case "data": + return api.DISK_TYPE_DATA + default: + return api.DISK_TYPE_DATA + } +} + +func (disk *SDisk) GetFsFormat() string { + return "" +} + +func (disk *SDisk) GetIsNonPersistent() bool { + return false +} + +func (disk *SDisk) GetDriver() string { + return "scsi" +} + +func (disk *SDisk) GetCacheMode() string { + return "none" +} + +func (disk *SDisk) GetMountpoint() string { + return "" +} + +func (disk *SDisk) GetISnapshot(snapshotId string) (cloudprovider.ICloudSnapshot, error) { + return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetISnapshot") +} + +func (disk *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { + return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetISnapshots") +} + +func (disk *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) { + return "", disk.storage.zone.region.resetDisk(disk.VolumeId, snapshotId) +} + +func (disk *SDisk) GetBillingType() string { + return convertChargeType(disk.BillingType) +} + +func (disk *SDisk) GetCreatedAt() time.Time { + return disk.CreatedAt +} + +func (disk *SDisk) GetExtSnapshotPolicyIds() ([]string, error) { + return nil, errors.ErrNotImplemented +} + +func (disk *SDisk) GetExpiredAt() time.Time { + return convertExpiredAt(disk.ExpiredTime) +} + +func (disk *SDisk) GetAccessPath() string { + return "" +} + +func (disk *SDisk) Rebuild(ctx context.Context) error { + return errors.Wrapf(cloudprovider.ErrNotImplemented, "Rebuild") + +} + +func (disk *SDisk) GetProjectId() string { + return "" +} + +// Snapshot API is not supported, refer to +// https://www.volcengine.com/docs/6460/195549 +func (disk *SDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) { + snapshotId, err := disk.storage.zone.region.CreateSnapshot(disk.VolumeId, name, desc) + if err != nil { + return nil, errors.Wrapf(err, "CreateSnapshot") + } + snapshot, err := disk.storage.zone.region.GetISnapshotById(snapshotId) + if err != nil { + return nil, errors.Wrapf(err, "getSnapshot(%s)", snapshotId) + } + err = cloudprovider.WaitStatus(snapshot, api.SNAPSHOT_READY, 15*time.Second, 3600*time.Second) + if err != nil { + return nil, errors.Wrapf(err, "cloudprovider.WaitStatus") + } + return snapshot, nil +} + +// region +func (region *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, pageNumber int, pageSize int) ([]SDisk, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + params := make(map[string]string) + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + + if len(instanceId) > 0 { + params["InstanceId"] = instanceId + } + if len(zoneId) > 0 { + params["ZoneId"] = zoneId + } + if len(category) > 0 { + params["VolumeType"] = category + } + if len(diskIds) > 0 { + for index, id := range diskIds { + key := fmt.Sprintf("VolumeIds.%d", index+1) + params[key] = id + } + } + + body, err := region.storageRequest("DescribeVolumes", params) + if err != nil { + return nil, 0, errors.Wrap(err, "GetDisks fail") + } + + disks := make([]SDisk, 0) + err = body.Unmarshal(&disks, "Result", "Volumes") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal disk details fail") + } + total, _ := body.Int("Result", "TotalCount") + return disks, int(total), nil +} + +func (region *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, desc string, projectId string) (string, error) { + params := make(map[string]string) + params["ZoneId"] = zoneId + params["VolumeName"] = name + if len(desc) > 0 { + params["Description"] = desc + } + params["VolumeType"] = category + // only data disk is supported + params["Kind"] = "data" + + params["Size"] = fmt.Sprintf("%d", sizeGb) + params["ClientToken"] = utils.GenRequestId(20) + + body, err := region.storageRequest("CreateVolume", params) + if err != nil { + return "", err + } + return body.GetString("Result", "VolumeId") +} + +func (region *SRegion) getDisk(diskId string) (*SDisk, error) { + disks, _, err := region.GetDisks("", "", "", []string{diskId}, 1, 50) + if err != nil { + return nil, errors.Wrapf(err, fmt.Sprint("%s not found", diskId)) + } + for _, disk := range disks { + if disk.VolumeId == diskId { + return &disk, nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, fmt.Sprint("%s not found", diskId)) +} + +func (region *SRegion) DeleteDisk(diskId string) error { + params := make(map[string]string) + params["VolumeId"] = diskId + + _, err := region.storageRequest("DeleteVolume", params) + return err +} + +func (region *SRegion) ResizeDisk(diskId string, sizeGb int64) error { + params := make(map[string]string) + params["VolumeId"] = diskId + params["NewSize"] = fmt.Sprintf("%d", sizeGb) + + _, err := region.storageRequest("ExtendVolume", params) + if err != nil { + return errors.Wrapf(err, "Resizing disk (%s) to %d GiB failed", diskId, sizeGb) + } + + return nil +} + +func (region *SRegion) resetDisk(diskId, snapshotId string) error { + // not supported API + return errors.Wrapf(cloudprovider.ErrNotImplemented, "resetDisk") +} + +func (region *SRegion) CreateSnapshot(diskId, name, desc string) (string, error) { + return "", errors.Wrapf(cloudprovider.ErrNotImplemented, "CreateSnapshot") +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/eip.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/eip.go new file mode 100644 index 0000000000..b410afb462 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/eip.go @@ -0,0 +1,388 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "strings" + "time" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" +) + +type TBillingType int + +const ( + BillingByPrePaid TBillingType = iota + 1 + BillingByBandwidth + BillingByTraffic +) + +const ( + EIP_STATUS_ATTACHING = "Attaching" + EIP_STATUS_DETACHING = "Detaching" + EIP_STATUS_ATTACHED = "Attached" + EIP_STATUS_AVAILABLE = "Available" + EIP_STATUS_DELETING = "Deleting" + + EIP_INSTANCE_TYPE_NAT = "Nat" // NAT网关 + EIP_INSTANCE_TYPE_ENI = "NetworkInterface" // 辅助网卡 + EIP_INSTANCE_TYPE_CLB = "ClbInstance" // 负载均衡 + EIP_INSTANCE_TYPE_ALB = "Albinstance" // 应用型负载均衡 + EIP_INSTANCE_TYPE_ECS = "EcsInstance" // 云服务器 + EIP_INSTANCE_TYPE_HAVIP = "HaVip" // 高可用虚拟IP +) + +type SEipAddress struct { + region *SRegion + multicloud.SEipBase + VolcEngineTags + + Name string + AllocationId string + + BillingType TBillingType + + EipAddress string + Status string + + InstanceType string + InstanceId string + Bandwidth int /* Mbps */ + + BusinessStatus string + AllocationTime time.Time + Description string + ISP string + + LockReason string + + ExpiredTime time.Time + ProjectName string +} + +func (eipaddr *SEipAddress) GetId() string { + return eipaddr.AllocationId +} + +func (eipaddr *SEipAddress) GetName() string { + if len(eipaddr.Name) > 0 { + return eipaddr.Name + } + return eipaddr.EipAddress +} + +func (eipaddr *SEipAddress) GetGlobalId() string { + return eipaddr.AllocationId +} + +func (eipaddr *SEipAddress) GetStatus() string { + switch eipaddr.Status { + case EIP_STATUS_ATTACHED, EIP_STATUS_AVAILABLE: + return api.EIP_STATUS_READY + case EIP_STATUS_ATTACHING: + return api.EIP_STATUS_ASSOCIATE + case EIP_STATUS_DETACHING: + return api.EIP_STATUS_DISSOCIATE + case EIP_STATUS_DELETING: + return api.EIP_STATUS_DEALLOCATE + default: + return api.EIP_STATUS_UNKNOWN + } +} + +func (eipaddr *SEipAddress) Refresh() error { + if eipaddr.IsEmulated() { + return nil + } + new, err := eipaddr.region.GetEip(eipaddr.AllocationId) + if err != nil { + return err + } + return jsonutils.Update(eipaddr, new) +} + +func (eipaddr *SEipAddress) GetProjectId() string { + return eipaddr.ProjectName +} + +func (eipaddr *SEipAddress) GetIpAddr() string { + return eipaddr.EipAddress +} + +func (eipaddr *SEipAddress) GetMode() string { + if eipaddr.InstanceId == eipaddr.AllocationId { + return api.EIP_MODE_INSTANCE_PUBLICIP + } else { + return api.EIP_MODE_STANDALONE_EIP + } +} + +func (eipaddr *SEipAddress) GetINetworkId() string { + return "" +} + +func (eipaddr *SEipAddress) GetAssociationType() string { + switch eipaddr.InstanceType { + case EIP_INSTANCE_TYPE_ECS, EIP_INSTANCE_TYPE_ENI: + return api.EIP_ASSOCIATE_TYPE_SERVER + case EIP_INSTANCE_TYPE_NAT: + return api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY + case EIP_INSTANCE_TYPE_ALB, EIP_INSTANCE_TYPE_CLB: + return api.EIP_ASSOCIATE_TYPE_LOADBALANCER + default: + return eipaddr.InstanceType + } +} + +func (eipaddr *SEipAddress) GetAssociationExternalId() string { + return eipaddr.InstanceId +} + +func (eipaddr *SEipAddress) GetBandwidth() int { + return eipaddr.Bandwidth +} + +func (eipaddr *SEipAddress) GetInternetChargeType() string { + switch eipaddr.BillingType { + case BillingByPrePaid, BillingByBandwidth: + return api.EIP_CHARGE_TYPE_BY_BANDWIDTH + case BillingByTraffic: + return api.EIP_CHARGE_TYPE_BY_TRAFFIC + default: + return api.EIP_CHARGE_TYPE_BY_BANDWIDTH + } +} + +func (eipaddr *SEipAddress) Delete() error { + return eipaddr.region.DeallocateEIP(eipaddr.AllocationId) +} + +func (eipaddr *SEipAddress) Associate(conf *cloudprovider.AssociateConfig) error { + _ = cloudprovider.Wait(20*time.Second, time.Minute, func() (bool, error) { + err := eipaddr.region.AssociateEip(eipaddr.AllocationId, conf.InstanceId) + if err != nil { + if isError(err, "IncorrectInstanceStatus") { + return false, nil + } + return false, errors.Wrapf(err, "region.AssociateEip") + } + return true, nil + }) + err := cloudprovider.WaitStatus(eipaddr, api.EIP_STATUS_READY, 10*time.Second, 180*time.Second) + return err +} + +func (eipaddr *SEipAddress) Dissociate() error { + err := eipaddr.region.DissociateEip(eipaddr.AllocationId, eipaddr.InstanceId) + if err != nil { + return err + } + err = cloudprovider.WaitStatus(eipaddr, api.EIP_STATUS_READY, 10*time.Second, 180*time.Second) + return err +} + +func (eipaddr *SEipAddress) ChangeBandwidth(bw int) error { + return eipaddr.region.UpdateEipBandwidth(eipaddr.AllocationId, bw) +} + +func getInstanceType(instanceId string) (string, error) { + prefixMap := map[string]string{ + "i-": EIP_INSTANCE_TYPE_ECS, + "clb-": EIP_INSTANCE_TYPE_CLB, + "alb-": EIP_INSTANCE_TYPE_ALB, + "ngw-": EIP_INSTANCE_TYPE_NAT, + "eni-": EIP_INSTANCE_TYPE_ENI, + "havip-": EIP_INSTANCE_TYPE_HAVIP, + } + for prefix, instanceType := range prefixMap { + if strings.HasPrefix(instanceId, prefix) { + return instanceType, nil + } + } + return "", errors.Errorf("Unknown instance type for %s", instanceId) +} + +func (region *SRegion) GetEips(eipIds []string, associatedId string, addresses []string, pageNumber int, pageSize int) ([]SEipAddress, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + + params := make(map[string]string) + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + + for index, addr := range addresses { + params[fmt.Sprintf("EipAddresses.%d", index+1)] = addr + } + + for index, eipId := range eipIds { + params[fmt.Sprintf("AllocationIds.%d", index+1)] = eipId + } + + if len(associatedId) > 0 { + params["AssociatedInstanceId"] = associatedId + associatedType, err := getInstanceType(associatedId) + if err != nil { + return nil, 0, errors.Wrapf(err, "Unknown associated type") + } + params["AssociatedInstanceType"] = associatedType + } + + body, err := region.vpcRequest("DescribeEipAddresses", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "DescribeEipAddresses fail") + } + + eips := make([]SEipAddress, 0) + err = body.Unmarshal(&eips, "Result", "EipAddresses") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal EipAddress details fail") + } + total, _ := body.Int("TotalCount") + for i := 0; i < len(eips); i += 1 { + eips[i].region = region + } + return eips, int(total), nil +} + +func (region *SRegion) GetEip(eipId string) (*SEipAddress, error) { + eips, _, err := region.GetEips([]string{eipId}, "", make([]string, 0), 1, 1) + if err != nil { + return nil, err + } + for i := range eips { + if eips[i].AllocationId == eipId { + eips[i].region = region + return &eips[i], nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, eipId) +} + +func (region *SRegion) AllocateEIP(opts *cloudprovider.SEip) (*SEipAddress, error) { + params := make(map[string]string) + if len(opts.Name) > 0 { + params["Name"] = opts.Name + } + params["Bandwidth"] = fmt.Sprintf("%d", opts.BandwidthMbps) + switch opts.ChargeType { + case api.EIP_CHARGE_TYPE_BY_TRAFFIC: + params["BillingType"] = fmt.Sprintf("%d", BillingByTraffic) + case api.EIP_CHARGE_TYPE_BY_BANDWIDTH: + params["BillingType"] = fmt.Sprintf("%d", BillingByBandwidth) + } + + params["ClientToken"] = utils.GenRequestId(20) + if len(opts.ProjectId) > 0 { + params["ProjectName"] = opts.ProjectId + } + params["ISP"] = "BGP" + + index := 1 + for key, value := range opts.Tags { + params[fmt.Sprintf("Tags.%d.Key", index)] = key + params[fmt.Sprintf("Tags.%d.Value", index)] = value + index++ + } + + body, err := region.vpcRequest("AllocateEipAddress", params) + if err != nil { + return nil, errors.Wrapf(err, "AllocateEipAddress fail") + } + + eipId, err := body.GetString("Result", "AllocationId") + if err != nil { + return nil, errors.Wrapf(err, "get AllocationId after created fail") + } + + err = cloudprovider.Wait(5*time.Second, time.Minute, func() (bool, error) { + _, err := region.GetEip(eipId) + if errors.Cause(err) == cloudprovider.ErrNotFound { + return false, nil + } else { + return true, err + } + }) + if err != nil { + return nil, errors.Wrapf(err, "cannot find eip after create") + } + return region.GetEip(eipId) +} + +func (region *SRegion) CreateEIP(opts *cloudprovider.SEip) (cloudprovider.ICloudEIP, error) { + return region.AllocateEIP(opts) +} + +func (region *SRegion) DeallocateEIP(eipId string) error { + params := make(map[string]string) + params["AllocationId"] = eipId + + _, err := region.vpcRequest("ReleaseEipAddress", params) + if err != nil { + err = errors.Wrapf(err, "ReleaseEipAddress fail") + } + return err +} + +func (region *SRegion) AssociateEip(eipId string, instanceId string) error { + params := make(map[string]string) + params["AllocationId"] = eipId + params["InstanceId"] = instanceId + + instanceType, err := getInstanceType(instanceId) + if err != nil { + return errors.Wrapf(err, "Unknown instance type") + } + params["InstanceType"] = instanceType + + _, err = region.vpcRequest("AssociateEipAddress", params) + return errors.Wrapf(err, "AssociateEipAddress fail") +} + +func (region *SRegion) DissociateEip(eipId string, instanceId string) error { + params := make(map[string]string) + params["AllocationId"] = eipId + params["InstanceId"] = instanceId + + instanceType, err := getInstanceType(instanceId) + if err != nil { + return errors.Wrapf(err, "Unknown instance type") + } + params["InstanceType"] = instanceType + + _, err = region.vpcRequest("DisassociateEipAddress", params) + if err != nil { + err = errors.Wrapf(err, "DisassociateEipAddress fail") + } + return err +} + +func (region *SRegion) UpdateEipBandwidth(eipId string, bw int) error { + params := make(map[string]string) + params["AllocationId"] = eipId + params["Bandwidth"] = fmt.Sprintf("%d", bw) + + _, err := region.vpcRequest("ModifyEipAddressAttributes", params) + if err != nil { + err = errors.Wrapf(err, "ModifyEipAddressAttributes fail") + } + return err +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/errors.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/errors.go new file mode 100644 index 0000000000..372f830874 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/errors.go @@ -0,0 +1,29 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "strings" +) + +func isError(err error, code string) bool { + errStr := fmt.Sprintf("%s", err) + if strings.Index(errStr, code) > 0 { + return true + } else { + return false + } +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/host.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/host.go new file mode 100644 index 0000000000..822f706c34 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/host.go @@ -0,0 +1,250 @@ +package volcengine + +import ( + "fmt" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/billing" +) + +type SHost struct { + multicloud.SHostBase + zone *SZone +} + +func (host *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { + return host.zone.GetIWires() +} + +func (host *SHost) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + return host.zone.GetIStorages() +} + +func (host *SHost) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + return host.zone.GetIStorageById(id) +} + +func (host *SHost) GetId() string { + return fmt.Sprintf("%s-%s", host.zone.region.client.cpcfg.Id, host.zone.GetId()) +} + +func (host *SHost) GetName() string { + return fmt.Sprintf("%s-%s", host.zone.region.client.cpcfg.Name, host.zone.GetId()) +} + +func (host *SHost) GetGlobalId() string { + return fmt.Sprintf("%s-%s", host.zone.region.client.cpcfg.Id, host.zone.GetId()) +} + +func (host *SHost) IsEmulated() bool { + return true +} + +func (host *SHost) GetStatus() string { + return api.HOST_STATUS_RUNNING +} + +func (host *SHost) Refresh() error { + return nil +} + +func (host *SHost) GetHostStatus() string { + return api.HOST_ONLINE +} + +func (host *SHost) GetEnabled() bool { + return true +} + +func (host *SHost) GetAccessIp() string { + return "" +} + +func (host *SHost) GetAccessMac() string { + return "" +} + +func (host *SHost) GetSysInfo() jsonutils.JSONObject { + info := jsonutils.NewDict() + info.Add(jsonutils.NewString(CLOUD_PROVIDER_VOLCENGINE), "manufacture") + return info +} + +func (host *SHost) GetSN() string { + return "" +} + +func (host *SHost) GetCpuCount() int { + return 0 +} + +func (host *SHost) GetNodeCount() int8 { + return 0 +} + +func (host *SHost) GetCpuDesc() string { + return "" +} + +func (host *SHost) GetCpuMhz() int { + return 0 +} + +func (host *SHost) GetMemSizeMB() int { + return 0 +} + +func (host *SHost) GetStorageSizeMB() int { + return 0 +} + +func (host *SHost) GetStorageType() string { + return api.DISK_TYPE_HYBRID +} + +func (host *SHost) GetHostType() string { + return api.HOST_TYPE_VOLCENGINE +} + +func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { + wires, err := host.zone.GetIWires() + if err != nil { + return nil, errors.Wrap(err, "GetIWires") + } + return cloudprovider.GetHostNetifs(host, wires), nil +} + +func (host *SHost) GetIsMaintenance() bool { + return false +} + +func (host *SHost) GetVersion() string { + return VOLCENGINE_API_VERSION +} + +func (host *SHost) GetIVMs() ([]cloudprovider.ICloudVM, error) { + vms := make([]SInstance, 0) + token := "" + for { + parts, nextToken, err := host.zone.region.GetInstances(host.zone.ZoneId, nil, 10, token) + if err != nil { + return nil, err + } + vms = append(vms, parts...) + if len(nextToken) == 0 { + break + } + token = nextToken + } + ivms := make([]cloudprovider.ICloudVM, len(vms)) + for i := 0; i < len(vms); i += 1 { + vms[i].host = host + ivms[i] = &vms[i] + } + return ivms, nil +} + +func (host *SHost) GetIVMById(gid string) (cloudprovider.ICloudVM, error) { + id := gid + parts, _, err := host.zone.region.GetInstances(host.zone.ZoneId, []string{id}, 1, "") + if err != nil { + return nil, err + } + if len(parts) == 0 { + return nil, cloudprovider.ErrNotFound + } + if len(parts) > 1 { + return nil, cloudprovider.ErrDuplicateId + } + parts[0].host = host + return &parts[0], nil +} + +func (host *SHost) GetInstanceById(instanceId string) (*SInstance, error) { + inst, err := host.zone.region.GetInstance(instanceId) + if err != nil { + return nil, err + } + inst.host = host + return inst, nil +} + +func (host *SHost) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (cloudprovider.ICloudVM, error) { + vmId, err := host._createVM(desc.Name, desc.Hostname, desc.ExternalImageId, desc.SysDisk, desc.Cpu, desc.MemoryMB, + desc.InstanceType, desc.ExternalNetworkId, desc.IpAddr, desc.Description, desc.Password, + desc.DataDisks, desc.PublicKey, desc.ExternalSecgroupId, desc.UserData, desc.BillingCycle, + desc.ProjectId, desc.Tags, desc.SPublicIpInfo) + if err != nil { + return nil, err + } + vm, err := host.GetInstanceById(vmId) + if err != nil { + return nil, errors.Wrapf(err, "GetInstanceById") + } + return vm, nil +} + +func (host *SHost) _createVM(name string, hostname string, imgId string, + sysDisk cloudprovider.SDiskInfo, cpu int, memMB int, instanceType string, + networkID string, ipAddr string, desc string, passwd string, + dataDisks []cloudprovider.SDiskInfo, publicKey string, secgroupId string, + userData string, bc *billing.SBillingCycle, projectId string, + tags map[string]string, publicIp cloudprovider.SPublicIpInfo, +) (string, error) { + var err error + keypair := "" + if len(publicKey) > 0 { + keypair, err = host.zone.region.syncKeypair(publicKey) + if err != nil { + return "", err + } + } + + img, err := host.zone.region.GetImage(imgId) + if err != nil { + return "", errors.Wrapf(err, "GetImage fail") + } + if img.Status != ImageStatusAvailable { + log.Errorf("image %s status %s", imgId, img.Status) + return "", fmt.Errorf("image not ready") + } + + disks := make([]SDisk, len(dataDisks)+1) + disks[0].Size = img.Size + if sysDisk.SizeGB > 0 && sysDisk.SizeGB > img.Size { + disks[0].Size = sysDisk.SizeGB + } + storage, err := host.zone.getStorageByCategory(sysDisk.StorageType) + if err != nil { + return "", fmt.Errorf("storage %s not avaiable: %s", sysDisk.StorageType, err) + } + disks[0].VolumeType = storage.storageType + + for i, dataDisk := range dataDisks { + disks[i+1].Size = dataDisk.SizeGB + storage, err := host.zone.getStorageByCategory(dataDisk.StorageType) + if err != nil { + return "", fmt.Errorf("storage %s not avaiable: %s", dataDisk.StorageType, err) + } + disks[i+1].VolumeType = storage.storageType + } + + _, err = host.zone.region.GetSecurityGroupDetails(secgroupId) + if err != nil { + return "", errors.Wrapf(err, "GetSecurityGroup fail") + } + + if len(instanceType) == 0 { + return "", fmt.Errorf("instance type must be specified") + } + vmId, err := host.zone.region.CreateInstance(name, hostname, imgId, instanceType, secgroupId, host.zone.ZoneId, desc, passwd, disks, networkID, ipAddr, keypair, userData, bc, projectId, tags) + if err != nil { + return "", errors.Wrapf(err, "Failed to create %s", instanceType) + } + return vmId, nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/image.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/image.go new file mode 100644 index 0000000000..a4573ec531 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/image.go @@ -0,0 +1,320 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "context" + "fmt" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/imagetools" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" +) + +type ImageStatusType string + +const ( + ImageStatusCreating ImageStatusType = "creating" + ImageStatusAvailable ImageStatusType = "available" + ImageStatusError ImageStatusType = "error" +) + +type ImageOwnerType string + +const ( + ImageOwnerPrivate ImageOwnerType = "private" + ImageOwnerShared ImageOwnerType = "shared" + ImageOwnerPublic ImageOwnerType = "public" +) + +type SImage struct { + multicloud.SImageBase + VolcEngineTags + storageCache *SStoragecache + + // normalized image info + imgInfo *imagetools.ImageInfo + + Architecture string + CreationTime time.Time + Description string + ImageId string + ImageName string + OSName string + OSType string + Visibility string + IsSupportCloudinit bool + IsSupportIoOptimized bool + Platform string + Size int + Status ImageStatusType + Usage string +} + +func (img *SImage) GetMinRamSizeMb() int { + return 0 +} + +func (img *SImage) GetId() string { + return img.ImageId +} + +func (img *SImage) GetName() string { + return img.ImageName +} + +func (img *SImage) Delete(ctx context.Context) error { + return img.storageCache.region.DeleteImage(img.ImageId) +} + +func (img *SImage) GetGlobalId() string { + return img.ImageId +} + +func (img *SImage) GetIStoragecache() cloudprovider.ICloudStoragecache { + return img.storageCache +} + +func (img *SImage) GetStatus() string { + switch img.Status { + case ImageStatusCreating: + return api.CACHED_IMAGE_STATUS_SAVING + case ImageStatusAvailable: + return api.CACHED_IMAGE_STATUS_ACTIVE + case ImageStatusError: + return api.CACHED_IMAGE_STATUS_CACHE_FAILED + default: + return api.CACHED_IMAGE_STATUS_CACHE_FAILED + } +} + +func (region *SRegion) ImportImage(name string, osArch string, osType string, platform, platformVersion string, bucket string, key string) (string, error) { + params := map[string]string{ + "Architecture": osArch, + "OsType": osType, + "Platform": platform, + "PlatformVersion": platformVersion, + "Tags.1.Key": "Name", + "Tags.2.Value": name, + "Url": fmt.Sprintf("https://%s.%s/%s", bucket, region.getS3Endpoint(), key), + } + body, err := region.ecsRequest("ImportImage", params) + if err != nil { + return "", errors.Wrapf(err, "ImportImage") + } + imageId, err := body.GetString("Result", "ImageId") + if err != nil { + return "", errors.Wrap(err, "Unmarsh imageId failed") + } + return imageId, nil +} + +func (region *SRegion) ExportImage(imageId, bucketName string) (string, error) { + params := make(map[string]string) + params["RegionId"] = region.RegionId + params["ImageId"] = imageId + params["OssBucket"] = bucketName + params["OssPrefix"] = fmt.Sprintf("%sexport", strings.Replace(imageId, "-", "", -1)) + + body, err := region.ecsRequest("ExportImage", params) + if err != nil { + return "", errors.Wrapf(err, "ExportImage") + } + taskId, err := body.GetString("Result", "TaskId") + if err != nil { + return "", errors.Wrapf(err, "Unmarshal") + } + return taskId, nil +} + +func (img *SImage) GetImageStatus() string { + switch img.Status { + case ImageStatusCreating: + return cloudprovider.IMAGE_STATUS_QUEUED + case ImageStatusAvailable: + return cloudprovider.IMAGE_STATUS_ACTIVE + case ImageStatusError: + return cloudprovider.IMAGE_STATUS_DELETED + default: + return cloudprovider.IMAGE_STATUS_KILLED + } +} + +func (img *SImage) Refresh() error { + new, err := img.storageCache.region.GetImage(img.ImageId) + if err != nil { + return err + } + return jsonutils.Update(img, new) +} + +func (img *SImage) GetImageType() cloudprovider.TImageType { + switch img.Visibility { + case string(ImageOwnerPublic): + return cloudprovider.ImageTypeSystem + case string(ImageOwnerPrivate), string(ImageOwnerShared): + return cloudprovider.ImageTypeCustomized + default: + return cloudprovider.ImageTypeCustomized + } +} + +func (img *SImage) GetSizeByte() int64 { + return int64(img.Size) * 1024 * 1024 * 1024 +} + +func (img *SImage) GetOsType() cloudprovider.TOsType { + return cloudprovider.TOsType(img.getNormalizedImageInfo().OsType) +} + +func (img *SImage) GetOsDist() string { + return img.getNormalizedImageInfo().OsDistro +} + +func (img *SImage) getNormalizedImageInfo() *imagetools.ImageInfo { + if img.imgInfo == nil { + imgInfo := imagetools.NormalizeImageInfo(img.OSName, img.Architecture, img.OSType, img.Platform, "") + img.imgInfo = &imgInfo + } + + return img.imgInfo +} + +func (img *SImage) GetFullOsName() string { + return img.OSName +} + +func (img *SImage) GetOsVersion() string { + return img.getNormalizedImageInfo().OsVersion +} + +func (img *SImage) GetOsLang() string { + return img.getNormalizedImageInfo().OsLang +} + +func (img *SImage) GetOsArch() string { + return img.getNormalizedImageInfo().OsArch +} + +func (img *SImage) GetBios() cloudprovider.TBiosType { + return cloudprovider.ToBiosType(img.getNormalizedImageInfo().OsBios) +} + +func (img *SImage) GetMinOsDiskSizeGb() int { + return 40 +} + +func (img *SImage) GetImageFormat() string { + return "vhd" +} + +func (img *SImage) GetCreatedAt() time.Time { + return img.CreationTime +} + +func (region *SRegion) GetImage(imageId string) (*SImage, error) { + images, _, err := region.GetImages("", "", []string{imageId}, "", 1, "") + if err != nil { + return nil, err + } + if len(images) == 0 { + return nil, cloudprovider.ErrNotFound + } + return &images[0], nil +} + +func (region *SRegion) GetImageByName(name string) (*SImage, error) { + images, _, err := region.GetImages("", "", nil, name, 1, "") + if err != nil { + return nil, err + } + if len(images) == 0 { + return nil, cloudprovider.ErrNotFound + } + return &images[0], nil +} + +func (region *SRegion) GetImageStatus(imageId string) (ImageStatusType, error) { + image, err := region.GetImage(imageId) + if err != nil { + return "", err + } + return image.Status, nil +} + +func (region *SRegion) GetImages(status ImageStatusType, owner ImageOwnerType, imageId []string, name string, limit int, token string) ([]SImage, string, error) { + if limit > 100 || limit <= 0 { + limit = 100 + } + params := make(map[string]string) + params["MaxResults"] = fmt.Sprintf("%d", limit) + if len(token) > 0 { + params["NextToken"] = token + } + if len(status) > 0 { + params["Status"] = string(status) + } else { + allStatus := []string{"available", "creating", "error"} + for idx, status := range allStatus { + params[fmt.Sprintf("Status.%d", idx+1)] = status + } + } + if len(imageId) > 0 { + params["ImageId"] = strings.Join(imageId, ",") + } + if len(owner) > 0 { + params["ImageOwnerAlias"] = string(owner) + } + + if len(name) > 0 { + params["ImageName"] = name + } + + return region.getImages(params) +} + +func (region *SRegion) getImages(params map[string]string) ([]SImage, string, error) { + body, err := region.ecsRequest("DescribeImages", params) + if err != nil { + return nil, "", errors.Wrapf(err, "DescribeImages fail") + } + + images := make([]SImage, 0) + err = body.Unmarshal(&images, "Result", "Images") + if err != nil { + return nil, "", errors.Wrapf(err, "Unmarshal images fail") + } + nextToken, _ := body.GetString("Result", "NextToken") + return images, nextToken, nil +} + +func (region *SRegion) DeleteImage(imageId string) error { + params := make(map[string]string) + params["RegionId"] = region.RegionId + params["ImageId"] = imageId + params["Force"] = "true" + + _, err := region.ecsRequest("DeleteImage", params) + if err != nil { + return errors.Wrapf(err, "DeleteImage fail") + } + return nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instance.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instance.go new file mode 100644 index 0000000000..30a40a9159 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instance.go @@ -0,0 +1,832 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "context" + "encoding/base64" + "fmt" + "strings" + "time" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/billing" + "yunion.io/x/pkg/util/cloudinit" + "yunion.io/x/pkg/util/imagetools" + "yunion.io/x/pkg/util/osprofile" + "yunion.io/x/pkg/utils" +) + +const ( + InstanceStatusCreating = "CREATING" + InstanceStatusRunning = "RUNNING" + InstanceStatusStopping = "STOPPING" + InstanceStatusStopped = "STOPPED" + InstanceStatusRebooting = "REBOOTING" + InstanceStatusStarting = "STARTING" + InstanceStatusRebuilding = "REBUILDING" + InstanceStatusResizing = "RESIZING" + InstanceStatusError = "ERROR" + InstanceStatusDeleting = "DELETING" +) + +type TChargeType string + +type SSecurityGroupIds []string + +type SRdmaIPAddress []string + +type SInstance struct { + multicloud.SInstanceBase + VolcEngineTags + + host *SHost + + osInfo *imagetools.ImageInfo + + CreatedAt time.Time + UpdatedAt time.Time + InstanceId string + ZoneId string + ImageId string + Status string + InstanceName string + Description string + Hostname string + VpcId string + InstanceTypeId string + Cpus int + MemorySize int + OsName string + OsType string + NetworkInterfaces []SNetworkInterface + RdmaIpAddress SRdmaIPAddress + KeyPairName string + KeyPairId string + InstanceChargeType TChargeType + StoppedMode string + SpotStrategy string + DeploymentSetId string + EipAddress SEipAddress + ExpiredAt time.Time + Uuid string + ProjectName string +} + +func billingCycle2Params(bc *billing.SBillingCycle, params map[string]string) error { + if bc.GetMonths() > 0 { + params["PeriodUnit"] = "Month" + params["Period"] = fmt.Sprintf("%d", bc.GetMonths()) + } else if bc.GetWeeks() > 0 { + params["PeriodUnit"] = "Week" + params["Period"] = fmt.Sprintf("%d", bc.GetWeeks()) + // renew by week is not currently supported + return fmt.Errorf("invalid renew time period %s", bc.String()) + } else { + return fmt.Errorf("invalid renew time period %s", bc.String()) + } + return nil +} + +func (instance *SInstance) UpdatePassword(passwd string) error { + params := make(map[string]string) + params["Password"] = passwd + return instance.host.zone.region.modifyInstanceAttribute(instance.InstanceId, params) +} + +func (instance *SInstance) UpdateUserData(userData string) error { + params := make(map[string]string) + params["UserData"] = userData + return instance.host.zone.region.modifyInstanceAttribute(instance.InstanceId, params) +} + +func (instance *SInstance) GetUserData() (string, error) { + params := make(map[string]string) + params["InstanceId"] = instance.InstanceId + body, err := instance.host.zone.region.ecsRequest("DescribeUserData", params) + if err != nil { + return "", errors.Wrapf(err, "GetUserData") + } + userData, err := body.GetString("Result", "UserData") + if err != nil { + return "", errors.Wrapf(err, "GetUserData") + } + return userData, nil +} + +func (region *SRegion) GetInstance(instanceId string) (*SInstance, error) { + instances, _, err := region.GetInstances("", []string{instanceId}, 1, "") + if err != nil { + return nil, err + } + if len(instances) == 0 { + return nil, cloudprovider.ErrNotFound + } + return &instances[0], nil +} + +func (region *SRegion) GetInstances(zoneId string, ids []string, limit int, token string) ([]SInstance, string, error) { + if limit > 10 || limit <= 0 { + limit = 10 + } + params := make(map[string]string) + params["MaxResults"] = fmt.Sprintf("%d", limit) + if len(token) > 0 { + params["NextToken"] = token + } + if len(zoneId) > 0 { + params["ZoneId"] = zoneId + } + if len(ids) > 0 { + for index, id := range ids { + key := fmt.Sprintf("InstanceIds.%d", index+1) + params[key] = id + } + } + body, err := region.ecsRequest("DescribeInstances", params) + if err != nil { + return nil, "", errors.Wrapf(err, "GetInstances fail") + } + instances := make([]SInstance, 0) + err = body.Unmarshal(&instances, "Result", "Instances") + if err != nil { + return nil, "", errors.Wrapf(err, "Unmarshal details fail") + } + nextToken, _ := body.GetString("Result", "NextToken") + return instances, nextToken, nil +} + +func (instance *SInstance) GetIHost() cloudprovider.ICloudHost { + return instance.host +} + +func (instance *SInstance) GetIHostId() string { + return instance.host.GetGlobalId() +} + +func (instance *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) { + pageNumber := 1 + disks := make([]SDisk, 0) + for { + parts, total, err := instance.host.zone.region.GetDisks(instance.InstanceId, "", "", nil, pageNumber, 50) + if err != nil { + return nil, err + } + disks = append(disks, parts...) + if len(disks) >= total { + break + } + pageNumber += 1 + } + + idisks := make([]cloudprovider.ICloudDisk, len(disks)) + for i := 0; i < len(disks); i += 1 { + store, err := instance.host.zone.getStorageByCategory(disks[i].VolumeType) + if err != nil { + return nil, errors.Wrap(err, "getStorageByCategory") + } + disks[i].storage = store + idisks[i] = &disks[i] + } + return idisks, nil +} + +func (instance *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) { + if len(instance.EipAddress.AllocationId) > 0 { + return instance.host.zone.region.GetEip(instance.EipAddress.AllocationId) + } + for _, nic := range instance.NetworkInterfaces { + if len(nic.AssociatedElasticIp.EipAddress) > 0 { + eip := SEipAddress{region: instance.host.zone.region} + eip.region = instance.host.zone.region + eip.EipAddress = nic.AssociatedElasticIp.EipAddress + eip.InstanceId = instance.InstanceId + eip.AllocationId = instance.InstanceId + return &eip, nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (instance *SInstance) GetINics() ([]cloudprovider.ICloudNic, error) { + networkInterfaces := instance.NetworkInterfaces + nics := make([]cloudprovider.ICloudNic, 0) + for _, ni := range networkInterfaces { + nic := SInstanceNic{ + instance: instance, + id: ni.NetworkInterfaceId, + ipAddr: ni.PrimaryIpAddress, + macAddr: ni.MacAddress, + } + nics = append(nics, &nic) + } + return nics, nil +} + +func (instance *SInstance) GetId() string { + return instance.InstanceId +} + +func (instance *SInstance) GetName() string { + if len(instance.InstanceName) > 0 { + return instance.InstanceName + } + return instance.InstanceId +} + +func (instance *SInstance) GetHostname() string { + return instance.Hostname +} + +func (instance *SInstance) GetGlobalId() string { + return instance.InstanceId +} + +func (instance *SInstance) GetInstanceType() string { + return instance.InstanceTypeId +} + +func (instance *SInstance) GetSecurityGroupIds() ([]string, error) { + ret := []string{} + for _, net := range instance.NetworkInterfaces { + ret = append(ret, net.SecurityGroupIds...) + } + return ret, nil +} + +func (instance *SInstance) GetVcpuCount() int { + return instance.Cpus +} + +func (instance *SInstance) GetVmemSizeMB() int { + return instance.MemorySize +} + +func (instance *SInstance) GetBootOrder() string { + return "dcn" +} + +func (instance *SInstance) GetVga() string { + return "std" +} + +func (instance *SInstance) GetVdi() string { + return "vnc" +} + +func (ins *SInstance) getNormalizedOsInfo() *imagetools.ImageInfo { + if ins.osInfo == nil { + osInfo := imagetools.NormalizeImageInfo(ins.OsName, "", ins.OsType, "", "") + ins.osInfo = &osInfo + } + return ins.osInfo +} + +func (instance *SInstance) GetOsType() cloudprovider.TOsType { + return cloudprovider.TOsType(osprofile.NormalizeOSType(instance.OsType)) +} + +func (instance *SInstance) GetFullOsName() string { + return instance.OsName +} + +func (ins *SInstance) GetBios() cloudprovider.TBiosType { + return cloudprovider.ToBiosType(ins.getNormalizedOsInfo().OsBios) +} + +func (ins *SInstance) GetOsArch() string { + return ins.getNormalizedOsInfo().OsArch +} + +func (ins *SInstance) GetOsDist() string { + return ins.getNormalizedOsInfo().OsDistro +} + +func (ins *SInstance) GetOsVersion() string { + return ins.getNormalizedOsInfo().OsVersion +} + +func (ins *SInstance) GetOsLang() string { + return ins.getNormalizedOsInfo().OsLang +} + +func (instance *SInstance) GetMachine() string { + return "pc" +} + +func (instance *SInstance) GetStatus() string { + switch instance.Status { + case InstanceStatusRunning: + return api.VM_RUNNING + case InstanceStatusStarting: + return api.VM_STARTING + case InstanceStatusStopping: + return api.VM_STOPPING + case InstanceStatusStopped: + return api.VM_READY + case InstanceStatusDeleting: + return api.VM_DELETING + default: + return api.VM_UNKNOWN + } +} + +func (instance *SInstance) Refresh() error { + ins, err := instance.host.zone.region.GetInstance(instance.InstanceId) + if err != nil { + return err + } + return jsonutils.Update(instance, ins) +} + +func (instance *SInstance) GetHypervisor() string { + return api.HYPERVISOR_VOLCENGINE +} + +func (instance *SInstance) GetCreatedAt() time.Time { + return instance.CreatedAt +} + +func (instance *SInstance) GetExpiredAt() time.Time { + // return instance.ExpiredAt + return time.Time{} +} + +func (instance *SInstance) AssignSecurityGroup(secgroupId string) error { + return errors.Wrapf(cloudprovider.ErrNotImplemented, "AssignSecurityGroup") +} + +func (instance *SInstance) SetSecurityGroups(secgroupIds []string) error { + return errors.Wrapf(cloudprovider.ErrNotImplemented, "SetSecurityGroups") +} + +func (instance *SInstance) GetError() error { + return nil +} + +func (instance *SInstance) ChangeConfig(ctx context.Context, config *cloudprovider.SManagedVMChangeConfig) error { + if config.InstanceType == "nil" { + return errors.Wrapf(cloudprovider.ErrInputParameter, "InstanceType") + } + return instance.host.zone.region.ChangeConfig(instance.InstanceId, config.InstanceType) +} + +func (instance *SInstance) GetVNCInfo(input *cloudprovider.ServerVncInput) (*cloudprovider.ServerVncOutput, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (instance *SInstance) StartVM(ctx context.Context) error { + err := instance.host.zone.region.StartVM(instance.InstanceId) + return err +} + +func (instance *SInstance) StopVM(ctx context.Context, opts *cloudprovider.ServerStopOptions) error { + err := instance.host.zone.region.StopVM(instance.InstanceId, opts.IsForce, opts.StopCharging) + if err != nil { + return err + } + return cloudprovider.WaitStatus(instance, api.VM_READY, 10*time.Second, 300*time.Second) +} + +func (instance *SInstance) DeleteVM(ctx context.Context) error { + for { + err := instance.host.zone.region.DeleteVM(instance.InstanceId) + if err != nil { + if isError(err, "IncorrectInstanceStatus.Initializing") { + log.Infof("The instance is initializing, try later ...") + time.Sleep(10 * time.Second) + } else { + return errors.Wrapf(err, "DeleteVM fail") + } + } else { + break + } + } + return cloudprovider.WaitDeleted(instance, 10*time.Second, 300*time.Second) +} + +func (instance *SInstance) UpdateVM(ctx context.Context, input cloudprovider.SInstanceUpdateOptions) error { + return instance.host.zone.region.UpdateVM(instance.InstanceId, input.NAME, input.Description) +} + +func (instance *SInstance) DeployVM(ctx context.Context, name string, username string, password string, publicKey string, deleteKeypair bool, description string) error { + var keypairName string + if len(publicKey) > 0 { + var err error + keypairName, err = instance.host.zone.region.syncKeypair(publicKey) + if err != nil { + return err + } + } + + return instance.host.zone.region.DeployVM(instance.InstanceId, name, password, keypairName, deleteKeypair, description) +} + +func (instance *SInstance) AttachDisk(ctx context.Context, diskId string) error { + return instance.host.zone.region.AttachDisk(instance.InstanceId, diskId) +} + +func (instance *SInstance) DetachDisk(ctx context.Context, diskId string) error { + return cloudprovider.RetryOnError( + func() error { + return instance.host.zone.region.DetachDisk(instance.InstanceId, diskId) + }, + []string{ + `"Code":"InvalidOperation.Conflict"`, + }, + 4) +} + +func (instance *SInstance) GetProjectId() string { + return instance.ProjectName +} + +func (instance *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { + udata, err := instance.GetUserData() + if err != nil { + return "", err + } + + image, err := instance.host.zone.region.GetImage(desc.ImageId) + if err != nil { + return "", errors.Wrapf(err, "GetImage fail") + } + + keypairName := instance.KeyPairName + if len(desc.PublicKey) > 0 { + keypairName, err = instance.host.zone.region.syncKeypair(desc.PublicKey) + if err != nil { + return "", fmt.Errorf("RebuildRoot.syncKeypair %s", err) + } + } + + userdata := "" + srcOsType := strings.ToLower(string(instance.GetOsType())) + destOsType := strings.ToLower(string(image.GetOsType())) + winOS := strings.ToLower(osprofile.OS_TYPE_WINDOWS) + + cloudconfig := &cloudinit.SCloudConfig{} + if srcOsType != winOS && len(udata) > 0 { + _cloudconfig, err := cloudinit.ParseUserDataBase64(udata) + if err != nil { + log.Debugf("RebuildRoot invalid instance user data %s", udata) + } else { + cloudconfig = _cloudconfig + } + } + + if (srcOsType != winOS && destOsType != winOS) || (srcOsType == winOS && destOsType != winOS) { + // linux/windows to linux + loginUser := cloudinit.NewUser(api.VM_AWS_DEFAULT_LOGIN_USER) + loginUser.SudoPolicy(cloudinit.USER_SUDO_NOPASSWD) + if len(desc.PublicKey) > 0 { + loginUser.SshKey(desc.PublicKey) + cloudconfig.MergeUser(loginUser) + } else if len(desc.Password) > 0 { + cloudconfig.SshPwauth = cloudinit.SSH_PASSWORD_AUTH_ON + loginUser.Password(desc.Password) + cloudconfig.MergeUser(loginUser) + } + + userdata = cloudconfig.UserDataBase64() + } else { + // linux/windows to windows + data := "" + if len(desc.Password) > 0 { + cloudconfig.SshPwauth = cloudinit.SSH_PASSWORD_AUTH_ON + loginUser := cloudinit.NewUser(api.VM_AWS_DEFAULT_WINDOWS_LOGIN_USER) + loginUser.SudoPolicy(cloudinit.USER_SUDO_NOPASSWD) + loginUser.Password(desc.Password) + cloudconfig.MergeUser(loginUser) + data = fmt.Sprintf("%s", cloudconfig.UserDataPowerShell()) + } else { + if len(udata) > 0 { + data = fmt.Sprintf("%s", udata) + } + } + + userdata = base64.StdEncoding.EncodeToString([]byte(data)) + } + + diskId, err := instance.host.zone.region.ReplaceSystemDisk(ctx, instance.InstanceId, desc.ImageId, desc.Password, keypairName, userdata) + if err != nil { + return "", err + } + + return diskId, nil +} + +func (instance *SInstance) SaveImage(opts *cloudprovider.SaveImageOptions) (cloudprovider.ICloudImage, error) { + image, err := instance.host.zone.region.SaveImage(instance.InstanceId, opts) + if err != nil { + return nil, errors.Wrapf(err, "SaveImage %s", opts.Name) + } + return image, nil +} + +// region +func (region *SRegion) CreateInstance( + name string, + hostname string, + imageId string, + instanceType string, + securityGroupId string, + zoneId string, + desc string, + passwd string, + disks []SDisk, + networkID string, + ipAddr string, + keypair string, + userData string, + bc *billing.SBillingCycle, + projectId string, + tags map[string]string, +) (string, error) { + params := make(map[string]string) + params["RegionId"] = region.RegionId + params["ImageId"] = imageId + params["InstanceType"] = instanceType + params["ZoneId"] = zoneId + params["InstanceName"] = name + params["ProjectName"] = projectId + if len(hostname) > 0 { + params["HostName"] = hostname + } + params["Description"] = desc + if len(passwd) > 0 { + params["Password"] = passwd + } else { + params["KeepImageCredential"] = "True" + } + if len(keypair) > 0 { + params["KeyPairName"] = keypair + } + + if len(userData) > 0 { + params["UserData"] = userData + } + + if len(tags) > 0 { + tagIdx := 1 + for k, v := range tags { + params[fmt.Sprintf("Tag.%d.Key", tagIdx)] = k + params[fmt.Sprintf("Tag.%d.Value", tagIdx)] = v + tagIdx += 1 + } + } + + if len(disks) > 0 { + for idx, disk := range disks { + diskIdx := idx + 1 + params[fmt.Sprintf("Volumes.%d.Size", diskIdx)] = fmt.Sprintf("%d", disk.Size) + params[fmt.Sprintf("Volumes.%d.VolumeType", diskIdx)] = disk.VolumeType + } + } + + params["NetworkInterfaces.1.SubnetId"] = ipAddr + // currently only support binding the first NetworkInterface securitygroup + params["NetworkInterfaces.1.SecurityGroupIds.1"] = securityGroupId + + if bc != nil { + params["InstanceChargeType"] = "PrePaid" + err := billingCycle2Params(bc, params) + if err != nil { + return "", err + } + if bc.AutoRenew { + params["AutoRenew"] = "true" + params["AutoRenewPeriod"] = "1" + } else { + params["AutoRenew"] = "False" + } + } else { + params["InstanceChargeType"] = "PostPaid" + params["SpotStrategy"] = "NoSpot" + } + + params["ClientToken"] = utils.GenRequestId(20) + + body, err := region.ecsRequest("CreateInstance", params) + if err != nil { + return "", errors.Wrapf(err, "CreateInstance fail") + } + instanceId, _ := body.GetString("InstanceId") + return instanceId, nil +} + +func (region *SRegion) RenewInstance(instanceId string, bc billing.SBillingCycle) error { + params := make(map[string]string) + params["InstanceId"] = instanceId + err := billingCycle2Params(&bc, params) + if err != nil { + return err + } + params["ClientToken"] = utils.GenRequestId(20) + _, err = region.ecsRequest("RenewInstance", params) + if err != nil { + return errors.Wrapf(err, "RenewInstance fail") + } + return nil +} + +func (region *SRegion) ChangeConfig(instanceId string, instanceTypeId string) error { + params := make(map[string]string) + params["InstanceTypeId"] = instanceTypeId + return region.instanceOperation(instanceId, "ModifyInstanceSpec", params) +} + +func (region *SRegion) StartVM(instanceId string) error { + status, err := region.GetInstanceStatus(instanceId) + if err != nil { + return errors.Wrapf(err, "Fail to get instance status on StartVM") + } + if status != InstanceStatusStopped { + return errors.Wrapf(cloudprovider.ErrInvalidStatus, "StartVM: vm status is %s expect %s", status, InstanceStatusStopped) + } + return region.doStartVM(instanceId) +} + +func (region *SRegion) StopVM(instanceId string, isForce, stopCharging bool) error { + status, err := region.GetInstanceStatus(instanceId) + if err != nil { + return errors.Wrapf(err, "Fail to get instance status on StopVM") + } + if status == InstanceStatusStopped { + return nil + } + if status != InstanceStatusRunning { + return errors.Wrapf(cloudprovider.ErrInvalidStatus, "StartVM: vm status is %s expect %s", status, InstanceStatusRunning) + } + return region.doStopVM(instanceId, isForce, stopCharging) +} + +func (region *SRegion) DeleteVM(instanceId string) error { + status, err := region.GetInstanceStatus(instanceId) + if err != nil { + return errors.Wrapf(err, "Fail to get instance status on DeleteVM") + } + log.Debugf("Instance status on delete is %s", status) + if status != InstanceStatusStopped { + log.Warningf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped) + } + return region.doDeleteVM(instanceId) +} + +func (region *SRegion) doStartVM(instanceId string) error { + return region.instanceOperation(instanceId, "StartInstance", nil) +} + +func (region *SRegion) doStopVM(instanceId string, isForce, stopCharging bool) error { + params := make(map[string]string) + if isForce { + params["ForceStop"] = "true" + } else { + params["ForceStop"] = "false" + } + params["StoppedMode"] = "KeepCharging" + if stopCharging { + params["StoppedMode"] = "StopCharging" + } + return region.instanceOperation(instanceId, "StopInstance", params) +} + +func (region *SRegion) doDeleteVM(instanceId string) error { + return region.instanceOperation(instanceId, "DeleteInstance", nil) +} + +func (region *SRegion) modifyInstanceAttribute(instanceId string, params map[string]string) error { + return region.instanceOperation(instanceId, "ModifyInstanceAttribute", params) +} + +func (region *SRegion) UpdateVM(instanceId string, name, description string) error { + params := make(map[string]string) + params["InstanceName"] = name + params["Description"] = description + return region.modifyInstanceAttribute(instanceId, params) +} + +func (region *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, deleteKeypair bool, description string) error { + instance, err := region.GetInstance(instanceId) + if err != nil { + return err + } + + if deleteKeypair { + err = region.DetachKeyPair(instanceId, instance.KeyPairName) + if err != nil { + return err + } + } + + if len(keypairName) > 0 { + err = region.AttachKeypair(instanceId, keypairName) + if err != nil { + return err + } + } + + params := make(map[string]string) + + if len(password) > 0 { + params["Password"] = password + } + + if len(name) > 0 && instance.InstanceName != name { + params["InstanceName"] = name + } + + if len(description) > 0 && instance.Description != description { + params["Description"] = description + } + + if len(params) > 0 { + return region.modifyInstanceAttribute(instanceId, params) + } else { + return nil + } +} + +func (region *SRegion) DetachDisk(instanceId string, diskId string) error { + params := make(map[string]string) + params["InstanceId"] = instanceId + params["VolumeId"] = diskId + log.Infof("Detach instance %s disk %s", instanceId, diskId) + _, err := region.storageRequest("DetachVolume", params) + if err != nil { + return errors.Wrap(err, "DetachDisk") + } + + return nil +} + +func (region *SRegion) AttachDisk(instanceId string, diskId string) error { + params := make(map[string]string) + params["InstanceId"] = instanceId + params["VolumeId"] = diskId + _, err := region.storageRequest("AttachVolume", params) + if err != nil { + return errors.Wrapf(err, "AttachDisk %s to %s fail", diskId, instanceId) + } + + return nil +} + +func (region *SRegion) ReplaceSystemDisk(ctx context.Context, instanceId string, imageId string, passwd string, keypairName string, userdata string) (string, error) { + params := make(map[string]string) + params["InstanceId"] = instanceId + params["ImageId"] = imageId + if len(passwd) > 0 { + params["Password"] = passwd + } else { + params["KeepImageCredential"] = "True" + } + if len(keypairName) > 0 { + params["KeyPairName"] = keypairName + } + _, err := region.ecsRequest("ReplaceSystemVolume", params) + if err != nil { + return "", err + } + // volcengine does not return volumeId + return "", nil +} + +func (region *SRegion) SaveImage(instanceId string, opts *cloudprovider.SaveImageOptions) (*SImage, error) { + params := map[string]string{ + "InstanceId": instanceId, + "ImageName": opts.Name, + "Description": opts.Notes, + "ClientToken": utils.GenRequestId(20), + } + body, err := region.ecsRequest("CreateImage", params) + if err != nil { + return nil, errors.Wrapf(err, "CreateImage") + } + imageId, err := body.GetString("Result", "IamgeId") + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal") + } + image, err := region.GetImage(imageId) + if err != nil { + return nil, errors.Wrapf(err, "GetImage %s", imageId) + } + return image, nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instancenic.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instancenic.go new file mode 100644 index 0000000000..590e0bef4f --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instancenic.go @@ -0,0 +1,122 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + + "github.com/golang-plus/errors" + "yunion.io/x/cloudmux/pkg/cloudprovider" +) + +type SInstanceNic struct { + cloudprovider.DummyICloudNic + + instance *SInstance + + id string + ipAddr string + macAddr string +} + +func (nic *SInstanceNic) GetId() string { + return nic.id +} + +func (nic *SInstanceNic) GetIP() string { + return nic.ipAddr +} + +func (nic *SInstanceNic) GetMAC() string { + return nic.macAddr +} + +func (nic *SInstanceNic) InClassicNetwork() bool { + return false +} + +func (nic *SInstanceNic) GetDriver() string { + return "virtio" +} + +func (nic *SInstanceNic) GetINetworkId() string { + return nic.instance.NetworkInterfaces[0].SubnetId +} + +func (nic *SInstanceNic) GetSubAddress() ([]string, error) { + return nic.instance.host.zone.region.GetSubAddress(nic.id) +} + +func (nic *SInstanceNic) AssignAddress(ipAddrs []string) error { + return nic.instance.host.zone.region.AssignAddres(nic.id, ipAddrs) +} + +func (nic *SInstanceNic) UnassignAddress(ipAddrs []string) error { + return nic.instance.host.zone.region.UnassignAddress(nic.id, ipAddrs) +} + +func (region *SRegion) GetSubAddress(nicId string) ([]string, error) { + params := map[string]string{ + "NetworkInterfaceId.1": nicId, + } + body, err := region.vpcRequest("DescribeNetworkInterfaces", params) + if err != nil { + return nil, errors.Wrapf(err, "DescribeNetworkInterfaces") + } + + interfaces := []SNetworkInterface{} + err = body.Unmarshal(&interfaces, "Result", "NetworkInterfaceSets") + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal") + } + ipAddrs := []string{} + for _, net := range interfaces { + if net.NetworkInterfaceId != nicId { + continue + } + for _, addr := range net.PrivateIpSets.PrivateIpSet { + if !addr.Primary { + ipAddrs = append(ipAddrs, addr.PrivateIpAddress) + } + } + } + return ipAddrs, nil +} + +func (region *SRegion) AssignAddres(nicId string, ipAddrs []string) error { + params := make(map[string]string) + params["NetworkInterfaceId"] = nicId + for idx, addr := range ipAddrs { + params[fmt.Sprintf("PrivateIpAddress.%d", idx+1)] = addr + } + _, err := region.vpcRequest("AssignPrivateIpAddresses", params) + if err != nil { + return errors.Wrapf(err, "AssignPrivateIpAddresses") + } + return nil +} + +func (region *SRegion) UnassignAddress(nicId string, ipAddrs []string) error { + params := make(map[string]string) + params["NetworkInterfaceId"] = nicId + for idx, addr := range ipAddrs { + params[fmt.Sprintf("PrivateIpAddress.%d", idx+1)] = addr + } + _, err := region.vpcRequest("UnassignPrivateAddress", params) + if err != nil { + return errors.Wrapf(err, "UnassignPrivateAdress") + } + return nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/keypairs.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/keypairs.go new file mode 100644 index 0000000000..ae3acfc97d --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/keypairs.go @@ -0,0 +1,143 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/aokoli/goutils" + "golang.org/x/crypto/ssh" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" +) + +type SKeypair struct { + KeyPairFingerPrint string + KeyPairName string +} + +func (region *SRegion) GetKeypairs(finger string, name string, limit int, token string) ([]SKeypair, string, error) { + if limit > 500 || limit <= 0 { + limit = 500 + } + params := make(map[string]string) + params["MaxResults"] = fmt.Sprintf("%d", limit) + if len(token) > 0 { + params["NextToken"] = token + } + if len(finger) > 0 { + params["FingerPrint"] = finger + } + if len(name) > 0 { + params["KeyPairName"] = name + } + + body, err := region.ecsRequest("DescribeKeyPairs", params) + if err != nil { + return nil, "", errors.Wrapf(err, "GetKeypairs fail") + } + + keypairs := make([]SKeypair, 0) + err = body.Unmarshal(&keypairs, "Result", "KeyPairs") + if err != nil { + return nil, "", errors.Wrapf(err, "Unmarshal keypair fail") + } + nextToken, _ := body.GetString("Result", "NextToken") + return keypairs, nextToken, nil +} + +func (region *SRegion) ImportKeypair(name string, pubKey string) (*SKeypair, error) { + params := make(map[string]string) + params["PublicKey"] = pubKey + params["KeyPairName"] = name + + body, err := region.ecsRequest("ImportKeyPair", params) + if err != nil { + return nil, errors.Wrapf(err, "ImportKeypair fail") + } + + log.Debugf("%s", body) + keypair := SKeypair{} + err = body.Unmarshal(&keypair, "Result") + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal keypair fail") + } + return &keypair, nil +} + +func (region *SRegion) AttachKeypair(instanceId string, name string) error { + params := make(map[string]string) + params["KeyPairName"] = name + params["InstanceIds.1"] = instanceId + _, err := region.ecsRequest("AttachKeyPair", params) + if err != nil { + return errors.Wrapf(err, "AttachKeyPair fail") + } + + return nil +} + +func (region *SRegion) DetachKeyPair(instanceId string, name string) error { + params := make(map[string]string) + params["KeyPairName"] = name + params["InstanceIds.1"] = instanceId + _, err := region.ecsRequest("DetachKeyPair", params) + if err != nil { + return errors.Wrapf(err, "DetachKeyPair fail") + } + + return nil +} + +func (region *SRegion) lookUpVolcEngineKeypair(publicKey string) (string, error) { + pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKey)) + if err != nil { + return "", fmt.Errorf("publicKey error %s", err) + } + + fingerprint := strings.Replace(ssh.FingerprintLegacyMD5(pk), ":", "", -1) + ks, _, err := region.GetKeypairs(fingerprint, "*", 0, "") + if len(ks) < 1 { + return "", fmt.Errorf("keypair not found %s", err) + } else { + return ks[0].KeyPairName, nil + } +} + +func (region *SRegion) importVolcEngineKeypair(publicKey string) (string, error) { + prefix, e := goutils.RandomAlphabetic(6) + if e != nil { + return "", fmt.Errorf("publicKey error %s", e) + } + + name := prefix + strconv.FormatInt(time.Now().Unix(), 10) + if k, e := region.ImportKeypair(name, publicKey); e != nil { + return "", fmt.Errorf("keypair import error %s", e) + } else { + return k.KeyPairName, nil + } +} + +func (region *SRegion) syncKeypair(publicKey string) (string, error) { + name, e := region.lookUpVolcEngineKeypair(publicKey) + if e == nil { + return name, nil + } + return region.importVolcEngineKeypair(publicKey) +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/latitude_and_longitude.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/latitude_and_longitude.go new file mode 100644 index 0000000000..58dc332d5e --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/latitude_and_longitude.go @@ -0,0 +1,26 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" +) + +var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{ + "cn-guangzhou": api.RegionGuangzhou, + "cn-beijing": api.RegionBeijing, + "cn-shanghai": api.RegionShanghai, +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natdtable.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natdtable.go new file mode 100644 index 0000000000..5d3fd120ed --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natdtable.go @@ -0,0 +1,176 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "strconv" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/pkg/errors" +) + +type SDNATEntry struct { + multicloud.SResourceBase + VolcEngineTags + nat *SNatGateway + + NatGatewayId string + DnatEntryId string + DnatEntryName string + Protocol string + InternalIp string + InternalPort string + ExternalIp string + ExternalPort string + Status string +} + +func (dentry *SDNATEntry) GetName() string { + if len(dentry.DnatEntryName) > 0 { + return dentry.DnatEntryName + } + return dentry.DnatEntryId +} + +func (dentry *SDNATEntry) GetId() string { + return dentry.DnatEntryId +} + +func (dentry *SDNATEntry) GetGlobalId() string { + return dentry.DnatEntryId +} + +func (dentry *SDNATEntry) GetStatus() string { + switch dentry.Status { + case "Creating": + return api.NAT_STATUS_ALLOCATE + case "Available": + return api.NAT_STAUTS_AVAILABLE + case "Deleteting": + return api.NAT_STATUS_DELETING + default: + return api.NAT_STATUS_UNKNOWN + } +} + +func (dentry *SDNATEntry) GetIpProtocol() string { + return dentry.Protocol +} + +func (dentry *SDNATEntry) GetExternalIp() string { + return dentry.ExternalIp +} + +func (dentry *SDNATEntry) GetExternalPort() int { + port, _ := strconv.Atoi(dentry.ExternalPort) + return port +} + +func (dentry *SDNATEntry) GetInternalIp() string { + return dentry.InternalIp +} + +func (dentry *SDNATEntry) GetInternalPort() int { + port, _ := strconv.Atoi(dentry.InternalPort) + return port +} + +func (dentry *SDNATEntry) Delete() error { + return dentry.nat.vpc.region.DeleteDnatEntry(dentry.DnatEntryId) +} + +func (nat *SNatGateway) getDnatEntries() ([]SDNATEntry, error) { + entries := make([]SDNATEntry, 0) + pageNumber := 1 + for { + parts, total, err := nat.vpc.region.GetDnatEntries(nat.NatGatewayId, pageNumber, 50) + if err != nil { + return nil, err + } + entries = append(entries, parts...) + if len(entries) >= total { + break + } + pageNumber += 1 + } + return entries, nil +} + +func (region *SRegion) GetDnatEntries(natGatewayId string, pageNumber int, pageSize int) ([]SDNATEntry, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + params := make(map[string]string) + params["NatGatewayId"] = natGatewayId + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + + body, err := region.natRequest("DescribeDnatEntries", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "DescribeDnatEntries fail") + } + entries := make([]SDNATEntry, 0) + err = body.Unmarshal(&entries, "Result", "DnatEntries") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal entries fail") + } + total, _ := body.Int("Result", "TotalCount") + return entries, int(total), nil +} + +func (region *SRegion) GetDnatEntry(natGatewayId string, dnatEntryID string) (SDNATEntry, error) { + params := make(map[string]string) + params["NatGatewayId"] = natGatewayId + params["DnatEntryIds.1"] = dnatEntryID + body, err := region.natRequest("DescribeDnatEntries", params) + if err != nil { + return SDNATEntry{}, errors.Wrapf(err, "DescribeDnatEntries fail") + } + entries := make([]SDNATEntry, 0) + err = body.Unmarshal(&entries, "Result", "DnatEntries") + if err != nil { + return SDNATEntry{}, errors.Wrapf(err, "Unmarshal entries fail") + } + if len(entries) == 0 { + return SDNATEntry{}, cloudprovider.ErrNotFound + } + return entries[0], nil +} + +func (region *SRegion) CreateDnatEntry(rule cloudprovider.SNatDRule, natGatewayId string) (string, error) { + params := make(map[string]string) + params["NatGatewayId"] = natGatewayId + params["ExternalIp"] = rule.ExternalIP + params["ExternalPort"] = strconv.Itoa(rule.ExternalPort) + params["InternalIp"] = rule.InternalIP + params["InternalPort"] = strconv.Itoa(rule.InternalPort) + params["Protocol"] = rule.Protocol + body, err := region.natRequest("CreateDnatEntry", params) + if err != nil { + return "", err + } + entryID, _ := body.GetString("Result", "DnatEntryId") + return entryID, nil +} + +func (region *SRegion) DeleteDnatEntry(dnatEntryId string) error { + params := make(map[string]string) + params["DnatEntryId"] = dnatEntryId + _, err := region.natRequest("DeleteDnatEntry", params) + return err +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natgateway.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natgateway.go new file mode 100644 index 0000000000..30432c40df --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natgateway.go @@ -0,0 +1,293 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "time" + + billing "yunion.io/x/cloudmux/pkg/apis/billing" + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" +) + +type SNatGateway struct { + multicloud.SNatGatewayBase + VolcEngineTags + + vpc *SVpc + + NatGatewayId string + NatGatewayName string + Description string + Spec string + BillingType int + VpcId string + SubnetId string + ZoneId string + NetworkInterfaceId string + ProjectName string + Status string + BusinessStatus string + LockReason string + CreationTime time.Time + UpdatedAt time.Time + ExpiredTime time.Time + OverdueTime time.Time + DeletedTime time.Time + PrivateIP string +} + +func (nat *SNatGateway) GetId() string { + return nat.NatGatewayId +} + +func (nat *SNatGateway) GetGlobalId() string { + return nat.NatGatewayId +} + +func (nat *SNatGateway) GetName() string { + if len(nat.NatGatewayName) > 0 { + return nat.NatGatewayName + } + return nat.NatGatewayId +} + +func (nat *SNatGateway) GetStatus() string { + switch nat.Status { + case "Creating": + return api.NAT_STATUS_ALLOCATE + case "Available": + return api.NAT_STAUTS_AVAILABLE + case "Pending": + return api.NAT_STATUS_DEPLOYING + case "Deleteting": + return api.NAT_STATUS_DELETING + default: + return api.NAT_STATUS_UNKNOWN + } +} + +func (nat *SNatGateway) GetINetworkId() string { + return nat.SubnetId +} + +func (nat *SNatGateway) GetIpAddr() string { + if len(nat.PrivateIP) > 0 { + return nat.PrivateIP + } + return "" +} + +func (nat *SNatGateway) GetBandwidthMb() int { + return 0 +} + +func (nat *SNatGateway) Delete() error { + return nat.vpc.region.DeleteNatGateway(nat.NatGatewayId, false) +} + +func (nat *SNatGateway) GetBillingType() string { + switch nat.BillingType { + case 1: + return billing.BILLING_TYPE_PREPAID + case 2: + return billing.BILLING_TYPE_POSTPAID + default: + return "" + } +} + +func (nat *SNatGateway) GetNatSpec() string { + if len(nat.Spec) == 0 { + return "Small" + } + return nat.Spec +} + +func (nat *SNatGateway) Refresh() error { + newNat, _, err := nat.vpc.region.GetNatGateways("", nat.NatGatewayId, 1, 1) + if err != nil { + return errors.Wrapf(err, "GetNatGateways") + } + for _, nt := range newNat { + if nt.NatGatewayId == nat.NatGatewayId { + return jsonutils.Update(nat, nt) + } + } + return errors.Wrapf(cloudprovider.ErrNotFound, "%s not found", nat.NatGatewayId) +} + +func (nat *SNatGateway) GetCreatedAt() time.Time { + return nat.CreationTime +} + +func (nat *SNatGateway) GetExpiredAt() time.Time { + return nat.ExpiredTime +} + +func (nat *SNatGateway) GetINatDTable() ([]cloudprovider.ICloudNatDEntry, error) { + stables, err := nat.getDnatEntries() + if err != nil { + return nil, err + } + itables := []cloudprovider.ICloudNatDEntry{} + for i := 0; i < len(stables); i++ { + stables[i].nat = nat + itables = append(itables, &stables[i]) + } + return itables, nil +} + +func (nat *SNatGateway) GetINatSTable() ([]cloudprovider.ICloudNatSEntry, error) { + stables, err := nat.getSnatEntries() + if err != nil { + return nil, err + } + itables := []cloudprovider.ICloudNatSEntry{} + for i := 0; i < len(stables); i++ { + stables[i].nat = nat + itables = append(itables, &stables[i]) + } + return itables, nil +} + +func (nat *SNatGateway) GetINatDEntryByID(id string) (cloudprovider.ICloudNatDEntry, error) { + dNATEntry, err := nat.vpc.region.GetDnatEntry(nat.NatGatewayId, id) + if err != nil { + return nil, cloudprovider.ErrNotFound + } + dNATEntry.nat = nat + return &dNATEntry, nil +} + +func (nat *SNatGateway) GetINatSEntryByID(id string) (cloudprovider.ICloudNatSEntry, error) { + sNATEntry, err := nat.vpc.region.GetSnatEntry(nat.NatGatewayId, id) + if err != nil { + return nil, cloudprovider.ErrNotFound + } + sNATEntry.nat = nat + return &sNATEntry, nil +} + +func (nat *SNatGateway) CreateINatDEntry(rule cloudprovider.SNatDRule) (cloudprovider.ICloudNatDEntry, error) { + entryID, err := nat.vpc.region.CreateDnatEntry(rule, nat.NatGatewayId) + if err != nil { + return nil, errors.Wrapf(err, `create dnat rule for nat gateway %q`, nat.GetId()) + } + return nat.GetINatDEntryByID(entryID) +} + +func (nat *SNatGateway) CreateINatSEntry(rule cloudprovider.SNatSRule) (cloudprovider.ICloudNatSEntry, error) { + entryID, err := nat.vpc.region.CreateSnatEntry(rule, nat.NatGatewayId) + if err != nil { + return nil, errors.Wrapf(err, `create snat rule for nat gateway %q`, nat.GetId()) + } + return nat.GetINatSEntryByID(entryID) +} + +func (region *SRegion) GetNatGateways(vpcId string, natGatewayId string, pageNumber int, pageSize int) ([]SNatGateway, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + params := make(map[string]string) + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + if len(vpcId) > 0 { + params["VpcId"] = vpcId + } + if len(natGatewayId) > 0 { + params["NatGatewayId.1"] = natGatewayId + } + body, err := region.natRequest("DescribeNatGateways", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "DescribeNatGateways") + } + gateways := make([]SNatGateway, 0) + err = body.Unmarshal(&gateways, "Result", "NatGateways") + if err != nil { + return nil, 0, errors.Wrapf(err, "body.Unmarshal") + } + total, _ := body.Int("Result", "TotalCount") + return gateways, int(total), nil +} + +func (region *SRegion) CreateNatGateway(opts *cloudprovider.NatGatewayCreateOptions) (*SNatGateway, error) { + params := map[string]string{ + "VpcId": opts.VpcId, + "SubnetId": opts.NetworkId, + "NatGatewayName": opts.Name, + "Description": opts.Desc, + "ClientToken": utils.GenRequestId(20), + "BillingType": fmt.Sprintf("%d", 2), + } + if len(opts.NatSpec) != 0 { + params["Spec"] = opts.NatSpec + } + + if opts.BillingCycle != nil { + params["BillingType"] = fmt.Sprintf("%d", 1) + params["Period"] = fmt.Sprintf("%d", 1) + params["PeriodUnit"] = "Month" + if opts.BillingCycle.GetYears() > 0 { + params["PeriodUnit"] = "Year" + params["Period"] = fmt.Sprintf("%d", opts.BillingCycle.GetYears()) + } else if opts.BillingCycle.GetMonths() > 0 { + params["PeriodUnit"] = "Month" + params["Period"] = fmt.Sprintf("%d", opts.BillingCycle.GetMonths()) + } + } + resp, err := region.natRequest("CreateNatGateway", params) + if err != nil { + return nil, errors.Wrapf(err, "CreateNatGateway") + } + natId, err := resp.GetString("Result", "NatGatewayId") + if err != nil { + return nil, errors.Wrapf(err, "resp.Get(NatGatewayId)") + } + if len(natId) == 0 { + return nil, errors.Errorf("empty NatGatewayId after created") + } + + err = cloudprovider.Wait(time.Second*5, time.Minute*15, func() (bool, error) { + _, _, err := region.GetNatGateways("", natId, 1, 1) + if errors.Cause(err) == cloudprovider.ErrNotFound { + return false, nil + } else { + return true, err + } + }) + if err != nil { + return nil, errors.Wrapf(err, "cannot find nat gateway after create") + } + nats, _, err := region.GetNatGateways("", natId, 1, 1) + for _, nat := range nats { + if nat.NatGatewayId == natId { + return &nat, nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s not found", natId) +} + +func (region *SRegion) DeleteNatGateway(natId string, isForce bool) error { + params := make(map[string]string) + params["NatGatewayId"] = natId + _, err := region.natRequest("DeleteNatGateway", params) + return errors.Wrapf(err, "DeleteNatGateway") +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natstable.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natstable.go new file mode 100644 index 0000000000..c5a8f5d8d8 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natstable.go @@ -0,0 +1,194 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" +) + +type SSNATEntry struct { + multicloud.SResourceBase + VolcEngineTags + nat *SNatGateway + + NatGatewayId string + SnatEntryId string + SnatEntryName string + SubnetId string + SourceCidr string + EipId string + EipAddress string + Status string +} + +func (sentry *SSNATEntry) GetName() string { + if len(sentry.SnatEntryName) > 0 { + return sentry.SnatEntryName + } + return sentry.SnatEntryId +} + +func (sentry *SSNATEntry) GetId() string { + return sentry.SnatEntryId +} + +func (sentry *SSNATEntry) GetGlobalId() string { + return sentry.SnatEntryId +} + +func (sentry *SSNATEntry) GetStatus() string { + switch sentry.Status { + case "Creating": + return api.NAT_STATUS_ALLOCATE + case "Available": + return api.NAT_STAUTS_AVAILABLE + case "Deleteting": + return api.NAT_STATUS_DELETING + default: + return api.NAT_STATUS_UNKNOWN + } +} + +func (sentry *SSNATEntry) GetIP() string { + return sentry.EipAddress +} + +func (sentry *SSNATEntry) GetSourceCIDR() string { + return sentry.SourceCidr +} + +func (sentry *SSNATEntry) GetNetworkId() string { + return sentry.SubnetId +} + +func (sentry *SSNATEntry) Delete() error { + return sentry.nat.vpc.region.DeleteSnatEntry(sentry.SnatEntryId) +} + +func (sentry *SSNATEntry) Refresh() error { + new, err := sentry.nat.vpc.region.GetSnatEntry(sentry.NatGatewayId, sentry.SnatEntryId) + if err != nil { + return err + } + return jsonutils.Update(sentry, new) +} + +func (nat *SNatGateway) getSnatEntries() ([]SSNATEntry, error) { + entries := make([]SSNATEntry, 0) + pageNumber := 1 + for { + parts, total, err := nat.vpc.region.GetSnatEntries(nat.NatGatewayId, pageNumber, 50) + if err != nil { + return nil, err + } + entries = append(entries, parts...) + if len(entries) >= total { + break + } + pageNumber += 1 + } + return entries, nil +} + +func (nat *SNatGateway) dissociateWithSubnet(subnetId string) error { + entries, err := nat.getSnatEntries() + if err != nil { + return err + } + for i := range entries { + if entries[i].SubnetId == subnetId { + err := nat.vpc.region.DeleteSnatEntry(entries[i].SnatEntryId) + if err != nil { + return nil + } + } + } + return nil +} + +func (region *SRegion) GetSnatEntries(natGatewayId string, pageNumber int, pageSize int) ([]SSNATEntry, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + params := make(map[string]string) + params["NatGatewayId"] = natGatewayId + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + + body, err := region.natRequest("DescribeSnatEntries", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "DescribeSNATEntries fail") + } + entries := make([]SSNATEntry, 0) + err = body.Unmarshal(&entries, "Result", "SnatEntries") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal entries fail") + } + total, _ := body.Int("Result", "TotalCount") + return entries, int(total), nil +} + +func (region *SRegion) GetSnatEntry(natGatewayId string, snatEntryID string) (SSNATEntry, error) { + params := make(map[string]string) + params["NatGatewayId"] = natGatewayId + params["SnatEntryIds.1"] = snatEntryID + body, err := region.natRequest("DescribeSnatEntries", params) + if err != nil { + return SSNATEntry{}, errors.Wrapf(err, "DescribeSnatEntries fail") + } + entries := make([]SSNATEntry, 0) + err = body.Unmarshal(&entries, "Result", "SnatEntries") + if err != nil { + return SSNATEntry{}, errors.Wrapf(err, "Unmarshal entries fail") + } + if len(entries) == 0 { + return SSNATEntry{}, cloudprovider.ErrNotFound + } + return entries[0], nil +} + +func (region *SRegion) CreateSnatEntry(rule cloudprovider.SNatSRule, natGatewayId string) (string, error) { + params := make(map[string]string) + params["NatGatewayId"] = natGatewayId + params["SubnetId"] = rule.NetworkID + eips, _, err := region.GetEips(nil, rule.ExternalIP, nil, 1, 1) + if err != nil { + return "", err + } + params["EipId"] = eips[0].AllocationId + if len(rule.SourceCIDR) != 0 { + params["SourceCidr"] = rule.SourceCIDR + } + body, err := region.natRequest("CreateSnatEntry", params) + if err != nil { + return "", err + } + + entryID, _ := body.GetString("Result", "SnatEntryId") + return entryID, nil +} + +func (region *SRegion) DeleteSnatEntry(snatEntryId string) error { + params := make(map[string]string) + params["SnatEntryId"] = snatEntryId + _, err := region.natRequest("DeleteSnatEntry", params) + return errors.Wrapf(err, "DeleteSnatEntry") +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/network.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/network.go new file mode 100644 index 0000000000..e2756981d7 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/network.go @@ -0,0 +1,255 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package volcengine + +import ( + "fmt" + "strings" + "time" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/netutils" + "yunion.io/x/pkg/util/rbacscope" + "yunion.io/x/pkg/utils" +) + +type SNetwork struct { + multicloud.SResourceBase + VolcEngineTags + wire *SWire + + AccountId string + SubnetId string + VpcId string + Status string + CidrBlock string + Ipv6CidrBlock string + ZoneId string + AvailableIpAddressCount int + Description string + SubnetName string + CreationTime time.Time + UpdateTime time.Time + TotalIpv4Count int + NetworkAclId string + IsDefault bool + RouteTable SRouteTable + ProjectName string +} + +func (subnet *SNetwork) GetId() string { + return subnet.SubnetId +} + +func (subnet *SNetwork) GetName() string { + if len(subnet.SubnetName) > 0 { + return subnet.SubnetName + } + return subnet.SubnetId +} + +func (subnet *SNetwork) GetGlobalId() string { + return subnet.SubnetId +} + +func (subnet *SNetwork) GetStatus() string { + return strings.ToLower(subnet.Status) +} + +func (subnet *SNetwork) Refresh() error { + log.Debugf("Subnet refresh %s", subnet.SubnetId) + new, err := subnet.wire.zone.region.GetSubnetAttributes(subnet.SubnetId) + if err != nil { + return err + } + return jsonutils.Update(subnet, new) +} + +func (subnet *SNetwork) GetIWire() cloudprovider.ICloudWire { + return subnet.wire +} + +func (subnet *SNetwork) GetProjectId() string { + return subnet.ProjectName +} + +func (subnet *SNetwork) GetIpStart() string { + pref, _ := netutils.NewIPV4Prefix(subnet.CidrBlock) + startIp := pref.Address.NetAddr(pref.MaskLen) + startIp = startIp.StepUp() + return startIp.String() +} + +func (subnet *SNetwork) GetIpEnd() string { + pref, _ := netutils.NewIPV4Prefix(subnet.CidrBlock) + endIp := pref.Address.BroadcastAddr(pref.MaskLen) + endIp = endIp.StepDown() + endIp = endIp.StepDown() + endIp = endIp.StepDown() + return endIp.String() +} + +func (subnet *SNetwork) GetIpMask() int8 { + pref, _ := netutils.NewIPV4Prefix(subnet.CidrBlock) + return pref.MaskLen +} + +func (subnet *SNetwork) GetGateway() string { + pref, _ := netutils.NewIPV4Prefix(subnet.CidrBlock) + endIp := pref.Address.BroadcastAddr(pref.MaskLen) + endIp = endIp.StepDown() + return endIp.String() +} + +func (subnet *SNetwork) GetServerType() string { + return api.NETWORK_TYPE_GUEST +} + +func (subnet *SNetwork) GetIsPublic() bool { + return subnet.IsDefault +} + +func (subnet *SNetwork) GetPublicScope() rbacscope.TRbacScope { + return rbacscope.ScopeDomain +} + +func (region *SRegion) CreateSubnet(zoneId string, vpcId string, name string, cidr string, desc string) (string, error) { + params := make(map[string]string) + params["ZoneId"] = zoneId + params["VpcId"] = vpcId + params["CidrBlock"] = cidr + params["SubnetName"] = name + if len(desc) > 0 { + params["Description"] = desc + } + params["ClientToken"] = utils.GenRequestId(20) + + body, err := region.vpcRequest("CreateSubnet", params) + if err != nil { + return "", err + } + return body.GetString("Result", "SubnetId") +} + +func (region *SRegion) DeleteSubnet(SubnetId string) error { + params := make(map[string]string) + params["SubnetId"] = SubnetId + + _, err := region.vpcRequest("DeleteSubnet", params) + return err +} + +func (subnet *SNetwork) dissociateWithSNAT() error { + natgatways, err := subnet.wire.vpc.getNatGateways() + if err != nil { + return err + } + for i := range natgatways { + err = natgatways[i].dissociateWithSubnet(subnet.SubnetId) + if err != nil { + return err + } + } + return nil +} + +func (subnet *SNetwork) Delete() error { + err := subnet.Refresh() + if err != nil { + log.Errorf("refresh Subnet fail %s", err) + return err + } + if len(subnet.RouteTable.RouteTableId) > 0 && !subnet.RouteTable.IsSystem() { + err = subnet.wire.zone.region.UnassociateRouteTable(subnet.RouteTable.RouteTableId, subnet.SubnetId) + if err != nil { + log.Errorf("unassociate routetable fail %s", err) + return err + } + } + err = subnet.dissociateWithSNAT() + if err != nil { + log.Errorf("fail to dissociateWithSNAT") + return err + } + err = cloudprovider.Wait(10*time.Second, time.Minute, func() (bool, error) { + err := subnet.wire.zone.region.DeleteSubnet(subnet.SubnetId) + if err != nil { + if isError(err, "DependencyViolation") { + return false, nil + } + return false, err + } else { + return true, nil + } + }) + return err +} + +func (subnet *SNetwork) GetAllocTimeoutSeconds() int { + return 120 +} + +func (region *SRegion) GetSubnets(ids []string, zoneId string, vpcId string, pageNumber int, pageSize int) ([]SNetwork, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + params := make(map[string]string) + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + for idx, id := range ids { + params[fmt.Sprintf("SubnetIds.%d", idx)] = id + } + if len(zoneId) > 0 { + params["ZoneId"] = zoneId + } + if len(vpcId) > 0 { + params["VpcId"] = vpcId + } + + body, err := region.vpcRequest("DescribeSubnets", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "GetSubnets fail") + } + + subnets := make([]SNetwork, 0) + err = body.Unmarshal(&subnets, "Result", "Subnets") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal subnets fail") + } + total, _ := body.Int("Result", "TotalCount") + return subnets, int(total), nil +} + +func (region *SRegion) GetSubnetAttributes(SubnetId string) (*SNetwork, error) { + params := make(map[string]string) + params["SubnetId"] = SubnetId + body, err := region.vpcRequest("DescribeSubnetAttributes", params) + if err != nil { + return nil, errors.Wrapf(err, "DescribeSubnetAttributes fail") + } + if region.client.debug { + log.Debugf("%s", body.PrettyString()) + } + subnet := SNetwork{} + err = body.Unmarshal(&subnet, "Result") + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal subnet fail") + } + return &subnet, nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/networkinterfaces.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/networkinterfaces.go new file mode 100644 index 0000000000..46bd176ad9 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/networkinterfaces.go @@ -0,0 +1,174 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "time" + + "yunion.io/x/pkg/errors" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" +) + +type SPrivateIp struct { + nic *SNetworkInterface + Primary bool + PrivateIpAddress string +} + +func (ip *SPrivateIp) GetGlobalId() string { + return ip.PrivateIpAddress +} + +func (ip *SPrivateIp) GetINetworkId() string { + return ip.nic.SubnetId +} + +func (ip *SPrivateIp) GetIP() string { + return ip.PrivateIpAddress +} + +func (ip *SPrivateIp) IsPrimary() bool { + return ip.Primary +} + +type SAssociatedElasticIp struct { + AllocationId string + EipAddress string +} + +type SPrivateIpSets struct { + PrivateIpSet []SPrivateIp +} + +type SNetworkInterface struct { + multicloud.SNetworkInterfaceBase + VolcEngineTags + region *SRegion + + InstanceId string + NetworkInterfaceId string + VpcId string + SubnetId string + PrimaryIpAddress string + Type string + MacAddress string + CreationTime time.Time + NetworkInterfaceName string + PrivateIpSets SPrivateIpSets + ResourceGroupId string + SecurityGroupIds SSecurityGroupIds + Status string + ZoneId string + PrivateIpAddresses []string + AssociatedElasticIp SAssociatedElasticIp +} + +func (nic *SNetworkInterface) GetName() string { + return nic.NetworkInterfaceName +} + +func (nic *SNetworkInterface) GetId() string { + return nic.NetworkInterfaceId +} + +func (nic *SNetworkInterface) GetGlobalId() string { + return nic.NetworkInterfaceId +} + +func (nic *SNetworkInterface) GetAssociateId() string { + return nic.InstanceId +} + +func (nic *SNetworkInterface) GetAssociateType() string { + return api.NETWORK_INTERFACE_ASSOCIATE_TYPE_SERVER +} + +func (nic *SNetworkInterface) GetMacAddress() string { + return nic.MacAddress +} + +func (nic *SNetworkInterface) GetStatus() string { + switch nic.Status { + case "Available": + return api.NETWORK_INTERFACE_STATUS_AVAILABLE + } + return nic.Status +} + +func (region *SRegion) GetINetworkInterfaces() ([]cloudprovider.ICloudNetworkInterface, error) { + interfaces := []SNetworkInterface{} + pageNumber := 1 + for { + parts, total, err := region.GetNetworkInterfaces("", pageNumber, 50) + if err != nil { + return nil, err + } + interfaces = append(interfaces, parts...) + if len(interfaces) >= total { + break + } + pageNumber += 1 + } + ret := []cloudprovider.ICloudNetworkInterface{} + for i := 0; i < len(interfaces); i++ { + if len(interfaces[i].InstanceId) == 0 { + interfaces[i].region = region + ret = append(ret, &interfaces[i]) + } + } + return ret, nil +} + +func (nic *SNetworkInterface) GetICloudInterfaceAddresses() ([]cloudprovider.ICloudInterfaceAddress, error) { + address := []cloudprovider.ICloudInterfaceAddress{} + for i := 0; i < len(nic.PrivateIpSets.PrivateIpSet); i++ { + nic.PrivateIpSets.PrivateIpSet[i].nic = nic + address = append(address, &nic.PrivateIpSets.PrivateIpSet[i]) + } + return address, nil +} + +func (region *SRegion) GetNetworkInterfaces(instanceId string, pageNumber int, pageSize int) ([]SNetworkInterface, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + + params := map[string]string{ + "RegionId": region.RegionId, + "PageSize": fmt.Sprintf("%d", pageSize), + "PageNumber": fmt.Sprintf("%d", pageNumber), + } + + if len(instanceId) > 0 { + params["InstanceId"] = instanceId + } + + body, err := region.vpcRequest("DescribeNetworkInterfaces", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "DescribeNetworkInterfaces") + } + + interfaces := []SNetworkInterface{} + err = body.Unmarshal(&interfaces, "Result", "NetworkInterfaceSets") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal") + } + total, _ := body.Int("Result", "TotalCount") + return interfaces, int(total), nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/objects.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/objects.go new file mode 100644 index 0000000000..2343af7e4a --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/objects.go @@ -0,0 +1,102 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "context" + "net/http" + + "github.com/volcengine/ve-tos-golang-sdk/v2/tos" + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" +) + +const ( + OSS_META_HEADER = "x-oss-meta-" +) + +type SObject struct { + bucket *SBucket + + cloudprovider.SBaseCloudObject +} + +func (obj *SObject) GetIBucket() cloudprovider.ICloudBucket { + return obj.bucket +} + +func (obj *SObject) GetAcl() cloudprovider.TBucketACLType { + acl := cloudprovider.ACLPrivate + toscli, err := obj.bucket.region.GetTosClient() + if err != nil { + log.Errorf("Get Client %s", err) + return acl + } + result, err := toscli.GetObjectACL(context.Background(), &tos.GetObjectACLInput{Bucket: obj.bucket.Name, Key: obj.Key}) + if err != nil { + log.Errorf("GetObjectAcl %s", err) + } + grants := result.Grants + return grantToCannedAcl(grants) +} + +func (obj *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error { + toscli, err := obj.bucket.region.GetTosClient() + if err != nil { + return errors.Wrap(err, "GetTosClient") + } + _, err = toscli.PutObjectACL(context.Background(), &tos.PutObjectACLInput{Key: obj.Key, ACL: enum.ACLType(aclStr)}) + if err != nil { + return errors.Wrapf(err, "PutObjectACL") + } + return nil +} + +func (obj *SObject) GetMeta() http.Header { + if obj.Meta != nil { + return obj.Meta + } + toscli, err := obj.bucket.region.GetTosClient() + if err != nil { + log.Errorf("Get Client %s", err) + return nil + } + result, err := toscli.GetObjectV2(context.Background(), &tos.GetObjectV2Input{Bucket: obj.bucket.Name, Key: obj.Key}) + if err != nil { + log.Errorf("Get Object error %s", err) + return nil + } + newHeader := http.Header{} + meta := result.GetObjectBasicOutput.ObjectMetaV2.Meta + for _, key := range meta.AllKeys() { + value, exist := meta.Get(key) + if !exist { + log.Errorf("Key missing in meta data %s", key) + } else { + newHeader.Add(key, value) + } + } + obj.Meta = cloudprovider.FetchMetaFromHttpHeader( + OSS_META_HEADER, + newHeader, + ) + return obj.Meta +} + +func (obj *SObject) SetMeta(ctx context.Context, meta http.Header) error { + return cloudprovider.ObjectSetMeta(ctx, obj.bucket, obj, meta) +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/project.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/project.go new file mode 100644 index 0000000000..2110229535 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/project.go @@ -0,0 +1,140 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "time" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" +) + +type SProject struct { + multicloud.SProjectBase + VolcEngineTags + client *SVolcEngineClient + + AccountId int + ProjectName string + ParentProjectName string + Path string + DisplayName string + Description string + CreateDate time.Time + UpdateDate time.Time + Status string +} + +func (project *SProject) GetGlobalId() string { + return project.ProjectName +} + +func (project *SProject) GetId() string { + return project.ProjectName +} + +func (project *SProject) GetName() string { + if len(project.DisplayName) > 0 { + return project.DisplayName + } + return project.ProjectName +} + +func (project *SProject) Refresh() error { + group, err := project.client.GetProject(project.ProjectName) + if err != nil { + return errors.Wrap(err, "GetProject") + } + return jsonutils.Update(project, group) +} + +func (project *SProject) GetStatus() string { + switch project.Status { + case "active": + return api.EXTERNAL_PROJECT_STATUS_AVAILABLE + default: + return api.EXTERNAL_PROJECT_STATUS_UNKNOWN + } +} + +func (client *SVolcEngineClient) GetProject(name string) (*SProject, error) { + params := map[string]string{ + "ProjectName": name, + } + body, err := client.iamRequest("", "GetProject", params) + if err != nil { + return nil, err + } + project := &SProject{client: client} + err = body.Unmarshal(project, "Result") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return project, nil +} + +func (client *SVolcEngineClient) ListProjects(limit int, offset int) ([]SProject, int, error) { + if limit > 50 || limit <= 0 { + limit = 50 + } + params := map[string]string{ + "Limit": fmt.Sprintf("%d", limit), + "Offset": fmt.Sprintf("%d", offset), + } + resp, err := client.iamRequest("", "ListProjects", params) + if err != nil { + return nil, 0, errors.Wrap(err, "iamRequest.ListProjects") + } + projects := []SProject{} + err = resp.Unmarshal(&projects, "Result", "Projects") + if err != nil { + return nil, 0, errors.Wrap(err, "resp.Unmarshal") + } + total, _ := resp.Int("Result", "Total") + return projects, int(total), nil +} + +func (client *SVolcEngineClient) CreateIProject(name string) (cloudprovider.ICloudProject, error) { + group, err := client.CreateProject(name) + if err != nil { + return nil, errors.Wrap(err, "CreateProject") + } + return group, nil +} + +func (client *SVolcEngineClient) CreateProject(name string) (*SProject, error) { + params := map[string]string{ + "DisplayName": name, + "ProjectName": name, + } + resp, err := client.iamRequest("", "CreateProject", params) + if err != nil { + return nil, errors.Wrap(err, "CreateProject") + } + group := &SProject{client: client} + err = resp.Unmarshal(group, "Project") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + err = cloudprovider.WaitStatus(group, api.EXTERNAL_PROJECT_STATUS_AVAILABLE, time.Second*5, time.Minute*3) + if err != nil { + return nil, errors.Wrap(err, "WaitStatus") + } + return group, nil +} diff --git a/pkg/keystone/cache/doc.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/doc.go similarity index 86% rename from pkg/keystone/cache/doc.go rename to vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/doc.go index 9fe843d1e1..6cef4bceb2 100644 --- a/pkg/keystone/cache/doc.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/doc.go @@ -1,4 +1,4 @@ -// Copyright 2019 Yunion +// Copyright 2023 Yunion // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,4 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache // import "yunion.io/x/onecloud/pkg/keystone/cache" +package volcengine diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/provider.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/provider.go new file mode 100644 index 0000000000..37b0e071ce --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/provider.go @@ -0,0 +1,189 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "context" + "strings" + + "github.com/pkg/errors" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud/volcengine" + "yunion.io/x/jsonutils" +) + +type SVolcEngineProviderFactory struct { + cloudprovider.SPublicCloudBaseProviderFactory +} + +func (self *SVolcEngineProviderFactory) GetId() string { + return volcengine.CLOUD_PROVIDER_VOLCENGINE +} + +func (self *SVolcEngineProviderFactory) GetName() string { + return volcengine.CLOUD_PROVIDER_VOLCENGINE_CN +} + +func (self *SVolcEngineProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) { + output := cloudprovider.SCloudaccount{} + if len(input.AccessKeyId) == 0 { + return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_id") + } + if len(input.AccessKeySecret) == 0 { + return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_secret") + } + output.Account = input.AccessKeyId + output.Secret = input.AccessKeySecret + return output, nil +} + +func (f *SVolcEngineProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) { + output := cloudprovider.SCloudaccount{} + if len(input.AccessKeyId) == 0 { + return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_id") + } + if len(input.AccessKeySecret) == 0 { + return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_secret") + } + output = cloudprovider.SCloudaccount{ + Account: input.AccessKeyId, + Secret: input.AccessKeySecret, + } + return output, nil +} + +func validateClientCloudenv(client *volcengine.SVolcEngineClient) error { + regions := client.GetIRegions() + if len(regions) == 0 { + return nil + } + return nil +} + +func parseAccount(account string) (accessKey string, projectId string) { + segs := strings.Split(account, "::") + if len(segs) == 2 { + accessKey = segs[0] + projectId = segs[1] + } else { + accessKey = account + projectId = "" + } + + return +} + +func (self *SVolcEngineProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + accessKey, accountId := parseAccount(cfg.Account) + client, err := volcengine.NewVolcEngineClient( + volcengine.NewVolcEngineClientConfig( + accessKey, + cfg.Secret, + ).CloudproviderConfig(cfg).AccountId(accountId), + ) + if err != nil { + return nil, err + } + + err = validateClientCloudenv(client) + if err != nil { + return nil, errors.Wrap(err, "validateClientCloudenv") + } + + return &SVolcEngineProvider{ + SBaseProvider: cloudprovider.NewBaseProvider(self), + client: client, + }, nil +} + +func (self *SVolcEngineProviderFactory) GetClientRC(info cloudprovider.SProviderInfo) (map[string]string, error) { + accessKey, accountId := parseAccount(info.Account) + return map[string]string{ + "VOLCENGINE_ACCESS_KEY": accessKey, + "VOLCENGINE_SECRET_KEY": info.Secret, + "VOLCENGINE_REGION": volcengine.VOLCENGINE_DEFAULT_REGION, + "VOLCENGINE_ACCOUNT_ID": accountId, + }, nil +} + +func init() { + factory := SVolcEngineProviderFactory{} + cloudprovider.RegisterFactory(&factory) +} + +type SVolcEngineProvider struct { + cloudprovider.SBaseProvider + + client *volcengine.SVolcEngineClient +} + +func (self *SVolcEngineProvider) GetAccountId() string { + return self.client.GetAccountId() +} + +func (self *SVolcEngineProvider) GetSysInfo() (jsonutils.JSONObject, error) { + regions := self.client.GetIRegions() + info := jsonutils.NewDict() + info.Add(jsonutils.NewInt(int64(len(regions))), "region_count") + info.Add(jsonutils.NewString(volcengine.VOLCENGINE_API_VERSION), "api_version") + return info, nil +} + +func (self *SVolcEngineProvider) GetBalance() (*cloudprovider.SBalanceInfo, error) { + // GetBalance is not currently open + return &cloudprovider.SBalanceInfo{ + Amount: 0.0, + Currency: "CNY", + Status: api.CLOUD_PROVIDER_HEALTH_NORMAL, + }, nil +} + +func (self *SVolcEngineProvider) GetBucketCannedAcls(regionId string) []string { + return nil +} + +func (self *SVolcEngineProvider) GetCapabilities() []string { + return self.client.GetCapabilities() +} + +func (self *SVolcEngineProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) { + return self.client.GetIProjects() +} + +func (self *SVolcEngineProvider) GetIRegionById(extId string) (cloudprovider.ICloudRegion, error) { + return self.client.GetIRegionById(extId) +} + +func (self *SVolcEngineProvider) GetIRegions() []cloudprovider.ICloudRegion { + return self.client.GetIRegions() +} + +func (self *SVolcEngineProvider) GetObjectCannedAcls(regionId string) []string { + return nil +} + +func (self *SVolcEngineProvider) GetStorageClasses(regionId string) []string { + return nil +} + +func (self *SVolcEngineProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { + return self.client.GetSubAccounts() +} + +func (self *SVolcEngineProvider) GetVersion() string { + return volcengine.VOLCENGINE_API_VERSION +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/region.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/region.go new file mode 100644 index 0000000000..3669802df2 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/region.go @@ -0,0 +1,680 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "context" + "fmt" + "strings" + "time" + + tos "github.com/volcengine/ve-tos-golang-sdk/v2/tos" + "github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum" + sdk "github.com/volcengine/volc-sdk-golang/base" + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" +) + +var RegionLocations = map[string]string{ + "cn-beijing": "中国(北京)", + "cn-shanghai": "中国(上海)", + "cn-guangzhou": "中国(广州)", +} + +var RegionLocationsEN = map[string]string{ + "cn-beijing": "China (Beijing)", + "cn-shanghai": "China (Shanghai)", + "cn-guangzhou": "China (Guangzhou)", +} + +var RegionEndpoint = map[string]string{ + "cn-beijing": "cn-beijing.volces.com", + "cn-shanghai": "cn-shanghai.volces.com", + "cn-guangzhou": "cn-beijing.volces.com", +} + +type SRegion struct { + multicloud.SRegion + multicloud.SNoLbRegion + + client *SVolcEngineClient + tosClient *tos.ClientV2 + RegionId string + + ivpcs []cloudprovider.ICloudVpc + + storageCache *SStoragecache +} + +func (region *SRegion) GetClient() *SVolcEngineClient { + return region.client +} + +func (region *SRegion) Refresh() error { + return nil +} + +func (region *SRegion) GetProvider() string { + return CLOUD_PROVIDER_VOLCENGINE +} + +func (region *SRegion) GetCloudEnv() string { + return region.client.cloudEnv +} + +func (region *SRegion) GetId() string { + return region.RegionId +} + +func (region *SRegion) GetName() string { + if localName, ok := RegionLocations[region.RegionId]; ok { + return fmt.Sprintf("%s %s", CLOUD_PROVIDER_VOLCENGINE_CN, localName) + } + return fmt.Sprintf("%s %s", CLOUD_PROVIDER_VOLCENGINE_CN, region.RegionId) +} + +func (region *SRegion) GetGlobalId() string { + return fmt.Sprintf("%s/%s", region.client.GetAccessEnv(), region.RegionId) +} + +func (region *SRegion) GetI18n() cloudprovider.SModelI18nTable { + var en string + if localName, ok := RegionLocationsEN[region.RegionId]; ok { + en = fmt.Sprintf("%s %s", CLOUD_PROVIDER_VOLCENGINE_EN, localName) + } else { + en = fmt.Sprintf("%s %s", CLOUD_PROVIDER_VOLCENGINE_EN, region.RegionId) + } + table := cloudprovider.SModelI18nTable{} + table["name"] = cloudprovider.NewSModelI18nEntry(region.GetName()).CN(region.GetName()).EN(en) + return table +} + +func (region *SRegion) GetStatus() string { + return api.CLOUD_REGION_STATUS_INSERVER +} + +func (region *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo { + if info, ok := LatitudeAndLongitude[region.RegionId]; ok { + return info + } + return cloudprovider.SGeographicInfo{} +} + +func (region *SRegion) getStoragecache() *SStoragecache { + if region.storageCache == nil { + region.storageCache = &SStoragecache{region: region} + } + return region.storageCache +} + +func (region *SRegion) GetZones(id string) ([]SZone, error) { + params := map[string]string{} + params["DestinationResource"] = "InstanceType" + // DedicatedHost is not supported + if len(id) > 0 { + params["ZoneId"] = id + } + body, err := region.ecsRequest("DescribeAvailableResource", params) + if err != nil { + return nil, err + } + ret := []SZone{} + err = body.Unmarshal(&ret, "Result", "AvailableZones") + if err != nil { + return nil, err + } + return ret, nil +} + +func (region *SRegion) GetIZones() ([]cloudprovider.ICloudZone, error) { + zones, err := region.GetZones("") + if err != nil { + return nil, errors.Wrapf(err, "GetZones") + } + ret := []cloudprovider.ICloudZone{} + for i := range zones { + zones[i].region = region + ret = append(ret, &zones[i]) + } + return ret, nil +} + +func (region *SRegion) GetIZoneById(id string) (cloudprovider.ICloudZone, error) { + zones, err := region.GetZones("") + if err != nil { + return nil, errors.Wrap(err, "GetZones") + } + + for i := range zones { + zones[i].region = region + if zones[i].GetId() == id || zones[i].GetGlobalId() == id { + return &zones[i], nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s", id) +} + +// vpc +func (region *SRegion) CreateIVpc(opts *cloudprovider.VpcCreateOptions) (cloudprovider.ICloudVpc, error) { + vpc, err := region.CreateVpc(opts) + if err != nil { + return nil, err + } + return vpc, nil +} + +func (region *SRegion) CreateVpc(opts *cloudprovider.VpcCreateOptions) (*SVpc, error) { + params := make(map[string]string) + if len(opts.CIDR) > 0 { + params["CidrBlock"] = opts.CIDR + } + if len(opts.NAME) > 0 { + params["VpcName"] = opts.NAME + } + if len(opts.Desc) > 0 { + params["Description"] = opts.Desc + } + params["ClientToken"] = utils.GenRequestId(20) + body, err := region.vpcRequest("CreateVpc", params) + if err != nil { + return nil, err + } + vpcId, err := body.GetString("Result", "VpcId") + if err != nil { + return nil, err + } + err = cloudprovider.Wait(5*time.Second, time.Minute, func() (bool, error) { + _, err = region.getVpc(vpcId) + if errors.Cause(err) == cloudprovider.ErrNotFound { + return false, nil + } else { + return true, err + } + }) + if err != nil { + return nil, errors.Wrapf(err, "cannot find networks after create") + } + return region.getVpc(vpcId) +} + +func (region *SRegion) DeleteVpc(vpcId string) error { + params := make(map[string]string) + params["VpcId"] = vpcId + + _, err := region.vpcRequest("DeleteVpc", params) + return err +} + +func (region *SRegion) getVpc(vpcId string) (*SVpc, error) { + vpcs, _, err := region.GetVpcs([]string{vpcId}, 1, 50) + if err != nil { + return nil, err + } + for _, vpc := range vpcs { + if vpc.VpcId == vpcId { + vpc.region = region + return &vpc, nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s not found", vpcId) +} + +func (region *SRegion) GetVpcs(vpcIds []string, pageNumber int, pageSize int) ([]SVpc, int, error) { + params := make(map[string]string) + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + if len(vpcIds) > 0 { + for index, id := range vpcIds { + key := fmt.Sprintf("VpcIds.%d", index+1) + params[key] = id + } + } + body, err := region.vpcRequest("DescribeVpcs", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "GetVpcs fail") + } + vpcs := make([]SVpc, 0) + err = body.Unmarshal(&vpcs, "Result", "Vpcs") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal vpcs fail") + } + total, _ := body.Int("Result", "TotalCount") + return vpcs, int(total), nil +} + +func (region *SRegion) GetIVpcs() ([]cloudprovider.ICloudVpc, error) { + if region.ivpcs == nil { + vpcs, err := region.GetAllVpcs() + if err != nil { + return nil, err + } + region.ivpcs = make([]cloudprovider.ICloudVpc, len(vpcs)) + for i := 0; i < len(vpcs); i += 1 { + vpcs[i].region = region + region.ivpcs[i] = &vpcs[i] + } + } + return region.ivpcs, nil +} + +func (region *SRegion) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { + ivpcs, err := region.GetIVpcs() + if err != nil { + return nil, err + } + for i := 0; i < len(ivpcs); i += 1 { + if ivpcs[i].GetGlobalId() == id { + return ivpcs[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (region *SRegion) GetAllVpcs() ([]SVpc, error) { + vpcs := make([]SVpc, 0) + pageNumber := 1 + for { + part, total, err := region.GetVpcs(nil, pageNumber, 50) + if err != nil { + return nil, err + } + vpcs = append(vpcs, part...) + if len(vpcs) >= total { + break + } + pageNumber += 1 + } + return vpcs, nil +} + +// EIP +func (region *SRegion) GetIEipById(eipId string) (cloudprovider.ICloudEIP, error) { + eip, err := region.GetEip(eipId) + if err != nil { + return nil, err + } + return eip, nil +} + +func (region *SRegion) GetIEips() ([]cloudprovider.ICloudEIP, error) { + pageNumber := 1 + eips, total, err := region.GetEips(make([]string, 0), "", make([]string, 0), pageNumber, 100) + if err != nil { + return nil, err + } + for len(eips) < total { + var parts []SEipAddress + pageNumber++ + parts, total, err = region.GetEips(make([]string, 0), "", make([]string, 0), pageNumber, 100) + if err != nil { + return nil, err + } + eips = append(eips, parts...) + } + ret := make([]cloudprovider.ICloudEIP, len(eips)) + for i := 0; i < len(eips); i += 1 { + ret[i] = &eips[i] + } + return ret, nil +} + +func (region *SRegion) FetchSubnets(ids []string, zoneId string, vpcId string) ([]SNetwork, error) { + pageNumber := 1 + nets := make([]SNetwork, 0) + for { + parts, total, err := region.GetSubnets(ids, zoneId, vpcId, pageNumber, 50) + if err != nil { + return nil, err + } + nets = append(nets, parts...) + if len(nets) >= total { + break + } + pageNumber += 1 + } + return nets, nil +} + +// IBucket +func (region *SRegion) IBucketExist(name string) (bool, error) { + toscli, err := region.GetTosClient() + if err != nil { + return false, errors.Wrap(err, "region.GetTosClient") + } + _, err = toscli.HeadBucket(context.Background(), &tos.HeadBucketInput{Bucket: name}) + if err != nil || tos.StatusCode(err) != 404 { + return false, errors.Wrap(err, "IsBucketExist") + } + return true, nil +} + +func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error { + toscli, err := region.GetTosClient() + if err != nil { + return errors.Wrap(err, "region.GetTosClient") + } + _, err = toscli.CreateBucketV2(context.Background(), &tos.CreateBucketV2Input{Bucket: name, ACL: enum.ACLType(aclStr), StorageClass: enum.StorageClassType(storageClassStr)}) + if err != nil { + return errors.Wrap(err, "tos.CreateBucketV2") + } + region.client.invalidateIBuckets() + return nil +} + +func (region *SRegion) DeleteIBucket(name string) error { + toscli, err := region.GetTosClient() + if err != nil { + return errors.Wrapf(err, "region.GetOssClient") + } + _, err = toscli.DeleteBucket(context.Background(), &tos.DeleteBucketInput{Bucket: name}) + if err != nil { + if tos.StatusCode(err) == 404 { + return nil + } + return errors.Wrap(err, "DeleteBucket") + } + region.client.invalidateIBuckets() + return nil +} + +func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { + toscli, err := region.GetTosClient() + if err != nil { + return nil, errors.Wrapf(err, "region.GetOssClient") + } + out, err := toscli.ListBuckets(context.Background(), &tos.ListBucketsInput{}) + if err != nil { + return nil, errors.Wrap(err, "ListBucket") + } + for _, bucket := range out.Buckets { + if bucket.Name == name { + t, err := time.Parse(time.RFC3339, bucket.CreationDate) + if err != nil { + return nil, errors.Wrapf(err, "Prase CreationDate error") + } + b := SBucket{ + region: region, + Name: name, + Location: bucket.Location, + CreationDate: t, + } + return &b, nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "Bucket Not Found") +} + +func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { + return region.GetIBucketById(name) +} + +func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { + iBuckets, err := region.client.getIBuckets() + if err != nil { + return nil, errors.Wrap(err, "getIBuckets") + } + ret := make([]cloudprovider.ICloudBucket, 0) + for i := range iBuckets { + if iBuckets[i].GetIRegion().GetId() != region.GetId() { + continue + } + ret = append(ret, iBuckets[i]) + } + return ret, nil +} + +func (region *SRegion) GetCapabilities() []string { + return region.client.GetCapabilities() +} + +// Security Group +func (region *SRegion) CreateISecurityGroup(conf *cloudprovider.SecurityGroupCreateInput) (cloudprovider.ICloudSecurityGroup, error) { + externalId, err := region.CreateSecurityGroup(conf.VpcId, conf.Name, conf.Desc, conf.ProjectId) + if err != nil { + return nil, err + } + if conf.OnCreated != nil { + conf.OnCreated(externalId) + } + outRules := conf.OutRules + if len(outRules) > 0 && outRules[0].String() == "out:allow any" { + outRules = outRules[1:] + } + rules := append(conf.InRules, outRules...) + for _, rule := range rules { + rule.Priority = 101 - rule.Priority + err = region.addSecurityGroupRule(externalId, rule) + if err != nil { + return nil, err + } + } + err = cloudprovider.Wait(5*time.Second, time.Minute, func() (bool, error) { + _, err := region.GetISecurityGroupById(externalId) + if errors.Cause(err) == cloudprovider.ErrNotFound { + return false, nil + } else { + return true, err + } + }) + if err != nil { + return nil, errors.Wrapf(err, "cannot find security group after create") + } + return region.GetISecurityGroupById(externalId) +} + +func (region *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.ICloudSecurityGroup, error) { + return region.GetSecurityGroupDetails(secgroupId) +} + +func (region *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + secgroups, _, err := region.GetSecurityGroups(opts.VpcId, opts.Name, nil, 1, 100) + if err != nil { + return nil, err + } + for _, secgroup := range secgroups { + if secgroup.SecurityGroupName == opts.Name { + secgroup.region = region + return &secgroup, nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s not found", opts.Name) +} + +func (region *SRegion) DeleteISecurityGroupById(secgroupId string) error { + return region.DeleteSecurityGroupById(secgroupId) +} + +func (region *SRegion) getSdkCredential(service string, token string) sdk.Credentials { + return region.client.getSdkCredential(region.RegionId, service, token) +} + +func (region *SRegion) ecsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cred := region.getSdkCredential(VOLCENGINE_SERVICE_ECS, "") + return region.client.jsonRequest(cred, VOLCENGINE_API, VOLCENGINE_API_VERSION, apiName, params) +} + +func (region *SRegion) vpcRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cred := region.getSdkCredential(VOLCENGINE_SERVICE_VPC, "") + return region.client.jsonRequest(cred, VOLCENGINE_API, VOLCENGINE_API_VERSION, apiName, params) +} + +func (region *SRegion) natRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cred := region.getSdkCredential(VOLCENGINE_SERVICE_NAT, "") + return region.client.jsonRequest(cred, VOLCENGINE_API, VOLCENGINE_API_VERSION, apiName, params) +} + +func (region *SRegion) storageRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cred := region.getSdkCredential(VOLCENGINE_SERVICE_STORAGE, "") + return region.client.jsonRequest(cred, VOLCENGINE_API, VOLCENGINE_API_VERSION, apiName, params) +} + +func (region *SRegion) GetTosClient() (*tos.ClientV2, error) { + if region.tosClient == nil { + cli, err := region.client.getTosClient(region.RegionId) + if err != nil { + return nil, errors.Wrap(err, "region.client.getOssClient") + } + region.tosClient = cli + } + return region.tosClient, nil +} + +func (region *SRegion) UpdateInstancePassword(instanceId string, passwd string) error { + params := make(map[string]string) + params["Password"] = passwd + return region.modifyInstanceAttribute(instanceId, params) +} + +func (region *SRegion) GetInstanceStatus(instanceId string) (string, error) { + instance, err := region.GetInstance(instanceId) + if err != nil { + return "", err + } + return instance.Status, nil +} + +func (region *SRegion) instanceOperation(instanceId string, apiName string, extra map[string]string) error { + params := make(map[string]string) + params["RegionId"] = region.RegionId + params["InstanceId"] = instanceId + if len(extra) > 0 { + for k, v := range extra { + params[k] = v + } + } + _, err := region.ecsRequest(apiName, params) + return err +} + +func (region *SRegion) getBaseEndpoint() string { + return RegionEndpoint[region.RegionId] +} + +func (region *SRegion) getS3Endpoint() string { + base := region.getBaseEndpoint() + if len(base) > 0 { + return "tos-s3-" + base + } + return "" +} + +func (region *SRegion) getTOSExternalDomain() string { + return getTOSExternalDomain(region.RegionId) +} + +func (region *SRegion) getTOSInternalDomain() string { + return getTOSInternalDomain(region.RegionId) +} + +func (region *SRegion) GetRouteTables(ids []string, pageNumber int, pageSize int) ([]SRouteTable, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + params := make(map[string]string) + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + if len(ids) > 0 { + params["RouteTableId"] = strings.Join(ids, ",") + } + body, err := region.vpcRequest("DescribeRouteTableList", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "GetRoutseTables fail") + } + routetables := make([]SRouteTable, 0) + err = body.Unmarshal(&routetables, "RouteTables", "RouteTable") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal routetables fail") + } + total, _ := body.Int("Result", "TotalCount") + return routetables, int(total), nil +} + +func (region *SRegion) GetIHostById(id string) (cloudprovider.ICloudHost, error) { + izones, err := region.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + ihost, err := izones[i].GetIHostById(id) + if err == nil { + return ihost, nil + } else if errors.Cause(err) != cloudprovider.ErrNotFound { + return nil, err + } + } + return nil, cloudprovider.ErrNotFound +} + +func (regioin *SRegion) GetIHosts() ([]cloudprovider.ICloudHost, error) { + iHosts := make([]cloudprovider.ICloudHost, 0) + + izones, err := regioin.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneHost, err := izones[i].GetIHosts() + if err != nil { + return nil, err + } + iHosts = append(iHosts, iZoneHost...) + } + return iHosts, nil +} + +func (region *SRegion) GetIDiskById(id string) (cloudprovider.ICloudDisk, error) { + return region.getDisk(id) +} + +func (region *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + izones, err := region.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + istore, err := izones[i].GetIStorageById(id) + if err == nil { + return istore, nil + } else if errors.Cause(err) != cloudprovider.ErrNotFound { + return nil, err + } + } + return nil, cloudprovider.ErrNotFound +} + +func (region *SRegion) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + iStores := make([]cloudprovider.ICloudStorage, 0) + + izones, err := region.GetIZones() + if err != nil { + return nil, err + } + for i := 0; i < len(izones); i += 1 { + iZoneStores, err := izones[i].GetIStorages() + if err != nil { + return nil, err + } + iStores = append(iStores, iZoneStores...) + } + return iStores, nil +} + +func (region *SRegion) GetIVMById(id string) (cloudprovider.ICloudVM, error) { + return region.GetInstance(id) +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/routetable.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/routetable.go new file mode 100644 index 0000000000..765b36829a --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/routetable.go @@ -0,0 +1,290 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package volcengine + +import ( + "fmt" + "strings" + "time" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/pkg/errors" +) + +type SRouteEntry struct { + multicloud.SResourceBase + VolcEngineTags + routeTable *SRouteTable + + Description string + DestinationCidrBlock string + RouteEntryId string + RouteEntryName string + RouteTableId string + Status string + Type string + VpcId string + NextHopId string + NextHopName string + NextHopType string +} + +type SRouteEntrys []*SRouteEntry + +type SubnetIds []string + +type SRouteTable struct { + multicloud.SResourceBase + VolcEngineTags + region *SRegion + vpc *SVpc + routes []cloudprovider.ICloudRoute + + Description string + RouteTableId string + RouteTableName string + RouteTableType string + VpcId string + VpcName string + CreationTime time.Time + UpdateTime time.Time + AccountId string + ProjectName string + SubnetIds SubnetIds + RouteEntrys SRouteEntrys +} + +func (route *SRouteEntry) GetId() string { + return fmt.Sprintf("%s-%s-%s", route.RouteTableId, route.DestinationCidrBlock, route.NextHopType) +} + +func (route *SRouteEntry) GetName() string { + return route.RouteEntryName +} + +func (route *SRouteEntry) GetGlobalId() string { + return route.GetId() +} + +func (route *SRouteEntry) GetStatus() string { + return api.ROUTE_ENTRY_STATUS_AVAILIABLE +} + +func (route *SRouteEntry) Refresh() error { + return nil +} + +func (route *SRouteEntry) GetType() string { + return route.Type +} + +func (route *SRouteEntry) GetCidr() string { + return route.DestinationCidrBlock +} + +func (route *SRouteEntry) GetNextHopType() string { + switch route.NextHopType { + case "Instance": + return api.NEXT_HOP_TYPE_INSTANCE + case "HaVip": + return api.NEXT_HOP_TYPE_HAVIP + case "VpnGW": + return api.NEXT_HOP_TYPE_VPN + case "NatGW": + return api.NEXT_HOP_TYPE_NAT + case "NetworkInterface": + return api.NEXT_HOP_TYPE_NETWORK + case "IPv6GW": + return api.NEXT_HOP_TYPE_IPV6 + case "TransitRouter": + return api.NEXT_HOP_TYPE_ROUTER + default: + return "" + } +} + +func (route *SRouteEntry) GetNextHop() string { + return route.NextHopId +} + +func (table *SRouteTable) GetDescription() string { + return table.Description +} + +func (table *SRouteTable) GetId() string { + return table.RouteTableId +} + +func (table *SRouteTable) GetGlobalId() string { + return table.RouteTableId +} + +func (table *SRouteTable) GetName() string { + return table.RouteTableName +} + +func (table *SRouteTable) GetRegionId() string { + return table.region.RegionId +} + +func (table *SRouteTable) GetType() cloudprovider.RouteTableType { + switch table.RouteTableType { + case "System": + return cloudprovider.RouteTableTypeSystem + case "Custom": + return cloudprovider.RouteTableTypeCustom + default: + return cloudprovider.RouteTableTypeSystem + } +} + +func (table *SRouteTable) GetVpcId() string { + return table.VpcId +} + +func (table *SRouteTable) GetStatus() string { + return api.ROUTE_TABLE_AVAILABLE +} + +func (table *SRouteTable) Refresh() error { + return nil +} + +func (routeTable *SRouteTable) IsSystem() bool { + return strings.ToLower(routeTable.RouteTableType) == "system" +} + +func (table *SRouteTable) RemoteGetRoutes(pageNumber int, pageSize int) ([]*SRouteEntry, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + params := make(map[string]string) + params["RouteTableId"] = table.RouteTableId + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + + body, err := table.region.vpcRequest("DescribeRouteEntryList", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "RemoteGetRoutes fail") + } + + entries := SRouteEntrys{} + err = body.Unmarshal(&entries, "Result", "RouteEntries") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal routeEntrys fail") + } + total, _ := body.Int("Result", "TotalCount") + return entries, int(total), nil +} + +func (table *SRouteTable) fetchRoutes() error { + routes := []*SRouteEntry{} + pageNumber := 1 + for { + parts, total, err := table.RemoteGetRoutes(pageNumber, 50) + if err != nil { + return err + } + routes = append(routes, parts...) + if len(routes) >= total { + break + } + pageNumber += 1 + } + table.routes = make([]cloudprovider.ICloudRoute, len(routes)) + for i := 0; i < len(routes); i++ { + routes[i].routeTable = table + table.routes[i] = routes[i] + } + return nil +} + +func (table *SRouteTable) GetIRoutes() ([]cloudprovider.ICloudRoute, error) { + if table.routes == nil { + err := table.fetchRoutes() + if err != nil { + return nil, err + } + } + return table.routes, nil +} + +func (table *SRouteTable) GetAssociations() []cloudprovider.RouteTableAssociation { + result := []cloudprovider.RouteTableAssociation{} + for i := range table.SubnetIds { + association := cloudprovider.RouteTableAssociation{ + AssociationId: table.RouteTableId + ":" + table.SubnetIds[i], + AssociationType: cloudprovider.RouteTableAssociaToSubnet, + AssociatedResourceId: table.SubnetIds[i], + } + result = append(result, association) + } + return result +} + +func (table *SRouteTable) CreateRoute(route cloudprovider.RouteSet) error { + return cloudprovider.ErrNotSupported +} + +func (table *SRouteTable) UpdateRoute(route cloudprovider.RouteSet) error { + return cloudprovider.ErrNotSupported +} + +func (table *SRouteTable) RemoveRoute(route cloudprovider.RouteSet) error { + return cloudprovider.ErrNotSupported +} + +func (vpc *SVpc) RemoteGetRouteTableList(pageNumber int, pageSize int) ([]*SRouteTable, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + params := make(map[string]string) + params["VpcId"] = vpc.VpcId + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + + body, err := vpc.region.vpcRequest("DescribeRouteTableList", params) + if err != nil { + return nil, 0, errors.Wrapf(err, "RemoteGetRouteTableList fail") + } + + routeTables := make([]*SRouteTable, 0) + err = body.Unmarshal(&routeTables, "Result", "RouterTableList") + if err != nil { + return nil, 0, errors.Wrapf(err, "Unmarshal routeTables fail") + } + for _, routeTable := range routeTables { + routeTable.region = vpc.region + } + total, _ := body.Int("Result", "TotalCount") + return routeTables, int(total), nil +} + +func (region *SRegion) AssociateRouteTable(rtableId string, SubnetId string) error { + params := make(map[string]string) + params["RouteTableId"] = rtableId + params["SubnetId"] = SubnetId + _, err := region.vpcRequest("AssociateRouteTable", params) + return err +} + +func (region *SRegion) UnassociateRouteTable(rtableId string, SubnetId string) error { + params := make(map[string]string) + params["RouteTableId"] = rtableId + params["SubnetId"] = SubnetId + _, err := region.vpcRequest("UnassociateRouteTable", params) + return err +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/securitygroup.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/securitygroup.go new file mode 100644 index 0000000000..a452a0e4d6 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/securitygroup.go @@ -0,0 +1,327 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "strings" + "time" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/secrules" + "yunion.io/x/pkg/utils" +) + +type SCidrList []string + +type SSecurityGroupRule struct { + CreationTime time.Time + UpdateTime time.Time + Description string + Direction string + Protocol string + Policy string + PortStart int + PortEnd int + CidrIp string + PrefixListId string + PrefixListCidrs SCidrList + Priority int + SourceGroupId string +} + +type SSecurityGroup struct { + multicloud.SSecurityGroup + VolcEngineTags + + region *SRegion + Description string + Permissions []SSecurityGroupRule + SecurityGroupId string + SecurityGroupName string + VpcId string + CreationTime time.Time + UpdateTime time.Time + Type string + ProjectName string + ServiceManaged bool + Status string +} + +func (region *SRegion) CreateSecurityGroup(vpcId string, name string, desc, projectName string) (string, error) { + params := make(map[string]string) + if len(vpcId) > 0 { + params["VpcId"] = vpcId + } + + if len(projectName) > 0 { + params["ProjectName"] = projectName + } + + if len(name) > 0 { + params["SecurityGroupName"] = name + } + if len(desc) > 0 { + params["Description"] = desc + } + params["ClientToken"] = utils.GenRequestId(20) + + body, err := region.vpcRequest("CreateSecurityGroup", params) + if err != nil { + return "", errors.Wrap(err, "CreateSecurityGroup") + } + return body.GetString("Result", "SecurityGroupId") +} + +func (region *SRegion) GetSecurityGroupDetails(secGroupId string) (*SSecurityGroup, error) { + params := make(map[string]string) + params["SecurityGroupId"] = secGroupId + body, err := region.vpcRequest("DescribeSecurityGroupAttributes", params) + + if err != nil { + return nil, err + } + securitygroup := SSecurityGroup{} + err = body.Unmarshal(&securitygroup, "Result") + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal security group details fail") + } + securitygroup.region = region + return &securitygroup, err +} + +func (region *SRegion) DeleteSecurityGroupById(secGroupId string) error { + params := make(map[string]string) + params["SecurityGroupId"] = secGroupId + _, err := region.vpcRequest("DeleteSecurityGroupId", params) + return err +} + +func (region *SRegion) AddSecurityGroupRules(secGrpId string, rule cloudprovider.SecurityRule) error { + if len(rule.Ports) != 0 { + for _, port := range rule.Ports { + rule.PortStart, rule.PortEnd = port, port + err := region.addSecurityGroupRule(secGrpId, rule) + if err != nil { + return errors.Wrapf(err, "addSecurityGroupRule %s", rule.String()) + } + } + return nil + } + return region.addSecurityGroupRule(secGrpId, rule) +} + +func (region *SRegion) addSecurityGroupRule(secGrpId string, rule cloudprovider.SecurityRule) error { + params := make(map[string]string) + params["RegionId"] = region.RegionId + params["SecurityGroupId"] = secGrpId + params["Description"] = rule.Description + params["PortStart"] = fmt.Sprintf("%d", rule.PortStart) + params["PortEnd"] = fmt.Sprintf("%d", rule.PortEnd) + protocol := rule.Protocol + if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY { + protocol = "all" + } + params["Protocol"] = protocol + if rule.Action == secrules.SecurityRuleAllow { + params["Policy"] = "accept" + } else { + params["Policy"] = "drop" + } + + params["Priority"] = fmt.Sprintf("%d", rule.Priority) + if rule.Direction == secrules.SecurityRuleIngress { + if rule.IPNet != nil { + params["CidrIp"] = rule.IPNet.String() + } else { + params["CidrIp"] = "0.0.0.0/0" + } + _, err := region.vpcRequest("AuthorizeSecurityGroupIngress", params) + return err + } else { + if rule.IPNet != nil { + params["CidrIp"] = rule.IPNet.String() + } else { + params["CidrIp"] = "0.0.0.0/0" + } + _, err := region.vpcRequest("AuthorizeSecurityGroupEgress", params) + return err + } +} + +func (region *SRegion) GetSecurityGroups(vpcId, name string, securityGroupIds []string, pageSize int, pageNumber int) ([]SSecurityGroup, int, error) { + if pageSize > 100 || pageSize <= 0 { + pageSize = 100 + } + params := make(map[string]string) + params["PageSize"] = fmt.Sprintf("%d", pageSize) + params["PageNumber"] = fmt.Sprintf("%d", pageNumber) + if len(vpcId) > 0 { + params["VpcId"] = vpcId + } + if len(name) > 0 { + params["SecurityGroupName"] = name + } + + if len(securityGroupIds) > 0 { + params["SecurityGroupIds"] = jsonutils.Marshal(securityGroupIds).String() + } + + body, err := region.vpcRequest("DescribeSecurityGroups", params) + if err != nil { + log.Errorf("GetSecurityGroups fail %s", err) + return nil, 0, err + } + + secgrps := make([]SSecurityGroup, 0) + err = body.Unmarshal(&secgrps, "Result", "SecurityGroups") + if err != nil { + log.Errorf("Unmarshal security groups fail %s", err) + return nil, 0, err + } + total, _ := body.Int("Result", "TotalCount") + return secgrps, int(total), nil +} + +func (rule *SSecurityGroupRule) toUniformRule() (cloudprovider.SecurityRule, error) { + uniformRule := cloudprovider.SecurityRule{ + SecurityRule: secrules.SecurityRule{ + Action: secrules.SecurityRuleDeny, + Direction: secrules.DIR_IN, + Priority: 101 - rule.Priority, + Description: rule.Description, + PortStart: -1, + PortEnd: -1, + }, + } + if strings.ToLower(rule.Policy) == "accept" { + uniformRule.Action = secrules.SecurityRuleAllow + } + cidr := rule.CidrIp + if rule.Direction == "egress" { + uniformRule.Direction = secrules.DIR_OUT + } + uniformRule.ParseCIDR(cidr) + switch strings.ToLower(rule.Protocol) { + case "tcp", "udp", "icmp": + uniformRule.Protocol = strings.ToLower(rule.Protocol) + case "all": + uniformRule.Protocol = secrules.PROTO_ANY + default: + return uniformRule, fmt.Errorf("unsupported protocal %s", rule.Protocol) + } + port := "" + if rule.PortStart == rule.PortEnd { + if rule.PortStart != -1 { + port = fmt.Sprintf("%d", rule.PortStart) + } + } else if rule.PortStart != -1 && rule.PortEnd != 65535 { + port = fmt.Sprintf("%d-%d", rule.PortStart, rule.PortEnd) + } + err := uniformRule.ParsePorts(port) + if err != nil { + return uniformRule, errors.Wrapf(err, "ParsePorts(%s)", port) + } + return uniformRule, nil +} + +func (secgroup *SSecurityGroup) GetId() string { + return secgroup.SecurityGroupId +} + +func (secgroup *SSecurityGroup) GetName() string { + return secgroup.SecurityGroupName +} + +func (secgroup *SSecurityGroup) GetGlobalId() string { + return secgroup.GetId() +} + +func (secgroup *SSecurityGroup) GetCreatedAt() time.Time { + return secgroup.CreationTime +} + +func (secgroup *SSecurityGroup) GetDescription() string { + return secgroup.Description +} + +func (secgroup *SSecurityGroup) GetStatus() string { + return secgroup.Status +} + +func (secgroup *SSecurityGroup) Refresh() error { + if body, err := secgroup.region.GetSecurityGroupDetails(secgroup.GetId()); err != nil { + return err + } else { + return jsonutils.Update(secgroup, body) + } +} + +func (secgroup *SSecurityGroup) GetProjectId() string { + return secgroup.ProjectName +} + +func (secgroup *SSecurityGroup) GetRules() ([]cloudprovider.SecurityRule, error) { + rules := make([]cloudprovider.SecurityRule, 0) + updatedSecgroup, err := secgroup.region.GetSecurityGroupDetails(secgroup.SecurityGroupId) + if err != nil { + return nil, err + } + outAllow := secrules.MustParseSecurityRule("out:allow any") + rules = append(rules, cloudprovider.SecurityRule{SecurityRule: *outAllow}) + for _, permission := range updatedSecgroup.Permissions { + if len(permission.SourceGroupId) > 0 { + continue + } + if !utils.IsInStringArray(strings.ToLower(permission.Protocol), []string{"tcp", "udp", "icmp", "all"}) { + continue + } + rule, err := permission.toUniformRule() + if err != nil { + log.Errorf("convert rule %s for group %s(%s) error: %v", permission.Description, secgroup.SecurityGroupName, secgroup.SecurityGroupId, err) + continue + } + rules = append(rules, rule) + } + return rules, nil +} + +func (secgroup *SSecurityGroup) GetVpcId() string { + return secgroup.VpcId +} + +func (secgroup *SSecurityGroup) GetReferences() ([]cloudprovider.SecurityGroupReference, error) { + ret := []cloudprovider.SecurityGroupReference{} + return ret, errors.Wrapf(errors.ErrNotImplemented, "GetReferences not supported") +} + +func (region *SRegion) DeleteSecurityGroup(secGrpId string) error { + params := make(map[string]string) + params["SecurityGroupId"] = secGrpId + + _, err := region.vpcRequest("DeleteSecurityGroup", params) + if err != nil { + return errors.Wrapf(err, "Delete security group fail") + } + return nil +} + +func (secgroup *SSecurityGroup) Delete() error { + return secgroup.region.DeleteSecurityGroupById(secgroup.SecurityGroupId) +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storage.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storage.go new file mode 100644 index 0000000000..8a9a3ab545 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storage.go @@ -0,0 +1,155 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "time" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" +) + +type SStorage struct { + multicloud.SStorageBase + VolcEngineTags + zone *SZone + storageType string +} + +func (storage *SStorage) GetId() string { + return fmt.Sprintf("%s-%s-%s", storage.zone.region.client.cpcfg.Id, storage.zone.GetId(), storage.storageType) +} + +func (storage *SStorage) GetName() string { + return fmt.Sprintf("%s-%s-%s", storage.zone.region.client.cpcfg.Name, storage.zone.GetId(), storage.storageType) +} + +func (storage *SStorage) GetGlobalId() string { + return fmt.Sprintf("%s-%s-%s", storage.zone.region.client.cpcfg.Id, storage.zone.GetGlobalId(), storage.storageType) +} + +func (storage *SStorage) IsEmulated() bool { + return true +} + +func (storage *SStorage) GetIZone() cloudprovider.ICloudZone { + return storage.zone +} + +func (storage *SStorage) GetIDisks() ([]cloudprovider.ICloudDisk, error) { + disks := make([]SDisk, 0) + pageNumber := 1 + storageType := storage.storageType + for { + parts, total, err := storage.zone.region.GetDisks("", storage.zone.GetId(), storageType, nil, pageNumber, 50) + if err != nil { + return nil, errors.Wrapf(err, "GetDisks") + } + disks = append(disks, parts...) + if len(parts) >= total { + break + } + pageNumber += 1 + } + idisks := make([]cloudprovider.ICloudDisk, len(disks)) + for i := 0; i < len(disks); i += 1 { + disks[i].storage = storage + idisks[i] = &disks[i] + } + return idisks, nil +} + +func (storage *SStorage) GetStorageType() string { + return storage.storageType +} + +func (storage *SStorage) GetCapacityMB() int64 { + return 0 +} + +func (storage *SStorage) GetCapacityUsedMB() int64 { + return 0 +} + +func (storage *SStorage) GetMediumType() string { + return api.DISK_TYPE_SSD +} + +func (storage *SStorage) GetStorageConf() jsonutils.JSONObject { + conf := jsonutils.NewDict() + return conf +} + +func (storage *SStorage) GetStatus() string { + return api.STORAGE_ONLINE +} + +func (storage *SStorage) Refresh() error { + return nil +} + +func (storage *SStorage) GetEnabled() bool { + return true +} + +func (storage *SStorage) GetIStoragecache() cloudprovider.ICloudStoragecache { + return storage.zone.region.getStoragecache() +} + +func (storage *SStorage) CreateIDisk(conf *cloudprovider.DiskCreateConfig) (cloudprovider.ICloudDisk, error) { + diskId, err := storage.zone.region.CreateDisk(storage.zone.ZoneId, storage.storageType, conf.Name, conf.SizeGb, conf.Desc, conf.ProjectId) + if err != nil { + log.Errorf("createDisk fail %s", err) + return nil, err + } + err = cloudprovider.Wait(5*time.Second, time.Minute, func() (bool, error) { + _, err := storage.zone.region.getDisk(diskId) + if errors.Cause(err) == cloudprovider.ErrNotFound { + return false, nil + } + return true, err + }) + if err != nil { + return nil, errors.Wrapf(err, "cannot find disk after create") + } + disk, err := storage.zone.region.getDisk(diskId) + if err != nil { + return nil, err + } + disk.storage = storage + return disk, nil +} + +func (storage *SStorage) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { + disk, err := storage.zone.region.getDisk(idStr) + if err != nil { + return nil, err + } + disk.storage = storage + return disk, nil +} + +func (storage *SStorage) GetMountPoint() string { + return "" +} + +func (storage *SStorage) IsSysDiskStore() bool { + return true +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storagecache.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storagecache.go new file mode 100644 index 0000000000..1ce571142f --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storagecache.go @@ -0,0 +1,174 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "context" + "fmt" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/qemuimgfmt" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" +) + +type SStoragecache struct { + multicloud.SResourceBase + VolcEngineTags + region *SRegion +} + +func GetBucketName(regionId string, imageId string) string { + return fmt.Sprintf("imgcache-%s-%s", strings.ToLower(regionId), imageId) +} + +func (scache *SStoragecache) GetId() string { + return fmt.Sprintf("%s-%s", scache.region.client.cpcfg.Id, scache.region.GetId()) +} + +func (scache *SStoragecache) GetName() string { + return fmt.Sprintf("%s-%s", scache.region.client.cpcfg.Name, scache.region.GetId()) +} + +func (scache *SStoragecache) GetStatus() string { + return "available" +} + +func (scache *SStoragecache) Refresh() error { + return nil +} + +func (scache *SStoragecache) GetGlobalId() string { + return fmt.Sprintf("%s-%s", scache.region.client.cpcfg.Id, scache.region.GetGlobalId()) +} + +func (scache *SStoragecache) GetICloudImages() ([]cloudprovider.ICloudImage, error) { + return nil, cloudprovider.ErrNotImplemented +} + +func (scache *SStoragecache) GetICustomizedCloudImages() ([]cloudprovider.ICloudImage, error) { + images := make([]SImage, 0) + token := "" + for { + parts, nextToken, err := scache.region.GetImages(ImageStatusType(""), ImageOwnerPrivate, nil, "", 50, token) + if err != nil { + return nil, errors.Wrapf(err, "GetImages") + } + images = append(images, parts...) + if len(nextToken) == 0 { + break + } + token = nextToken + } + ret := []cloudprovider.ICloudImage{} + for i := range images { + images[i].storageCache = scache + ret = append(ret, &images[i]) + } + return ret, nil +} + +func (scache *SStoragecache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) { + img, err := scache.region.GetImage(extId) + if err != nil { + return nil, err + } + img.storageCache = scache + return img, nil +} + +func (scache *SStoragecache) GetPath() string { + return "" +} + +func (scache *SStoragecache) UploadImage(ctx context.Context, image *cloudprovider.SImageCreateOption, callback func(progress float32)) (string, error) { + return scache.uploadImage(ctx, image, callback) +} + +func (scache *SStoragecache) uploadImage(ctx context.Context, image *cloudprovider.SImageCreateOption, callback func(progress float32)) (string, error) { + bucketName := GetBucketName(scache.region.GetId(), image.ImageId) + exist, err := scache.region.IBucketExist(bucketName) + if err != nil { + return "", errors.Wrapf(err, "IBucketExist") + } + if !exist { + err = scache.region.CreateIBucket(bucketName, "", "") + if err != nil { + return "", errors.Wrapf(err, "CreateIBucket") + } + } + defer scache.region.DeleteIBucket(bucketName) + + reader, sizeBytes, err := image.GetReader(image.ImageId, string(qemuimgfmt.VMDK)) + if err != nil { + return "", errors.Wrapf(err, "GetReader") + } + bucket, err := scache.region.GetIBucketByName(bucketName) + if err != nil { + return "", errors.Wrap(err, "GetIBucketByName") + } + body := multicloud.NewProgress(sizeBytes, 80, reader, callback) + err = cloudprovider.UploadObject(ctx, bucket, image.ImageId, 0, body, sizeBytes, "", "", nil, false) + if err != nil { + return "", errors.Wrap(err, "cloudprovider.UploadObject") + } + defer bucket.DeleteObject(ctx, image.ImageId) + + imageBaseName := image.ImageId + if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' { + imageBaseName = fmt.Sprintf("img%s", image.ImageId) + } + imageName := imageBaseName + nameIdx := 1 + + for { + _, err = scache.region.GetImageByName(imageName) + if err != nil { + if errors.Cause(err) == cloudprovider.ErrNotFound { + break + } else { + return "", err + } + } + + imageName = fmt.Sprintf("%s-%d", imageBaseName, nameIdx) + nameIdx += 1 + log.Debugf("uploadImage Match remote name %s", imageName) + } + + log.Debugf("Import image %s", imageName) + imageId, err := scache.region.ImportImage(imageName, image.OsArch, image.OsType, image.OsDistribution, image.OsVersion, bucketName, image.ImageId) + + if err != nil { + return "", errors.Wrapf(err, "ImportImage %s %s", image.ImageId, bucketName) + } + return imageId, nil +} + +func (region *SRegion) GetIStoragecaches() ([]cloudprovider.ICloudStoragecache, error) { + storageCache := region.getStoragecache() + return []cloudprovider.ICloudStoragecache{storageCache}, nil +} + +func (region *SRegion) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) { + storageCache := region.getStoragecache() + if id == storageCache.GetGlobalId() { + return storageCache, nil + } + return nil, cloudprovider.ErrNotFound +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/tag_base.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/tag_base.go new file mode 100644 index 0000000000..57e32a3348 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/tag_base.go @@ -0,0 +1,62 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "strings" + + "yunion.io/x/pkg/errors" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" +) + +type VolcEngineTags struct { + Tags []multicloud.STag +} + +func (itag *VolcEngineTags) GetTags() (map[string]string, error) { + ret := map[string]string{} + for _, tag := range itag.Tags { + if len(tag.TagKey) > 0 { + ret[tag.TagKey] = tag.TagValue + } else if len(tag.Key) > 0 { + ret[tag.Key] = tag.Value + } + } + return ret, nil +} + +func (itag *VolcEngineTags) GetSysTags() map[string]string { + ret := map[string]string{} + prefix := "volc:" + for _, tag := range itag.Tags { + if len(tag.TagKey) > 0 { + if strings.HasPrefix(tag.TagKey, prefix) { + ret[tag.TagKey] = tag.TagValue + } + } + if len(tag.Key) > 0 { + if strings.HasPrefix(tag.Key, prefix) { + ret[tag.Key] = tag.Value + } + } + } + return ret +} + +func (itag *VolcEngineTags) SetTags(tags map[string]string, replace bool) error { + return errors.Wrap(cloudprovider.ErrNotImplemented, "SetTags") +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/user.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/user.go new file mode 100644 index 0000000000..b3adca797e --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/user.go @@ -0,0 +1,63 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "time" + + "yunion.io/x/pkg/errors" +) + +type SUser struct { + Id int + CreateDate time.Time + UpdateDate time.Time + Status string + AccountId string + UserName string + Description string + DisplayName string + Email string + EmailIsVerify bool + MobilePhone string + MobilePhoneIsVerify bool + Trn string + Source string +} + +type SCallerIdentity struct { + AccountId string + UserId string + RoleId string + PrincipalId string + IdentityType string +} + +func (client *SVolcEngineClient) GetCallerIdentity() (*SCallerIdentity, error) { + // sys is not currently supported + params := map[string]string{} + body, err := client.iamRequest("", "ListUsers", params) + if err != nil { + return nil, err + } + id := &SCallerIdentity{} + users := []SUser{} + err = body.Unmarshal(&users, "Result", "UserMetadata") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + id.AccountId = users[0].AccountId + return id, nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/volcengine.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/volcengine.go new file mode 100644 index 0000000000..6d15fb35ad --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/volcengine.go @@ -0,0 +1,390 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + tos "github.com/volcengine/ve-tos-golang-sdk/v2/tos" + sdk "github.com/volcengine/volc-sdk-golang/base" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/httputils" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" +) + +const ( + CLOUD_PROVIDER_VOLCENGINE = api.CLOUD_PROVIDER_VOLCENGINE + CLOUD_PROVIDER_VOLCENGINE_CN = "火山云" + CLOUD_PROVIDER_VOLCENGINE_EN = "VolcEngine" + + VOLCENGINE_API_VERSION = "2020-04-01" + VOLCENGINE_IAM_API_VERSION = "2021-08-01" + + VOLCENGINE_API = "open.volcengineapi.com" + VOLCENGINE_IAM_API = "iam.volcengineapi.com" + VOLCENGINE_TOS_API = "tos-cn-beijing.volces.com" + + VOLCENGINE_SERVICE_ECS = "ecs" + VOLCENGINE_SERVICE_VPC = "vpc" + VOLCENGINE_SERVICE_NAT = "natgateway" + VOLCENGINE_SERVICE_STORAGE = "storage_ebs" + VOLCENGINE_SERVICE_IAM = "iam" + VOLCENGINE_SERVICE_TOS = "tos" + VOLCENGINE_DEFAULT_REGION = "cn-beijing" +) + +type VolcEngineClientConfig struct { + cpcfg cloudprovider.ProviderConfig + cloudEnv string + accessKey string + secretKey string + accountId string + debug bool +} + +func NewVolcEngineClientConfig(accessKey, secretKey string) *VolcEngineClientConfig { + cfg := &VolcEngineClientConfig{ + accessKey: accessKey, + secretKey: secretKey, + } + return cfg +} + +func (cfg *VolcEngineClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *VolcEngineClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *VolcEngineClientConfig) AccountId(id string) *VolcEngineClientConfig { + cfg.accountId = id + return cfg +} + +func (cfg *VolcEngineClientConfig) Debug(debug bool) *VolcEngineClientConfig { + cfg.debug = debug + return cfg +} + +func (cfg VolcEngineClientConfig) Copy() VolcEngineClientConfig { + return cfg +} + +type SVolcEngineClient struct { + *VolcEngineClientConfig + + ownerId string + + projects []SProject + iregions []cloudprovider.ICloudRegion + iBuckets []cloudprovider.ICloudBucket +} + +func NewVolcEngineClient(cfg *VolcEngineClientConfig) (*SVolcEngineClient, error) { + client := SVolcEngineClient{ + VolcEngineClientConfig: cfg, + } + err := client.fetchRegions() + if err != nil { + return nil, errors.Wrap(err, "fetchReginos") + } + return &client, nil +} + +// Regions +func (client *SVolcEngineClient) fetchRegions() error { + body, err := client.ecsRequest("", "DescribeRegions", nil) + if err != nil { + return errors.Wrapf(err, "DescribeRegions") + } + regions := make([]SRegion, 0) + err = body.Unmarshal(®ions, "Result", "Regions") + if err != nil { + return errors.Wrapf(err, "resp.Unmarshal") + } + client.iregions = make([]cloudprovider.ICloudRegion, len(regions)) + for i := 0; i < len(regions); i += 1 { + regions[i].client = client + client.iregions[i] = ®ions[i] + } + return nil +} + +func (client *SVolcEngineClient) GetRegions() []SRegion { + regions := make([]SRegion, len(client.iregions)) + for i := 0; i < len(regions); i += 1 { + region := client.iregions[i].(*SRegion) + regions[i] = *region + } + return regions +} + +func (client *SVolcEngineClient) GetRegion(regionId string) *SRegion { + if len(regionId) == 0 { + regionId = VOLCENGINE_DEFAULT_REGION + } + for i := 0; i < len(client.iregions); i += 1 { + if client.iregions[i].GetId() == regionId { + return client.iregions[i].(*SRegion) + } + } + return nil +} + +func (client *SVolcEngineClient) GetIRegions() []cloudprovider.ICloudRegion { + return client.iregions +} + +func (client *SVolcEngineClient) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) { + for i := 0; i < len(client.iregions); i += 1 { + if (client.iregions[i].GetId() == id) || (client.iregions[i].GetGlobalId() == id) { + return client.iregions[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (client *SVolcEngineClient) GetAccountId() string { + if len(client.ownerId) > 0 { + return client.ownerId + } + caller, err := client.GetCallerIdentity() + if err != nil { + return "" + } + client.ownerId = caller.AccountId + return client.ownerId +} + +func (client *SVolcEngineClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { + err := client.fetchRegions() + if err != nil { + return nil, err + } + subAccount := cloudprovider.SSubAccount{} + subAccount.Name = client.cpcfg.Name + subAccount.Account = client.accessKey + subAccount.HealthStatus = api.CLOUD_PROVIDER_HEALTH_NORMAL + subAccount.DefaultProjectId = "default" + return []cloudprovider.SSubAccount{subAccount}, nil +} + +func (client *SVolcEngineClient) GetIProjects() ([]cloudprovider.ICloudProject, error) { + projects, err := client.GetProjects() + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudProject{} + for i := range projects { + ret = append(ret, &projects[i]) + } + return ret, nil +} + +func (client *SVolcEngineClient) GetProjects() ([]SProject, error) { + if len(client.projects) > 0 { + return client.projects, nil + } + limit, offset := 50, 0 + client.projects = []SProject{} + for { + parts, total, err := client.ListProjects(limit, offset) + if err != nil { + return nil, errors.Wrap(err, "GetProjects") + } + client.projects = append(client.projects, parts...) + if len(client.projects) >= total { + break + } + offset += total + } + return client.projects, nil +} + +func (client *SVolcEngineClient) jsonRequest(cred sdk.Credentials, domain string, apiVersion string, apiName string, params map[string]string) (jsonutils.JSONObject, error) { + + query := url.Values{ + "Action": []string{apiName}, + "Version": []string{apiVersion}, + } + for k, v := range params { + query.Set(k, v) + } + + u := url.URL{ + Scheme: "http", + Host: domain, + Path: "/", + RawQuery: query.Encode(), + } + method := httputils.GET + for prefix, _method := range map[string]httputils.THttpMethod{ + "Get": httputils.GET, + "Describe": httputils.GET, + "List": httputils.GET, + "Delete": httputils.GET, + "Put": httputils.PUT, + } { + if strings.HasPrefix(apiName, prefix) { + method = _method + break + } + } + if strings.HasPrefix(domain, "bucketname") { + if strings.HasPrefix(apiName, "Delete") { + method = httputils.DELETE + } + } + + req, err := http.NewRequest(string(method), u.String(), nil) + if err != nil { + fmt.Println("Failed to build request:", err) + return nil, err + } + req = cred.Sign(req) + resp, err := http.DefaultClient.Do(req) + rbody, _ := io.ReadAll(resp.Body) + resp.Body = io.NopCloser(bytes.NewBuffer(rbody)) + _, result, err := httputils.ParseJSONResponse("", resp, err, client.debug) + if err != nil { + jrbody, _ := jsonutils.Parse(rbody) + errorCode, _ := jrbody.GetString("ResponseMetadata", "Error", "Code") + errorMessage, _ := jrbody.GetString("ResponseMetadata", "Error", "Message") + return nil, errors.Wrapf(err, errorCode, errorMessage) + } + return result, nil +} + +func (client *SVolcEngineClient) getSdkCredential(region string, service string, token string) sdk.Credentials { + cred := sdk.Credentials{ + AccessKeyID: client.accessKey, + SecretAccessKey: client.secretKey, + Region: region, + Service: service, + SessionToken: token, + } + return cred +} + +func (client *SVolcEngineClient) getDefaultCredential(region string, service string) sdk.Credentials { + if region == "" { + region = VOLCENGINE_DEFAULT_REGION + } + cred := sdk.Credentials{ + AccessKeyID: client.accessKey, + SecretAccessKey: client.secretKey, + Region: region, + Service: service, + SessionToken: "", + } + return cred +} + +func (client *SVolcEngineClient) ecsRequest(region string, apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cred := client.getDefaultCredential(region, VOLCENGINE_SERVICE_ECS) + return client.jsonRequest(cred, VOLCENGINE_API, VOLCENGINE_API_VERSION, apiName, params) +} + +func (client *SVolcEngineClient) iamRequest(region string, apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cred := client.getDefaultCredential(region, VOLCENGINE_SERVICE_IAM) + return client.jsonRequest(cred, VOLCENGINE_IAM_API, VOLCENGINE_IAM_API_VERSION, apiName, params) +} + +func (client *SVolcEngineClient) getTosClient(regionId string) (*tos.ClientV2, error) { + tosClient, err := tos.NewClientV2(VOLCENGINE_TOS_API, tos.WithRegion(regionId), tos.WithCredentials(tos.NewStaticCredentials(client.accessKey, client.secretKey))) + return tosClient, err +} + +// Buckets +func (client *SVolcEngineClient) invalidateIBuckets() { + client.iBuckets = nil +} + +func (client *SVolcEngineClient) getIBuckets() ([]cloudprovider.ICloudBucket, error) { + if client.iBuckets == nil { + err := client.fetchBuckets() + if err != nil { + return nil, errors.Wrap(err, "fetchBuckets") + } + } + return client.iBuckets, nil +} + +func (client *SVolcEngineClient) fetchBuckets() error { + toscli, err := client.getTosClient(VOLCENGINE_DEFAULT_REGION) + if err != nil { + return errors.Wrap(err, "client.getOssClient") + } + out, err := toscli.ListBuckets(context.Background(), &tos.ListBucketsInput{}) + if err != nil { + return errors.Wrap(err, "tos.ListBuckets") + } + + ret := make([]cloudprovider.ICloudBucket, 0) + for _, bucket := range out.Buckets { + regionId := bucket.Location + region, err := client.GetIRegionById(regionId) + if err != nil { + log.Errorf("cannot find bucket's region %s", regionId) + continue + } + t, err := time.Parse(time.RFC3339, bucket.CreationDate) + if err != nil { + return errors.Wrapf(err, "Prase CreationDate error") + } + b := SBucket{ + region: region.(*SRegion), + Name: bucket.Name, + Location: bucket.Location, + CreationDate: t, + } + ret = append(ret, &b) + } + client.iBuckets = ret + return nil +} + +func getTOSExternalDomain(regionId string) string { + return "tos-cn-beijing.volces.com" +} + +func getTOSInternalDomain(regionId string) string { + return "tos-cn-beijing.ivolces.com" +} + +func (region *SVolcEngineClient) GetCapabilities() []string { + caps := []string{ + cloudprovider.CLOUD_CAPABILITY_PROJECT, + cloudprovider.CLOUD_CAPABILITY_COMPUTE, + cloudprovider.CLOUD_CAPABILITY_NETWORK, + cloudprovider.CLOUD_CAPABILITY_EIP, + cloudprovider.CLOUD_CAPABILITY_OBJECTSTORE, + } + return caps +} + +func (client *SVolcEngineClient) GetAccessEnv() string { + return api.CLOUD_ACCESS_ENV_VOLCENGINE_CHINA +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/vpc.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/vpc.go new file mode 100644 index 0000000000..98d6051c4f --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/vpc.go @@ -0,0 +1,261 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "strings" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" +) + +type SUserCIDRs []string + +type SVpc struct { + multicloud.SVpc + VolcEngineTags + + region *SRegion + + secgroups []cloudprovider.ICloudSecurityGroup + routeTables []cloudprovider.ICloudRouteTable + + RegionId string + VpcId string + VpcName string + CidrBlock string + CidrBlockAssociationSet []string + IsDefault bool + Status string + InstanceTenancy string +} + +func (vpc *SVpc) GetId() string { + return vpc.VpcId +} + +func (vpc *SVpc) GetName() string { + if len(vpc.VpcName) > 0 { + return vpc.VpcName + } + return vpc.VpcId +} + +func (vpc *SVpc) GetGlobalId() string { + return vpc.VpcId +} + +func (vpc *SVpc) IsEmulated() bool { + return false +} + +func (vpc *SVpc) GetIsDefault() bool { + return vpc.IsDefault +} + +func (vpc *SVpc) GetCidrBlock() string { + return vpc.CidrBlock +} + +func (vpc *SVpc) GetStatus() string { + return strings.ToLower(vpc.Status) +} + +func (vpc *SVpc) Refresh() error { + new, err := vpc.region.getVpc(vpc.VpcId) + if err != nil { + return err + } + return jsonutils.Update(vpc, new) +} + +func (vpc *SVpc) GetRegion() cloudprovider.ICloudRegion { + return vpc.region +} + +func (vpc *SVpc) GetIWires() ([]cloudprovider.ICloudWire, error) { + zones, err := vpc.region.GetZones("") + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudWire{} + for i := range zones { + zones[i].region = vpc.region + ret = append(ret, &SWire{zone: &zones[i], vpc: vpc}) + } + return ret, nil +} + +func (vpc *SVpc) GetIWireById(wireId string) (cloudprovider.ICloudWire, error) { + wires, err := vpc.GetIWires() + if err != nil { + return nil, err + } + for i := range wires { + if wires[i].GetGlobalId() == wireId { + return wires[i], nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, wireId) +} + +func (vpc *SVpc) fetchSecurityGroups() error { + secgroups := make([]SSecurityGroup, 0) + pageNumber := 1 + for { + parts, total, err := vpc.region.GetSecurityGroups(vpc.VpcId, "", nil, 50, pageNumber) + if err != nil { + return err + } + secgroups = append(secgroups, parts...) + if len(secgroups) >= total { + break + } + pageNumber += 1 + } + vpc.secgroups = make([]cloudprovider.ICloudSecurityGroup, len(secgroups)) + for i := 0; i < len(secgroups); i++ { + secgroups[i].region = vpc.region + vpc.secgroups[i] = &secgroups[i] + } + return nil +} + +func (vpc *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, error) { + if vpc.secgroups == nil { + err := vpc.fetchSecurityGroups() + if err != nil { + return nil, err + } + } + return vpc.secgroups, nil +} + +func (vpc *SVpc) fetchRouteTables() error { + routeTables := make([]*SRouteTable, 0) + pageNumber := 1 + for { + parts, total, err := vpc.RemoteGetRouteTableList(pageNumber, 50) + if err != nil { + return err + } + routeTables = append(routeTables, parts...) + if len(routeTables) >= total { + break + } + pageNumber += 1 + } + vpc.routeTables = make([]cloudprovider.ICloudRouteTable, len(routeTables)) + for i := 0; i < len(routeTables); i++ { + routeTables[i].vpc = vpc + vpc.routeTables[i] = routeTables[i] + } + return nil +} + +func (vpc *SVpc) GetIRouteTables() ([]cloudprovider.ICloudRouteTable, error) { + if vpc.routeTables == nil { + err := vpc.fetchRouteTables() + if err != nil { + return nil, err + } + } + return vpc.routeTables, nil +} + +func (vpc *SVpc) GetIRouteTableById(routeTableId string) (cloudprovider.ICloudRouteTable, error) { + tables, err := vpc.GetIRouteTables() + if err != nil { + return nil, errors.Wrapf(err, "GetIRouteTables") + } + for i := range tables { + if tables[i].GetGlobalId() == routeTableId { + return tables[i], nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, routeTableId) +} + +func (vpc *SVpc) Delete() error { + err := vpc.fetchSecurityGroups() + if err != nil { + return errors.Wrapf(err, "fetchSecurityGroup for VPC delete fail") + } + for i := 0; i < len(vpc.secgroups); i += 1 { + secgroup := vpc.secgroups[i].(*SSecurityGroup) + err := vpc.region.DeleteSecurityGroup(secgroup.SecurityGroupId) + if err != nil { + return errors.Wrapf(err, "deleteSecurityGroup for VPC delete fail") + } + } + return vpc.region.DeleteVpc(vpc.VpcId) +} + +func (vpc *SVpc) getNatGateways() ([]SNatGateway, error) { + nats := make([]SNatGateway, 0) + pageNumber := 1 + for { + parts, total, err := vpc.region.GetNatGateways(vpc.VpcId, "", pageNumber, 50) + if err != nil { + return nil, err + } + nats = append(nats, parts...) + if len(nats) >= total { + break + } + pageNumber += 1 + } + for i := 0; i < len(nats); i += 1 { + nats[i].vpc = vpc + } + return nats, nil +} + +func (vpc *SVpc) getINatGateways() ([]cloudprovider.ICloudNatGateway, error) { + nats := make([]SNatGateway, 0) + pageNumber := 1 + for { + parts, total, err := vpc.region.GetNatGateways(vpc.VpcId, "", pageNumber, 50) + if err != nil { + return nil, err + } + nats = append(nats, parts...) + if len(nats) >= total { + break + } + pageNumber += 1 + } + inats := []cloudprovider.ICloudNatGateway{} + for i := 0; i < len(nats); i++ { + nats[i].vpc = vpc + inats = append(inats, &nats[i]) + } + return inats, nil +} + +func (vpc *SVpc) CreateINatGateway(opts *cloudprovider.NatGatewayCreateOptions) (cloudprovider.ICloudNatGateway, error) { + nat, err := vpc.region.CreateNatGateway(opts) + if err != nil { + return nil, errors.Wrapf(err, "CreateNatGateway") + } + nat.vpc = vpc + return nat, nil +} + +func (vpc *SVpc) GetAuthorityOwnerId() string { + return vpc.region.client.ownerId +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/wire.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/wire.go new file mode 100644 index 0000000000..98a88425e6 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/wire.go @@ -0,0 +1,142 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + "time" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" +) + +type SWire struct { + multicloud.SResourceBase + VolcEngineTags + + zone *SZone + vpc *SVpc + inetworks []cloudprovider.ICloudNetwork +} + +func (wire *SWire) GetId() string { + return fmt.Sprintf("%s-%s", wire.vpc.GetId(), wire.zone.GetId()) +} + +func (wire *SWire) GetName() string { + return wire.GetId() +} + +func (wire *SWire) IsEmulated() bool { + return true +} + +func (wire *SWire) GetStatus() string { + return api.WIRE_STATUS_AVAILABLE +} + +func (wire *SWire) Refresh() error { + return nil +} + +func (wire *SWire) GetGlobalId() string { + return fmt.Sprintf("%s-%s", wire.vpc.GetGlobalId(), wire.zone.GetGlobalId()) +} + +func (wire *SWire) GetIVpc() cloudprovider.ICloudVpc { + return wire.vpc +} + +func (wire *SWire) GetIZone() cloudprovider.ICloudZone { + return wire.zone +} + +func (wire *SWire) GetBandwidth() int { + return 10000 +} + +func (wire *SWire) GetINetworks() ([]cloudprovider.ICloudNetwork, error) { + networks, err := wire.vpc.region.FetchSubnets(nil, wire.zone.ZoneId, wire.vpc.VpcId) + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudNetwork{} + for i := range networks { + networks[i].wire = wire + ret = append(ret, &networks[i]) + } + return ret, nil +} + +func (wire *SWire) getNetworkById(SubnetId string) (*SNetwork, error) { + networks, err := wire.GetINetworks() + if err != nil { + return nil, err + } + log.Debugf("search for networks %d", len(networks)) + for i := 0; i < len(networks); i += 1 { + log.Debugf("search %s", networks[i].GetName()) + network := networks[i].(*SNetwork) + if network.SubnetId == SubnetId { + return network, nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (wire *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + subnetId, err := wire.zone.region.CreateSubnet(wire.zone.ZoneId, wire.vpc.VpcId, opts.Name, opts.Cidr, opts.Desc) + if err != nil { + log.Errorf("createSubnet error %s", err) + return nil, err + } + err = cloudprovider.Wait(5*time.Second, time.Minute, func() (bool, error) { + _, err = wire.getNetworkById(subnetId) + if errors.Cause(err) == cloudprovider.ErrNotFound { + return false, nil + } else { + return true, err + } + }) + if err != nil { + return nil, errors.Wrapf(err, "cannot find subnet after create") + } + subnet, err := wire.getNetworkById(subnetId) + if err != nil { + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s not found", subnetId) + } + subnet.wire = wire + if wire.inetworks == nil { + wire.inetworks = []cloudprovider.ICloudNetwork{} + } + wire.inetworks = append(wire.inetworks, subnet) + return subnet, nil +} + +func (wire *SWire) GetINetworkById(netid string) (cloudprovider.ICloudNetwork, error) { + networks, err := wire.GetINetworks() + if err != nil { + return nil, err + } + for i := 0; i < len(networks); i += 1 { + if networks[i].GetGlobalId() == netid { + return networks[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/zone.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/zone.go new file mode 100644 index 0000000000..2c304b603b --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/zone.go @@ -0,0 +1,183 @@ +// Copyright 2023 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package volcengine + +import ( + "fmt" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/pkg/errors" +) + +var StorageTypes = []string{ + api.STORAGE_VOLCENGINE_FlexPL, + api.STORAGE_VOLCENGINE_PL0, + api.STORAGE_VOLCENGINE_PTSSD, +} + +type SSupportedResource struct { + Status string + Value string +} + +type SAvailableResource struct { + Type string + SupportedResources []SSupportedResource +} + +type SZone struct { + multicloud.SResourceBase + VolcEngineTags + region *SRegion + + host *SHost + + istorages []cloudprovider.ICloudStorage + Status string + AvailableResources []SAvailableResource + + storageTypes []string + + ZoneId string + RegionId string + LocalName string +} + +func (zone *SZone) GetIWires() ([]cloudprovider.ICloudWire, error) { + vpcs, err := zone.region.GetAllVpcs() + if err != nil { + return nil, errors.Wrapf(err, "GetVpcs") + } + ret := []cloudprovider.ICloudWire{} + for i := range vpcs { + vpcs[i].region = zone.region + ret = append(ret, &SWire{zone: zone, vpc: &vpcs[i]}) + } + return ret, nil +} + +func (zone *SZone) GetId() string { + return zone.ZoneId +} + +func (zone *SZone) GetGlobalId() string { + return fmt.Sprintf("%s/%s", zone.region.GetGlobalId(), zone.ZoneId) +} + +func (zone *SZone) GetName() string { + return fmt.Sprintf("%s", zone.ZoneId) +} + +func (zone *SZone) GetI18n() cloudprovider.SModelI18nTable { + table := cloudprovider.SModelI18nTable{} + table["name"] = cloudprovider.NewSModelI18nEntry(zone.GetName()).CN(zone.GetName()) + return table +} + +func (zone *SZone) GetStatus() string { + return api.ZONE_ENABLE +} + +func (zone *SZone) Refresh() error { + return nil +} + +func (zone *SZone) GetIRegion() cloudprovider.ICloudRegion { + return zone.region +} + +// Host +func (zone *SZone) getHost() *SHost { + if zone.host == nil { + zone.host = &SHost{zone: zone} + } + return zone.host +} + +func (zone *SZone) GetIHosts() ([]cloudprovider.ICloudHost, error) { + return []cloudprovider.ICloudHost{zone.getHost()}, nil +} + +func (zone *SZone) GetIHostById(id string) (cloudprovider.ICloudHost, error) { + hosts, err := zone.GetIHosts() + if err != nil { + return nil, err + } + for i := range hosts { + if hosts[i].GetGlobalId() == id { + return hosts[i], nil + } + } + return nil, errors.Wrap(cloudprovider.ErrNotFound, "GetIHostById") +} + +// Storage +func (zone *SZone) getStorageType() { + if len(zone.storageTypes) == 0 { + zone.storageTypes = StorageTypes + } +} + +func (zone *SZone) fetchStorages() error { + zone.getStorageType() + zone.istorages = make([]cloudprovider.ICloudStorage, len(zone.storageTypes)) + + for i, sc := range zone.storageTypes { + storage := SStorage{zone: zone, storageType: sc} + zone.istorages[i] = &storage + } + return nil +} + +func (zone *SZone) getStorageByCategory(category string) (*SStorage, error) { + storages, err := zone.GetIStorages() + if err != nil { + return nil, err + } + for i := 0; i < len(storages); i += 1 { + storage := storages[i].(*SStorage) + if storage.storageType == category { + return storage, nil + } + } + return nil, fmt.Errorf("no such storage %s", category) +} + +func (zone *SZone) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + if zone.istorages == nil { + err := zone.fetchStorages() + if err != nil { + return nil, errors.Wrapf(err, "fetchStorages") + } + } + return zone.istorages, nil +} + +func (zone *SZone) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + if zone.istorages == nil { + err := zone.fetchStorages() + if err != nil { + return nil, errors.Wrapf(err, "fetchStorages") + } + } + for i := 0; i < len(zone.istorages); i += 1 { + if zone.istorages[i].GetGlobalId() == id { + return zone.istorages[i], nil + } + } + return nil, cloudprovider.ErrNotFound +}