From 43c7b02b08f2a6db2d7c78d05c0673a914a27742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 24 Sep 2025 17:20:31 +0800 Subject: [PATCH] fix(region): vendor update (#23367) --- go.mod | 7 +- go.sum | 10 +- pkg/apis/monitor/unifiedmonitor_query.go | 28 +- pkg/mcclient/cloudpods/instance.go | 58 - pkg/mcclient/cloudpods/monitor.go | 131 + pkg/mcclient/cloudpods/shell/monitor.go | 8 + .../ks3sdklib/aws-sdk-go/LICENSE.txt | 202 + .../ks3sdklib/aws-sdk-go/aws/awserr/error.go | 105 + .../ks3sdklib/aws-sdk-go/aws/awserr/types.go | 135 + .../ks3sdklib/aws-sdk-go/aws/awsutil/copy.go | 96 + .../aws-sdk-go/aws/awsutil/path_value.go | 175 + .../aws-sdk-go/aws/awsutil/string_value.go | 103 + .../ks3sdklib/aws-sdk-go/aws/awsutil/util.go | 151 + .../ks3sdklib/aws-sdk-go/aws/config.go | 303 + .../ks3sdklib/aws-sdk-go/aws/context.go | 22 + .../aws/credentials/chain_provider.go | 81 + .../aws-sdk-go/aws/credentials/credentials.go | 178 + .../aws/credentials/ec2_role_provider.go | 173 + .../aws/credentials/env_provider.go | 67 + .../aws-sdk-go/aws/credentials/example.ini | 8 + .../aws-sdk-go/aws/credentials/ini.go | 123 + .../shared_credentials_provider.go | 131 + .../aws/credentials/static_provider.go | 42 + .../aws-sdk-go/aws/default_redirect.go | 22 + .../ks3sdklib/aws-sdk-go/aws/dns_resolver.go | 155 + .../aws-sdk-go/aws/handler_functions.go | 161 + .../ks3sdklib/aws-sdk-go/aws/handlers.go | 88 + .../aws-sdk-go/aws/param_validator.go | 102 + .../ks3sdklib/aws-sdk-go/aws/progress.go | 96 + .../ks3sdklib/aws-sdk-go/aws/request.go | 326 + .../aws-sdk-go/aws/retry/exponential_retry.go | 37 + .../aws-sdk-go/aws/retry/fixed_retry.go | 26 + .../aws-sdk-go/aws/retry/linear_retry.go | 36 + .../aws-sdk-go/aws/retry/no_delay_retry.go | 18 + .../aws-sdk-go/aws/retry/random_retry.go | 39 + .../ks3sdklib/aws-sdk-go/aws/retry/retry.go | 19 + .../aws-sdk-go/aws/retry/retryable_error.go | 69 + .../ks3sdklib/aws-sdk-go/aws/service.go | 112 + .../ks3sdklib/aws-sdk-go/aws/types.go | 167 + .../ks3sdklib/aws-sdk-go/aws/version.go | 7 + .../aws-sdk-go/internal/apierr/error.go | 139 + .../aws-sdk-go/internal/crc/crc64.go | 127 + .../internal/endpoints/endpoints.go | 31 + .../internal/endpoints/endpoints_map.go | 89 + .../aws-sdk-go/internal/protocol/body/body.go | 37 + .../internal/protocol/json/jsonutil/build.go | 199 + .../protocol/json/jsonutil/unmarshal.go | 214 + .../internal/protocol/jsonrpc/jsonrpc.go | 121 + .../internal/protocol/query/build.go | 33 + .../protocol/query/queryutil/queryutil.go | 223 + .../internal/protocol/query/unmarshal.go | 29 + .../protocol/query/unmarshal_error.go | 108 + .../internal/protocol/rest/build.go | 397 ++ .../internal/protocol/rest/payload.go | 63 + .../internal/protocol/rest/unmarshal.go | 180 + .../internal/protocol/restjson/restjson.go | 77 + .../internal/protocol/restxml/restxml.go | 66 + .../internal/protocol/xml/xmlutil/build.go | 287 + .../protocol/xml/xmlutil/unmarshal.go | 266 + .../protocol/xml/xmlutil/xml_to_struct.go | 105 + .../internal/signer/open_api/open_api.go | 77 + .../aws-sdk-go/internal/signer/v2/v2.go | 371 ++ .../aws-sdk-go/internal/signer/v4/v4.go | 369 ++ .../ks3sdklib/aws-sdk-go/service/s3/api.go | 5720 +++++++++++++++++ .../aws-sdk-go/service/s3/api_object.go | 169 + .../service/s3/bucket_accessmonitor.go | 134 + .../service/s3/bucket_encryption.go | 194 + .../aws-sdk-go/service/s3/bucket_location.go | 27 + .../aws-sdk-go/service/s3/bucket_qos.go | 200 + .../aws-sdk-go/service/s3/bucket_tagging.go | 179 + .../s3/bucket_transfer_acceleration.go | 133 + .../aws-sdk-go/service/s3/checkpoint.go | 520 ++ .../ks3sdklib/aws-sdk-go/service/s3/const.go | 97 + .../ks3sdklib/aws-sdk-go/service/s3/copier.go | 804 +++ .../ks3sdklib/aws-sdk-go/service/s3/cors.go | 196 + .../aws-sdk-go/service/s3/crc_check.go | 42 + .../service/s3/data_redundancy_switch.go | 126 + .../aws-sdk-go/service/s3/decompresspolicy.go | 219 + .../aws-sdk-go/service/s3/downloader.go | 542 ++ .../ks3sdklib/aws-sdk-go/service/s3/errors.go | 60 + .../service/s3/host_style_bucket.go | 53 + .../aws-sdk-go/service/s3/inventory.go | 306 + .../aws-sdk-go/service/s3/ks3bill.go | 443 ++ .../aws-sdk-go/service/s3/lifecycle.go | 261 + .../ks3sdklib/aws-sdk-go/service/s3/mirror.go | 214 + .../aws-sdk-go/service/s3/replication.go | 190 + .../aws-sdk-go/service/s3/requester_qos.go | 194 + .../aws-sdk-go/service/s3/retention.go | 386 ++ .../aws-sdk-go/service/s3/service.go | 70 + .../ks3sdklib/aws-sdk-go/service/s3/try.go | 35 + .../aws-sdk-go/service/s3/uploader.go | 603 ++ .../ks3sdklib/aws-sdk-go/service/s3/util.go | 108 + .../aws-sdk-go/service/s3/vpc_access_block.go | 206 + vendor/modules.txt | 27 +- .../pkg/multicloud/aws/dbinstance_cluster.go | 11 +- .../aws/dbinstance_cluster_snapshots.go | 172 + .../pkg/multicloud/aws/dbinstance_snapshot.go | 8 + .../pkg/multicloud/google/instance.go | 2 +- .../x/cloudmux/pkg/multicloud/ksyun/bucket.go | 599 ++ .../x/cloudmux/pkg/multicloud/ksyun/ksyun.go | 5 + .../x/cloudmux/pkg/multicloud/ksyun/object.go | 84 + .../pkg/multicloud/objectstore/objectstore.go | 7 + .../pkg/multicloud/qcloud/dbinstance.go | 7 + .../cloudmux/pkg/multicloud/qcloud/monitor.go | 94 +- .../cloudmux/pkg/multicloud/qcloud/qcloud.go | 18 +- .../pkg/multicloud/qcloud/rds_sqlserver.go | 459 ++ .../pkg/multicloud/qcloud/tag_base.go | 2 +- .../x/sqlchemy/backends/dameng/sync.go | 10 + vendor/yunion.io/x/sqlchemy/reflect.go | 113 + vendor/yunion.io/x/sqlchemy/update.go | 3 + 110 files changed, 21393 insertions(+), 84 deletions(-) create mode 100644 pkg/mcclient/cloudpods/monitor.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/LICENSE.txt create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/awserr/error.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/awserr/types.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/copy.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/path_value.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/string_value.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/util.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/config.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/context.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/chain_provider.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/credentials.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/ec2_role_provider.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/env_provider.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/example.ini create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/ini.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/shared_credentials_provider.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/static_provider.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/default_redirect.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/dns_resolver.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/handler_functions.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/handlers.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/param_validator.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/progress.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/request.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/exponential_retry.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/fixed_retry.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/linear_retry.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/no_delay_retry.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/random_retry.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/retry.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/retryable_error.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/service.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/types.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/aws/version.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/apierr/error.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/crc/crc64.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/endpoints/endpoints.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/endpoints/endpoints_map.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/body/body.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil/build.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil/unmarshal.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/jsonrpc/jsonrpc.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/build.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/queryutil/queryutil.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/unmarshal.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/unmarshal_error.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/build.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/payload.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/unmarshal.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/restjson/restjson.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/restxml/restxml.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/build.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/unmarshal.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/xml_to_struct.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/open_api/open_api.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/v2/v2.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/v4/v4.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/api.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/api_object.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_accessmonitor.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_encryption.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_location.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_qos.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_tagging.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_transfer_acceleration.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/checkpoint.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/const.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/copier.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/cors.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/crc_check.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/data_redundancy_switch.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/decompresspolicy.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/downloader.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/errors.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/host_style_bucket.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/inventory.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/ks3bill.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/lifecycle.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/mirror.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/replication.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/requester_qos.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/retention.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/service.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/try.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/uploader.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/util.go create mode 100644 vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/vpc_access_block.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_cluster_snapshots.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/bucket.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/object.go create mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/rds_sqlserver.go diff --git a/go.mod b/go.mod index 1e1a77e90b..ee83b854f8 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,6 @@ require ( github.com/go-yaml/yaml v2.1.0+incompatible github.com/golang-plus/uuid v1.0.0 github.com/golang/mock v1.4.4 - github.com/golang/protobuf v1.5.3 github.com/google/cadvisor v0.38.5 github.com/google/gopacket v1.1.17 github.com/google/uuid v1.6.0 @@ -97,14 +96,14 @@ require ( k8s.io/cri-api v0.22.17 k8s.io/klog/v2 v2.20.0 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250915054625-7251d9eeceec + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250924031606-a2597eb1f66f yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 yunion.io/x/pkg v1.10.4-0.20250820035218-6008459eb4f0 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 - yunion.io/x/sqlchemy v1.1.3-0.20250902120005-3b18bc6371c7 + yunion.io/x/sqlchemy v1.1.3-0.20250923143347-b98e89dbc372 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c ) @@ -206,6 +205,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-plus/errors v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/go-querystring v1.0.0 // indirect @@ -229,6 +229,7 @@ require ( github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/karrick/godirwalk v1.16.1 // indirect github.com/kr/fs v0.1.0 // indirect + github.com/ks3sdklib/aws-sdk-go v1.8.1 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/lestrrat-go/iter v0.0.0-20200422075355-fc1769541911 // indirect github.com/lestrrat-go/pdebug v0.0.0-20200204225717-4d6bd78da58d // indirect diff --git a/go.sum b/go.sum index 92fbd40594..62a2e63462 100644 --- a/go.sum +++ b/go.sum @@ -588,6 +588,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ks3sdklib/aws-sdk-go v1.8.1 h1:/3mYWujGfr4K/+mAdESbi9u3T4VDuHYMYFk1/OL0JWE= +github.com/ks3sdklib/aws-sdk-go v1.8.1/go.mod h1:jGcsV0dJgMmStAyqjkKVUu6F167pAXYZAS3LqoZMmtM= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lestrrat-go/iter v0.0.0-20200422075355-fc1769541911 h1:FvnrqecqX4zT0wOIbYK1gNgTm0677INEWiFY8UEYggY= @@ -1424,8 +1426,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250915054625-7251d9eeceec h1:GvDds+zC42TTTFoxui2/Y8mquJQKZ0ay858+/VabUlE= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250915054625-7251d9eeceec/go.mod h1:7P/TJZk8o4JjhFnF1nGZcsPg+sIpMoV0dWPPuG6yGLg= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250924031606-a2597eb1f66f h1:aLijaM2lCiOSvU4aMHKmViaHvi7xP3h0WNilHVVxXJg= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250924031606-a2597eb1f66f/go.mod h1:R5iP/4nGCluuekoa30B5hM/49IfPpGHXwFK3yT7m6Vw= yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 h1:msG4SiDSVU7CrXH06WuHlNEZXIooTcmNbfrIGHuIHBU= yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= @@ -1443,7 +1445,7 @@ yunion.io/x/pkg v1.10.4-0.20250820035218-6008459eb4f0 h1:/vHcYqzQInxqR1jjS+e7A0o yunion.io/x/pkg v1.10.4-0.20250820035218-6008459eb4f0/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v1.1.3-0.20250902120005-3b18bc6371c7 h1:0OccQYkey6OsNUZNxd6CpCB/espBPnwlLT/gjl9YbXM= -yunion.io/x/sqlchemy v1.1.3-0.20250902120005-3b18bc6371c7/go.mod h1:vCIZpqhZ5Jzaq3tFyrti/vv8BijQKtkzSgNT/uH4H5A= +yunion.io/x/sqlchemy v1.1.3-0.20250923143347-b98e89dbc372 h1:8lR9HzYTxSgxcVv6uC2Ogkk1Y3FEEanABfgeleCzhfA= +yunion.io/x/sqlchemy v1.1.3-0.20250923143347-b98e89dbc372/go.mod h1:vCIZpqhZ5Jzaq3tFyrti/vv8BijQKtkzSgNT/uH4H5A= yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ= yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/apis/monitor/unifiedmonitor_query.go b/pkg/apis/monitor/unifiedmonitor_query.go index 0e88056b90..295a36d99e 100644 --- a/pkg/apis/monitor/unifiedmonitor_query.go +++ b/pkg/apis/monitor/unifiedmonitor_query.go @@ -185,11 +185,35 @@ func (p TimePoint) IsValids() bool { } func (p TimePoint) Value() float64 { - return *(p[0].(*float64)) + v := p[0] + if fval, ok := v.(*float64); ok { + return *fval + } + if ival, ok := v.(*int64); ok { + return float64(*ival) + } + if t, ok := v.(float64); ok { + return t + } + if t, ok := v.(int64); ok { + return float64(t) + } + return 0 } func (p TimePoint) Timestamp() float64 { - return p[len(p)-1].(float64) + v := p[len(p)-1] + if t, ok := v.(float64); ok { + return t + } + if t, ok := v.(int64); ok { + return float64(t) + } + return 0 +} + +func (p TimePoint) Time() time.Time { + return time.UnixMilli(int64(p.Timestamp())) } func (p TimePoint) Values() []float64 { diff --git a/pkg/mcclient/cloudpods/instance.go b/pkg/mcclient/cloudpods/instance.go index 09ed56c406..3e0c6039df 100644 --- a/pkg/mcclient/cloudpods/instance.go +++ b/pkg/mcclient/cloudpods/instance.go @@ -30,7 +30,6 @@ import ( modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" "yunion.io/x/onecloud/pkg/mcclient/modules/identity" "yunion.io/x/onecloud/pkg/mcclient/modules/logger" - "yunion.io/x/onecloud/pkg/mcclient/modules/monitor" "yunion.io/x/onecloud/pkg/mcclient/modules/webconsole" ) @@ -565,63 +564,6 @@ func (self *SRegion) CreateInstance(hostId, hypervisor string, opts *cloudprovid return ins, self.create(&modules.Servers, input, ins) } -type SMetricData struct { - Id string `json:"id"` - Time time.Time `json:"time"` - Value float64 `json:"value"` -} - -func (cli *SCloudpodsClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { - brandArr := []string{"OneCloud"} - metrics := []SMetricData{} - usefulResourceType := []cloudprovider.TResourceType{cloudprovider.METRIC_RESOURCE_TYPE_HOST, cloudprovider.METRIC_RESOURCE_TYPE_SERVER} - isUse, _ := utils.InArray(opts.ResourceType, usefulResourceType) - if !isUse { - return nil, nil - } - for i := 0; i < len(brandArr); i++ { - params := map[string]interface{}{ - "metric_name": opts.MetricType, - "start_time": opts.StartTime, - "end_time": opts.EndTime, - "interval": "1m", - "tag_pairs": map[string]interface{}{ - "brand": brandArr[i], - }, - } - onecloudObj, err := monitor.UnifiedMonitorManager.Get(cli.s, "simple-query", jsonutils.Marshal(params)) - if err != nil { - return nil, err - } - tmp := []SMetricData{} - onecloudObj.Unmarshal(&tmp, "values") - metrics = append(metrics, tmp...) - } - - idWithMetric := map[string][]cloudprovider.MetricValue{} - for _, v := range metrics { - if _, isExist := idWithMetric[v.Id]; !isExist { - idWithMetric[v.Id] = []cloudprovider.MetricValue{{ - Timestamp: v.Time, - Value: v.Value}} - } else { - idWithMetric[v.Id] = append(idWithMetric[v.Id], cloudprovider.MetricValue{ - Timestamp: v.Time, - Value: v.Value}) - } - } - - res := []cloudprovider.MetricValues{} - for _, metric := range metrics { - res = append(res, cloudprovider.MetricValues{ - Id: metric.Id, - MetricType: opts.MetricType, - Values: idWithMetric[metric.Id], - }) - } - return res, nil -} - func (self *SInstance) CreateDisk(ctx context.Context, opts *cloudprovider.GuestDiskCreateOptions) (string, error) { diskIds := []string{} for _, disk := range self.DisksInfo { diff --git a/pkg/mcclient/cloudpods/monitor.go b/pkg/mcclient/cloudpods/monitor.go new file mode 100644 index 0000000000..f213a7d18d --- /dev/null +++ b/pkg/mcclient/cloudpods/monitor.go @@ -0,0 +1,131 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudpods + +import ( + "crypto/sha256" + "fmt" + "strings" + "time" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + monitorapi "yunion.io/x/onecloud/pkg/apis/monitor" + "yunion.io/x/onecloud/pkg/mcclient/modules/monitor" +) + +func (cli *SCloudpodsClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + usefulResourceType := []cloudprovider.TResourceType{cloudprovider.METRIC_RESOURCE_TYPE_HOST, cloudprovider.METRIC_RESOURCE_TYPE_SERVER} + isUse, _ := utils.InArray(opts.ResourceType, usefulResourceType) + if !isUse { + return nil, nil + } + + info := strings.Split(string(opts.MetricType), ".") + if len(info) != 2 { + return nil, errors.Errorf("invalid metric type: %s", opts.MetricType) + } + measurement := info[0] + field := info[1] + from := fmt.Sprintf("now-%dm", int(time.Now().Sub(opts.StartTime).Minutes())) + to := fmt.Sprintf("now-%dm", int(time.Now().Sub(opts.EndTime).Minutes())) + + query := &monitorapi.AlertQuery{ + Model: monitorapi.MetricQuery{ + Database: "telegraf", + Measurement: measurement, + Selects: []monitorapi.MetricQuerySelect{ + { + { + Type: "field", + Params: []string{field}, + }, + }, + }, + Tags: []monitorapi.MetricQueryTag{ + { + Key: "brand", + Operator: "=", + Value: "OneCloud", + }, + }, + }, + From: from, + To: to, + } + + input := monitorapi.MetricQueryInput{ + From: from, + To: to, + Scope: "system", + Interval: "1m", + MetricQuery: []*monitorapi.AlertQuery{ + query, + }, + SkipCheckSeries: true, + } + if cli.debug { + input.ShowMeta = true + } + + data := jsonutils.Marshal(input).(*jsonutils.JSONDict) + sum256 := sha256.Sum256([]byte(data.String())) + data.Set("signature", jsonutils.NewString(fmt.Sprintf("%x", sum256))) + resp, err := monitor.UnifiedMonitorManager.PerformClassAction(cli.s, "query", data) + if err != nil { + return nil, errors.Wrapf(err, "query metric") + } + + metrics := &monitorapi.MetricsQueryResult{} + err = resp.Unmarshal(metrics) + if err != nil { + return nil, errors.Wrapf(err, "query metric Unmarshal") + } + + res := []cloudprovider.MetricValues{} + for _, serie := range metrics.Series { + if len(serie.Tags) == 0 || (len(serie.Tags["vm_id"]) == 0 && len(serie.Tags["host_id"]) == 0) { + continue + } + id := serie.Tags["vm_id"] + if len(id) == 0 { + id = serie.Tags["host_id"] + } + metric := cloudprovider.MetricValues{ + Id: id, + MetricType: opts.MetricType, + Values: []cloudprovider.MetricValue{}, + } + values := []cloudprovider.MetricValue{} + for _, point := range serie.Points { + if len(point) != 2 { + continue + } + values = append(values, cloudprovider.MetricValue{ + Timestamp: point.Time(), + Value: point.Value(), + }) + } + if len(values) == 0 { + continue + } + metric.Values = values + res = append(res, metric) + } + return res, nil +} diff --git a/pkg/mcclient/cloudpods/shell/monitor.go b/pkg/mcclient/cloudpods/shell/monitor.go index 4ac600f335..381bac1474 100644 --- a/pkg/mcclient/cloudpods/shell/monitor.go +++ b/pkg/mcclient/cloudpods/shell/monitor.go @@ -15,6 +15,8 @@ package shell import ( + "time" + "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/log" "yunion.io/x/pkg/util/shellutils" @@ -24,6 +26,12 @@ import ( func init() { shellutils.R(&cloudprovider.MetricListOptions{}, "metric-list", "List metrics in a namespace", func(cli *cloudpods.SRegion, args *cloudprovider.MetricListOptions) error { + if args.StartTime.IsZero() { + args.StartTime = time.Now().Add(-time.Minute * 10) + } + if args.EndTime.IsZero() { + args.EndTime = time.Now().Add(-time.Minute * 1) + } metrics, err := cli.GetClient().GetMetrics(args) if err != nil { return err diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/LICENSE.txt b/vendor/github.com/ks3sdklib/aws-sdk-go/LICENSE.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awserr/error.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awserr/error.go new file mode 100644 index 0000000000..eb86cd7a29 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awserr/error.go @@ -0,0 +1,105 @@ +// Package awserr represents API error interface accessors for the SDK. +package awserr + +// An Error wraps lower level errors with code, message and an original error. +// The underlying concrete error type may also satisfy other interfaces which +// can be to used to obtain more specific information about the error. +// +// Calling Error() or String() will always include the full information about +// an error based on its underlying type. +// +// Example: +// +// output, err := s3manage.Upload(svc, input, opts) +// if err != nil { +// if awsErr, ok := err.(awserr.Error); ok { +// // Get error details +// log.Println("Error:", err.Code(), err.Message()) +// +// Prints out full error message, including original error if there was one. +// log.Println("Error:", err.Error()) +// +// // Get original error +// if origErr := err.Err(); origErr != nil { +// // operate on original error. +// } +// } else { +// fmt.Println(err.Error()) +// } +// } +// +type Error interface { + // Satisfy the generic error interface. + error + + // Returns the short phrase depicting the classification of the error. + Code() string + + // Returns the error details message. + Message() string + + // Returns the original error if one was set. Nil is returned if not set. + OrigErr() error +} + +// A RequestFailure is an interface to extract request failure information from +// an Error such as the request ID of the failed request returned by a service. +// RequestFailures may not always have a requestID value if the request failed +// prior to reaching the service such as a connection error. +// +// Example: +// +// output, err := s3manage.Upload(svc, input, opts) +// if err != nil { +// if reqerr, ok := err.(RequestFailure); ok { +// log.Printf("Request failed", reqerr.Code(), reqerr.Message(), reqerr.RequestID()) +// } else { +// log.Printf("Error:", err.Error() +// } +// } +// +// Combined with awserr.Error: +// +// output, err := s3manage.Upload(svc, input, opts) +// if err != nil { +// if awsErr, ok := err.(awserr.Error); ok { +// // Generic AWS Error with Code, Message, and original error (if any) +// fmt.Println(awsErr.Code(), awsErr.Message(), awsErr.OrigErr()) +// +// if reqErr, ok := err.(awserr.RequestFailure); ok { +// // A service error occurred +// fmt.Println(reqErr.StatusCode(), reqErr.RequestID()) +// } +// } else { +// fmt.Println(err.Error()) +// } +// } +// +type RequestFailure interface { + Error + + // The status code of the HTTP response. + StatusCode() int + + // The request ID returned by the service for a request failure. This will + // be empty if no request ID is available such as the request failed due + // to a connection error. + RequestID() string +} + +// New returns an Error object described by the code, message, and origErr. +// +// If origErr satisfies the Error interface it will not be wrapped within a new +// Error object and will instead be returned. +func New(code, message string, origErr error) Error { + if e, ok := origErr.(Error); ok && e != nil { + return e + } + return newBaseError(code, message, origErr) +} + +// NewRequestFailure returns a new request error wrapper for the given Error +// provided. +func NewRequestFailure(err Error, statusCode int, reqID string) RequestFailure { + return newRequestError(err, statusCode, reqID) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awserr/types.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awserr/types.go new file mode 100644 index 0000000000..418fc4c14b --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awserr/types.go @@ -0,0 +1,135 @@ +package awserr + +import "fmt" + +// SprintError returns a string of the formatted error code. +// +// Both extra and origErr are optional. If they are included their lines +// will be added, but if they are not included their lines will be ignored. +func SprintError(code, message, extra string, origErr error) string { + msg := fmt.Sprintf("%s: %s", code, message) + if extra != "" { + msg = fmt.Sprintf("%s\n\t%s", msg, extra) + } + if origErr != nil { + msg = fmt.Sprintf("%s\ncaused by: %s", msg, origErr.Error()) + } + return msg +} + +// A baseError wraps the code and message which defines an error. It also +// can be used to wrap an original error object. +// +// Should be used as the root for errors satisfying the awserr.Error. Also +// for any error which does not fit into a specific error wrapper type. +type baseError struct { + // Classification of error + code string + + // Detailed information about error + message string + + // Optional original error this error is based off of. Allows building + // chained errors. + origErr error +} + +// newBaseError returns an error object for the code, message, and err. +// +// code is a short no whitespace phrase depicting the classification of +// the error that is being created. +// +// message is the free flow string containing detailed information about the error. +// +// origErr is the error object which will be nested under the new error to be returned. +func newBaseError(code, message string, origErr error) *baseError { + return &baseError{ + code: code, + message: message, + origErr: origErr, + } +} + +// Error returns the string representation of the error. +// +// See ErrorWithExtra for formatting. +// +// Satisfies the error interface. +func (b baseError) Error() string { + return SprintError(b.code, b.message, "", b.origErr) +} + +// String returns the string representation of the error. +// Alias for Error to satisfy the stringer interface. +func (b baseError) String() string { + return b.Error() +} + +// Code returns the short phrase depicting the classification of the error. +func (b baseError) Code() string { + return b.code +} + +// Message returns the error details message. +func (b baseError) Message() string { + return b.message +} + +// OrigErr returns the original error if one was set. Nil is returned if no error +// was set. +func (b baseError) OrigErr() error { + return b.origErr +} + +// So that the Error interface type can be included as an anonymous field +// in the requestError struct and not conflict with the error.Error() method. +type awsError Error + +// A requestError wraps a request or service error. +// +// Composed of baseError for code, message, and original error. +type requestError struct { + awsError + statusCode int + requestID string +} + +// newRequestError returns a wrapped error with additional information for request +// status code, and service requestID. +// +// Should be used to wrap all request which involve service requests. Even if +// the request failed without a service response, but had an HTTP status code +// that may be meaningful. +// +// Also wraps original errors via the baseError. +func newRequestError(err Error, statusCode int, requestID string) *requestError { + return &requestError{ + awsError: err, + statusCode: statusCode, + requestID: requestID, + } +} + +// Error returns the string representation of the error. +// Satisfies the error interface. +func (r requestError) Error() string { + extra := fmt.Sprintf("status code: %d, request id: [%s]", + r.statusCode, r.requestID) + return SprintError(r.Code(), r.Message(), extra, r.OrigErr()) +} + +// String returns the string representation of the error. +// Alias for Error to satisfy the stringer interface. +func (r requestError) String() string { + return r.Error() +} + +// StatusCode returns the wrapped status code for the error +func (r requestError) StatusCode() int { + return r.statusCode +} + +// RequestID returns the wrapped requestID +func (r requestError) RequestID() string { + return r.requestID +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/copy.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/copy.go new file mode 100644 index 0000000000..a0d7986e7f --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/copy.go @@ -0,0 +1,96 @@ +package awsutil + +import ( + "io" + "reflect" +) + +// Copy deeply copies a src structure to dst. Useful for copying request and +// response structures. +// +// Can copy between structs of different type, but will only copy fields which +// are assignable, and exist in both structs. Fields which are not assignable, +// or do not exist in both structs are ignored. +func Copy(dst, src interface{}) { + dstval := reflect.ValueOf(dst) + if !dstval.IsValid() { + panic("Copy dst cannot be nil") + } + + rcopy(dstval, reflect.ValueOf(src), true) +} + +// CopyOf returns a copy of src while also allocating the memory for dst. +// src must be a pointer type or this operation will fail. +func CopyOf(src interface{}) (dst interface{}) { + dsti := reflect.New(reflect.TypeOf(src).Elem()) + dst = dsti.Interface() + rcopy(dsti, reflect.ValueOf(src), true) + return +} + +// rcopy performs a recursive copy of values from the source to destination. +// +// root is used to skip certain aspects of the copy which are not valid +// for the root node of a object. +func rcopy(dst, src reflect.Value, root bool) { + if !src.IsValid() { + return + } + + switch src.Kind() { + case reflect.Ptr: + if _, ok := src.Interface().(io.Reader); ok { + if dst.Kind() == reflect.Ptr && dst.Elem().CanSet() { + dst.Elem().Set(src) + } else if dst.CanSet() { + dst.Set(src) + } + } else { + e := src.Type().Elem() + if dst.CanSet() && !src.IsNil() { + dst.Set(reflect.New(e)) + } + if src.Elem().IsValid() { + // Keep the current root state since the depth hasn't changed + rcopy(dst.Elem(), src.Elem(), root) + } + } + case reflect.Struct: + if !root { + dst.Set(reflect.New(src.Type()).Elem()) + } + + t := dst.Type() + for i := 0; i < t.NumField(); i++ { + name := t.Field(i).Name + srcVal := src.FieldByName(name) + dstVal := dst.FieldByName(name) + if srcVal.IsValid() && dstVal.CanSet() { + rcopy(dstVal, srcVal, false) + } + } + case reflect.Slice: + s := reflect.MakeSlice(src.Type(), src.Len(), src.Cap()) + dst.Set(s) + for i := 0; i < src.Len(); i++ { + rcopy(dst.Index(i), src.Index(i), false) + } + case reflect.Map: + s := reflect.MakeMap(src.Type()) + dst.Set(s) + for _, k := range src.MapKeys() { + v := src.MapIndex(k) + v2 := reflect.New(v.Type()).Elem() + rcopy(v2, v, false) + dst.SetMapIndex(k, v2) + } + default: + // Assign the value if possible. If its not assignable, the value would + // need to be converted and the impact of that may be unexpected, or is + // not compatible with the dst type. + if src.Type().AssignableTo(dst.Type()) { + dst.Set(src) + } + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/path_value.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/path_value.go new file mode 100644 index 0000000000..7ae01efe42 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/path_value.go @@ -0,0 +1,175 @@ +package awsutil + +import ( + "reflect" + "regexp" + "strconv" + "strings" +) + +var indexRe = regexp.MustCompile(`(.+)\[(-?\d+)?\]$`) + +// rValuesAtPath returns a slice of values found in value v. The values +// in v are explored recursively so all nested values are collected. +func rValuesAtPath(v interface{}, path string, create bool, caseSensitive bool) []reflect.Value { + pathparts := strings.Split(path, "||") + if len(pathparts) > 1 { + for _, pathpart := range pathparts { + vals := rValuesAtPath(v, pathpart, create, caseSensitive) + if vals != nil && len(vals) > 0 { + return vals + } + } + return nil + } + + values := []reflect.Value{reflect.Indirect(reflect.ValueOf(v))} + components := strings.Split(path, ".") + for len(values) > 0 && len(components) > 0 { + var index *int64 + var indexStar bool + c := strings.TrimSpace(components[0]) + if c == "" { // no actual component, illegal syntax + return nil + } else if caseSensitive && c != "*" && strings.ToLower(c[0:1]) == c[0:1] { + // TODO normalize case for user + return nil // don't support unexported fields + } + + // parse this component + if m := indexRe.FindStringSubmatch(c); m != nil { + c = m[1] + if m[2] == "" { + index = nil + indexStar = true + } else { + i, _ := strconv.ParseInt(m[2], 10, 32) + index = &i + indexStar = false + } + } + + nextvals := []reflect.Value{} + for _, value := range values { + // pull component name out of struct member + if value.Kind() != reflect.Struct { + continue + } + + if c == "*" { // pull all members + for i := 0; i < value.NumField(); i++ { + if f := reflect.Indirect(value.Field(i)); f.IsValid() { + nextvals = append(nextvals, f) + } + } + continue + } + + value = value.FieldByNameFunc(func(name string) bool { + if c == name { + return true + } else if !caseSensitive && strings.ToLower(name) == strings.ToLower(c) { + return true + } + return false + }) + + if create && value.Kind() == reflect.Ptr && value.IsNil() { + value.Set(reflect.New(value.Type().Elem())) + value = value.Elem() + } else { + value = reflect.Indirect(value) + } + + if value.IsValid() { + nextvals = append(nextvals, value) + } + } + values = nextvals + + if indexStar || index != nil { + nextvals = []reflect.Value{} + for _, value := range values { + value := reflect.Indirect(value) + if value.Kind() != reflect.Slice { + continue + } + + if indexStar { // grab all indices + for i := 0; i < value.Len(); i++ { + idx := reflect.Indirect(value.Index(i)) + if idx.IsValid() { + nextvals = append(nextvals, idx) + } + } + continue + } + + // pull out index + i := int(*index) + if i >= value.Len() { // check out of bounds + if create { + // TODO resize slice + } else { + continue + } + } else if i < 0 { // support negative indexing + i = value.Len() + i + } + value = reflect.Indirect(value.Index(i)) + + if value.IsValid() { + nextvals = append(nextvals, value) + } + } + values = nextvals + } + + components = components[1:] + } + return values +} + +// ValuesAtPath returns a list of objects at the lexical path inside of a structure +func ValuesAtPath(i interface{}, path string) []interface{} { + if rvals := rValuesAtPath(i, path, false, true); rvals != nil { + vals := make([]interface{}, len(rvals)) + for i, rval := range rvals { + vals[i] = rval.Interface() + } + return vals + } + return nil +} + +// ValuesAtAnyPath returns a list of objects at the case-insensitive lexical +// path inside of a structure +func ValuesAtAnyPath(i interface{}, path string) []interface{} { + if rvals := rValuesAtPath(i, path, false, false); rvals != nil { + vals := make([]interface{}, len(rvals)) + for i, rval := range rvals { + vals[i] = rval.Interface() + } + return vals + } + return nil +} + +// SetValueAtPath sets an object at the lexical path inside of a structure +func SetValueAtPath(i interface{}, path string, v interface{}) { + if rvals := rValuesAtPath(i, path, true, true); rvals != nil { + for _, rval := range rvals { + rval.Set(reflect.ValueOf(v)) + } + } +} + +// SetValueAtAnyPath sets an object at the case insensitive lexical path inside +// of a structure +func SetValueAtAnyPath(i interface{}, path string, v interface{}) { + if rvals := rValuesAtPath(i, path, true, false); rvals != nil { + for _, rval := range rvals { + rval.Set(reflect.ValueOf(v)) + } + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/string_value.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/string_value.go new file mode 100644 index 0000000000..09673a12ed --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/string_value.go @@ -0,0 +1,103 @@ +package awsutil + +import ( + "bytes" + "fmt" + "io" + "reflect" + "strings" +) + +// StringValue returns the string representation of a value. +func StringValue(i interface{}) string { + var buf bytes.Buffer + stringValue(reflect.ValueOf(i), 0, &buf) + return buf.String() +} + +// stringValue will recursively walk value v to build a textual +// representation of the value. +func stringValue(v reflect.Value, indent int, buf *bytes.Buffer) { + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + + switch v.Kind() { + case reflect.Struct: + strtype := v.Type().String() + if strtype == "time.Time" { + fmt.Fprintf(buf, "%s", v.Interface()) + break + } else if strings.HasPrefix(strtype, "io.") { + buf.WriteString("") + break + } + + buf.WriteString("{\n") + + names := []string{} + for i := 0; i < v.Type().NumField(); i++ { + name := v.Type().Field(i).Name + f := v.Field(i) + if name[0:1] == strings.ToLower(name[0:1]) { + continue // ignore unexported fields + } + if (f.Kind() == reflect.Ptr || f.Kind() == reflect.Slice || f.Kind() == reflect.Map) && f.IsNil() { + continue // ignore unset fields + } + names = append(names, name) + } + + for i, n := range names { + val := v.FieldByName(n) + buf.WriteString(strings.Repeat(" ", indent+2)) + buf.WriteString(n + ": ") + stringValue(val, indent+2, buf) + + if i < len(names)-1 { + buf.WriteString(",\n") + } + } + + buf.WriteString("\n" + strings.Repeat(" ", indent) + "}") + case reflect.Slice: + nl, id, id2 := "", "", "" + if v.Len() > 3 { + nl, id, id2 = "\n", strings.Repeat(" ", indent), strings.Repeat(" ", indent+2) + } + buf.WriteString("[" + nl) + for i := 0; i < v.Len(); i++ { + buf.WriteString(id2) + stringValue(v.Index(i), indent+2, buf) + + if i < v.Len()-1 { + buf.WriteString("," + nl) + } + } + + buf.WriteString(nl + id + "]") + case reflect.Map: + buf.WriteString("{\n") + + for i, k := range v.MapKeys() { + buf.WriteString(strings.Repeat(" ", indent+2)) + buf.WriteString(k.String() + ": ") + stringValue(v.MapIndex(k), indent+2, buf) + + if i < v.Len()-1 { + buf.WriteString(",\n") + } + } + + buf.WriteString("\n" + strings.Repeat(" ", indent) + "}") + default: + format := "%v" + switch v.Interface().(type) { + case string: + format = "%q" + case io.ReadSeeker, io.Reader: + format = "buffer(%p)" + } + fmt.Fprintf(buf, format, v.Interface()) + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/util.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/util.go new file mode 100644 index 0000000000..1b16617e65 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/awsutil/util.go @@ -0,0 +1,151 @@ +package awsutil + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/xml" + "fmt" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil" + "go/format" + "io" + "os" + "path/filepath" + "reflect" + "regexp" + "strings" +) + +// GoFmt returns the Go formated string of the input. +// +// Panics if the format fails. +func GoFmt(buf string) string { + formatted, err := format.Source([]byte(buf)) + if err != nil { + panic(fmt.Errorf("%s\nOriginal code:\n%s", err.Error(), buf)) + } + return string(formatted) +} + +var reTrim = regexp.MustCompile(`\s{2,}`) + +// Trim removes all leading and trailing white space. +// +// All consecutive spaces will be reduced to a single space. +func Trim(s string) string { + return strings.TrimSpace(reTrim.ReplaceAllString(s, " ")) +} + +// Capitalize capitalizes the first character of the string. +func Capitalize(s string) string { + if len(s) == 1 { + return strings.ToUpper(s) + } + return strings.ToUpper(s[0:1]) + s[1:] +} + +// SortXML sorts the reader's XML elements +func SortXML(r io.Reader) string { + var buf bytes.Buffer + d := xml.NewDecoder(r) + root, _ := xmlutil.XMLToStruct(d, nil) + e := xml.NewEncoder(&buf) + xmlutil.StructToXML(e, root, true) + return buf.String() +} + +// PrettyPrint generates a human readable representation of the value v. +// All values of v are recursively found and pretty printed also. +func PrettyPrint(v interface{}) string { + value := reflect.ValueOf(v) + switch value.Kind() { + case reflect.Struct: + str := fullName(value.Type()) + "{\n" + for i := 0; i < value.NumField(); i++ { + l := string(value.Type().Field(i).Name[0]) + if strings.ToUpper(l) == l { + str += value.Type().Field(i).Name + ": " + str += PrettyPrint(value.Field(i).Interface()) + str += ",\n" + } + } + str += "}" + return str + case reflect.Map: + str := "map[" + fullName(value.Type().Key()) + "]" + fullName(value.Type().Elem()) + "{\n" + for _, k := range value.MapKeys() { + str += "\"" + k.String() + "\": " + str += PrettyPrint(value.MapIndex(k).Interface()) + str += ",\n" + } + str += "}" + return str + case reflect.Ptr: + if e := value.Elem(); e.IsValid() { + return "&" + PrettyPrint(e.Interface()) + } + return "nil" + case reflect.Slice: + str := "[]" + fullName(value.Type().Elem()) + "{\n" + for i := 0; i < value.Len(); i++ { + str += PrettyPrint(value.Index(i).Interface()) + str += ",\n" + } + str += "}" + return str + default: + return fmt.Sprintf("%#v", v) + } +} + +func pkgName(t reflect.Type) string { + pkg := t.PkgPath() + c := strings.Split(pkg, "/") + return c[len(c)-1] +} + +func fullName(t reflect.Type) string { + if pkg := pkgName(t); pkg != "" { + return pkg + "." + t.Name() + } + return t.Name() +} + +//获取指定目录及所有子目录下的所有文件，可以匹配后缀过滤。 +func WalkDir(dirPth, suffix string) (files []string, err error) { + files = make([]string, 0, 30) + suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写 + + err = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error { //遍历目录 + //if err != nil { //忽略错误 + // return err + //} + + if fi.IsDir() { // 忽略目录 + return nil + } + + if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) { + files = append(files, filename) + } + + return nil + }) + return files, err +} +func IsHidden(path string) bool { + fileInfo, err := os.Stat(path) + if err != nil { + return false + } + return fileInfo.Name()[0] == '.' +} +func ComputeMD5Hash(input []byte) []byte { + h := md5.New() + h.Write(input) + return h.Sum(nil) +} + +func EncodeAsString(bytes []byte) string { + return base64.StdEncoding.EncodeToString(bytes) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/config.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/config.go new file mode 100644 index 0000000000..c6a0a02f46 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/config.go @@ -0,0 +1,303 @@ +package aws + +import ( + "bytes" + "fmt" + "github.com/ks3sdklib/aws-sdk-go/aws/retry" + "io" + "net/http" + "os" + "time" + + "github.com/ks3sdklib/aws-sdk-go/aws/credentials" +) + +// DefaultChainCredentials is a Credentials which will find the first available +// credentials Value from the list of Providers. +// +// This should be used in the default case. Once the type of credentials are +// known switching to the specific Credentials will be more efficient. +var DefaultChainCredentials = credentials.NewChainCredentials( + []credentials.Provider{ + &credentials.EnvProvider{}, + &credentials.SharedCredentialsProvider{Filename: "", Profile: ""}, + &credentials.EC2RoleProvider{ExpiryWindow: 5 * time.Minute}, + }) + +// DefaultMaxRetries is the default number of retries for a service. +const DefaultMaxRetries = 3 + +// DefaultConfig is the default all service configuration will be based off of. +var DefaultConfig = &Config{ + Credentials: DefaultChainCredentials, + Endpoint: "", + Ks3BillEndpoint: "ks3bill.api.ksyun.com", + Region: "", + DisableSSL: false, + ManualSend: false, + HTTPClient: http.DefaultClient, + LogHTTPBody: false, + LogLevel: Off, + Logger: os.Stdout, + MaxRetries: DefaultMaxRetries, + RetryRule: retry.DefaultExponentialRetryRule, + ShouldRetry: retry.ShouldRetry, + DisableParamValidation: false, + DisableComputeChecksums: false, + S3ForcePathStyle: false, + DomainMode: false, + SignerVersion: "V2", + CrcCheckEnabled: false, + DisableRestProtocolURICleaning: true, + DisableDnsCache: false, +} + +// A Config provides service configuration +type Config struct { + Credentials *credentials.Credentials + Endpoint string + Ks3BillEndpoint string + Region string + DisableSSL bool + ManualSend bool + HTTPClient *http.Client + LogHTTPBody bool + LogLevel uint + Logger io.Writer + MaxRetries int // 重试次数 + RetryRule retry.RetryRule // 重试规则 + ShouldRetry func(error) bool // 是否需要重试 + DisableParamValidation bool + DisableComputeChecksums bool + S3ForcePathStyle bool + DomainMode bool + SignerVersion string + CrcCheckEnabled bool // 允许crc64校验，默认为false + DisableRestProtocolURICleaning bool // 禁用path clean，默认为true + DisableDnsCache bool // 禁用DNS缓存，默认为false +} + +// Copy will return a shallow copy of the Config object. +func (c Config) Copy() Config { + dst := Config{} + dst.Credentials = c.Credentials + dst.Endpoint = c.Endpoint + dst.Ks3BillEndpoint = c.Ks3BillEndpoint + dst.Region = c.Region + dst.DisableSSL = c.DisableSSL + dst.ManualSend = c.ManualSend + dst.HTTPClient = c.HTTPClient + dst.LogHTTPBody = c.LogHTTPBody + dst.LogLevel = c.LogLevel + dst.Logger = c.Logger + dst.MaxRetries = c.MaxRetries + dst.RetryRule = c.RetryRule + dst.ShouldRetry = c.ShouldRetry + dst.DisableParamValidation = c.DisableParamValidation + dst.DisableComputeChecksums = c.DisableComputeChecksums + dst.S3ForcePathStyle = c.S3ForcePathStyle + dst.DomainMode = c.DomainMode + dst.SignerVersion = c.SignerVersion + dst.CrcCheckEnabled = c.CrcCheckEnabled + dst.DisableRestProtocolURICleaning = c.DisableRestProtocolURICleaning + dst.DisableDnsCache = c.DisableDnsCache + return dst +} + +// Merge merges the newcfg attribute values into this Config. Each attribute +// will be merged into this config if the newcfg attribute's value is non-zero. +// Due to this, newcfg attributes with zero values cannot be merged in. For +// example bool attributes cannot be cleared using Merge, and must be explicitly +// set on the Config structure. +func (c Config) Merge(newcfg *Config) *Config { + if newcfg == nil { + return &c + } + + cfg := Config{} + + if newcfg.Credentials != nil { + cfg.Credentials = newcfg.Credentials + } else { + cfg.Credentials = c.Credentials + } + + if newcfg.Endpoint != "" { + cfg.Endpoint = newcfg.Endpoint + } else { + cfg.Endpoint = c.Endpoint + } + + if newcfg.Ks3BillEndpoint != "" { + cfg.Ks3BillEndpoint = newcfg.Ks3BillEndpoint + } else { + cfg.Ks3BillEndpoint = c.Ks3BillEndpoint + } + + if newcfg.Region != "" { + cfg.Region = newcfg.Region + } else { + cfg.Region = c.Region + } + + if newcfg.DisableSSL { + cfg.DisableSSL = newcfg.DisableSSL + } else { + cfg.DisableSSL = c.DisableSSL + } + + if newcfg.ManualSend { + cfg.ManualSend = newcfg.ManualSend + } else { + cfg.ManualSend = c.ManualSend + } + + if newcfg.DisableDnsCache { + cfg.DisableDnsCache = newcfg.DisableDnsCache + } else { + cfg.DisableDnsCache = c.DisableDnsCache + } + + if newcfg.HTTPClient != nil { + cfg.HTTPClient = newcfg.HTTPClient + } else { + cfg.HTTPClient = c.HTTPClient + if !cfg.DisableDnsCache { + cfg.HTTPClient.Transport = DnsCacheTransport + } + } + defaultHTTPRedirect(cfg.HTTPClient) + + if newcfg.LogHTTPBody { + cfg.LogHTTPBody = newcfg.LogHTTPBody + } else { + cfg.LogHTTPBody = c.LogHTTPBody + } + + if newcfg.LogLevel != 0 { + cfg.LogLevel = newcfg.LogLevel + } else { + cfg.LogLevel = c.LogLevel + } + + if newcfg.Logger != nil { + cfg.Logger = newcfg.Logger + } else { + cfg.Logger = c.Logger + } + + if newcfg.MaxRetries != 0 { + cfg.MaxRetries = newcfg.MaxRetries + } else { + cfg.MaxRetries = c.MaxRetries + } + + if newcfg.RetryRule != nil { + cfg.RetryRule = newcfg.RetryRule + } else { + cfg.RetryRule = c.RetryRule + } + + if newcfg.ShouldRetry != nil { + cfg.ShouldRetry = newcfg.ShouldRetry + } else { + cfg.ShouldRetry = c.ShouldRetry + } + + if newcfg.DisableParamValidation { + cfg.DisableParamValidation = newcfg.DisableParamValidation + } else { + cfg.DisableParamValidation = c.DisableParamValidation + } + + if newcfg.DisableComputeChecksums { + cfg.DisableComputeChecksums = newcfg.DisableComputeChecksums + } else { + cfg.DisableComputeChecksums = c.DisableComputeChecksums + } + + if newcfg.S3ForcePathStyle { + cfg.S3ForcePathStyle = newcfg.S3ForcePathStyle + } else { + cfg.S3ForcePathStyle = c.S3ForcePathStyle + } + + if newcfg.DomainMode { + cfg.DomainMode = newcfg.DomainMode + } else { + cfg.DomainMode = c.DomainMode + } + if newcfg.SignerVersion != "" { + cfg.SignerVersion = newcfg.SignerVersion + } else { + cfg.SignerVersion = c.SignerVersion + } + if newcfg.CrcCheckEnabled { + cfg.CrcCheckEnabled = newcfg.CrcCheckEnabled + } else { + cfg.CrcCheckEnabled = c.CrcCheckEnabled + } + if newcfg.DisableRestProtocolURICleaning { + cfg.DisableRestProtocolURICleaning = newcfg.DisableRestProtocolURICleaning + } else { + cfg.DisableRestProtocolURICleaning = c.DisableRestProtocolURICleaning + } + + return &cfg +} + +// Define the level of the output log +const ( + Off = iota + Error + Warn + Info + Debug +) + +// LogTag Tag for each level of log +var LogTag = []string{"[error]", "[warn]", "[info]", "[debug]"} + +func (c Config) writeLog(level uint, format string, a ...interface{}) { + if c.LogLevel < level { + return + } + + var logBuffer bytes.Buffer + logBuffer.WriteString(time.Now().Format("2006/01/02 15:04:05")) + logBuffer.WriteString(" ") + logBuffer.WriteString(LogTag[level-1]) + logBuffer.WriteString(fmt.Sprintf(format, a...)) + if logBuffer.Bytes()[logBuffer.Len()-1] != '\n' { + logBuffer.WriteString("\n") + } + fmt.Fprintf(c.Logger, "%s", logBuffer.String()) +} + +func (c Config) LogError(format string, a ...interface{}) { + if c.LogLevel < Error { + return + } + c.writeLog(Error, format, a...) +} + +func (c Config) LogWarn(format string, a ...interface{}) { + if c.LogLevel < Warn { + return + } + c.writeLog(Warn, format, a...) +} + +func (c Config) LogInfo(format string, a ...interface{}) { + if c.LogLevel < Info { + return + } + c.writeLog(Info, format, a...) +} + +func (c Config) LogDebug(format string, a ...interface{}) { + if c.LogLevel < Debug { + return + } + c.writeLog(Debug, format, a...) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/context.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/context.go new file mode 100644 index 0000000000..52300747b7 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/context.go @@ -0,0 +1,22 @@ +package aws + +import ( + "context" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +type Context = context.Context + +// SetContext adds a Context to the current request that can be used to cancel +func (r *Request) SetContext(ctx Context) { + if ctx == nil { + r.Error = apierr.New("InvalidParameter", "context cannot be nil", nil) + } + r.context = ctx + r.HTTPRequest = r.HTTPRequest.WithContext(ctx) +} + +// BackgroundContext returns a context that will never be canceled +func BackgroundContext() Context { + return context.Background() +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/chain_provider.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/chain_provider.go new file mode 100644 index 0000000000..679719ae68 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/chain_provider.go @@ -0,0 +1,81 @@ +package credentials + +import ( + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +var ( + // ErrNoValidProvidersFoundInChain Is returned when there are no valid + // providers in the ChainProvider. + ErrNoValidProvidersFoundInChain = apierr.New("NoCredentialProviders", "no valid providers in chain", nil) +) + +// A ChainProvider will search for a provider which returns credentials +// and cache that provider until Retrieve is called again. +// +// The ChainProvider provides a way of chaining multiple providers together +// which will pick the first available using priority order of the Providers +// in the list. +// +// If none of the Providers retrieve valid credentials Value, ChainProvider's +// Retrieve() will return the error ErrNoValidProvidersFoundInChain. +// +// If a Provider is found which returns valid credentials Value ChainProvider +// will cache that Provider for all calls to IsExpired(), until Retrieve is +// called again. +// +// Example of ChainProvider to be used with an EnvProvider and EC2RoleProvider. +// In this example EnvProvider will first check if any credentials are available +// vai the environment variables. If there are none ChainProvider will check +// the next Provider in the list, EC2RoleProvider in this case. If EC2RoleProvider +// does not return any credentials ChainProvider will return the error +// ErrNoValidProvidersFoundInChain +// +// creds := NewChainCredentials( +// []Provider{ +// &EnvProvider{}, +// &EC2RoleProvider{}, +// }) +// creds.Retrieve() +// +type ChainProvider struct { + Providers []Provider + curr Provider +} + +// NewChainCredentials returns a pointer to a new Credentials object +// wrapping a chain of providers. +func NewChainCredentials(providers []Provider) *Credentials { + return NewCredentials(&ChainProvider{ + Providers: append([]Provider{}, providers...), + }) +} + +// Retrieve returns the credentials value or error if no provider returned +// without error. +// +// If a provider is found it will be cached and any calls to IsExpired() +// will return the expired state of the cached provider. +func (c *ChainProvider) Retrieve() (Value, error) { + for _, p := range c.Providers { + if creds, err := p.Retrieve(); err == nil { + c.curr = p + return creds, nil + } + } + c.curr = nil + + // TODO better error reporting. maybe report error for each failed retrieve? + + return Value{}, ErrNoValidProvidersFoundInChain +} + +// IsExpired will returned the expired state of the currently cached provider +// if there is one. If there is no current provider, true will be returned. +func (c *ChainProvider) IsExpired() bool { + if c.curr != nil { + return c.curr.IsExpired() + } + + return true +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/credentials.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/credentials.go new file mode 100644 index 0000000000..6f6ec03127 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/credentials.go @@ -0,0 +1,178 @@ +// Package credentials provides credential retrieval and management +// +// The Credentials is the primary method of getting access to and managing +// credentials Values. Using dependency injection retrieval of the credential +// values is handled by a object which satisfies the Provider interface. +// +// By default the Credentials.Get() will cache the successful result of a +// Provider's Retrieve() until Provider.IsExpired() returns true. At which +// point Credentials will call Provider's Retrieve() to get new credential Value. +// +// The Provider is responsible for determining when credentials Value have expired. +// It is also important to note that Credentials will always call Retrieve the +// first time Credentials.Get() is called. +// +// Example of using the environment variable credentials. +// +// creds := NewEnvCredentials() +// +// // Retrieve the credentials value +// credValue, err := creds.Get() +// if err != nil { +// // handle error +// } +// +// Example of forcing credentials to expire and be refreshed on the next Get(). +// This may be helpful to proactively expire credentials and refresh them sooner +// than they would naturally expire on their own. +// +// creds := NewCredentials(&EC2RoleProvider{}) +// creds.Expire() +// credsValue, err := creds.Get() +// // New credentials will be retrieved instead of from cache. +// +// +// Custom Provider +// +// Each Provider built into this package also provides a helper method to generate +// a Credentials pointer setup with the provider. To use a custom Provider just +// create a type which satisfies the Provider interface and pass it to the +// NewCredentials method. +// +// type MyProvider struct{} +// func (m *MyProvider) Retrieve() (Value, error) {...} +// func (m *MyProvider) IsExpired() bool {...} +// +// creds := NewCredentials(&MyProvider{}) +// credValue, err := creds.Get() +// +package credentials + +import ( + "sync" + "time" +) + +// Create an empty Credential object that can be used as dummy placeholder +// credentials for requests that do not need signed. +// +// This Credentials can be used to configure a service to not sign requests +// when making service API calls. For example, when accessing public +// s3 buckets. +// +// svc := s3.New(&aws.Config{Credentials: AnonymousCredentials}) +// // Access public S3 buckets. +// +var AnonymousCredentials = NewStaticCredentials("", "", "") + +// A Value is the AWS credentials value for individual credential fields. +type Value struct { + // AWS Access key ID + AccessKeyID string + + // AWS Secret Access Key + SecretAccessKey string + + // AWS Session Token + SessionToken string +} + +// A Provider is the interface for any component which will provide credentials +// Value. A provider is required to manage its own Expired state, and what to +// be expired means. +// +// The Provider should not need to implement its own mutexes, because +// that will be managed by Credentials. +type Provider interface { + // Refresh returns nil if it successfully retrieved the value. + // Error is returned if the value were not obtainable, or empty. + Retrieve() (Value, error) + + // IsExpired returns if the credentials are no longer valid, and need + // to be retrieved. + IsExpired() bool +} + +// A Credentials provides synchronous safe retrieval of AWS credentials Value. +// Credentials will cache the credentials value until they expire. Once the value +// expires the next Get will attempt to retrieve valid credentials. +// +// Credentials is safe to use across multiple goroutines and will manage the +// synchronous state so the Providers do not need to implement their own +// synchronization. +// +// The first Credentials.Get() will always call Provider.Retrieve() to get the +// first instance of the credentials Value. All calls to Get() after that +// will return the cached credentials Value until IsExpired() returns true. +type Credentials struct { + creds Value + forceRefresh bool + m sync.Mutex + + provider Provider +} + +// NewCredentials returns a pointer to a new Credentials with the provider set. +func NewCredentials(provider Provider) *Credentials { + return &Credentials{ + provider: provider, + forceRefresh: true, + } +} + +// Get returns the credentials value, or error if the credentials Value failed +// to be retrieved. +// +// Will return the cached credentials Value if it has not expired. If the +// credentials Value has expired the Provider's Retrieve() will be called +// to refresh the credentials. +// +// If Credentials.Expire() was called the credentials Value will be force +// expired, and the next call to Get() will cause them to be refreshed. +func (c *Credentials) Get() (Value, error) { + c.m.Lock() + defer c.m.Unlock() + + if c.isExpired() { + creds, err := c.provider.Retrieve() + if err != nil { + return Value{}, err + } + c.creds = creds + c.forceRefresh = false + } + + return c.creds, nil +} + +// Expire expires the credentials and forces them to be retrieved on the +// next call to Get(). +// +// This will override the Provider's expired state, and force Credentials +// to call the Provider's Retrieve(). +func (c *Credentials) Expire() { + c.m.Lock() + defer c.m.Unlock() + + c.forceRefresh = true +} + +// IsExpired returns if the credentials are no longer valid, and need +// to be retrieved. +// +// If the Credentials were forced to be expired with Expire() this will +// reflect that override. +func (c *Credentials) IsExpired() bool { + c.m.Lock() + defer c.m.Unlock() + + return c.isExpired() +} + +// isExpired helper method wrapping the definition of expired credentials. +func (c *Credentials) isExpired() bool { + return c.forceRefresh || c.provider.IsExpired() +} + +// Provide a stub-able time.Now for unit tests so expiry can be tested. +var currentTime = time.Now diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/ec2_role_provider.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/ec2_role_provider.go new file mode 100644 index 0000000000..639ae43033 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/ec2_role_provider.go @@ -0,0 +1,173 @@ +package credentials + +import ( + "bufio" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +const metadataCredentialsEndpoint = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + +// A EC2RoleProvider retrieves credentials from the EC2 service, and keeps track if +// those credentials are expired. +// +// Example how to configure the EC2RoleProvider with custom http Client, Endpoint +// or ExpiryWindow +// +// p := &credentials.EC2RoleProvider{ +// // Pass in a custom timeout to be used when requesting +// // IAM EC2 Role credentials. +// Client: &http.Client{ +// Timeout: 10 * time.Second, +// }, +// // Use default EC2 Role metadata endpoint, Alternate endpoints can be +// // specified setting Endpoint to something else. +// Endpoint: "", +// // Do not use early expiry of credentials. If a non zero value is +// // specified the credentials will be expired early +// ExpiryWindow: 0, +// } +// +type EC2RoleProvider struct { + // Endpoint must be fully quantified URL + Endpoint string + + // HTTP client to use when connecting to EC2 service + Client *http.Client + + // ExpiryWindow will allow the credentials to trigger refreshing prior to + // the credentials actually expiring. This is beneficial so race conditions + // with expiring credentials do not cause request to fail unexpectedly + // due to ExpiredTokenException exceptions. + // + // So a ExpiryWindow of 10s would cause calls to IsExpired() to return true + // 10 seconds before the credentials are actually expired. + // + // If ExpiryWindow is 0 or less it will be ignored. + ExpiryWindow time.Duration + + // The date/time at which the credentials expire. + expiresOn time.Time +} + +// NewEC2RoleCredentials returns a pointer to a new Credentials object +// wrapping the EC2RoleProvider. +// +// Takes a custom http.Client which can be configured for custom handling of +// things such as timeout. +// +// Endpoint is the URL that the EC2RoleProvider will connect to when retrieving +// role and credentials. +// +// Window is the expiry window that will be subtracted from the expiry returned +// by the role credential request. This is done so that the credentials will +// expire sooner than their actual lifespan. +func NewEC2RoleCredentials(client *http.Client, endpoint string, window time.Duration) *Credentials { + return NewCredentials(&EC2RoleProvider{ + Endpoint: endpoint, + Client: client, + ExpiryWindow: window, + }) +} + +// Retrieve retrieves credentials from the EC2 service. +// Error will be returned if the request fails, or unable to extract +// the desired credentials. +func (m *EC2RoleProvider) Retrieve() (Value, error) { + if m.Client == nil { + m.Client = http.DefaultClient + } + if m.Endpoint == "" { + m.Endpoint = metadataCredentialsEndpoint + } + + credsList, err := requestCredList(m.Client, m.Endpoint) + if err != nil { + return Value{}, err + } + + if len(credsList) == 0 { + return Value{}, apierr.New("EmptyEC2RoleList", "empty EC2 Role list", nil) + } + credsName := credsList[0] + + roleCreds, err := requestCred(m.Client, m.Endpoint, credsName) + if err != nil { + return Value{}, err + } + + m.expiresOn = roleCreds.Expiration + if m.ExpiryWindow > 0 { + // Offset based on expiry window if set. + m.expiresOn = m.expiresOn.Add(-m.ExpiryWindow) + } + + return Value{ + AccessKeyID: roleCreds.AccessKeyID, + SecretAccessKey: roleCreds.SecretAccessKey, + SessionToken: roleCreds.Token, + }, nil +} + +// IsExpired returns if the credentials are expired. +func (m *EC2RoleProvider) IsExpired() bool { + return m.expiresOn.Before(currentTime()) +} + +// A ec2RoleCredRespBody provides the shape for deserializing credential +// request responses. +type ec2RoleCredRespBody struct { + Expiration time.Time + AccessKeyID string + SecretAccessKey string + Token string +} + +// requestCredList requests a list of credentials from the EC2 service. +// If there are no credentials, or there is an error making or receiving the request +func requestCredList(client *http.Client, endpoint string) ([]string, error) { + resp, err := client.Get(endpoint) + if err != nil { + return nil, apierr.New("ListEC2Role", "failed to list EC2 Roles", err) + } + defer resp.Body.Close() + + credsList := []string{} + s := bufio.NewScanner(resp.Body) + for s.Scan() { + credsList = append(credsList, s.Text()) + } + + if err := s.Err(); err != nil { + return nil, apierr.New("ReadEC2Role", "failed to read list of EC2 Roles", err) + } + + return credsList, nil +} + +// requestCred requests the credentials for a specific credentials from the EC2 service. +// +// If the credentials cannot be found, or there is an error reading the response +// and error will be returned. +func requestCred(client *http.Client, endpoint, credsName string) (*ec2RoleCredRespBody, error) { + resp, err := client.Get(endpoint + credsName) + if err != nil { + return nil, apierr.New("GetEC2RoleCredentials", + fmt.Sprintf("failed to get %s EC2 Role credentials", credsName), + err) + } + defer resp.Body.Close() + + respCreds := &ec2RoleCredRespBody{} + if err := json.NewDecoder(resp.Body).Decode(respCreds); err != nil { + return nil, apierr.New("DecodeEC2RoleCredentials", + fmt.Sprintf("failed to decode %s EC2 Role credentials", credsName), + err) + } + + return respCreds, nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/env_provider.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/env_provider.go new file mode 100644 index 0000000000..75e7f60f91 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/env_provider.go @@ -0,0 +1,67 @@ +package credentials + +import ( + "os" + + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +var ( + // ErrAccessKeyIDNotFound is returned when the AWS Access Key ID can't be + // found in the process's environment. + ErrAccessKeyIDNotFound = apierr.New("EnvAccessKeyNotFound", "AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment", nil) + // ErrSecretAccessKeyNotFound is returned when the AWS Secret Access Key + // can't be found in the process's environment. + ErrSecretAccessKeyNotFound = apierr.New("EnvSecretNotFound", "AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment", nil) +) + +// A EnvProvider retrieves credentials from the environment variables of the +// running process. Environment credentials never expire. +// +// Environment variables used: +// - Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY +// - Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY +type EnvProvider struct { + retrieved bool +} + +// NewEnvCredentials returns a pointer to a new Credentials object +// wrapping the environment variable provider. +func NewEnvCredentials() *Credentials { + return NewCredentials(&EnvProvider{}) +} + +// Retrieve retrieves the keys from the environment. +func (e *EnvProvider) Retrieve() (Value, error) { + e.retrieved = false + + id := os.Getenv("AWS_ACCESS_KEY_ID") + if id == "" { + id = os.Getenv("AWS_ACCESS_KEY") + } + + secret := os.Getenv("AWS_SECRET_ACCESS_KEY") + if secret == "" { + secret = os.Getenv("AWS_SECRET_KEY") + } + + if id == "" { + return Value{}, ErrAccessKeyIDNotFound + } + + if secret == "" { + return Value{}, ErrSecretAccessKeyNotFound + } + + e.retrieved = true + return Value{ + AccessKeyID: id, + SecretAccessKey: secret, + SessionToken: os.Getenv("AWS_SESSION_TOKEN"), + }, nil +} + +// IsExpired returns if the credentials have been retrieved. +func (e *EnvProvider) IsExpired() bool { + return !e.retrieved +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/example.ini b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/example.ini new file mode 100644 index 0000000000..aa2dc506ad --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/example.ini @@ -0,0 +1,8 @@ +[default] +aws_access_key_id = accessKey +aws_secret_access_key = secret +aws_session_token = token + +[no_token] +aws_access_key_id = accessKey +aws_secret_access_key = secret diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/ini.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/ini.go new file mode 100644 index 0000000000..199df17755 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/ini.go @@ -0,0 +1,123 @@ +// Package ini provides functions for parsing INI configuration files. +package credentials + +import ( + "bufio" + "fmt" + "io" + "os" + "regexp" + "strings" +) + +var ( + sectionRegex = regexp.MustCompile(`^\[(.*)\]$`) + assignRegex = regexp.MustCompile(`^([^=]+)=(.*)$`) +) + +// ErrSyntax is returned when there is a syntax error in an INI file. +type ErrSyntax struct { + Line int + Source string // The contents of the erroneous line, without leading or trailing whitespace +} + +func (e ErrSyntax) Error() string { + return fmt.Sprintf("invalid INI syntax on line %d: %s", e.Line, e.Source) +} + +// A File represents a parsed INI file. +type File map[string]Section + +// A Section represents a single section of an INI file. +type Section map[string]string + +// Returns a named Section. A Section will be created if one does not already exist for the given name. +func (f File) Section(name string) Section { + section := f[name] + if section == nil { + section = make(Section) + f[name] = section + } + return section +} + +// Looks up a value for a key in a section and returns that value, along with a boolean result similar to a map lookup. +func (f File) Get(section, key string) (value string, ok bool) { + if s := f[section]; s != nil { + value, ok = s[key] + } + return +} + +// Loads INI data from a reader and stores the data in the File. +func (f File) Load(in io.Reader) (err error) { + bufin, ok := in.(*bufio.Reader) + if !ok { + bufin = bufio.NewReader(in) + } + return parseFile(bufin, f) +} + +// Loads INI data from a named file and stores the data in the File. +func (f File) LoadFile(file string) (err error) { + in, err := os.Open(file) + if err != nil { + return + } + defer in.Close() + return f.Load(in) +} + +func parseFile(in *bufio.Reader, file File) (err error) { + section := "" + lineNum := 0 + for done := false; !done; { + var line string + if line, err = in.ReadString('\n'); err != nil { + if err == io.EOF { + done = true + } else { + return + } + } + lineNum++ + line = strings.TrimSpace(line) + if len(line) == 0 { + // Skip blank lines + continue + } + if line[0] == ';' || line[0] == '#' { + // Skip comments + continue + } + + if groups := assignRegex.FindStringSubmatch(line); groups != nil { + key, val := groups[1], groups[2] + key, val = strings.TrimSpace(key), strings.TrimSpace(val) + file.Section(section)[key] = val + } else if groups := sectionRegex.FindStringSubmatch(line); groups != nil { + name := strings.TrimSpace(groups[1]) + section = name + // Create the section if it does not exist + file.Section(section) + } else { + return ErrSyntax{lineNum, line} + } + + } + return nil +} + +// Loads and returns a File from a reader. +func Load(in io.Reader) (File, error) { + file := make(File) + err := file.Load(in) + return file, err +} + +// Loads and returns an INI File from a file on disk. +func LoadFile(filename string) (File, error) { + file := make(File) + err := file.LoadFile(filename) + return file, err +} \ No newline at end of file diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/shared_credentials_provider.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/shared_credentials_provider.go new file mode 100644 index 0000000000..f1f38e9f33 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/shared_credentials_provider.go @@ -0,0 +1,131 @@ +package credentials + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +var ( + // ErrSharedCredentialsHomeNotFound is emitted when the user directory cannot be found. + ErrSharedCredentialsHomeNotFound = apierr.New("UserHomeNotFound", "user home directory not found.", nil) +) + +// A SharedCredentialsProvider retrieves credentials from the current user's home +// directory, and keeps track if those credentials are expired. +// +// Profile ini file example: $HOME/.aws/credentials +type SharedCredentialsProvider struct { + // Path to the shared credentials file. If empty will default to current user's + // home directory. + Filename string + + // AWS Profile to extract credentials from the shared credentials file. If empty + // will default to environment variable "AWS_PROFILE" or "default" if + // environment variable is also not set. + Profile string + + // retrieved states if the credentials have been successfully retrieved. + retrieved bool +} + +// NewSharedCredentials returns a pointer to a new Credentials object +// wrapping the Profile file provider. +func NewSharedCredentials(filename, profile string) *Credentials { + return NewCredentials(&SharedCredentialsProvider{ + Filename: filename, + Profile: profile, + }) +} + +// Retrieve reads and extracts the shared credentials from the current +// users home directory. +func (p *SharedCredentialsProvider) Retrieve() (Value, error) { + p.retrieved = false + + filename, err := p.filename() + if err != nil { + return Value{}, err + } + + creds, err := loadProfile(filename, p.profile()) + if err != nil { + return Value{}, err + } + + p.retrieved = true + return creds, nil +} + +// IsExpired returns if the shared credentials have expired. +func (p *SharedCredentialsProvider) IsExpired() bool { + return !p.retrieved +} + +// loadProfiles loads from the file pointed to by shared credentials filename for profile. +// The credentials retrieved from the profile will be returned or error. Error will be +// returned if it fails to read from the file, or the data is invalid. +func loadProfile(filename, profile string) (Value, error) { + config, err := LoadFile(filename) + if err != nil { + return Value{}, apierr.New("SharedCredsLoad", "failed to load shared credentials file", err) + } + iniProfile := config.Section(profile) + + id, ok := iniProfile["aws_access_key_id"] + if !ok { + return Value{}, apierr.New("SharedCredsAccessKey", + fmt.Sprintf("shared credentials %s in %s did not contain aws_access_key_id", profile, filename), + nil) + } + + secret, ok := iniProfile["aws_secret_access_key"] + if !ok { + return Value{}, apierr.New("SharedCredsSecret", + fmt.Sprintf("shared credentials %s in %s did not contain aws_secret_access_key", profile, filename), + nil) + } + + token := iniProfile["aws_session_token"] + + return Value{ + AccessKeyID: id, + SecretAccessKey: secret, + SessionToken: token, + }, nil +} + +// filename returns the filename to use to read AWS shared credentials. +// +// Will return an error if the user's home directory path cannot be found. +func (p *SharedCredentialsProvider) filename() (string, error) { + if p.Filename == "" { + homeDir := os.Getenv("HOME") // *nix + if homeDir == "" { // Windows + homeDir = os.Getenv("USERPROFILE") + } + if homeDir == "" { + return "", ErrSharedCredentialsHomeNotFound + } + + p.Filename = filepath.Join(homeDir, ".aws", "credentials") + } + + return p.Filename, nil +} + +// profile returns the AWS shared credentials profile. If empty will read +// environment variable "AWS_PROFILE". If that is not set profile will +// return "default". +func (p *SharedCredentialsProvider) profile() string { + if p.Profile == "" { + p.Profile = os.Getenv("AWS_PROFILE") + } + if p.Profile == "" { + p.Profile = "default" + } + + return p.Profile +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/static_provider.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/static_provider.go new file mode 100644 index 0000000000..253d79b32a --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/credentials/static_provider.go @@ -0,0 +1,42 @@ +package credentials + +import ( + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +var ( + // ErrStaticCredentialsEmpty is emitted when static credentials are empty. + ErrStaticCredentialsEmpty = apierr.New("EmptyStaticCreds", "static credentials are empty", nil) +) + +// A StaticProvider is a set of credentials which are set pragmatically, +// and will never expire. +type StaticProvider struct { + Value +} + +// NewStaticCredentials returns a pointer to a new Credentials object +// wrapping a static credentials value provider. +func NewStaticCredentials(id, secret, token string) *Credentials { + return NewCredentials(&StaticProvider{Value: Value{ + AccessKeyID: id, + SecretAccessKey: secret, + SessionToken: token, + }}) +} + +// Retrieve returns the credentials or error if the credentials are invalid. +func (s *StaticProvider) Retrieve() (Value, error) { + if s.AccessKeyID == "" || s.SecretAccessKey == "" { + return Value{}, ErrStaticCredentialsEmpty + } + + return s.Value, nil +} + +// IsExpired returns if the credentials are expired. +// +// For StaticProvider, the credentials never expired. +func (s *StaticProvider) IsExpired() bool { + return false +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/default_redirect.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/default_redirect.go new file mode 100644 index 0000000000..71c37aa564 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/default_redirect.go @@ -0,0 +1,22 @@ +package aws + +import ( + "errors" + "net/http" +) + +func defaultHTTPRedirect(client *http.Client) { + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + + // use prev Authorization if request has no Authorization + if req.Header.Get("Authorization") == "" { + prevAuth := via[len(via)-1].Header.Get("Authorization") + req.Header.Set("Authorization", prevAuth) + } + + return nil + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/dns_resolver.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/dns_resolver.go new file mode 100644 index 0000000000..8db3670889 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/dns_resolver.go @@ -0,0 +1,155 @@ +package aws + +import ( + "container/list" + "context" + "net" + "net/http" + "sync" + "time" +) + +var DnsCacheTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: DnsCacheTransportDialContext(&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }, NewDnsResolver(100)), + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, +} + +func DnsCacheTransportDialContext(dialer *net.Dialer, resolver *DnsResolver) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + conn, err := dialer.DialContext(ctx, network, address) + if err != nil { + remoteAddr, exists := resolver.cache.Get(address) + if exists { + return dialer.DialContext(ctx, network, remoteAddr) + } + return conn, err + } + + if conn.RemoteAddr().String() != "" { + resolver.cache.Set(address, conn.RemoteAddr().String()) + } + + return conn, err + } +} + +type DnsResolver struct { + cache *FIFOCache +} + +// NewDnsResolver 创建一个新的 DNS 解析器，使用固定长度的 FIFO 缓存 +func NewDnsResolver(maxSize int) *DnsResolver { + return &DnsResolver{ + cache: NewFIFOCache(maxSize), + } +} + +// FIFOCache 实现一个固定大小的并发安全FIFO（先进先出）缓存 +// 当缓存满时，添加新条目会自动淘汰最早插入的条目 +type FIFOCache struct { + maxSize int // 缓存的最大容量 + cache map[string]string // 存储键值对的映射 + keys *list.List // 使用链表维护键的插入顺序（FIFO） + rwMutex sync.RWMutex // 读写锁，保证并发安全 +} + +// NewFIFOCache 创建并返回一个新的FIFOCache实例 +// 参数: +// +// maxSize: 缓存的最大容量，必须大于0，否则默认为100 +// +// 返回值: +// +// *FIFOCache: 新创建的缓存实例 +func NewFIFOCache(maxSize int) *FIFOCache { + if maxSize < 1 { + maxSize = 100 + } + return &FIFOCache{ + maxSize: maxSize, + cache: make(map[string]string), + keys: list.New(), + } +} + +// Set 添加或更新一个键值对到缓存中 +// 如果缓存已满且键不存在，会淘汰最早插入的条目 +// 参数: +// +// key: 要添加或更新的键 +// value: 要添加或更新的值 +// +// 返回值: +// +// string: 如果键已存在，返回被替换的旧值；否则返回空字符串 +// bool: 表示是否替换了现有值（true表示替换，false表示新增） +func (c *FIFOCache) Set(key, value string) (string, bool) { + c.rwMutex.Lock() + defer c.rwMutex.Unlock() + + // 检查键是否已存在 + oldValue, exists := c.cache[key] + if exists { + // 更新现有键的值 + c.cache[key] = value + return oldValue, true + } + + // 缓存已满，淘汰最旧的条目 + if c.keys.Len() >= c.maxSize { + oldest := c.keys.Front() + if oldest != nil { + oldestKey := oldest.Value.(string) + delete(c.cache, oldestKey) + c.keys.Remove(oldest) + } + } + + // 添加新条目 + c.cache[key] = value + c.keys.PushBack(key) + return "", false +} + +// Get 从缓存中获取指定键对应的值 +// 参数: +// +// key: 要查找的键 +// +// 返回值: +// +// string: 找到的值（如果键不存在则返回空字符串） +// bool: 表示键是否存在（true表示存在，false表示不存在） +func (c *FIFOCache) Get(key string) (string, bool) { + c.rwMutex.RLock() + defer c.rwMutex.RUnlock() + + value, exists := c.cache[key] + return value, exists +} + +// Size 返回缓存中当前存储的条目数量 +// 返回值: +// +// int: 当前缓存中的条目数量 +func (c *FIFOCache) Size() int { + c.rwMutex.RLock() + defer c.rwMutex.RUnlock() + return len(c.cache) +} + +// GetMaxSize 返回缓存的最大容量 +// 返回值: +// +// int: 缓存的最大容量 +func (c *FIFOCache) GetMaxSize() int { + return c.maxSize +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/handler_functions.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/handler_functions.go new file mode 100644 index 0000000000..2e93333cdc --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/handler_functions.go @@ -0,0 +1,161 @@ +package aws + +import ( + "bytes" + "fmt" + "github.com/ks3sdklib/aws-sdk-go/aws/retry" + "io" + "io/ioutil" + "net/http" + "net/url" + "regexp" + "strconv" + "time" + + "github.com/ks3sdklib/aws-sdk-go/aws/awserr" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +var sleepDelay = func(delay time.Duration) { + time.Sleep(delay) +} + +// Interface for matching types which also have a Len method. +type lener interface { + Len() int +} + +// BuildContentLength builds the content length of a request based on the body, +// or will use the HTTPRequest.Header's "Content-Length" if defined. If unable +// to determine request body length and no "Content-Length" was specified it will panic. +func BuildContentLength(r *Request) { + if slength := r.HTTPRequest.Header.Get("Content-Length"); slength != "" { + length, _ := strconv.ParseInt(slength, 10, 64) + r.HTTPRequest.ContentLength = length + return + } + + var length int64 + switch body := r.Body.(type) { + case nil: + length = 0 + case lener: + length = int64(body.Len()) + case io.Seeker: + r.bodyStart, _ = body.Seek(0, 1) + end, _ := body.Seek(0, 2) + body.Seek(r.bodyStart, 0) // make sure to seek back to original location + length = end - r.bodyStart + default: + panic("Cannot get length of body, must provide `ContentLength`") + } + + r.HTTPRequest.ContentLength = length + r.HTTPRequest.Header.Set("Content-Length", fmt.Sprintf("%d", length)) +} + +// UserAgentHandler is a request handler for injecting User agent into requests. +func UserAgentHandler(r *Request) { + r.HTTPRequest.Header.Set("User-Agent", SDKName+"/"+SDKVersion) +} +func ContentTypeHandler(r *Request) { + if len(r.HTTPRequest.Header["Content-Type"]) == 0 { + r.HTTPRequest.Header.Set("Content-Type", "application/xml") + } +} + +var reStatusCode = regexp.MustCompile(`^(\d+)`) + +// SendHandler is a request handler to send service request using HTTP client. +func SendHandler(r *Request) { + + var err error + if r.HTTPRequest.ContentLength <= 0 { + r.HTTPRequest.Body = http.NoBody + } + r.HTTPResponse, err = r.Service.Config.HTTPClient.Do(r.HTTPRequest) + if err != nil { + // Capture the case where url.Error is returned for error processing + // response. e.g. 301 without location header comes back as string + // error and r.HTTPResponse is nil. Other url redirect errors will + // comeback in a similar method. + if e, ok := err.(*url.Error); ok { + if s := reStatusCode.FindStringSubmatch(e.Error()); s != nil { + code, _ := strconv.ParseInt(s[1], 10, 64) + r.HTTPResponse = &http.Response{ + StatusCode: int(code), + Status: http.StatusText(int(code)), + Body: ioutil.NopCloser(bytes.NewReader([]byte{})), + } + return + } + } + // Catch all other request errors. + r.Error = apierr.New("RequestError", "send request failed", err) + r.Retryable.Set(true) // network errors are retryable + } +} + +// ValidateResponseHandler is a request handler to validate service response. +func ValidateResponseHandler(r *Request) { + if r.HTTPResponse.StatusCode == 0 || r.HTTPResponse.StatusCode >= 300 { + // this may be replaced by an UnmarshalError handler + r.Error = apierr.New("UnknownError", "unknown error", nil) + } +} + +// AfterRetryHandler performs final checks to determine if the request should +// be retried and how long to delay. +func AfterRetryHandler(r *Request) { + // If one of the other handlers already set the retry state + // we don't want to override it based on the service's state + if !r.Retryable.IsSet() { + r.Retryable.Set(r.Service.ShouldRetry(r.Error)) + } + + if r.WillRetry() { + r.RetryCount++ + delay := r.Service.RetryRule.GetDelay(int(r.RetryCount)) + if delay < 0 { + delay = 0 + } + r.RetryDelay = delay + + r.Config.LogWarn("Tried %d times, will retry in %d ms.", r.RetryCount, r.RetryDelay.Milliseconds()) + sleepDelay(r.RetryDelay) + + // when the expired token exception occurs the credentials + // need to be expired locally so that the next request to + // get credentials will trigger a credentials refresh. + if r.Error != nil { + if err, ok := r.Error.(awserr.Error); ok { + if retry.IsCodeExpiredCreds(err.Code()) { + r.Config.Credentials.Expire() + } + } + } + + r.Error = nil + } +} + +var ( + // ErrMissingRegion is an error that is returned if region configuration is + // not found. + ErrMissingRegion error = apierr.New("MissingRegion", "could not find region configuration", nil) + + // ErrMissingEndpoint is an error that is returned if an endpoint cannot be + // resolved for a service. + ErrMissingEndpoint error = apierr.New("MissingEndpoint", "'Endpoint' configuration is required for this service", nil) +) + +// ValidateEndpointHandler is a request handler to validate a request had the +// appropriate Region and Endpoint set. Will set r.Error if the endpoint or +// region is not valid. +func ValidateEndpointHandler(r *Request) { + if r.Service.SigningRegion == "" && r.Service.Config.Region == "" { + r.Error = ErrMissingRegion + } else if r.Service.Endpoint == "" { + r.Error = ErrMissingEndpoint + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/handlers.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/handlers.go new file mode 100644 index 0000000000..318cba5c3a --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/handlers.go @@ -0,0 +1,88 @@ +package aws + +// A Handlers provides a collection of request handlers for various +// stages of handling requests. +type Handlers struct { + Validate HandlerList + Build HandlerList + Sign HandlerList + Send HandlerList + ValidateResponse HandlerList + Unmarshal HandlerList + UnmarshalMeta HandlerList + UnmarshalError HandlerList + Retry HandlerList + AfterRetry HandlerList + CheckCrc64 HandlerList +} + +// copy returns of this handler's lists. +func (h *Handlers) copy() Handlers { + return Handlers{ + Validate: h.Validate.copy(), + Build: h.Build.copy(), + Sign: h.Sign.copy(), + Send: h.Send.copy(), + ValidateResponse: h.ValidateResponse.copy(), + Unmarshal: h.Unmarshal.copy(), + UnmarshalError: h.UnmarshalError.copy(), + UnmarshalMeta: h.UnmarshalMeta.copy(), + Retry: h.Retry.copy(), + AfterRetry: h.AfterRetry.copy(), + CheckCrc64: h.CheckCrc64.copy(), + } +} + +// Clear removes callback functions for all handlers +func (h *Handlers) Clear() { + h.Validate.Clear() + h.Build.Clear() + h.Send.Clear() + h.Sign.Clear() + h.Unmarshal.Clear() + h.UnmarshalMeta.Clear() + h.UnmarshalError.Clear() + h.ValidateResponse.Clear() + h.Retry.Clear() + h.AfterRetry.Clear() + h.CheckCrc64.Clear() +} + +// A HandlerList manages zero or more handlers in a list. +type HandlerList struct { + list []func(*Request) +} + +// copy creates a copy of the handler list. +func (l *HandlerList) copy() HandlerList { + var n HandlerList + n.list = append([]func(*Request){}, l.list...) + return n +} + +// Clear clears the handler list. +func (l *HandlerList) Clear() { + l.list = []func(*Request){} +} + +// Len returns the number of handlers in the list. +func (l *HandlerList) Len() int { + return len(l.list) +} + +// PushBack pushes handlers f to the back of the handler list. +func (l *HandlerList) PushBack(f ...func(*Request)) { + l.list = append(l.list, f...) +} + +// PushFront pushes handlers f to the front of the handler list. +func (l *HandlerList) PushFront(f ...func(*Request)) { + l.list = append(f, l.list...) +} + +// Run executes all handlers in the list with a given request object. +func (l *HandlerList) Run(r *Request) { + for _, f := range l.list { + f(r) + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/param_validator.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/param_validator.go new file mode 100644 index 0000000000..5a293352bb --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/param_validator.go @@ -0,0 +1,102 @@ +package aws + +import ( + "fmt" + "reflect" + "strings" + + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +// ValidateParameters is a request handler to validate the input parameters. +// Validating parameters only has meaning if done prior to the request being sent. +func ValidateParameters(r *Request) { + if r.ParamsFilled() { + v := validator{errors: []string{}} + v.validateAny(reflect.ValueOf(r.Params), "") + + if count := len(v.errors); count > 0 { + format := "%d validation errors:\n- %s" + msg := fmt.Sprintf(format, count, strings.Join(v.errors, "\n- ")) + r.Error = apierr.New("InvalidParameter", msg, nil) + } + } +} + +// A validator validates values. Collects validations errors which occurs. +type validator struct { + errors []string +} + +// validateAny will validate any struct, slice or map type. All validations +// are also performed recursively for nested types. +func (v *validator) validateAny(value reflect.Value, path string) { + value = reflect.Indirect(value) + if !value.IsValid() { + return + } + + switch value.Kind() { + case reflect.Struct: + v.validateStruct(value, path) + case reflect.Slice: + for i := 0; i < value.Len(); i++ { + v.validateAny(value.Index(i), path+fmt.Sprintf("[%d]", i)) + } + case reflect.Map: + for _, n := range value.MapKeys() { + v.validateAny(value.MapIndex(n), path+fmt.Sprintf("[%q]", n.String())) + } + } +} + +// validateStruct will validate the struct value's fields. If the structure has +// nested types those types will be validated also. +func (v *validator) validateStruct(value reflect.Value, path string) { + prefix := "." + if path == "" { + prefix = "" + } + + for i := 0; i < value.Type().NumField(); i++ { + f := value.Type().Field(i) + if strings.ToLower(f.Name[0:1]) == f.Name[0:1] { + continue + } + fvalue := value.FieldByName(f.Name) + + notset := false + notval := false + if f.Tag.Get("required") != "" { + switch fvalue.Kind() { + case reflect.Ptr: + if fvalue.IsNil() { + notset = true + } else { + elem := fvalue.Elem() + if elem.Kind() == reflect.String && elem.Len() == 0 { + notval = true + } + } + case reflect.Slice, reflect.Map: + if fvalue.IsNil() { + notset = true + } + default: + if !fvalue.IsValid() { + notset = true + } + } + } + + if notset { + msg := "missing required parameter: " + path + prefix + f.Name + v.errors = append(v.errors, msg) + } else if notval { + msg := "input member " + path + prefix + f.Name + " must not be empty" + v.errors = append(v.errors, msg) + } else { + v.validateAny(fvalue, path+prefix+f.Name) + } + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/progress.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/progress.go new file mode 100644 index 0000000000..412db11ee4 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/progress.go @@ -0,0 +1,96 @@ +package aws + +import ( + "bytes" + "io" + "os" + "strings" +) + +type ProgressFunc func(increment, completed, total int64) + +type teeReader struct { + reader io.Reader + writer io.Writer + tracker *readerTracker +} + +type readerTracker struct { + completedBytes int64 + totalBytes int64 + progressFunc ProgressFunc +} + +// TeeReader returns a Reader that writes to w what it reads from r. +// All reads from r performed through it are matched with +// corresponding writes to w. There is no internal buffering - +// to write must complete before the read completes. +// Any error encountered while writing is reported as a read error. +func TeeReader(reader io.Reader, writer io.Writer, totalBytes int64, progressFunc ProgressFunc) io.ReadCloser { + return &teeReader{ + reader: reader, + writer: writer, + tracker: &readerTracker{ + completedBytes: 0, + totalBytes: totalBytes, + progressFunc: progressFunc, + }, + } +} + +func (t *teeReader) Read(p []byte) (n int, err error) { + n, err = t.reader.Read(p) + + // Read encountered error + if err != nil && err != io.EOF { + return + } + + if n > 0 { + // update completedBytes + t.tracker.completedBytes += int64(n) + + if t.tracker.progressFunc != nil { + // report progress + t.tracker.progressFunc(int64(n), t.tracker.completedBytes, t.tracker.totalBytes) + } + // CRC + if t.writer != nil { + if n, err := t.writer.Write(p[:n]); err != nil { + return n, err + } + } + } + + return +} + +func (t *teeReader) Close() error { + if rc, ok := t.reader.(io.ReadCloser); ok { + return rc.Close() + } + return nil +} + +// GetReaderLen returns the length of the reader +func GetReaderLen(reader io.Reader) int64 { + var contentLength int64 + switch v := reader.(type) { + case *bytes.Buffer: + contentLength = int64(v.Len()) + case *bytes.Reader: + contentLength = int64(v.Len()) + case *strings.Reader: + contentLength = int64(v.Len()) + case *os.File: + fileInfo, err := v.Stat() + if err != nil { + contentLength = 0 + } else { + contentLength = fileInfo.Size() + } + default: + contentLength = 0 + } + return contentLength +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/request.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/request.go new file mode 100644 index 0000000000..22a5ea34c0 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/request.go @@ -0,0 +1,326 @@ +package aws + +import ( + "bytes" + "github.com/ks3sdklib/aws-sdk-go/aws/awsutil" + "github.com/ks3sdklib/aws-sdk-go/internal/crc" + "hash" + "io" + "net/http" + "net/url" + "reflect" + "strings" + "time" +) + +// A Request is the service request to be made. +type Request struct { + *Service + Handlers Handlers + Time time.Time + ExpireTime int64 + Operation *Operation + HTTPRequest *http.Request + HTTPResponse *http.Response + Body io.ReadSeeker + bodyStart int64 // offset from beginning of Body that the request body starts + Params interface{} + Error error + Data interface{} + RequestID string + RetryCount uint + Retryable SettableBool + RetryDelay time.Duration + built bool + context Context + Crc64 hash.Hash64 + ProgressFn ProgressFunc + ContentType string + RequestType string +} + +// An Operation is the service API operation to be made. +type Operation struct { + Name string + HTTPMethod string + HTTPPath string + *Paginator +} + +// Paginator keeps track of pagination configuration for an API operation. +type Paginator struct { + InputTokens []string + OutputTokens []string + LimitToken string + TruncationToken string +} + +// NewRequest returns a new Request pointer for the service API +// operation and parameters. +// +// Params is any value of input parameters to be the request payload. +// Data is pointer value to an object which the request's response +// payload will be deserialized to. +func NewRequest(service *Service, operation *Operation, params interface{}, data interface{}) *Request { + method := operation.HTTPMethod + if method == "" { + method = "POST" + } + p := operation.HTTPPath + if p == "" { + p = "/" + } + + httpReq, _ := http.NewRequest(method, "", nil) + httpReq.URL, _ = url.Parse(service.Endpoint + p) + + r := &Request{ + Service: service, + Handlers: service.Handlers.copy(), + Time: time.Now(), + ExpireTime: 0, + Operation: operation, + HTTPRequest: httpReq, + Body: nil, + Params: params, + Error: nil, + Data: data, + } + r.SetBufferBody([]byte{}) + return r +} + +func NewRequestV2(service *Service, method string, params interface{}) *Request { + + httpReq, _ := http.NewRequest(method, "", nil) + httpReq.URL, _ = url.Parse(service.Endpoint) + r := &Request{ + Service: service, + ExpireTime: 0, + HTTPRequest: httpReq, + Params: params, + Error: nil, + } + return r +} + +// WillRetry returns if the request's can be retried. +func (r *Request) WillRetry() bool { + return r.Error != nil && r.Retryable.Get() && int(r.RetryCount) < r.Service.MaxRetries +} + +// ParamsFilled returns if the request's parameters have been populated +// and the parameters are valid. False is returned if no parameters are +// provided or invalid. +func (r *Request) ParamsFilled() bool { + return r.Params != nil && reflect.ValueOf(r.Params).Elem().IsValid() +} + +// DataFilled returns true if the request's data for response deserialization +// target has been set and is a valid. False is returned if data is not +// set, or is invalid. +func (r *Request) DataFilled() bool { + return r.Data != nil && reflect.ValueOf(r.Data).Elem().IsValid() +} + +// SetBufferBody will set the request's body bytes that will be sent to +// the service API. +func (r *Request) SetBufferBody(buf []byte) { + r.SetReaderBody(bytes.NewReader(buf)) +} + +// SetStringBody sets the body of the request to be backed by a string. +func (r *Request) SetStringBody(s string) { + r.SetReaderBody(strings.NewReader(s)) +} + +// SetReaderBody will set the request's body reader. +func (r *Request) SetReaderBody(reader io.ReadSeeker) { + if r.Config.CrcCheckEnabled { + r.Crc64 = crc.NewCRC(crc.CrcTable(), 0) + } + if r.Config.CrcCheckEnabled || r.ProgressFn != nil { + r.HTTPRequest.Body = TeeReader(reader, r.Crc64, GetReaderLen(reader), r.ProgressFn) + } else { + r.HTTPRequest.Body = io.NopCloser(reader) + } + r.Body = reader +} + +// Build will build the request's object, so it can be signed and sent +// to the service. Build will also validate all the request's parameters. +// Anny additional build Handlers set on this request will be run +// in the order they were set. +// +// The request will only be built once. Multiple calls to build will have +// no effect. +// +// If any Validate or Build errors occur the build will stop and the error +// which occurred will be returned. +func (r *Request) Build() error { + if !r.built { + r.Error = nil + r.Handlers.Validate.Run(r) + if r.Error != nil { + return r.Error + } + r.Handlers.Build.Run(r) + r.built = true + } + + return r.Error +} + +// Sign will sign the request retuning error if errors are encountered. +// +// Send will build the request prior to signing. All Sign Handlers will +// be executed in the order they were set. +func (r *Request) Sign() error { + r.Build() + if r.Error != nil { + return r.Error + } + + r.Handlers.Sign.Run(r) + return r.Error +} + +// Send will send the request returning error if errors are encountered. +// +// Send will sign the request prior to sending. All Send Handlers will +// be executed in the order they were set. +func (r *Request) Send() error { + for { + r.Sign() + + if r.Error != nil { + return r.Error + } + if r.Retryable.Get() { + // Re-seek the body back to the original point in for a retry so that + // send will send the body's contents again in the upcoming request. + r.Body.Seek(r.bodyStart, 0) + } + r.Retryable.Reset() + + r.Handlers.Send.Run(r) + if r.Error != nil { + r.Handlers.Retry.Run(r) + r.Handlers.AfterRetry.Run(r) + if r.Error != nil { + return r.Error + } + continue + } + + r.Handlers.UnmarshalMeta.Run(r) + r.Handlers.ValidateResponse.Run(r) + if r.Error != nil { + r.Handlers.UnmarshalError.Run(r) + r.Handlers.Retry.Run(r) + r.Handlers.AfterRetry.Run(r) + if r.Error != nil { + return r.Error + } + continue + } + + r.Handlers.Unmarshal.Run(r) + r.Handlers.CheckCrc64.Run(r) + if r.Error != nil { + r.Handlers.Retry.Run(r) + r.Handlers.AfterRetry.Run(r) + if r.Error != nil { + return r.Error + } + continue + } + + break + } + + return nil +} + +// HasNextPage returns true if this request has more pages of data available. +func (r *Request) HasNextPage() bool { + return r.nextPageTokens() != nil +} + +// nextPageTokens returns the tokens to use when asking for the next page of +// data. +func (r *Request) nextPageTokens() []interface{} { + if r.Operation.Paginator == nil { + return nil + } + + if r.Operation.TruncationToken != "" { + tr := awsutil.ValuesAtAnyPath(r.Data, r.Operation.TruncationToken) + if tr == nil || len(tr) == 0 { + return nil + } + switch v := tr[0].(type) { + case bool: + if v == false { + return nil + } + } + } + + found := false + tokens := make([]interface{}, len(r.Operation.OutputTokens)) + + for i, outtok := range r.Operation.OutputTokens { + v := awsutil.ValuesAtAnyPath(r.Data, outtok) + if v != nil && len(v) > 0 { + found = true + tokens[i] = v[0] + } + } + + if found { + return tokens + } + return nil +} + +// NextPage returns a new Request that can be executed to return the next +// page of result data. Call .Send() on this request to execute it. +func (r *Request) NextPage() *Request { + tokens := r.nextPageTokens() + if tokens == nil { + return nil + } + + data := reflect.New(reflect.TypeOf(r.Data).Elem()).Interface() + nr := NewRequest(r.Service, r.Operation, awsutil.CopyOf(r.Params), data) + for i, intok := range nr.Operation.InputTokens { + awsutil.SetValueAtAnyPath(nr.Params, intok, tokens[i]) + } + return nr +} + +// EachPage iterates over each page of a paginated request object. The fn +// parameter should be a function with the following sample signature: +// +// func(page *T, lastPage bool) bool { +// return true // return false to stop iterating +// } +// +// Where "T" is the structure type matching the output structure of the given +// operation. For example, a request object generated by +// DynamoDB.ListTablesRequest() would expect to see dynamodb.ListTablesOutput +// as the structure "T". The lastPage value represents whether the page is +// the last page of data or not. The return value of this function should +// return true to keep iterating or false to stop. +func (r *Request) EachPage(fn func(data interface{}, isLastPage bool) (shouldContinue bool)) error { + for page := r; page != nil; page = page.NextPage() { + page.Send() + shouldContinue := fn(page.Data, !page.HasNextPage()) + if page.Error != nil || !shouldContinue { + return page.Error + } + } + + return nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/exponential_retry.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/exponential_retry.go new file mode 100644 index 0000000000..52e188eb5a --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/exponential_retry.go @@ -0,0 +1,37 @@ +package retry + +import ( + "math" + "time" +) + +// ExponentialRetryRule 指数级增长等待时间重试规则 +type ExponentialRetryRule struct { + baseDelay time.Duration // 基础等待时间 + maxDelay time.Duration // 单次最大等待时间 +} + +var DefaultExponentialRetryRule = NewExponentialRetryRule(DefaultBaseDelay, DefaultMaxDelay) + +func NewExponentialRetryRule(baseDelay time.Duration, maxDelay time.Duration) ExponentialRetryRule { + if baseDelay < 0 { + baseDelay = DefaultBaseDelay + } + + if maxDelay < 0 { + maxDelay = DefaultMaxDelay + } + + return ExponentialRetryRule{ + baseDelay: baseDelay, + maxDelay: maxDelay, + } +} + +func (r ExponentialRetryRule) GetDelay(attempts int) time.Duration { + delay := r.baseDelay * time.Duration(math.Pow(2, float64(attempts-1))) + if delay > r.maxDelay { + return r.maxDelay + } + return delay +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/fixed_retry.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/fixed_retry.go new file mode 100644 index 0000000000..c97d619e19 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/fixed_retry.go @@ -0,0 +1,26 @@ +package retry + +import ( + "time" +) + +// FixedRetryRule 固定等待时间重试规则 +type FixedRetryRule struct { + baseDelay time.Duration // 基础等待时间 +} + +var DefaultFixedRetryRule = NewFixedRetryRule(DefaultBaseDelay) + +func NewFixedRetryRule(baseDelay time.Duration) FixedRetryRule { + if baseDelay < 0 { + baseDelay = DefaultBaseDelay + } + + return FixedRetryRule{ + baseDelay: baseDelay, + } +} + +func (r FixedRetryRule) GetDelay(attempts int) time.Duration { + return r.baseDelay +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/linear_retry.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/linear_retry.go new file mode 100644 index 0000000000..01e87eea12 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/linear_retry.go @@ -0,0 +1,36 @@ +package retry + +import ( + "time" +) + +// LinearRetryRule 线性增长等待时间重试规则 +type LinearRetryRule struct { + baseDelay time.Duration // 基础等待时间 + maxDelay time.Duration // 单次最大等待时间 +} + +var DefaultLinearRetryRule = NewLinearRetryRule(DefaultBaseDelay, DefaultMaxDelay) + +func NewLinearRetryRule(baseDelay time.Duration, maxDelay time.Duration) LinearRetryRule { + if baseDelay < 0 { + baseDelay = DefaultBaseDelay + } + + if maxDelay < 0 { + maxDelay = DefaultMaxDelay + } + + return LinearRetryRule{ + baseDelay: baseDelay, + maxDelay: maxDelay, + } +} + +func (r LinearRetryRule) GetDelay(attempts int) time.Duration { + delay := r.baseDelay * time.Duration(attempts) + if delay > r.maxDelay { + return r.maxDelay + } + return delay +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/no_delay_retry.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/no_delay_retry.go new file mode 100644 index 0000000000..4696a26368 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/no_delay_retry.go @@ -0,0 +1,18 @@ +package retry + +import ( + "time" +) + +// NoDelayRetryRule 不等待重试规则（立即重试） +type NoDelayRetryRule struct{} + +var DefaultNoDelayRetryRule = NewNoDelayRetryRule() + +func NewNoDelayRetryRule() NoDelayRetryRule { + return NoDelayRetryRule{} +} + +func (r NoDelayRetryRule) GetDelay(attempts int) time.Duration { + return 0 +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/random_retry.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/random_retry.go new file mode 100644 index 0000000000..be67250472 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/random_retry.go @@ -0,0 +1,39 @@ +package retry + +import ( + "math/rand" + "time" +) + +// RandomRetryRule 随机等待时间重试规则 +type RandomRetryRule struct { + minDelay time.Duration // 最小随机等待时间 + maxDelay time.Duration // 最大随机等待时间 +} + +var DefaultRandomRetryRule = NewRandomRetryRule(DefaultRandomMinDelay, DefaultRandomMaxDelay) + +func NewRandomRetryRule(minDelay time.Duration, maxDelay time.Duration) RandomRetryRule { + if minDelay < 0 { + minDelay = DefaultRandomMinDelay + } + + if maxDelay < 0 { + maxDelay = DefaultRandomMaxDelay + } + + if maxDelay < minDelay { + minDelay = DefaultRandomMinDelay + maxDelay = DefaultRandomMaxDelay + } + + return RandomRetryRule{ + minDelay: minDelay, + maxDelay: maxDelay, + } +} + +func (r RandomRetryRule) GetDelay(attempts int) time.Duration { + delay := r.minDelay + time.Duration(rand.Int63n(int64(r.maxDelay-r.minDelay+1))) + return delay +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/retry.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/retry.go new file mode 100644 index 0000000000..73c8a13bc4 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/retry.go @@ -0,0 +1,19 @@ +package retry + +import "time" + +const ( + // DefaultMaxDelay 默认最大等待时间 + DefaultMaxDelay = 20 * time.Second + // DefaultBaseDelay 默认基础等待时间 + DefaultBaseDelay = 200 * time.Millisecond + // DefaultRandomMinDelay 默认随机最小等待时间 + DefaultRandomMinDelay = 0 + // DefaultRandomMaxDelay 默认随机最大等待时间 + DefaultRandomMaxDelay = 200 * time.Millisecond +) + +// RetryRule 重试等待规则 +type RetryRule interface { + GetDelay(attempts int) time.Duration +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/retryable_error.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/retryable_error.go new file mode 100644 index 0000000000..f42acecb89 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/retry/retryable_error.go @@ -0,0 +1,69 @@ +package retry + +import ( + "errors" + "github.com/ks3sdklib/aws-sdk-go/aws/awserr" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +// ShouldRetry 判断是否需要重试 +// 重试条件： +// 1.状态码为5xx +// 2.状态码在retryErrorCodes中 +// 3.错误码在retryableCodes中 +func ShouldRetry(err error) bool { + var requestError *apierr.RequestError + if errors.As(err, &requestError) { + if requestError.StatusCode() >= 500 { + return true + } + + for _, code := range retryErrorCodes { + if requestError.StatusCode() == code { + return true + } + } + } + + if err, ok := err.(awserr.Error); ok { + return IsCodeRetryable(err.Code()) + } + + return false +} + +// 重试错误码 +var retryErrorCodes = []int{ + 408, // RequestTimeout + 429, // TooManyRequests +} + +// retryableCodes is a collection of service response codes which are retry-able +// without any further action. +var retryableCodes = map[string]struct{}{ + "RequestError": {}, + "ProvisionedThroughputExceededException": {}, + "Throttling": {}, +} + +// credsExpiredCodes is a collection of error codes which signify the credentials +// need to be refreshed. Expired tokens require refreshing of credentials, and +// resigning before the request can be retried. +var credsExpiredCodes = map[string]struct{}{ + "ExpiredToken": {}, + "ExpiredTokenException": {}, + "RequestExpired": {}, +} + +func IsCodeRetryable(code string) bool { + if _, ok := retryableCodes[code]; ok { + return true + } + + return IsCodeExpiredCreds(code) +} + +func IsCodeExpiredCreds(code string) bool { + _, ok := credsExpiredCodes[code] + return ok +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/service.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/service.go new file mode 100644 index 0000000000..aff6158677 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/service.go @@ -0,0 +1,112 @@ +package aws + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws/retry" + "github.com/ks3sdklib/aws-sdk-go/internal/endpoints" + "net/http" + "net/http/httputil" + "regexp" +) + +// A Service implements the base service request and response handling +// used by all services. +type Service struct { + Config *Config + Handlers Handlers + ManualSend bool + ServiceName string + APIVersion string + Endpoint string + SigningName string + SigningRegion string + JSONVersion string + TargetPrefix string + RetryRule retry.RetryRule + ShouldRetry func(error) bool + MaxRetries int +} + +var schemeRE = regexp.MustCompile("^([^:]+)://") + +// NewService will return a pointer to a new Server object initialized. +func NewService(config *Config) *Service { + svc := &Service{Config: config} + svc.Initialize() + return svc +} + +// Initialize initializes the service. +func (s *Service) Initialize() { + if s.Config == nil { + s.Config = &Config{} + } + if s.Config.HTTPClient == nil { + s.Config.HTTPClient = http.DefaultClient + } + + if s.RetryRule == nil { + s.RetryRule = s.Config.RetryRule + } + + if s.ShouldRetry == nil { + s.ShouldRetry = s.Config.ShouldRetry + } + + s.MaxRetries = s.Config.MaxRetries + s.Handlers.Validate.PushBack(ValidateEndpointHandler) + s.Handlers.Build.PushBack(UserAgentHandler) + s.Handlers.Sign.PushBack(BuildContentLength) + s.Handlers.Send.PushBack(SendHandler) + s.Handlers.AfterRetry.PushBack(AfterRetryHandler) + s.Handlers.ValidateResponse.PushBack(ValidateResponseHandler) + s.AddDebugHandlers() + s.buildEndpoint() + + if !s.Config.DisableParamValidation { + s.Handlers.Validate.PushBack(ValidateParameters) + } +} + +// buildEndpoint builds the endpoint values the service will use to make requests with. +func (s *Service) buildEndpoint() { + if s.Config.Endpoint != "" { + s.Endpoint = s.Config.Endpoint + } else { + s.Endpoint, s.SigningRegion = + endpoints.EndpointForRegion(s.ServiceName, s.Config.Region) + } + if s.Endpoint != "" && !schemeRE.MatchString(s.Endpoint) { + scheme := "https" + if s.Config.DisableSSL { + scheme = "http" + } + s.Endpoint = scheme + "://" + s.Endpoint + } +} + +// AddDebugHandlers injects debug logging handlers into the service to log request +// debug information. +func (s *Service) AddDebugHandlers() { + if s.Config.LogLevel < Debug { + return + } + + s.Handlers.Send.PushFront(func(r *Request) { + logBody := r.Config.LogHTTPBody + dumpedBody, _ := httputil.DumpRequestOut(r.HTTPRequest, logBody) + r.Config.LogDebug("---[ REQUEST ]-----------------------------") + r.Config.LogDebug("%s", string(dumpedBody)) + r.Config.LogDebug("-----------------------------------------------------") + }) + s.Handlers.Send.PushBack(func(r *Request) { + r.Config.LogDebug("---[ RESPONSE ]--------------------------------------") + if r.HTTPResponse != nil { + logBody := r.Config.LogHTTPBody + dumpedBody, _ := httputil.DumpResponse(r.HTTPResponse, logBody) + r.Config.LogDebug("%s", string(dumpedBody)) + } else if r.Error != nil { + r.Config.LogDebug("%s", r.Error.Error()) + } + r.Config.LogDebug("-----------------------------------------------------") + }) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/types.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/types.go new file mode 100644 index 0000000000..5ad6a1f669 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/types.go @@ -0,0 +1,167 @@ +package aws + +import ( + "fmt" + "io" + "time" +) + +// String converts a Go string into a string pointer. +func String(v string) *string { + return &v +} + +// ToString converts a string pointer into a string. +func ToString(p *string) (v string) { + if p == nil { + return v + } + + return *p +} + +// Boolean converts a Go bool into a boolean pointer. +func Boolean(v bool) *bool { + return &v +} + +// ToBoolean converts a boolean pointer into a bool. +func ToBoolean(p *bool) (v bool) { + if p == nil { + return v + } + + return *p +} + +// Long converts a Go int64 into a long pointer. +func Long(v int64) *int64 { + return &v +} + +// ToLong converts a long pointer into a int64. +func ToLong(p *int64) (v int64) { + if p == nil { + return v + } + + return *p +} + +// Double converts a Go float64 into a double pointer. +func Double(v float64) *float64 { + return &v +} + +// ToDouble converts a double pointer into a float64. +func ToDouble(p *float64) (v float64) { + if p == nil { + return v + } + + return *p +} + +// Time converts a Go Time into a Time pointer +func Time(t time.Time) *time.Time { + return &t +} + +// ReadSeekCloser wraps a io.Reader returning a ReaderSeakerCloser +func ReadSeekCloser(r io.Reader) ReaderSeekerCloser { + return ReaderSeekerCloser{r} +} + +// ReaderSeekerCloser represents a reader that can also delegate io.Seeker and +// io.Closer interfaces to the underlying object if they are available. +type ReaderSeekerCloser struct { + r io.Reader +} + +// Read reads from the reader up to size of p. The number of bytes read, and +// error if it occurred will be returned. +// +// If the reader is not an io.Reader zero bytes read, and nil error will be returned. +// +// Performs the same functionality as io.Reader Read +func (r ReaderSeekerCloser) Read(p []byte) (int, error) { + switch t := r.r.(type) { + case io.Reader: + return t.Read(p) + } + return 0, nil +} + +// Seek sets the offset for the next Read to offset, interpreted according to +// whence: 0 means relative to the origin of the file, 1 means relative to the +// current offset, and 2 means relative to the end. Seek returns the new offset +// and an error, if any. +// +// If the ReaderSeekerCloser is not an io.Seeker nothing will be done. +func (r ReaderSeekerCloser) Seek(offset int64, whence int) (int64, error) { + switch t := r.r.(type) { + case io.Seeker: + return t.Seek(offset, whence) + } + return int64(0), nil +} + +// Close closes the ReaderSeekerCloser. +// +// If the ReaderSeekerCloser is not an io.Closer nothing will be done. +func (r ReaderSeekerCloser) Close() error { + switch t := r.r.(type) { + case io.Closer: + return t.Close() + } + return nil +} + +// A SettableBool provides a boolean value which includes the state if +// the value was set or unset. The set state is in addition to the value's +// value(true|false) +type SettableBool struct { + value bool + set bool +} + +// SetBool returns a SettableBool with a value set +func SetBool(value bool) SettableBool { + return SettableBool{value: value, set: true} +} + +// Get returns the value. Will always be false if the SettableBool was not set. +func (b *SettableBool) Get() bool { + if !b.set { + return false + } + return b.value +} + +// Set sets the value and updates the state that the value has been set. +func (b *SettableBool) Set(value bool) { + b.value = value + b.set = true +} + +// IsSet returns if the value has been set +func (b *SettableBool) IsSet() bool { + return b.set +} + +// Reset resets the state and value of the SettableBool to its initial default +// state of not set and zero value. +func (b *SettableBool) Reset() { + b.value = false + b.set = false +} + +// String returns the string representation of the value if set. Zero if not set. +func (b *SettableBool) String() string { + return fmt.Sprintf("%t", b.Get()) +} + +// GoString returns the string representation of the SettableBool value and state +func (b *SettableBool) GoString() string { + return fmt.Sprintf("Bool{value:%t, set:%t}", b.value, b.set) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/aws/version.go b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/version.go new file mode 100644 index 0000000000..636e447bc1 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/aws/version.go @@ -0,0 +1,7 @@ +package aws + +// SDKName is the name of this KS3 SDK +const SDKName = "ks3-sdk-go" + +// SDKVersion is the version of this SDK +const SDKVersion = "1.8.1" diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/apierr/error.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/apierr/error.go new file mode 100644 index 0000000000..c45555a9f0 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/apierr/error.go @@ -0,0 +1,139 @@ +// Package apierr represents API error types. +package apierr + +import "fmt" + +// A BaseError wraps the code and message which defines an error. It also +// can be used to wrap an original error object. +// +// Should be used as the root for errors satisfying the awserr.Error. Also +// for any error which does not fit into a specific error wrapper type. +type BaseError struct { + // Classification of error + code string + + // Detailed information about error + message string + + // Optional original error this error is based off of. Allows building + // chained errors. + origErr error +} + +// New returns an error object for the code, message, and err. +// +// code is a short no whitespace phrase depicting the classification of +// the error that is being created. +// +// message is the free flow string containing detailed information about the error. +// +// origErr is the error object which will be nested under the new error to be returned. +func New(code, message string, origErr error) *BaseError { + return &BaseError{ + code: code, + message: message, + origErr: origErr, + } +} + +// Error returns the string representation of the error. +// +// See ErrorWithExtra for formatting. +// +// Satisfies the error interface. +func (b *BaseError) Error() string { + return b.ErrorWithExtra("") +} + +// String returns the string representation of the error. +// Alias for Error to satisfy the stringer interface. +func (b *BaseError) String() string { + return b.Error() +} + +// Code returns the short phrase depicting the classification of the error. +func (b *BaseError) Code() string { + return b.code +} + +// Message returns the error details message. +func (b *BaseError) Message() string { + return b.message +} + +// OrigErr returns the original error if one was set. Nil is returned if no error +// was set. +func (b *BaseError) OrigErr() error { + return b.origErr +} + +// ErrorWithExtra is a helper method to add an extra string to the stratified +// error message. The extra message will be added on the next line below the +// error message like the following: +// +// : +// +// +// If there is a original error the error will be included on a new line. +// +// : +// +// caused by: +func (b *BaseError) ErrorWithExtra(extra string) string { + msg := fmt.Sprintf("%s: %s", b.code, b.message) + if extra != "" { + msg = fmt.Sprintf("%s\n\t%s", msg, extra) + } + if b.origErr != nil { + msg = fmt.Sprintf("%s\ncaused by: %s", msg, b.origErr.Error()) + } + return msg +} + +// A RequestError wraps a request or service error. +// +// Composed of BaseError for code, message, and original error. +type RequestError struct { + *BaseError + statusCode int + requestID string +} + +// NewRequestError returns a wrapped error with additional information for request +// status code, and service requestID. +// +// Should be used to wrap all request which involve service requests. Even if +// the request failed without a service response, but had an HTTP status code +// that may be meaningful. +// +// Also wraps original errors via the BaseError. +func NewRequestError(base *BaseError, statusCode int, requestID string) *RequestError { + return &RequestError{ + BaseError: base, + statusCode: statusCode, + requestID: requestID, + } +} + +// Error returns the string representation of the error. +// Satisfies the error interface. +func (r *RequestError) Error() string { + return r.ErrorWithExtra(fmt.Sprintf("status code: %d, request id: [%s]", + r.statusCode, r.requestID)) +} + +// String returns the string representation of the error. +// Alias for Error to satisfy the stringer interface. +func (r *RequestError) String() string { + return r.Error() +} + +// StatusCode returns the wrapped status code for the error +func (r *RequestError) StatusCode() int { + return r.statusCode +} + +// RequestID returns the wrapped requestID +func (r *RequestError) RequestID() string { + return r.requestID +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/crc/crc64.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/crc/crc64.go new file mode 100644 index 0000000000..5a4d728f9a --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/crc/crc64.go @@ -0,0 +1,127 @@ +package crc + +import ( + "hash" + "hash/crc64" +) + +// digest represents the partial evaluation of a checksum. +type digest struct { + crc uint64 + tab *crc64.Table +} + +// NewCRC creates a new hash.Hash64 computing the CRC64 checksum +// using the polynomial represented by the Table. +func NewCRC(tab *crc64.Table, init uint64) hash.Hash64 { return &digest{init, tab} } + +var CrcTable = func() *crc64.Table { + return crc64.MakeTable(crc64.ECMA) +} + +// Size returns the number of bytes sum will return. +func (d *digest) Size() int { return crc64.Size } + +// BlockSize returns the hash's underlying block size. +// The Write method must be able to accept any amount +// of data, but it may operate more efficiently if all writes +// are a multiple of the block size. +func (d *digest) BlockSize() int { return 1 } + +// Reset resets the hash to its initial state. +func (d *digest) Reset() { d.crc = 0 } + +// Write (via the embedded io.Writer interface) adds more data to the running hash. +// It never returns an error. +func (d *digest) Write(p []byte) (n int, err error) { + d.crc = crc64.Update(d.crc, d.tab, p) + return len(p), nil +} + +// Sum64 returns CRC64 value. +func (d *digest) Sum64() uint64 { return d.crc } + +// Sum returns hash value. +func (d *digest) Sum(in []byte) []byte { + s := d.Sum64() + return append(in, byte(s>>56), byte(s>>48), byte(s>>40), byte(s>>32), byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) +} + +// gf2Dim dimension of GF(2) vectors (length of CRC) +const gf2Dim int = 64 + +func gf2MatrixTimes(mat []uint64, vec uint64) uint64 { + var sum uint64 + for i := 0; vec != 0; i++ { + if vec&1 != 0 { + sum ^= mat[i] + } + + vec >>= 1 + } + return sum +} + +func gf2MatrixSquare(square []uint64, mat []uint64) { + for n := 0; n < gf2Dim; n++ { + square[n] = gf2MatrixTimes(mat, mat[n]) + } +} + +// CRC64Combine combines CRC64 +func CRC64Combine(crc1 uint64, crc2 uint64, len2 uint64) uint64 { + var even [gf2Dim]uint64 // Even-power-of-two zeros operator + var odd [gf2Dim]uint64 // Odd-power-of-two zeros operator + + // Degenerate case + if len2 == 0 { + return crc1 + } + + // Put operator for one zero bit in odd + odd[0] = crc64.ECMA // CRC64 polynomial + var row uint64 = 1 + for n := 1; n < gf2Dim; n++ { + odd[n] = row + row <<= 1 + } + + // Put operator for two zero bits in even + gf2MatrixSquare(even[:], odd[:]) + + // Put operator for four zero bits in odd + gf2MatrixSquare(odd[:], even[:]) + + // Apply len2 zeros to crc1, first square will put the operator for one zero byte, eight zero bits, in even + for { + // Apply zeros operator for this bit of len2 + gf2MatrixSquare(even[:], odd[:]) + + if len2&1 != 0 { + crc1 = gf2MatrixTimes(even[:], crc1) + } + + len2 >>= 1 + + // If no more bits set, then done + if len2 == 0 { + break + } + + // Another iteration of the loop with odd and even swapped + gf2MatrixSquare(odd[:], even[:]) + if len2&1 != 0 { + crc1 = gf2MatrixTimes(odd[:], crc1) + } + len2 >>= 1 + + // If no more bits set, then done + if len2 == 0 { + break + } + } + + // Return combined CRC + crc1 ^= crc2 + return crc1 +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/endpoints/endpoints.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/endpoints/endpoints.go new file mode 100644 index 0000000000..d040cccd57 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/endpoints/endpoints.go @@ -0,0 +1,31 @@ +// Package endpoints validates regional endpoints for services. +package endpoints + +//go:generate go run ../model/cli/gen-endpoints/main.go endpoints.json endpoints_map.go +//go:generate gofmt -s -w endpoints_map.go + +import "strings" + +// EndpointForRegion returns an endpoint and its signing region for a service and region. +// if the service and region pair are not found endpoint and signingRegion will be empty. +func EndpointForRegion(svcName, region string) (endpoint, signingRegion string) { + derivedKeys := []string{ + region + "/" + svcName, + region + "/*", + "*/" + svcName, + "*/*", + } + + for _, key := range derivedKeys { + if val, ok := endpointsMap.Endpoints[key]; ok { + ep := val.Endpoint + ep = strings.Replace(ep, "{region}", region, -1) + ep = strings.Replace(ep, "{service}", svcName, -1) + + endpoint = ep + signingRegion = val.SigningRegion + return + } + } + return +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/endpoints/endpoints_map.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/endpoints/endpoints_map.go new file mode 100644 index 0000000000..894c1a6434 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/endpoints/endpoints_map.go @@ -0,0 +1,89 @@ +package endpoints + +// THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT. + +type endpointStruct struct { + Version int + Endpoints map[string]endpointEntry +} + +type endpointEntry struct { + Endpoint string + SigningRegion string +} + +var endpointsMap = endpointStruct{ + Version: 2, + Endpoints: map[string]endpointEntry{ + "*/*": { + Endpoint: "{service}.{region}.amazonaws.com", + }, + "*/cloudfront": { + Endpoint: "cloudfront.amazonaws.com", + SigningRegion: "us-east-1", + }, + "*/cloudsearchdomain": { + Endpoint: "", + SigningRegion: "us-east-1", + }, + "*/iam": { + Endpoint: "iam.amazonaws.com", + SigningRegion: "us-east-1", + }, + "*/importexport": { + Endpoint: "importexport.amazonaws.com", + SigningRegion: "us-east-1", + }, + "*/route53": { + Endpoint: "route53.amazonaws.com", + SigningRegion: "us-east-1", + }, + "*/sts": { + Endpoint: "sts.amazonaws.com", + SigningRegion: "us-east-1", + }, + "ap-northeast-1/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + "ap-southeast-1/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + "ap-southeast-2/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + "cn-north-1/*": { + Endpoint: "{service}.{region}.amazonaws.com.cn", + }, + "eu-central-1/s3": { + Endpoint: "{service}.{region}.amazonaws.com", + }, + "eu-west-1/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + "sa-east-1/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + "us-east-1/s3": { + Endpoint: "s3.amazonaws.com", + }, + "us-east-1/sdb": { + Endpoint: "sdb.amazonaws.com", + SigningRegion: "us-east-1", + }, + "us-gov-west-1/iam": { + Endpoint: "iam.us-gov.amazonaws.com", + }, + "us-gov-west-1/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + "us-gov-west-1/sts": { + Endpoint: "sts.us-gov-west-1.amazonaws.com", + }, + "us-west-1/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + "us-west-2/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + }, +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/body/body.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/body/body.go new file mode 100644 index 0000000000..ec8cc39d8c --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/body/body.go @@ -0,0 +1,37 @@ +package body + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/restjson" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/restxml" +) + +// Build builds the REST component of a service request. +func Build(r *aws.Request) { + if r.ContentType == "application/json" { + restjson.Build(r) + } else { + restxml.Build(r) + } +} + +// UnmarshalBody unmarshal a response body for the REST protocol. +func UnmarshalBody(r *aws.Request) { + rest.Unmarshal(r) + if r.ContentType == "application/json" { + restjson.Unmarshal(r) + } else { + restxml.Unmarshal(r) + } +} + +// UnmarshalMeta unmarshal response headers for the REST protocol. +func UnmarshalMeta(r *aws.Request) { + rest.UnmarshalMeta(r) +} + +// UnmarshalError unmarshal a response error for the REST protocol. +func UnmarshalError(r *aws.Request) { + restxml.UnmarshalError(r) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil/build.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil/build.go new file mode 100644 index 0000000000..58b65fb8fb --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil/build.go @@ -0,0 +1,199 @@ +// Package jsonutil provides JSON serialisation of AWS requests and responses. +package jsonutil + +import ( + "bytes" + "encoding/base64" + "fmt" + "reflect" + "sort" + "strconv" + "strings" + "time" +) + +// BuildJSON builds a JSON string for a given object v. +func BuildJSON(v interface{}) ([]byte, error) { + var buf bytes.Buffer + + err := buildAny(reflect.ValueOf(v), &buf, "") + return buf.Bytes(), err +} + +func buildAny(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + value = reflect.Indirect(value) + if !value.IsValid() { + return nil + } + + vtype := value.Type() + + t := tag.Get("type") + if t == "" { + switch vtype.Kind() { + case reflect.Struct: + // also it can't be a time object + if _, ok := value.Interface().(time.Time); !ok { + t = "structure" + } + case reflect.Slice: + // also it can't be a byte slice + if _, ok := value.Interface().([]byte); !ok { + t = "list" + } + case reflect.Map: + t = "map" + } + } + + switch t { + case "structure": + if field, ok := vtype.FieldByName("SDKShapeTraits"); ok { + tag = field.Tag + } + return buildStruct(value, buf, tag) + case "list": + return buildList(value, buf, tag) + case "map": + return buildMap(value, buf, tag) + default: + return buildScalar(value, buf, tag) + } +} + +func buildStruct(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + if !value.IsValid() { + return nil + } + + buf.WriteString("{") + + t, fields := value.Type(), []*reflect.StructField{} + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + member := value.FieldByName(field.Name) + if (member.Kind() == reflect.Ptr || member.Kind() == reflect.Slice || member.Kind() == reflect.Map) && member.IsNil() { + continue // ignore unset fields + } + if c := field.Name[0:1]; strings.ToLower(c) == c { + continue // ignore unexported fields + } + if field.Tag.Get("location") != "" { + continue // ignore non-body elements + } + + fields = append(fields, &field) + } + + for i, field := range fields { + member := value.FieldByName(field.Name) + + // figure out what this field is called + name := field.Name + if locName := field.Tag.Get("locationName"); locName != "" { + name = locName + } + + buf.WriteString(fmt.Sprintf("%q:", name)) + + err := buildAny(member, buf, field.Tag) + if err != nil { + return err + } + + if i < len(fields)-1 { + buf.WriteString(",") + } + } + + buf.WriteString("}") + + return nil +} + +func buildList(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + buf.WriteString("[") + + for i := 0; i < value.Len(); i++ { + buildAny(value.Index(i), buf, "") + + if i < value.Len()-1 { + buf.WriteString(",") + } + } + + buf.WriteString("]") + + return nil +} + +func buildMap(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + buf.WriteString("{") + + keys := make([]string, value.Len()) + for i, n := range value.MapKeys() { + keys[i] = n.String() + } + sort.Strings(keys) + + for i, k := range keys { + buf.WriteString(fmt.Sprintf("%q:", k)) + buildAny(value.MapIndex(reflect.ValueOf(k)), buf, "") + + if i < len(keys)-1 { + buf.WriteString(",") + } + } + + buf.WriteString("}") + + return nil +} + +func buildScalar(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { + switch converted := value.Interface().(type) { + case string: + writeString(converted, buf) + case []byte: + if !value.IsNil() { + buf.WriteString(fmt.Sprintf("%q", base64.StdEncoding.EncodeToString(converted))) + } + case bool: + buf.WriteString(strconv.FormatBool(converted)) + case int64: + buf.WriteString(strconv.FormatInt(converted, 10)) + case float64: + buf.WriteString(strconv.FormatFloat(converted, 'f', -1, 64)) + case time.Time: + buf.WriteString(strconv.FormatInt(converted.UTC().Unix(), 10)) + default: + return fmt.Errorf("unsupported JSON value %v (%s)", value.Interface(), value.Type()) + } + return nil +} + +func writeString(s string, buf *bytes.Buffer) { + buf.WriteByte('"') + for _, r := range s { + if r == '"' { + buf.WriteString(`\"`) + } else if r == '\\' { + buf.WriteString(`\\`) + } else if r == '\b' { + buf.WriteString(`\b`) + } else if r == '\f' { + buf.WriteString(`\f`) + } else if r == '\r' { + buf.WriteString(`\r`) + } else if r == '\t' { + buf.WriteString(`\t`) + } else if r == '\n' { + buf.WriteString(`\n`) + } else if r < 32 { + fmt.Fprintf(buf, "\\u%0.4x", r) + } else { + buf.WriteRune(r) + } + } + buf.WriteByte('"') +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil/unmarshal.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil/unmarshal.go new file mode 100644 index 0000000000..730dcd0f80 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil/unmarshal.go @@ -0,0 +1,214 @@ +package jsonutil + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "reflect" + "strings" + "time" +) + +// UnmarshalJSON reads a stream and unmarshals the results in object v. +func UnmarshalJSON(v interface{}, stream io.Reader) error { + var out interface{} + + b, err := ioutil.ReadAll(stream) + if err != nil { + return err + } + + if len(b) == 0 { + return nil + } + + if err := json.Unmarshal(b, &out); err != nil { + return err + } + + return unmarshalAny(reflect.ValueOf(v), out, "") +} + +func unmarshalAny(value reflect.Value, data interface{}, tag reflect.StructTag) error { + vtype := value.Type() + if vtype.Kind() == reflect.Ptr { + vtype = vtype.Elem() // check kind of actual element type + } + + t := tag.Get("type") + if t == "" { + switch vtype.Kind() { + case reflect.Struct: + // also it can't be a time object + if _, ok := value.Interface().(*time.Time); !ok { + t = "structure" + } + case reflect.Slice: + // also it can't be a byte slice + if _, ok := value.Interface().([]byte); !ok { + t = "list" + } + case reflect.Map: + t = "map" + } + } + + switch t { + case "structure": + if field, ok := vtype.FieldByName("SDKShapeTraits"); ok { + tag = field.Tag + } + return unmarshalStruct(value, data, tag) + case "list": + return unmarshalList(value, data, tag) + case "map": + return unmarshalMap(value, data, tag) + default: + return unmarshalScalar(value, data, tag) + } +} + +func unmarshalStruct(value reflect.Value, data interface{}, tag reflect.StructTag) error { + if data == nil { + return nil + } + mapData, ok := data.(map[string]interface{}) + if !ok { + return fmt.Errorf("JSON value is not a structure (%#v)", data) + } + + t := value.Type() + if value.Kind() == reflect.Ptr { + if value.IsNil() { // create the structure if it's nil + s := reflect.New(value.Type().Elem()) + value.Set(s) + value = s + } + + value = value.Elem() + t = t.Elem() + } + + // unwrap any payloads + if payload := tag.Get("payload"); payload != "" { + field, _ := t.FieldByName(payload) + return unmarshalAny(value.FieldByName(payload), data, field.Tag) + } + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if c := field.Name[0:1]; strings.ToLower(c) == c { + continue // ignore unexported fields + } + + // figure out what this field is called + name := field.Name + if locName := field.Tag.Get("locationName"); locName != "" { + name = locName + } + + member := value.FieldByName(field.Name) + err := unmarshalAny(member, mapData[name], field.Tag) + if err != nil { + return err + } + } + return nil +} + +func unmarshalList(value reflect.Value, data interface{}, tag reflect.StructTag) error { + if data == nil { + return nil + } + listData, ok := data.([]interface{}) + if !ok { + return fmt.Errorf("JSON value is not a list (%#v)", data) + } + + if value.IsNil() { + l := len(listData) + value.Set(reflect.MakeSlice(value.Type(), l, l)) + } + + for i, c := range listData { + err := unmarshalAny(value.Index(i), c, "") + if err != nil { + return err + } + } + + return nil +} + +func unmarshalMap(value reflect.Value, data interface{}, tag reflect.StructTag) error { + if data == nil { + return nil + } + mapData, ok := data.(map[string]interface{}) + if !ok { + return fmt.Errorf("JSON value is not a map (%#v)", data) + } + + if value.IsNil() { + value.Set(reflect.MakeMap(value.Type())) + } + + for k, v := range mapData { + kvalue := reflect.ValueOf(k) + vvalue := reflect.New(value.Type().Elem()).Elem() + + unmarshalAny(vvalue, v, "") + value.SetMapIndex(kvalue, vvalue) + } + + return nil +} + +func unmarshalScalar(value reflect.Value, data interface{}, tag reflect.StructTag) error { + errf := func() error { + return fmt.Errorf("unsupported value: %v (%s)", value.Interface(), value.Type()) + } + + switch d := data.(type) { + case nil: + return nil // nothing to do here + case string: + switch value.Interface().(type) { + case *string: + value.Set(reflect.ValueOf(&d)) + case []byte: + b, err := base64.StdEncoding.DecodeString(d) + if err != nil { + return err + } + value.Set(reflect.ValueOf(b)) + default: + return errf() + } + case float64: + switch value.Interface().(type) { + case *int64: + di := int64(d) + value.Set(reflect.ValueOf(&di)) + case *float64: + value.Set(reflect.ValueOf(&d)) + case *time.Time: + t := time.Unix(int64(d), 0).UTC() + value.Set(reflect.ValueOf(&t)) + default: + return errf() + } + case bool: + switch value.Interface().(type) { + case *bool: + value.Set(reflect.ValueOf(&d)) + default: + return errf() + } + default: + return fmt.Errorf("unsupported JSON value (%v)", data) + } + return nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/jsonrpc/jsonrpc.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/jsonrpc/jsonrpc.go new file mode 100644 index 0000000000..14953dae8d --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/jsonrpc/jsonrpc.go @@ -0,0 +1,121 @@ +// Package jsonrpc provides JSON RPC utilities for serialisation of AWS +// requests and responses. +package jsonrpc + +//go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/input/json.json build_test.go +//go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/output/json.json unmarshal_test.go + +import ( + "encoding/json" + "io/ioutil" + "reflect" + "strings" + + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil" +) + +var emptyJSON = []byte("{}") + +// Build builds a JSON payload for a JSON RPC request. +func Build(req *aws.Request) { + var buf []byte + var err error + if req.ParamsFilled() { + v := reflect.ValueOf(req.Params).Elem() + if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { + if payloadName := field.Tag.Get("payload"); payloadName != "" { + pfield, _ := v.Type().FieldByName(payloadName) + if ptag := pfield.Tag.Get("type"); ptag == "" || ptag == "structure" { + payload := reflect.Indirect(v.FieldByName(payloadName)) + if payload.IsValid() && payload.Interface() != nil { + buf, err = jsonutil.BuildJSON(payload.Interface()) + } + } + } + } + if err != nil { + req.Error = apierr.New("Marshal", "failed encoding JSON RPC request", err) + return + } + } else { + buf = emptyJSON + } + + if req.Service.TargetPrefix != "" || string(buf) != "{}" { + req.SetBufferBody(buf) + } + + if req.Service.TargetPrefix != "" { + target := req.Service.TargetPrefix + "." + req.Operation.Name + req.HTTPRequest.Header.Add("X-Amz-Target", target) + } + if req.Service.JSONVersion != "" { + jsonVersion := req.Service.JSONVersion + req.HTTPRequest.Header.Add("Content-Type", "application/x-amz-json-"+jsonVersion) + } +} + +// Unmarshal unmarshals a response for a JSON RPC service. +func Unmarshal(req *aws.Request) { + defer req.HTTPResponse.Body.Close() + if req.DataFilled() { + v := reflect.ValueOf(req.Data).Elem() + if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { + if payloadName := field.Tag.Get("payload"); payloadName != "" { + pfield, _ := v.Type().FieldByName(payloadName) + if ptag := pfield.Tag.Get("type"); ptag == "" || ptag == "structure" { + payload := v.FieldByName(payloadName) + if payload.IsValid() && payload.Interface() != nil { + err := jsonutil.UnmarshalJSON(payload.Interface(), req.HTTPResponse.Body) + if err != nil { + req.Error = apierr.New("Unmarshal", "failed decoding JSON RPC response", err) + } + } + } + } + } + } + return +} + +// UnmarshalMeta unmarshals headers from a response for a JSON RPC service. +func UnmarshalMeta(req *aws.Request) { + req.RequestID = req.HTTPResponse.Header.Get("x-amzn-requestid") +} + +// UnmarshalError unmarshals an error response for a JSON RPC service. +func UnmarshalError(req *aws.Request) { + defer req.HTTPResponse.Body.Close() + bodyBytes, err := ioutil.ReadAll(req.HTTPResponse.Body) + if err != nil { + req.Error = apierr.New("Unmarshal", "failed reading JSON RPC error response", err) + return + } + if len(bodyBytes) == 0 { + req.Error = apierr.NewRequestError( + apierr.New("Unmarshal", req.HTTPResponse.Status, nil), + req.HTTPResponse.StatusCode, + "", + ) + return + } + var jsonErr jsonErrorResponse + if err := json.Unmarshal(bodyBytes, &jsonErr); err != nil { + req.Error = apierr.New("Unmarshal", "failed decoding JSON RPC error response", err) + return + } + + codes := strings.SplitN(jsonErr.Code, "#", 2) + req.Error = apierr.NewRequestError( + apierr.New(codes[len(codes)-1], jsonErr.Message, nil), + req.HTTPResponse.StatusCode, + "", + ) +} + +type jsonErrorResponse struct { + Code string `json:"__type"` + Message string `json:"message"` +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/build.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/build.go new file mode 100644 index 0000000000..0d3c0af42c --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/build.go @@ -0,0 +1,33 @@ +// Package query provides serialisation of AWS query requests, and responses. +package query + +//go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/input/query.json build_test.go + +import ( + "net/url" + + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/queryutil" +) + +// Build builds a request for an AWS Query service. +func Build(r *aws.Request) { + body := url.Values{ + "Action": {r.Operation.Name}, + "Version": {r.Service.APIVersion}, + } + if err := queryutil.Parse(body, r.Params, false); err != nil { + r.Error = apierr.New("Marshal", "failed encoding Query request", err) + return + } + + if r.ExpireTime == 0 { + r.HTTPRequest.Method = "POST" + r.HTTPRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8") + r.SetBufferBody([]byte(body.Encode())) + } else { // This is a pre-signed request + r.HTTPRequest.Method = "GET" + r.HTTPRequest.URL.RawQuery = body.Encode() + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/queryutil/queryutil.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/queryutil/queryutil.go new file mode 100644 index 0000000000..3b417a89f7 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/queryutil/queryutil.go @@ -0,0 +1,223 @@ +package queryutil + +import ( + "encoding/base64" + "fmt" + "net/url" + "reflect" + "sort" + "strconv" + "strings" + "time" +) + +// Parse parses an object i and fills a url.Values object. The isEC2 flag +// indicates if this is the EC2 Query sub-protocol. +func Parse(body url.Values, i interface{}, isEC2 bool) error { + q := queryParser{isEC2: isEC2} + return q.parseValue(body, reflect.ValueOf(i), "", "") +} + +func elemOf(value reflect.Value) reflect.Value { + for value.Kind() == reflect.Ptr { + value = value.Elem() + } + return value +} + +type queryParser struct { + isEC2 bool +} + +func (q *queryParser) parseValue(v url.Values, value reflect.Value, prefix string, tag reflect.StructTag) error { + value = elemOf(value) + + // no need to handle zero values + if !value.IsValid() { + return nil + } + + t := tag.Get("type") + if t == "" { + switch value.Kind() { + case reflect.Struct: + t = "structure" + case reflect.Slice: + t = "list" + case reflect.Map: + t = "map" + } + } + + switch t { + case "structure": + return q.parseStruct(v, value, prefix) + case "list": + return q.parseList(v, value, prefix, tag) + case "map": + return q.parseMap(v, value, prefix, tag) + default: + return q.parseScalar(v, value, prefix, tag) + } +} + +func (q *queryParser) parseStruct(v url.Values, value reflect.Value, prefix string) error { + if !value.IsValid() { + return nil + } + + t := value.Type() + for i := 0; i < value.NumField(); i++ { + if c := t.Field(i).Name[0:1]; strings.ToLower(c) == c { + continue // ignore unexported fields + } + + value := elemOf(value.Field(i)) + field := t.Field(i) + var name string + + if q.isEC2 { + name = field.Tag.Get("queryName") + } + if name == "" { + if field.Tag.Get("flattened") != "" && field.Tag.Get("locationNameList") != "" { + name = field.Tag.Get("locationNameList") + } else if locName := field.Tag.Get("locationName"); locName != "" { + name = locName + } + if name != "" && q.isEC2 { + name = strings.ToUpper(name[0:1]) + name[1:] + } + } + if name == "" { + name = field.Name + } + + if prefix != "" { + name = prefix + "." + name + } + + if err := q.parseValue(v, value, name, field.Tag); err != nil { + return err + } + } + return nil +} + +func (q *queryParser) parseList(v url.Values, value reflect.Value, prefix string, tag reflect.StructTag) error { + // If it's empty, generate an empty value + if !value.IsNil() && value.Len() == 0 { + v.Set(prefix, "") + return nil + } + + // check for unflattened list member + if !q.isEC2 && tag.Get("flattened") == "" { + prefix += ".member" + } + + for i := 0; i < value.Len(); i++ { + slicePrefix := prefix + if slicePrefix == "" { + slicePrefix = strconv.Itoa(i + 1) + } else { + slicePrefix = slicePrefix + "." + strconv.Itoa(i+1) + } + if err := q.parseValue(v, value.Index(i), slicePrefix, ""); err != nil { + return err + } + } + return nil +} + +func (q *queryParser) parseMap(v url.Values, value reflect.Value, prefix string, tag reflect.StructTag) error { + // If it's empty, generate an empty value + if !value.IsNil() && value.Len() == 0 { + v.Set(prefix, "") + return nil + } + + // check for unflattened list member + if !q.isEC2 && tag.Get("flattened") == "" { + prefix += ".entry" + } + + // sort keys for improved serialization consistency. + // this is not strictly necessary for protocol support. + mapKeyValues := value.MapKeys() + mapKeys := map[string]reflect.Value{} + mapKeyNames := make([]string, len(mapKeyValues)) + for i, mapKey := range mapKeyValues { + name := mapKey.String() + mapKeys[name] = mapKey + mapKeyNames[i] = name + } + sort.Strings(mapKeyNames) + + for i, mapKeyName := range mapKeyNames { + mapKey := mapKeys[mapKeyName] + mapValue := value.MapIndex(mapKey) + + kname := tag.Get("locationNameKey") + if kname == "" { + kname = "key" + } + vname := tag.Get("locationNameValue") + if vname == "" { + vname = "value" + } + + // serialize key + var keyName string + if prefix == "" { + keyName = strconv.Itoa(i+1) + "." + kname + } else { + keyName = prefix + "." + strconv.Itoa(i+1) + "." + kname + } + + if err := q.parseValue(v, mapKey, keyName, ""); err != nil { + return err + } + + // serialize value + var valueName string + if prefix == "" { + valueName = strconv.Itoa(i+1) + "." + vname + } else { + valueName = prefix + "." + strconv.Itoa(i+1) + "." + vname + } + + if err := q.parseValue(v, mapValue, valueName, ""); err != nil { + return err + } + } + + return nil +} + +func (q *queryParser) parseScalar(v url.Values, r reflect.Value, name string, tag reflect.StructTag) error { + switch value := r.Interface().(type) { + case string: + v.Set(name, value) + case []byte: + if !r.IsNil() { + v.Set(name, base64.StdEncoding.EncodeToString(value)) + } + case bool: + v.Set(name, strconv.FormatBool(value)) + case int64: + v.Set(name, strconv.FormatInt(value, 10)) + case int: + v.Set(name, strconv.Itoa(value)) + case float64: + v.Set(name, strconv.FormatFloat(value, 'f', -1, 64)) + case float32: + v.Set(name, strconv.FormatFloat(float64(value), 'f', -1, 32)) + case time.Time: + const ISO8601UTC = "2006-01-02T15:04:05Z" + v.Set(name, value.UTC().Format(ISO8601UTC)) + default: + return fmt.Errorf("unsupported value for param %s: %v (%s)", name, r.Interface(), r.Type().Name()) + } + return nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/unmarshal.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/unmarshal.go new file mode 100644 index 0000000000..db688671a3 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/unmarshal.go @@ -0,0 +1,29 @@ +package query + +//go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/output/query.json unmarshal_test.go + +import ( + "encoding/xml" + + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil" +) + +// Unmarshal unmarshals a response for an AWS Query service. +func Unmarshal(r *aws.Request) { + defer r.HTTPResponse.Body.Close() + if r.DataFilled() { + decoder := xml.NewDecoder(r.HTTPResponse.Body) + err := xmlutil.UnmarshalXML(r.Data, decoder, r.Operation.Name+"Result") + if err != nil { + r.Error = apierr.New("Unmarshal", "failed decoding Query response", err) + return + } + } +} + +// UnmarshalMeta unmarshals header response values for an AWS Query service. +func UnmarshalMeta(r *aws.Request) { + // TODO implement unmarshaling of request IDs +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/unmarshal_error.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/unmarshal_error.go new file mode 100644 index 0000000000..dfeda4a553 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/unmarshal_error.go @@ -0,0 +1,108 @@ +package query + +import ( + "encoding/json" + "encoding/xml" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" + "io" + "regexp" + "strings" +) + +type XmlErrorResponse struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + StatusCode int `xml:"StatusCode"` + Message string `xml:"Message"` + Resource string `xml:"Resource"` + RequestID string `xml:"RequestId"` +} + +type Ks3BillJsonErrorResponse struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + RequestID string `xml:"RequestId"` +} + +type Ks3BillXmlErrorResponse struct { + XMLName xml.Name `xml:"ErrorResponse"` + RequestID string `xml:"RequestId"` + Error Ks3BillXmlError `xml:"Error"` +} + +type Ks3BillXmlError struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} + +// UnmarshalError unmarshal an error response for an AWS Query service. +func UnmarshalError(r *aws.Request) { + defer r.HTTPResponse.Body.Close() + + body, err := io.ReadAll(r.HTTPResponse.Body) + if err != nil { + r.Error = apierr.New("Unmarshal", "failed to read body", err) + return + } + + if r.RequestType == "ks3bill" { + if strings.Contains(r.HTTPResponse.Header.Get("Content-Type"), "application/xml") { + resp := &Ks3BillXmlErrorResponse{} + err = xml.Unmarshal(body, &resp) + if err != nil && err != io.EOF { + r.Error = apierr.New("Unmarshal", "failed to decode ks3bill xml error response", err) + return + } + + r.Error = apierr.NewRequestError(apierr.New(resp.Error.Code, resp.Error.Message, nil), r.HTTPResponse.StatusCode, resp.RequestID) + return + } + + resp := &Ks3BillJsonErrorResponse{} + err = json.Unmarshal(body, &resp) + if err != nil && err != io.EOF { + r.Error = apierr.New("Unmarshal", "failed to decode ks3bill json error response", err) + return + } + + r.Error = apierr.NewRequestError(apierr.New(resp.Code, resp.Message, nil), r.HTTPResponse.StatusCode, resp.RequestID) + return + } + + resp := &XmlErrorResponse{} + // 如果响应类型是html，则解析html文本 + if strings.Contains(r.HTTPResponse.Header.Get("Content-Type"), "text/html") { + // 获取HTML文本中title标签的内容 + re := regexp.MustCompile(`(.*?)`) + matches := re.FindStringSubmatch(string(body)) + + title := "" + if len(matches) > 1 { + title = matches[1] + } + + r.Error = apierr.NewRequestError(apierr.New(title, "", nil), r.HTTPResponse.StatusCode, "") + return + } + + err = xml.Unmarshal(body, &resp) + resp.StatusCode = r.HTTPResponse.StatusCode + + // head请求无法从body中获取request id，如果是head请求，则从header中获取 + if resp.RequestID == "" && r.HTTPRequest.Method == "HEAD" { + resp.RequestID = r.HTTPResponse.Header.Get("X-Kss-Request-Id") + } + + if err != nil && err != io.EOF { + r.Error = apierr.New("Unmarshal", "failed to decode query XML error response", err) + } else { + r.Error = apierr.NewRequestError( + apierr.New(resp.Code, resp.Message, nil), + r.HTTPResponse.StatusCode, + resp.RequestID, + ) + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/build.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/build.go new file mode 100644 index 0000000000..c0c02cf411 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/build.go @@ -0,0 +1,397 @@ +// Package rest provides RESTful serialisation of AWS requests and responses. +package rest + +import ( + "bytes" + "encoding/base64" + "fmt" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" + "io" + "net/url" + "path" + "reflect" + "strconv" + "strings" + "time" +) + +// RFC822 returns an RFC822 formatted timestamp for AWS protocols +const RFC822 = "Mon, 2 Jan 2006 15:04:05 GMT" + +// Whether the byte value can be sent without escaping in AWS URLs +var noEscape [256]bool + +func init() { + for i := 0; i < len(noEscape); i++ { + // AWS expects every character except these to be escaped + noEscape[i] = (i >= 'A' && i <= 'Z') || + (i >= 'a' && i <= 'z') || + (i >= '0' && i <= '9') || + i == '-' || + i == '.' || + i == '_' || + i == '~' + } +} + +var isTokenTable = [256]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, +} + +func ValidHeaderFieldName(v string) bool { + if len(v) == 0 { + return false + } + for i := 0; i < len(v); i++ { + if !isTokenTable[v[i]] { + return false + } + } + return true +} + +// Build builds the REST component of a service request. +func Build(r *aws.Request) { + if r.ParamsFilled() { + v := reflect.ValueOf(r.Params).Elem() + buildLocationElements(r, v) + buildBody(r, v) + } +} + +func buildLocationElements(r *aws.Request, v reflect.Value) { + query := r.HTTPRequest.URL.Query() + + for i := 0; i < v.NumField(); i++ { + m := v.Field(i) + if n := v.Type().Field(i).Name; n[0:1] == strings.ToLower(n[0:1]) { + continue + } + + if m.IsValid() { + field := v.Type().Field(i) + name := field.Tag.Get("locationName") + if name == "" { + name = field.Name + } + if m.Kind() == reflect.Ptr { + m = m.Elem() + } + if !m.IsValid() { + continue + } + + switch field.Tag.Get("location") { + case "headers": // header maps + buildHeaderMap(r, m, field.Tag.Get("locationName")) + case "header": + buildHeader(r, m, name) + case "uri": + buildURI(r, m, name) + case "querystring": + buildQueryString(r, m, name, query) + case "querystrings": + buildQueryStrings(r, m, name, query) + } + } + if r.Error != nil { + return + } + } + + buildExtendHeaders(r, v) + buildExtendQueryParams(v, query) + + r.HTTPRequest.URL.RawQuery = query.Encode() + updatePath(r.HTTPRequest.URL, r.Config) +} + +func buildBody(r *aws.Request, v reflect.Value) { + if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { + if payloadName := field.Tag.Get("payload"); payloadName != "" { + pfield, _ := v.Type().FieldByName(payloadName) + if ptag := pfield.Tag.Get("type"); ptag != "" && ptag != "structure" { + payload := reflect.Indirect(v.FieldByName(payloadName)) + if payload.IsValid() && payload.Interface() != nil { + switch reader := payload.Interface().(type) { + case io.ReadSeeker: + r.SetReaderBody(reader) + case []byte: + r.SetBufferBody(reader) + case string: + r.SetStringBody(reader) + default: + r.Error = apierr.New("Marshal", + "failed to encode REST request", + fmt.Errorf("unknown payload type %s", payload.Type())) + } + } + } + } + } +} + +func buildHeader(r *aws.Request, v reflect.Value, name string) { + str, err := convertType(v) + if err != nil { + r.Error = apierr.New("Marshal", "failed to encode REST request", err) + } else if str != nil { + r.HTTPRequest.Header.Add(name, *str) + } +} + +func buildHeaderMap(r *aws.Request, v reflect.Value, prefix string) { + for _, key := range v.MapKeys() { + str, err := convertType(v.MapIndex(key)) + if err != nil { + r.Error = apierr.New("Marshal", "failed to encode REST request", err) + } else if str != nil { + if strings.HasPrefix(strings.ToLower(key.String()), strings.ToLower(prefix)) { + r.HTTPRequest.Header.Add(key.String(), *str) + } else { + r.HTTPRequest.Header.Add(prefix+key.String(), *str) + } + } + } +} + +func buildExtendHeaders(r *aws.Request, v reflect.Value) { + extendHeaders := v.FieldByName("ExtendHeaders") + if extendHeaders.IsValid() { + iter := extendHeaders.MapRange() + for iter.Next() { + key := iter.Key().String() + value := iter.Value() + if ValidHeaderFieldName(key) { + if !value.IsNil() { + r.HTTPRequest.Header.Set(key, value.Elem().String()) + } else { + r.HTTPRequest.Header.Set(key, "") + } + } else { + r.Error = apierr.New("Marshal", fmt.Sprintf("invalid extend header field name \"%s\"", key), nil) + return + } + } + } +} + +func buildExtendQueryParams(v reflect.Value, query url.Values) { + extendQueryParams := v.FieldByName("ExtendQueryParams") + if extendQueryParams.IsValid() { + iter := extendQueryParams.MapRange() + for iter.Next() { + key := iter.Key().String() + if key == "" { + continue + } + value := iter.Value() + if !value.IsNil() { + query.Set(key, value.Elem().String()) + } else { + query.Set(key, "") + } + } + } +} + +func buildURI(r *aws.Request, v reflect.Value, name string) { + value, err := convertType(v) + if err != nil { + r.Error = apierr.New("Marshal", "failed to encode REST request", err) + } else if value != nil { + uri := r.HTTPRequest.URL.Path + uri = strings.Replace(uri, "{"+name+"}", EscapePath(*value, true), -1) + uri = strings.Replace(uri, "{"+name+"+}", EscapePath(*value, false), -1) + r.HTTPRequest.URL.Path = uri + } +} + +func buildQueryString(r *aws.Request, v reflect.Value, name string, query url.Values) { + str, err := convertType(v) + if err != nil { + r.Error = apierr.New("Marshal", "failed to encode REST request", err) + } else if str != nil { + query.Set(name, *str) + } else if str == nil { + query.Set(name, "") + } +} + +func buildQueryStrings(r *aws.Request, v reflect.Value, name string, query url.Values) { + if v.Kind() == reflect.Slice { + var valSlice []string + for i := 0; i < v.Len(); i++ { + str, err := convertType(v.Index(i)) + if err != nil { + r.Error = apierr.New("Marshal", "failed to encode REST request", err) + return + } + valSlice = append(valSlice, *str) + } + if len(valSlice) > 0 { + query.Set(name, strings.Join(valSlice, ",")) + } + } +} + +func updatePath(url *url.URL, cfg *aws.Config) { + urlPath := url.Path + scheme, query := url.Scheme, url.RawQuery + + // path.Clean will remove duplicate leading / + // this will make deleting / started key impossible + // so escape it here first + urlPath = strings.Replace(urlPath, "//", "/%2F", -1) + + // 新增参数控制path clean，默认值为true + if !cfg.DisableRestProtocolURICleaning { + urlPath = cleanPath(urlPath) + } + + // get formatted URL minus scheme, so we can build this into Opaque + url.Scheme, url.Path, url.RawQuery = "", "", "" + s := url.String() + url.Scheme = scheme + url.RawQuery = query + + // build opaque URI + url.Opaque = s + urlPath +} + +func cleanPath(urlPath string) string { + // path.Clean会去掉最后的斜杠，导致无法创建目录。所以添加以下逻辑 + add := false + if urlPath[len(urlPath)-1] == '/' && len(urlPath) > 1 { + add = true + } + + // clean up path + urlPath = path.Clean(urlPath) + if add { + urlPath += "/" + } + + return urlPath +} + +// EscapePath escapes part of a URL path in Amazon style +// +// path The path segment to escape +// encodeSep If true, '/' will be encoded, otherwise they will not +func EscapePath(path string, encodeSep bool) string { + var buf bytes.Buffer + for i := 0; i < len(path); i++ { + c := path[i] + if noEscape[c] || (c == '/' && !encodeSep) { + buf.WriteByte(c) + } else { + fmt.Fprintf(&buf, "%%%02X", c) + } + } + return buf.String() +} + +func convertType(v reflect.Value) (*string, error) { + v = reflect.Indirect(v) + if !v.IsValid() { + return nil, nil + } + + var str string + switch value := v.Interface().(type) { + case string: + str = value + case []byte: + str = base64.StdEncoding.EncodeToString(value) + case bool: + str = strconv.FormatBool(value) + case int64: + str = strconv.FormatInt(value, 10) + case float64: + str = strconv.FormatFloat(value, 'f', -1, 64) + case time.Time: + str = value.UTC().Format(RFC822) + default: + err := fmt.Errorf("unsupported value for param %v (%s)", v.Interface(), v.Type()) + return nil, err + } + return &str, nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/payload.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/payload.go new file mode 100644 index 0000000000..c4ce64acc3 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/payload.go @@ -0,0 +1,63 @@ +package rest + +import ( + "reflect" +) + +// PayloadMember returns the payload field member of i if there is one, or nil. +func PayloadMember(i interface{}) interface{} { + if i == nil { + return nil + } + + v := reflect.ValueOf(i).Elem() + if !v.IsValid() { + return nil + } + if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { + if payloadName := field.Tag.Get("payload"); payloadName != "" { + field, _ := v.Type().FieldByName(payloadName) + if field.Tag.Get("type") != "structure" { + return nil + } + + payload := v.FieldByName(payloadName) + if payload.IsValid() || (payload.Kind() == reflect.Ptr && !payload.IsNil()) { + return payload.Interface() + } + } + } + return nil +} + +// PayloadType returns the type of a payload field member of i if there is one, or "". +func PayloadType(i interface{}) string { + v := reflect.Indirect(reflect.ValueOf(i)) + if !v.IsValid() { + return "" + } + if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { + if payloadName := field.Tag.Get("payload"); payloadName != "" { + if payloadName == "GetBucketCORSInput" { + return field.Tag.Get("type") + } + if member, ok := v.Type().FieldByName(payloadName); ok { + return member.Tag.Get("type") + } + } + } + return "" +} + +// PayloadMd5 判断给定结构体 i 中是否有 AutoFillMD5 字段 +func PayloadMd5(i interface{}) (hasField bool) { + // 获取结构体指针的 Value + v := reflect.Indirect(reflect.ValueOf(i)) + // 如果结构体不存在或为空，则直接返回 false + if !v.IsValid() { + return + } + // 判断是否存在 AutoFillMD5 字段 + _, hasField = v.Type().FieldByName("AutoFillMD5") + return +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/unmarshal.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/unmarshal.go new file mode 100644 index 0000000000..009b57c656 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest/unmarshal.go @@ -0,0 +1,180 @@ +package rest + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "reflect" + "strconv" + "strings" + "time" + + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" +) + +// Unmarshal unmarshals the REST component of a response in a REST service. +func Unmarshal(r *aws.Request) { + if r.DataFilled() { + v := reflect.Indirect(reflect.ValueOf(r.Data)) + unmarshalBody(r, v) + } +} + +func UnmarshalMeta(r *aws.Request) { + if r.DataFilled() { + v := reflect.Indirect(reflect.ValueOf(r.Data)) + unmarshalLocationElements(r, v) + } +} + +func unmarshalBody(r *aws.Request, v reflect.Value) { + if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { + if payloadName := field.Tag.Get("payload"); payloadName != "" { + pfield, _ := v.Type().FieldByName(payloadName) + if ptag := pfield.Tag.Get("type"); ptag != "" && ptag != "structure" { + payload := v.FieldByName(payloadName) + if payload.IsValid() { + switch payload.Interface().(type) { + case []byte: + b, err := ioutil.ReadAll(r.HTTPResponse.Body) + if err != nil { + r.Error = apierr.New("Unmarshal", "failed to decode REST response", err) + } else { + payload.Set(reflect.ValueOf(b)) + } + case *string: + b, err := ioutil.ReadAll(r.HTTPResponse.Body) + if err != nil { + r.Error = apierr.New("Unmarshal", "failed to decode REST response", err) + } else { + str := string(b) + payload.Set(reflect.ValueOf(&str)) + } + default: + switch payload.Type().String() { + case "io.ReadSeeker": + payload.Set(reflect.ValueOf(aws.ReadSeekCloser(r.HTTPResponse.Body))) + case "aws.ReadSeekCloser", "io.ReadCloser": + payload.Set(reflect.ValueOf(r.HTTPResponse.Body)) + default: + r.Error = apierr.New("Unmarshal", + "failed to decode REST response", + fmt.Errorf("unknown payload type %s", payload.Type())) + } + } + } + } + } + } +} + +func unmarshalLocationElements(r *aws.Request, v reflect.Value) { + for i := 0; i < v.NumField(); i++ { + m, field := v.Field(i), v.Type().Field(i) + if n := field.Name; n[0:1] == strings.ToLower(n[0:1]) { + continue + } + + if m.IsValid() { + name := field.Tag.Get("locationName") + if name == "" { + name = field.Name + } + + switch field.Tag.Get("location") { + case "statusCode": + unmarshalStatusCode(m, r.HTTPResponse.StatusCode) + case "header": + err := unmarshalHeader(m, r.HTTPResponse.Header.Get(name)) + if err != nil { + r.Error = apierr.New("Unmarshal", "failed to decode REST response", err) + break + } + case "headers": + prefix := field.Tag.Get("locationName") + err := unmarshalHeaderMap(m, r.HTTPResponse.Header, prefix) + if err != nil { + r.Error = apierr.New("Unmarshal", "failed to decode REST response", err) + break + } + } + } + if r.Error != nil { + return + } + } +} + +func unmarshalStatusCode(v reflect.Value, statusCode int) { + if !v.IsValid() { + return + } + + switch v.Interface().(type) { + case *int64: + s := int64(statusCode) + v.Set(reflect.ValueOf(&s)) + } +} + +func unmarshalHeaderMap(r reflect.Value, headers http.Header, prefix string) error { + switch r.Interface().(type) { + case map[string]*string: // we only support string map value types + out := map[string]*string{} + for k, v := range headers { + k = http.CanonicalHeaderKey(k) + if strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) { + out[k[len(prefix):]] = &v[0] + } + } + r.Set(reflect.ValueOf(out)) + } + return nil +} + +func unmarshalHeader(v reflect.Value, header string) error { + if !v.IsValid() || (header == "" && v.Elem().Kind() != reflect.String) { + return nil + } + + switch v.Interface().(type) { + case *string: + v.Set(reflect.ValueOf(&header)) + case []byte: + b, err := base64.StdEncoding.DecodeString(header) + if err != nil { + return err + } + v.Set(reflect.ValueOf(&b)) + case *bool: + b, err := strconv.ParseBool(header) + if err != nil { + return err + } + v.Set(reflect.ValueOf(&b)) + case *int64: + i, err := strconv.ParseInt(header, 10, 64) + if err != nil { + return err + } + v.Set(reflect.ValueOf(&i)) + case *float64: + f, err := strconv.ParseFloat(header, 64) + if err != nil { + return err + } + v.Set(reflect.ValueOf(&f)) + case *time.Time: + t, err := time.Parse(RFC822, header) + if err != nil { + return err + } + v.Set(reflect.ValueOf(&t)) + default: + err := fmt.Errorf("Unsupported value for param %v (%s)", v.Interface(), v.Type()) + return err + } + return nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/restjson/restjson.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/restjson/restjson.go new file mode 100644 index 0000000000..98d863fda0 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/restjson/restjson.go @@ -0,0 +1,77 @@ +// Package restjson provides RESTful JSON serialisation of AWS +// requests and responses. +package restjson + +//go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/input/rest-json.json build_test.go +//go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/output/rest-json.json unmarshal_test.go + +import ( + "encoding/json" + "io/ioutil" + "strings" + + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/jsonrpc" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest" +) + +// Build builds a request for the REST JSON protocol. +func Build(r *aws.Request) { + rest.Build(r) + + if t := rest.PayloadType(r.Params); t == "structure" || t == "" { + jsonrpc.Build(r) + } +} + +// Unmarshal unmarshals a response body for the REST JSON protocol. +func Unmarshal(r *aws.Request) { + if t := rest.PayloadType(r.Data); t == "structure" || t == "" { + jsonrpc.Unmarshal(r) + } +} + +// UnmarshalMeta unmarshals response headers for the REST JSON protocol. +func UnmarshalMeta(r *aws.Request) { + rest.Unmarshal(r) +} + +// UnmarshalError unmarshals a response error for the REST JSON protocol. +func UnmarshalError(r *aws.Request) { + code := r.HTTPResponse.Header.Get("X-Amzn-Errortype") + bodyBytes, err := ioutil.ReadAll(r.HTTPResponse.Body) + if err != nil { + r.Error = apierr.New("Unmarshal", "failed reading REST JSON error response", err) + return + } + if len(bodyBytes) == 0 { + r.Error = apierr.NewRequestError( + apierr.New("Unmarshal", r.HTTPResponse.Status, nil), + r.HTTPResponse.StatusCode, + "", + ) + return + } + var jsonErr jsonErrorResponse + if err := json.Unmarshal(bodyBytes, &jsonErr); err != nil { + r.Error = apierr.New("Unmarshal", "failed decoding REST JSON error response", err) + return + } + + if code == "" { + code = jsonErr.Code + } + + codes := strings.SplitN(code, ":", 2) + r.Error = apierr.NewRequestError( + apierr.New(codes[0], jsonErr.Message, nil), + r.HTTPResponse.StatusCode, + "", + ) +} + +type jsonErrorResponse struct { + Code string `json:"code"` + Message string `json:"message"` +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/restxml/restxml.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/restxml/restxml.go new file mode 100644 index 0000000000..8597f9f444 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/restxml/restxml.go @@ -0,0 +1,66 @@ +// Package restxml provides RESTful XML serialisation of AWS +// requests and responses. +package restxml + +//go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/input/rest-xml.json build_test.go +//go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/output/rest-xml.json unmarshal_test.go + +import ( + "bytes" + "encoding/xml" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/aws/awsutil" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/query" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil" + "io/ioutil" +) + +// Build builds a request payload for the REST XML protocol. +func Build(r *aws.Request) { + rest.Build(r) + + if t := rest.PayloadType(r.Params); t == "structure" || t == "" { + var buf bytes.Buffer + err := xmlutil.BuildXML(r.Params, xml.NewEncoder(&buf)) + if err != nil { + r.Error = apierr.New("Marshal", "failed to encode rest XML request", err) + return + } + r.SetBufferBody(buf.Bytes()) + if rest.PayloadMd5(r.Params) { + //增加md5 + r.HTTPRequest.Header.Set("Content-MD5", awsutil.EncodeAsString(awsutil.ComputeMD5Hash(buf.Bytes()))) + } + } +} + +// Unmarshal unmarshals a payload response for the REST XML protocol. +func Unmarshal(r *aws.Request) { + if t := rest.PayloadType(r.Data); t == "structure" || t == "" { + defer r.HTTPResponse.Body.Close() + data, err := ioutil.ReadAll(r.HTTPResponse.Body) + if err != nil { + r.Error = apierr.New("ReadBody", "failed to read response body", err) + return + } + decoder := xml.NewDecoder(bytes.NewReader(data)) + err = xmlutil.UnmarshalXML(r.Data, decoder, "") + if err != nil { + r.Error = apierr.New("Unmarshal", "failed to decode REST XML response", err) + return + } + return + } +} + +// UnmarshalMeta unmarshals response headers for the REST XML protocol. +func UnmarshalMeta(r *aws.Request) { + rest.Unmarshal(r) +} + +// UnmarshalError unmarshals a response error for the REST XML protocol. +func UnmarshalError(r *aws.Request) { + query.UnmarshalError(r) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/build.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/build.go new file mode 100644 index 0000000000..d3db250231 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/build.go @@ -0,0 +1,287 @@ +// Package xmlutil provides XML serialisation of AWS requests and responses. +package xmlutil + +import ( + "encoding/base64" + "encoding/xml" + "fmt" + "reflect" + "sort" + "strconv" + "strings" + "time" +) + +// BuildXML will serialize params into an xml.Encoder. +// Error will be returned if the serialization of any of the params or nested values fails. +func BuildXML(params interface{}, e *xml.Encoder) error { + b := xmlBuilder{encoder: e, namespaces: map[string]string{}} + root := NewXMLElement(xml.Name{}) + if err := b.buildValue(reflect.ValueOf(params), root, ""); err != nil { + return err + } + for _, c := range root.Children { + for _, v := range c { + return StructToXML(e, v, false) + } + } + return nil +} + +// Returns the reflection element of a value, if it is a pointer. +func elemOf(value reflect.Value) reflect.Value { + for value.Kind() == reflect.Ptr { + value = value.Elem() + } + return value +} + +// A xmlBuilder serializes values from Go code to XML +type xmlBuilder struct { + encoder *xml.Encoder + namespaces map[string]string +} + +// buildValue generic XMLNode builder for any type. Will build value for their specific type +// struct, list, map, scalar. +// +// Also takes a "type" tag value to set what type a value should be converted to XMLNode as. If +// type is not provided reflect will be used to determine the value's type. +func (b *xmlBuilder) buildValue(value reflect.Value, current *XMLNode, tag reflect.StructTag) error { + value = elemOf(value) + if !value.IsValid() { // no need to handle zero values + return nil + } else if tag.Get("location") != "" { // don't handle non-body location values + return nil + } + + t := tag.Get("type") + if t == "" { + switch value.Kind() { + case reflect.Struct: + t = "structure" + case reflect.Slice: + t = "list" + case reflect.Map: + t = "map" + } + } + + switch t { + case "structure": + if field, ok := value.Type().FieldByName("SDKShapeTraits"); ok { + tag = tag + reflect.StructTag(" ") + field.Tag + } + return b.buildStruct(value, current, tag) + case "list": + return b.buildList(value, current, tag) + case "map": + return b.buildMap(value, current, tag) + default: + return b.buildScalar(value, current, tag) + } +} + +// buildStruct adds a struct and its fields to the current XMLNode. All fields any any nested +// types are converted to XMLNodes also. +func (b *xmlBuilder) buildStruct(value reflect.Value, current *XMLNode, tag reflect.StructTag) error { + if !value.IsValid() { + return nil + } + + fieldAdded := false + + // unwrap payloads + if payload := tag.Get("payload"); payload != "" { + field, _ := value.Type().FieldByName(payload) + tag = field.Tag + value = elemOf(value.FieldByName(payload)) + + if !value.IsValid() { + return nil + } + } + + child := NewXMLElement(xml.Name{Local: tag.Get("locationName")}) + + // there is an xmlNamespace associated with this struct + if prefix, uri := tag.Get("xmlPrefix"), tag.Get("xmlURI"); uri != "" { + ns := xml.Attr{ + Name: xml.Name{Local: "xmlns"}, + Value: uri, + } + if prefix != "" { + b.namespaces[prefix] = uri // register the namespace + ns.Name.Local = "xmlns:" + prefix + } + + child.Attr = append(child.Attr, ns) + } + + t := value.Type() + for i := 0; i < value.NumField(); i++ { + if c := t.Field(i).Name[0:1]; strings.ToLower(c) == c { + continue // ignore unexported fields + } + + member := elemOf(value.Field(i)) + field := t.Field(i) + mTag := field.Tag + + if mTag.Get("location") != "" { // skip non-body members + continue + } + + memberName := mTag.Get("locationName") + if memberName == "" { + memberName = field.Name + mTag = reflect.StructTag(string(mTag) + ` locationName:"` + memberName + `"`) + } + if err := b.buildValue(member, child, mTag); err != nil { + return err + } + + fieldAdded = true + } + + if fieldAdded { // only append this child if we have one ore more valid members + current.AddChild(child) + } + + return nil +} + +// buildList adds the value's list items to the current XMLNode as children nodes. All +// nested values in the list are converted to XMLNodes also. +func (b *xmlBuilder) buildList(value reflect.Value, current *XMLNode, tag reflect.StructTag) error { + if value.IsNil() { // don't build omitted lists + return nil + } + + // check for unflattened list member + flattened := tag.Get("flattened") != "" + + xname := xml.Name{Local: tag.Get("locationName")} + if flattened { + for i := 0; i < value.Len(); i++ { + child := NewXMLElement(xname) + current.AddChild(child) + if err := b.buildValue(value.Index(i), child, ""); err != nil { + return err + } + } + } else { + list := NewXMLElement(xname) + current.AddChild(list) + + for i := 0; i < value.Len(); i++ { + iname := tag.Get("locationNameList") + if iname == "" { + iname = "member" + } + + child := NewXMLElement(xml.Name{Local: iname}) + list.AddChild(child) + if err := b.buildValue(value.Index(i), child, ""); err != nil { + return err + } + } + } + + return nil +} + +// buildMap adds the value's key/value pairs to the current XMLNode as children nodes. All +// nested values in the map are converted to XMLNodes also. +// +// Error will be returned if it is unable to build the map's values into XMLNodes +func (b *xmlBuilder) buildMap(value reflect.Value, current *XMLNode, tag reflect.StructTag) error { + if value.IsNil() { // don't build omitted maps + return nil + } + + maproot := NewXMLElement(xml.Name{Local: tag.Get("locationName")}) + current.AddChild(maproot) + current = maproot + + kname, vname := "key", "value" + if n := tag.Get("locationNameKey"); n != "" { + kname = n + } + if n := tag.Get("locationNameValue"); n != "" { + vname = n + } + + // sorting is not required for compliance, but it makes testing easier + keys := make([]string, value.Len()) + for i, k := range value.MapKeys() { + keys[i] = k.String() + } + sort.Strings(keys) + + for _, k := range keys { + v := value.MapIndex(reflect.ValueOf(k)) + + mapcur := current + if tag.Get("flattened") == "" { // add "entry" tag to non-flat maps + child := NewXMLElement(xml.Name{Local: "entry"}) + mapcur.AddChild(child) + mapcur = child + } + + kchild := NewXMLElement(xml.Name{Local: kname}) + kchild.Text = k + vchild := NewXMLElement(xml.Name{Local: vname}) + mapcur.AddChild(kchild) + mapcur.AddChild(vchild) + + if err := b.buildValue(v, vchild, ""); err != nil { + return err + } + } + + return nil +} + +// buildScalar will convert the value into a string and append it as a attribute or child +// of the current XMLNode. +// +// The value will be added as an attribute if tag contains a "xmlAttribute" attribute value. +// +// Error will be returned if the value type is unsupported. +func (b *xmlBuilder) buildScalar(value reflect.Value, current *XMLNode, tag reflect.StructTag) error { + var str string + switch converted := value.Interface().(type) { + case string: + str = converted + case []byte: + if !value.IsNil() { + str = base64.StdEncoding.EncodeToString(converted) + } + case bool: + str = strconv.FormatBool(converted) + case int64: + str = strconv.FormatInt(converted, 10) + case int: + str = strconv.Itoa(converted) + case float64: + str = strconv.FormatFloat(converted, 'f', -1, 64) + case float32: + str = strconv.FormatFloat(float64(converted), 'f', -1, 32) + case time.Time: + const ISO8601UTC = "2006-01-02T15:04:05Z" + str = converted.UTC().Format(ISO8601UTC) + default: + return fmt.Errorf("unsupported value for param %s: %v (%s)", + tag.Get("locationName"), value.Interface(), value.Type().Name()) + } + + xname := xml.Name{Local: tag.Get("locationName")} + if tag.Get("xmlAttribute") != "" { // put into current node's attribute list + attr := xml.Attr{Name: xname, Value: str} + current.Attr = append(current.Attr, attr) + } else { // regular text node + current.AddChild(&XMLNode{Name: xname, Text: str}) + } + return nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/unmarshal.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/unmarshal.go new file mode 100644 index 0000000000..478ba9b41a --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/unmarshal.go @@ -0,0 +1,266 @@ +package xmlutil + +import ( + "encoding/base64" + "encoding/xml" + "fmt" + "io" + "reflect" + "strconv" + "strings" + "time" +) + +// UnmarshalXML deserializes an xml.Decoder into the container v. V +// needs to match the shape of the XML expected to be decoded. +// If the shape doesn't match unmarshaling will fail. +func UnmarshalXML(v interface{}, d *xml.Decoder, wrapper string) error { + n, _ := XMLToStruct(d, nil) + if n.Children != nil { + for _, root := range n.Children { + for _, c := range root { + if wrappedChild, ok := c.Children[wrapper]; ok { + c = wrappedChild[0] // pull out wrapped element + } + + err := parse(reflect.ValueOf(v), c, "") + if err != nil { + if err == io.EOF { + return nil + } + return err + } + } + } + return nil + } + return nil +} + +// parse deserializes any value from the XMLNode. The type tag is used to infer the type, or reflect +// will be used to determine the type from r. +func parse(r reflect.Value, node *XMLNode, tag reflect.StructTag) error { + rtype := r.Type() + if rtype.Kind() == reflect.Ptr { + rtype = rtype.Elem() // check kind of actual element type + } + + t := tag.Get("type") + if t == "" { + switch rtype.Kind() { + case reflect.Struct: + t = "structure" + case reflect.Slice: + t = "list" + case reflect.Map: + t = "map" + } + } + + switch t { + case "structure": + if field, ok := rtype.FieldByName("SDKShapeTraits"); ok { + tag = field.Tag + } + return parseStruct(r, node, tag) + case "list": + return parseList(r, node, tag) + case "map": + return parseMap(r, node, tag) + default: + return parseScalar(r, node, tag) + } +} + +// parseStruct deserializes a structure and its fields from an XMLNode. Any nested +// types in the structure will also be deserialized. +func parseStruct(r reflect.Value, node *XMLNode, tag reflect.StructTag) error { + t := r.Type() + if r.Kind() == reflect.Ptr { + if r.IsNil() { // create the structure if it's nil + s := reflect.New(r.Type().Elem()) + r.Set(s) + r = s + } + + r = r.Elem() + t = t.Elem() + } + + // unwrap any payloads + if payload := tag.Get("payload"); payload != "" { + field, _ := t.FieldByName(payload) + return parseStruct(r.FieldByName(payload), node, field.Tag) + } + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if c := field.Name[0:1]; strings.ToLower(c) == c { + continue // ignore unexported fields + } + + // figure out what this field is called + name := field.Name + if field.Tag.Get("flattened") != "" && field.Tag.Get("locationNameList") != "" { + name = field.Tag.Get("locationNameList") + } else if locName := field.Tag.Get("locationName"); locName != "" { + name = locName + } + + // try to find the field by name in elements + elems := node.Children[name] + + if elems == nil { // try to find the field in attributes + for _, a := range node.Attr { + if name == a.Name.Local { + // turn this into a text node for de-serializing + elems = []*XMLNode{{Text: a.Value}} + } + } + } + + member := r.FieldByName(field.Name) + for _, elem := range elems { + err := parse(member, elem, field.Tag) + if err != nil { + return err + } + } + } + return nil +} + +// parseList deserializes a list of values from an XML node. Each list entry +// will also be deserialized. +func parseList(r reflect.Value, node *XMLNode, tag reflect.StructTag) error { + t := r.Type() + + if tag.Get("flattened") == "" { // look at all item entries + mname := "member" + if name := tag.Get("locationNameList"); name != "" { + mname = name + } + + if Children, ok := node.Children[mname]; ok { + if r.IsNil() { + r.Set(reflect.MakeSlice(t, len(Children), len(Children))) + } + + for i, c := range Children { + err := parse(r.Index(i), c, "") + if err != nil { + return err + } + } + } + } else { // flattened list means this is a single element + if r.IsNil() { + r.Set(reflect.MakeSlice(t, 0, 0)) + } + + childR := reflect.Zero(t.Elem()) + r.Set(reflect.Append(r, childR)) + err := parse(r.Index(r.Len()-1), node, "") + if err != nil { + return err + } + } + + return nil +} + +// parseMap deserializes a map from an XMLNode. The direct children of the XMLNode +// will also be deserialized as map entries. +func parseMap(r reflect.Value, node *XMLNode, tag reflect.StructTag) error { + if r.IsNil() { + r.Set(reflect.MakeMap(r.Type())) + } + + if tag.Get("flattened") == "" { // look at all child entries + for _, entry := range node.Children["entry"] { + parseMapEntry(r, entry, tag) + } + } else { // this element is itself an entry + parseMapEntry(r, node, tag) + } + + return nil +} + +// parseMapEntry deserializes a map entry from a XML node. +func parseMapEntry(r reflect.Value, node *XMLNode, tag reflect.StructTag) error { + kname, vname := "key", "value" + if n := tag.Get("locationNameKey"); n != "" { + kname = n + } + if n := tag.Get("locationNameValue"); n != "" { + vname = n + } + + keys, ok := node.Children[kname] + values := node.Children[vname] + if ok { + for i, key := range keys { + keyR := reflect.ValueOf(key.Text) + value := values[i] + valueR := reflect.New(r.Type().Elem()).Elem() + + parse(valueR, value, "") + r.SetMapIndex(keyR, valueR) + } + } + return nil +} + +// parseScaller deserializes an XMLNode value into a concrete type based on the +// interface type of r. +// +// Error is returned if the deserialization fails due to invalid type conversion, +// or unsupported interface type. +func parseScalar(r reflect.Value, node *XMLNode, tag reflect.StructTag) error { + switch r.Interface().(type) { + case *string: + r.Set(reflect.ValueOf(&node.Text)) + return nil + case string: + r.Set(reflect.ValueOf(node.Text)) + return nil + case []byte: + b, err := base64.StdEncoding.DecodeString(node.Text) + if err != nil { + return err + } + r.Set(reflect.ValueOf(b)) + case *bool: + v, err := strconv.ParseBool(node.Text) + if err != nil { + return err + } + r.Set(reflect.ValueOf(&v)) + case *int64: + v, err := strconv.ParseInt(node.Text, 10, 64) + if err != nil { + return err + } + r.Set(reflect.ValueOf(&v)) + case *float64: + v, err := strconv.ParseFloat(node.Text, 64) + if err != nil { + return err + } + r.Set(reflect.ValueOf(&v)) + case *time.Time: + const ISO8601UTC = "2006-01-02T15:04:05Z" + t, err := time.Parse(ISO8601UTC, node.Text) + if err != nil { + t, err = time.Parse("2006-01-02T15:04:05-07:00", node.Text) + if err != nil { + return err + } + } + r.Set(reflect.ValueOf(&t)) + default: + return fmt.Errorf("unsupported value: %v (%s)", r.Interface(), r.Type()) + } + return nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/xml_to_struct.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/xml_to_struct.go new file mode 100644 index 0000000000..3aa92f1bc0 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil/xml_to_struct.go @@ -0,0 +1,105 @@ +package xmlutil + +import ( + "encoding/xml" + "io" + "sort" +) + +// A XMLNode contains the values to be encoded or decoded. +type XMLNode struct { + Name xml.Name `json:",omitempty"` + Children map[string][]*XMLNode `json:",omitempty"` + Text string `json:",omitempty"` + Attr []xml.Attr `json:",omitempty"` +} + +// NewXMLElement returns a pointer to a new XMLNode initialized to default values. +func NewXMLElement(name xml.Name) *XMLNode { + return &XMLNode{ + Name: name, + Children: map[string][]*XMLNode{}, + Attr: []xml.Attr{}, + } +} + +// AddChild adds child to the XMLNode. +func (n *XMLNode) AddChild(child *XMLNode) { + if _, ok := n.Children[child.Name.Local]; !ok { + n.Children[child.Name.Local] = []*XMLNode{} + } + n.Children[child.Name.Local] = append(n.Children[child.Name.Local], child) +} + +// XMLToStruct converts a xml.Decoder stream to XMLNode with nested values. +func XMLToStruct(d *xml.Decoder, s *xml.StartElement) (*XMLNode, error) { + out := &XMLNode{} + for { + tok, err := d.Token() + if tok == nil || err == io.EOF { + break + } + if err != nil { + return out, err + } + + switch typed := tok.(type) { + case xml.CharData: + out.Text = string(typed.Copy()) + case xml.StartElement: + el := typed.Copy() + out.Attr = el.Attr + if out.Children == nil { + out.Children = map[string][]*XMLNode{} + } + + name := typed.Name.Local + slice := out.Children[name] + if slice == nil { + slice = []*XMLNode{} + } + node, e := XMLToStruct(d, &el) + if e != nil { + return out, e + } + node.Name = typed.Name + slice = append(slice, node) + out.Children[name] = slice + case xml.EndElement: + if s != nil && s.Name.Local == typed.Name.Local { // matching end token + return out, nil + } + } + } + return out, nil +} + +// StructToXML writes an XMLNode to a xml.Encoder as tokens. +func StructToXML(e *xml.Encoder, node *XMLNode, sorted bool) error { + e.EncodeToken(xml.StartElement{Name: node.Name, Attr: node.Attr}) + + if node.Text != "" { + e.EncodeToken(xml.CharData(node.Text)) + } else if sorted { + sortedNames := []string{} + for k := range node.Children { + sortedNames = append(sortedNames, k) + } + sort.Strings(sortedNames) + + for _, k := range sortedNames { + for _, v := range node.Children[k] { + StructToXML(e, v, sorted) + } + } + } else { + for _, c := range node.Children { + for _, v := range c { + StructToXML(e, v, sorted) + } + } + } + + e.EncodeToken(xml.EndElement{Name: node.Name}) + return e.Flush() +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/open_api/open_api.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/open_api/open_api.go new file mode 100644 index 0000000000..e531e31fda --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/open_api/open_api.go @@ -0,0 +1,77 @@ +package open_api + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/aws/credentials" + "net/url" + "sort" + "time" +) + +func Sign(req *aws.Request) { + if req.Config.Credentials == credentials.AnonymousCredentials { + return + } + + CredValues, err := req.Config.Credentials.Get() + if err != nil { + req.Error = err + return + } + + params := map[string]string{ + //固定参数 + "Accesskey": CredValues.AccessKeyID, + "Service": "ks3bill", + "Version": "v1", + "Timestamp": time.Now().UTC().Format("2006-01-02T15:04:05Z"), + "SignatureVersion": "1.0", + "SignatureMethod": "HMAC-SHA256", + } + + query := req.HTTPRequest.URL.Query() + for k, v := range params { + query.Set(k, v) + } + + queryString := getCanonicalizedQueryString(query) + cfg := req.Config + cfg.LogDebug("%s", "---[ QUERY STRING ]--------------------------------") + cfg.LogDebug("%s", queryString) + cfg.LogDebug("-----------------------------------------------------") + + signature := getSignature(queryString, CredValues.SecretAccessKey) + query.Set("Signature", signature) + req.HTTPRequest.URL.RawQuery = query.Encode() + + return +} + +// getCanonicalizedQueryString 构建规范化查询字符串 +func getCanonicalizedQueryString(params url.Values) string { + //对参数键进行排序 + keys := make([]string, 0, len(params)) + for k := range params { + keys = append(keys, k) + } + sort.Strings(keys) + + //构建待签名字符串 + var strEncode string + for _, k := range keys { + strEncode += url.QueryEscape(k) + "=" + url.QueryEscape(params.Get(k)) + "&" + } + strEncode = strEncode[:len(strEncode)-1] + return strEncode +} + +// getSignature 简易签名函数，使用HMAC-SHA256算法 +func getSignature(queryString, secretKey string) string { + //生成HMAC-SHA256签名 + h := hmac.New(sha256.New, []byte(secretKey)) + h.Write([]byte(queryString)) + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/v2/v2.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/v2/v2.go new file mode 100644 index 0000000000..79243c4e11 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/v2/v2.go @@ -0,0 +1,371 @@ +package v2 + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "github.com/ks3sdklib/aws-sdk-go/aws/awsutil" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/ks3sdklib/aws-sdk-go/aws/credentials" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest" + + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +const ( + authHeaderPrefix = "AWS" + timeFormat = "Mon, 02 Jan 2006 15:04:05 GMT" +) + +var signQuerys = map[string]bool{ + "acl": true, + "lifecycle": true, + "location": true, + "logging": true, + "notification": true, + "policy": true, + "requestPayment": true, + "torrent": true, + "uploadId": true, + "uploads": true, + "versionId": true, + "versioning": true, + "versions": true, + "website": true, + "delete": true, + "thumbnail": true, + "cors": true, + "pfop": true, + "querypfop": true, + "partNumber": true, + "response-content-type": true, + "response-content-language": true, + "response-expires": true, + "response-cache-control": true, + "response-content-disposition": true, + "response-content-encoding": true, + "tagging": true, + "fetch": true, + "copy": true, + "mirror": true, + "restore": true, + "append": true, + "position": true, + "decompresspolicy": true, + "retention": true, + "crr": true, + "inventory": true, + "recycle": true, + "recover": true, + "clear": true, + "id": true, + "continuation-token": true, + "dataRedundancySwitch": true, + "bucketqos": true, + "requesterqos": true, + "encryption": true, + "accessmonitor": true, + "transferAcceleration": true, + "VpcAccessBlock": true, +} + +type signer struct { + Service *aws.Service + Request *http.Request + Time time.Time + ExpireTime int64 + ServiceName string + Region string + CredValues credentials.Value + Credentials *credentials.Credentials + Query url.Values + Body io.ReadSeeker + Debug uint + Logger io.Writer + + isPresign bool + formattedTime string + + canonicalHeaders string + canonicalResource string + stringToSign string + signature string + authorization string + awsRequest *aws.Request +} + +func Sign(req *aws.Request) { + if req.Service.Config.Credentials == credentials.AnonymousCredentials { + return + } + + region := req.Service.SigningRegion + if region == "" { + region = req.Service.Config.Region + } + + name := req.Service.SigningName + if name == "" { + name = req.Service.ServiceName + } + + s := signer{ + Service: req.Service, + Request: req.HTTPRequest, + Time: req.Time, + ExpireTime: req.ExpireTime, + Query: req.HTTPRequest.URL.Query(), + Body: req.Body, + ServiceName: name, + Region: region, + Credentials: req.Service.Config.Credentials, + Debug: req.Service.Config.LogLevel, + Logger: req.Service.Config.Logger, + awsRequest: req, + } + + req.Error = s.sign() +} + +func (v2 *signer) sign() error { + if v2.ExpireTime != 0 { + v2.isPresign = true + } + + if v2.isRequestSigned() { + if !v2.Credentials.IsExpired() { + // If the request is already signed, and the credentials have not + // expired yet ignore the signing request. + return nil + } + + // The credentials have expired for this request. The current signing + // is invalid, and needs to be request because the request will fail. + if v2.isPresign { + v2.removePresign() + // Update the request's query string to ensure the values stays in + // sync in the case retrieving the new credentials fails. + v2.Request.URL.RawQuery = v2.Query.Encode() + } + } + + var err error + v2.CredValues, err = v2.Credentials.Get() + if err != nil { + return err + } + + v2.build() + + v2.logSigningInfo() + + return nil +} + +func (v2 *signer) logSigningInfo() { + cfg := v2.Service.Config + cfg.LogDebug("%s", "---[ STRING TO SIGN ]--------------------------------") + cfg.LogDebug("%s", v2.stringToSign) + if v2.isPresign { + cfg.LogDebug("---[ SIGNED URL ]--------------------------------") + cfg.LogDebug("%s", v2.Request.URL) + } + cfg.LogDebug("-----------------------------------------------------") +} + +func (v2 *signer) build() { + v2.buildTime() // no depends + v2.buildCanonicalHeaders() // depends on cred string + v2.buildCanonicalResource() // depends on canon headers / signed headers + v2.buildStringToSign() // depends on canon string + v2.buildSignature() // depends on string to sign + + if v2.isPresign { + var querys url.Values + querys = make(map[string][]string) + querys.Add("Signature", v2.signature) + querys.Add("AWSAccessKeyId", v2.CredValues.AccessKeyID) + v2.Request.URL.RawQuery += "&" + querys.Encode() + } else { + v2.Request.Header.Set("Authorization", "AWS "+v2.CredValues.AccessKeyID+":"+v2.signature) + } +} + +func (v2 *signer) buildTime() { + v2.formattedTime = v2.Time.UTC().Format(timeFormat) + + if v2.isPresign { + duration := int64(v2.ExpireTime) + v2.Query.Set("Expires", strconv.FormatInt(duration, 10)) + } else { + v2.Request.Header.Set("Date", v2.formattedTime) + } +} + +func (v2 *signer) buildCanonicalHeaders() { + var headers []string + for k, v := range v2.Request.Header { + if strings.HasPrefix(k, "X-Amz-") { + key := strings.ToLower(http.CanonicalHeaderKey(k)) + v2.Request.Header[key] = v + v2.Request.Header.Del(http.CanonicalHeaderKey(k)) + headers = append(headers, key) + } + } + sort.Strings(headers) + + headerValues := make([]string, len(headers)) + for i, k := range headers { + key := strings.ToLower(http.CanonicalHeaderKey(k)) + headerValues[i] = key + ":" + strings.Join(v2.Request.Header[key], ",") + } + + v2.canonicalHeaders = strings.Join(headerValues, "\n") +} + +func (v2 *signer) buildCanonicalResource() { + endpoint := v2.Service.Endpoint + + v2.Request.URL.RawQuery = strings.Replace(v2.Query.Encode(), "+", "%20", -1) + url := v2.Request.URL.String() + //在aws.service.go,buildEndpoint会把sheme也加上 + pathStyle := strings.HasPrefix(url, endpoint) + uri := v2.Request.URL.Opaque + bucketInHost := "" + if !pathStyle { + if strings.HasPrefix(url, "http://") { + url = url[7:] + endpoint = endpoint[7:] + } else if strings.HasPrefix(url, "https://") { + url = url[8:] + endpoint = endpoint[8:] + } + bucketInHost = url[0 : strings.Index(url, endpoint)-1] + } + if uri != "" { + uris := strings.Split(uri, "/")[3:] + append := false + if len(uris) == 1 && uris[0] != "" && bucketInHost == "" { + //只有bucket + append = true + } else if len(uris) == 0 && bucketInHost != "" { + append = true + } + uri = "/" + strings.Join(strings.Split(uri, "/")[3:], "/") + if bucketInHost != "" { + uri = "/" + bucketInHost + uri + } + if v2.awsRequest.Config.DomainMode { + b := awsutil.ValuesAtPath(v2.awsRequest.Params, "Bucket") + bucket := b[0].(string) + uri = "/" + bucket + uri + append = false + } + if append { + uri += "/" + } + } else { + uri = v2.Request.URL.Path + } + if uri == "" { + uri = "/" + } + + if v2.ServiceName != "s3" { + uri = rest.EscapePath(uri, false) + } + + var querys []string + for k := range v2.Query { + if _, ok := signQuerys[k]; ok { + querys = append(querys, k) + } + } + sort.Strings(querys) + + queryValues := make([]string, len(querys)) + for i, k := range querys { + v := v2.Query[k] + vString := strings.Join(v, ",") + if vString != "" { + queryValues[i] = k + "=" + vString + } else { + queryValues[i] = k + } + } + queryString := strings.Join(queryValues, "&") + if queryString == "" { + v2.canonicalResource = uri + } else { + v2.canonicalResource = uri + "?" + queryString + } +} + +func (v2 *signer) buildStringToSign() { + md5list := v2.Request.Header["Content-Md5"] + md5 := "" + if len(md5list) > 0 { + md5 = v2.Request.Header["Content-Md5"][0] + } + + typelist := v2.Request.Header["Content-Type"] + contenttype := "" + if len(typelist) > 0 { + contenttype = v2.Request.Header["Content-Type"][0] + } + signItems := []string{v2.Request.Method, md5, contenttype} + if v2.isPresign { + signItems = append(signItems, v2.Query["Expires"][0]) + } else { + signItems = append(signItems, v2.formattedTime) + } + if v2.canonicalHeaders != "" { + signItems = append(signItems, v2.canonicalHeaders) + } + signItems = append(signItems, v2.canonicalResource) + + v2.stringToSign = strings.Join(signItems, "\n") + +} + +func (v2 *signer) buildSignature() { + secret := v2.CredValues.SecretAccessKey + signature := string(base64Encode(makeHmac([]byte(secret), []byte(v2.stringToSign)))) + v2.signature = signature +} + +// isRequestSigned returns if the request is currently signed or presigned +func (v2 *signer) isRequestSigned() bool { + if v2.isPresign && v2.Query.Get("Signature") != "" { + return true + } + if v2.Request.Header.Get("Authorization") != "" { + return true + } + + return false +} + +// unsign removes signing flags for both signed and presigned requests. +func (v2 *signer) removePresign() { + v2.Query.Del("AWSAccessKeyId") + v2.Query.Del("Signature") + v2.Query.Del("Expires") + +} + +func makeHmac(key []byte, data []byte) []byte { + hash := hmac.New(sha1.New, key) + hash.Write(data) + return hash.Sum(nil) +} +func base64Encode(src []byte) []byte { + return []byte(base64.StdEncoding.EncodeToString(src)) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/v4/v4.go b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/v4/v4.go new file mode 100644 index 0000000000..480e095f08 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/internal/signer/v4/v4.go @@ -0,0 +1,369 @@ +// Package v4 implements signing for AWS V4 signer +package v4 + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/ks3sdklib/aws-sdk-go/aws/credentials" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest" + + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +const ( + authHeaderPrefix = "AWS4-HMAC-SHA256" + timeFormat = "20060102T150405Z" + shortTimeFormat = "20060102" +) + +var ignoredHeaders = map[string]bool{ + "Authorization": true, + "Content-Type": true, + "Content-Length": true, + "User-Agent": true, +} + +type signer struct { + Service *aws.Service + Request *http.Request + Time time.Time + ExpireTime int64 + ServiceName string + Region string + CredValues credentials.Value + Credentials *credentials.Credentials + Query url.Values + Body io.ReadSeeker + Debug uint + Logger io.Writer + + isPresign bool + isSignBody bool + formattedTime string + formattedShortTime string + + signedHeaders string + canonicalHeaders string + canonicalString string + credentialString string + stringToSign string + signature string + authorization string +} + +// Sign requests with signature version 4. +// +// Will sign the requests with the service config's Credentials object +// Signing is skipped if the credentials is the credentials.AnonymousCredentials +// object. +func Sign(req *aws.Request) { + // If the request does not need to be signed ignore the signing of the + // request if the AnonymousCredentials object is used. + if req.Service.Config.Credentials == credentials.AnonymousCredentials { + return + } + + region := req.Service.SigningRegion + if region == "" { + region = req.Service.Config.Region + } + + name := req.Service.SigningName + if name == "" { + name = req.Service.ServiceName + } + + s := signer{ + Service: req.Service, + Request: req.HTTPRequest, + Time: req.Time, + ExpireTime: req.ExpireTime, + Query: req.HTTPRequest.URL.Query(), + Body: req.Body, + ServiceName: name, + Region: region, + Credentials: req.Service.Config.Credentials, + Debug: req.Service.Config.LogLevel, + Logger: req.Service.Config.Logger, + } + if req.Service.Config.SignerVersion == "V4_UNSIGNED_PAYLOAD_SIGNER" { + s.isSignBody = false + } else { + s.isSignBody = true + } + req.Error = s.sign() +} + +func (v4 *signer) sign() error { + if v4.ExpireTime != 0 { + v4.isPresign = true + } + + if v4.isRequestSigned() { + if !v4.Credentials.IsExpired() { + // If the request is already signed, and the credentials have not + // expired yet ignore the signing request. + return nil + } + + // The credentials have expired for this request. The current signing + // is invalid, and needs to be request because the request will fail. + if v4.isPresign { + v4.removePresign() + // Update the request's query string to ensure the values stays in + // sync in the case retrieving the new credentials fails. + v4.Request.URL.RawQuery = v4.Query.Encode() + } + } + + var err error + v4.CredValues, err = v4.Credentials.Get() + if err != nil { + return err + } + + if v4.isPresign { + v4.Query.Set("X-Amz-Algorithm", authHeaderPrefix) + if v4.CredValues.SessionToken != "" { + v4.Query.Set("X-Amz-Security-Token", v4.CredValues.SessionToken) + } else { + v4.Query.Del("X-Amz-Security-Token") + } + } else if v4.CredValues.SessionToken != "" { + v4.Request.Header.Set("X-Amz-Security-Token", v4.CredValues.SessionToken) + } + + v4.build() + + v4.logSigningInfo() + + return nil +} + +func (v4 *signer) logSigningInfo() { + cfg := v4.Service.Config + cfg.LogDebug("%s", "---[ CANONICAL STRING ]-----------------------------") + cfg.LogDebug("%s", v4.canonicalString) + cfg.LogDebug("%s", "---[ STRING TO SIGN ]--------------------------------") + cfg.LogDebug("%s", v4.stringToSign) + if v4.isPresign { + cfg.LogDebug("---[ SIGNED URL ]--------------------------------") + cfg.LogDebug("%s", v4.Request.URL) + } + cfg.LogDebug("-----------------------------------------------------") +} + +func (v4 *signer) build() { + v4.buildTime() // no depends + v4.buildCredentialString() // no depends + if v4.isPresign { + v4.buildQuery() // no depends + } + v4.buildCanonicalHeaders() // depends on cred string + v4.buildCanonicalString() // depends on canon headers / signed headers + v4.buildStringToSign() // depends on canon string + v4.buildSignature() // depends on string to sign + + if v4.isPresign { + v4.Request.URL.RawQuery += "&X-Amz-Signature=" + v4.signature + } else { + parts := []string{ + authHeaderPrefix + " Credential=" + v4.CredValues.AccessKeyID + "/" + v4.credentialString, + "SignedHeaders=" + v4.signedHeaders, + "Signature=" + v4.signature, + } + v4.Request.Header.Set("Authorization", strings.Join(parts, ", ")) + } +} + +func (v4 *signer) buildTime() { + v4.formattedTime = v4.Time.UTC().Format(timeFormat) + v4.formattedShortTime = v4.Time.UTC().Format(shortTimeFormat) + + if v4.isPresign { + duration := int64(v4.ExpireTime) + v4.Query.Set("X-Amz-Date", v4.formattedTime) + v4.Query.Set("X-Amz-Expires", strconv.FormatInt(duration, 10)) + } else { + v4.Request.Header.Set("X-Amz-Date", v4.formattedTime) + } +} + +func (v4 *signer) buildCredentialString() { + v4.credentialString = strings.Join([]string{ + v4.formattedShortTime, + v4.Region, + v4.ServiceName, + "aws4_request", + }, "/") + + if v4.isPresign { + v4.Query.Set("X-Amz-Credential", v4.CredValues.AccessKeyID+"/"+v4.credentialString) + } +} + +func (v4 *signer) buildQuery() { + for k, h := range v4.Request.Header { + if strings.HasPrefix(http.CanonicalHeaderKey(k), "X-Amz-") { + continue // never hoist x-amz-* headers, they must be signed + } + if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { + continue // never hoist ignored headers + } + + v4.Request.Header.Del(k) + v4.Query.Del(k) + for _, v := range h { + v4.Query.Add(k, v) + } + } +} + +func (v4 *signer) buildCanonicalHeaders() { + var headers []string + headers = append(headers, "host") + for k := range v4.Request.Header { + if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { + continue // ignored header + } + headers = append(headers, strings.ToLower(k)) + } + sort.Strings(headers) + + v4.signedHeaders = strings.Join(headers, ";") + + if v4.isPresign { + v4.Query.Set("X-Amz-SignedHeaders", v4.signedHeaders) + } + + headerValues := make([]string, len(headers)) + for i, k := range headers { + if k == "host" { + headerValues[i] = "host:" + v4.Request.URL.Host + } else { + headerValues[i] = k + ":" + + strings.Join(v4.Request.Header[http.CanonicalHeaderKey(k)], ",") + } + } + + v4.canonicalHeaders = strings.Join(headerValues, "\n") +} + +func (v4 *signer) buildCanonicalString() { + v4.Request.URL.RawQuery = strings.Replace(v4.Query.Encode(), "+", "%20", -1) + uri := strings.Replace(v4.Request.URL.Opaque, "%2F", "/", -1) + if uri != "" { + uri = "/" + strings.Join(strings.Split(uri, "/")[3:], "/") + } else { + uri = v4.Request.URL.Path + } + if uri == "" { + uri = "/" + } + + if v4.ServiceName != "s3" { + uri = rest.EscapePath(uri, false) + } + + v4.canonicalString = strings.Join([]string{ + v4.Request.Method, + uri, + v4.Request.URL.RawQuery, + v4.canonicalHeaders + "\n", + v4.signedHeaders, + v4.bodyDigest(), + }, "\n") +} + +func (v4 *signer) buildStringToSign() { + v4.stringToSign = strings.Join([]string{ + authHeaderPrefix, + v4.formattedTime, + v4.credentialString, + hex.EncodeToString(makeSha256([]byte(v4.canonicalString))), + }, "\n") +} + +func (v4 *signer) buildSignature() { + secret := v4.CredValues.SecretAccessKey + date := makeHmac([]byte("AWS4"+secret), []byte(v4.formattedShortTime)) + region := makeHmac(date, []byte(v4.Region)) + service := makeHmac(region, []byte(v4.ServiceName)) + credentials := makeHmac(service, []byte("aws4_request")) + signature := makeHmac(credentials, []byte(v4.stringToSign)) + v4.signature = hex.EncodeToString(signature) +} + +func (v4 *signer) bodyDigest() string { + hash := v4.Request.Header.Get("X-Amz-Content-Sha256") + if hash == "" { + if v4.isPresign && v4.ServiceName == "s3" { + hash = "UNSIGNED-PAYLOAD" + } else { + if v4.isSignBody { + if v4.Body == nil { + hash = hex.EncodeToString(makeSha256([]byte{})) + } else { + hash = hex.EncodeToString(makeSha256Reader(v4.Body)) + } + } else { + hash = "UNSIGNED-PAYLOAD" + } + } + v4.Request.Header.Add("X-Amz-Content-Sha256", hash) + } + return hash +} + +// isRequestSigned returns if the request is currently signed or presigned +func (v4 *signer) isRequestSigned() bool { + if v4.isPresign && v4.Query.Get("X-Amz-Signature") != "" { + return true + } + if v4.Request.Header.Get("Authorization") != "" { + return true + } + + return false +} + +// unsign removes signing flags for both signed and presigned requests. +func (v4 *signer) removePresign() { + v4.Query.Del("X-Amz-Algorithm") + v4.Query.Del("X-Amz-Signature") + v4.Query.Del("X-Amz-Security-Token") + v4.Query.Del("X-Amz-Date") + v4.Query.Del("X-Amz-Expires") + v4.Query.Del("X-Amz-Credential") + v4.Query.Del("X-Amz-SignedHeaders") +} + +func makeHmac(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + hash.Write(data) + return hash.Sum(nil) +} + +func makeSha256(data []byte) []byte { + hash := sha256.New() + hash.Write(data) + return hash.Sum(nil) +} + +func makeSha256Reader(reader io.ReadSeeker) []byte { + hash := sha256.New() + start, _ := reader.Seek(0, 1) + defer reader.Seek(start, 0) + + io.Copy(hash, reader) + return hash.Sum(nil) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/api.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/api.go new file mode 100644 index 0000000000..629c6acb81 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/api.go @@ -0,0 +1,5720 @@ +// THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT. + +// Package s3 provides a client for Amazon Simple Storage Service. +package s3 + +import ( + "context" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/aws/awserr" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" + "github.com/ks3sdklib/aws-sdk-go/internal/crc" + "hash" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +// AbortMultipartUploadRequest generates a request for the AbortMultipartUpload operation. +func (c *S3) AbortMultipartUploadRequest(input *AbortMultipartUploadInput) (req *aws.Request, output *AbortMultipartUploadOutput) { + op := &aws.Operation{ + Name: "AbortMultipartUpload", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &AbortMultipartUploadInput{} + } + + req = c.newRequest(op, input, output) + output = &AbortMultipartUploadOutput{} + req.Data = output + return +} + +// AbortMultipartUpload Aborts a multipart upload. +// +// To verify that all parts have been removed, so you don't get charged for +// the part storage, you should call the List Parts operation and ensure the +// parts list is empty. +func (c *S3) AbortMultipartUpload(input *AbortMultipartUploadInput) (*AbortMultipartUploadOutput, error) { + req, out := c.AbortMultipartUploadRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) AbortMultipartUploadWithContext(ctx aws.Context, input *AbortMultipartUploadInput) (*AbortMultipartUploadOutput, error) { + req, out := c.AbortMultipartUploadRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// CompleteMultipartUploadRequest generates a request for the CompleteMultipartUpload operation. +func (c *S3) CompleteMultipartUploadRequest(input *CompleteMultipartUploadInput) (req *aws.Request, output *CompleteMultipartUploadOutput) { + op := &aws.Operation{ + Name: "CompleteMultipartUpload", + HTTPMethod: "POST", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &CompleteMultipartUploadInput{} + } + + req = c.newRequest(op, input, output) + output = &CompleteMultipartUploadOutput{} + req.Data = output + return +} + +// CompleteMultipartUpload Completes a multipart upload by assembling previously uploaded parts. +func (c *S3) CompleteMultipartUpload(input *CompleteMultipartUploadInput) (*CompleteMultipartUploadOutput, error) { + req, out := c.CompleteMultipartUploadRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) CompleteMultipartUploadWithContext(ctx aws.Context, input *CompleteMultipartUploadInput) (*CompleteMultipartUploadOutput, error) { + req, out := c.CompleteMultipartUploadRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// CopyObjectRequest generates a request for the CopyObject operation. +func (c *S3) CopyObjectRequest(input *CopyObjectInput) (req *aws.Request, output *CopyObjectOutput) { + op := &aws.Operation{ + Name: "CopyObject", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &CopyObjectInput{} + } + + // URL encode the copy source + if input.CopySource == nil { + input.CopySource = aws.String(BuildCopySource(input.SourceBucket, input.SourceKey)) + } + req = c.newRequest(op, input, output) + output = &CopyObjectOutput{} + req.Data = output + return +} + +// CopyObject Creates a copy of an object that is already stored in KS3. +func (c *S3) CopyObject(input *CopyObjectInput) (*CopyObjectOutput, error) { + req, out := c.CopyObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) CopyObjectWithContext(ctx aws.Context, input *CopyObjectInput) (*CopyObjectOutput, error) { + req, out := c.CopyObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// CreateBucketRequest generates a request for the CreateBucket operation. +func (c *S3) CreateBucketRequest(input *CreateBucketInput) (req *aws.Request, output *CreateBucketOutput) { + var op *aws.Operation + if input.ProjectId == nil { + op = &aws.Operation{ + Name: "CreateBucket", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}", + } + } else { + op = &aws.Operation{ + Name: "CreateBucket", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?projectId=" + *input.ProjectId, + } + } + + if input == nil { + input = &CreateBucketInput{} + } + + req = c.newRequest(op, input, output) + output = &CreateBucketOutput{} + req.Data = output + return +} + +// CreateBucket Creates a new bucket. +func (c *S3) CreateBucket(input *CreateBucketInput) (*CreateBucketOutput, error) { + req, out := c.CreateBucketRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) CreateBucketWithContext(ctx aws.Context, input *CreateBucketInput) (*CreateBucketOutput, error) { + req, out := c.CreateBucketRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// CreateMultipartUploadRequest generates a request for the CreateMultipartUpload operation. +func (c *S3) CreateMultipartUploadRequest(input *CreateMultipartUploadInput) (req *aws.Request, output *CreateMultipartUploadOutput) { + op := &aws.Operation{ + Name: "CreateMultipartUpload", + HTTPMethod: "POST", + HTTPPath: "/{Bucket}/{Key+}?uploads", + } + + if input == nil { + input = &CreateMultipartUploadInput{} + } + + req = c.newRequest(op, input, output) + output = &CreateMultipartUploadOutput{} + req.Data = output + return +} + +// CreateMultipartUpload Initiates a multipart upload and returns an upload ID. +// +// Note: After you initiate multipart upload and upload one or more parts, +// you must either complete or abort multipart upload in order to stop getting +// charged for storage of the uploaded parts. Only after you either complete +// or abort multipart upload, Amazon S3 frees up the parts storage and stops +// charging you for the parts storage. +func (c *S3) CreateMultipartUpload(input *CreateMultipartUploadInput) (*CreateMultipartUploadOutput, error) { + req, out := c.CreateMultipartUploadRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) CreateMultipartUploadWithContext(ctx aws.Context, input *CreateMultipartUploadInput) (*CreateMultipartUploadOutput, error) { + req, out := c.CreateMultipartUploadRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// DeleteBucketRequest generates a request for the DeleteBucket operation. +func (c *S3) DeleteBucketRequest(input *DeleteBucketInput) (req *aws.Request, output *DeleteBucketOutput) { + op := &aws.Operation{ + Name: "DeleteBucket", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}", + } + + if input == nil { + input = &DeleteBucketInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketOutput{} + req.Data = output + return +} + +// DeleteBucket Deletes the bucket. All objects (including all object versions and Delete +// Markers) in the bucket must be deleted before the bucket itself can be deleted. +func (c *S3) DeleteBucket(input *DeleteBucketInput) (*DeleteBucketOutput, error) { + req, out := c.DeleteBucketRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketWithContext(ctx aws.Context, input *DeleteBucketInput) (*DeleteBucketOutput, error) { + req, out := c.DeleteBucketRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// DeleteBucketPolicyRequest generates a request for the DeleteBucketPolicy operation. +func (c *S3) DeleteBucketPolicyRequest(input *DeleteBucketPolicyInput) (req *aws.Request, output *DeleteBucketPolicyOutput) { + op := &aws.Operation{ + Name: "DeleteBucketPolicy", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?policy", + } + + if input == nil { + input = &DeleteBucketPolicyInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketPolicyOutput{} + req.Data = output + return +} + +// DeleteBucketPolicy Deletes the policy from the bucket. +func (c *S3) DeleteBucketPolicy(input *DeleteBucketPolicyInput) (*DeleteBucketPolicyOutput, error) { + req, out := c.DeleteBucketPolicyRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketPolicyWithContext(ctx aws.Context, input *DeleteBucketPolicyInput) (*DeleteBucketPolicyOutput, error) { + req, out := c.DeleteBucketPolicyRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// DeleteBucketWebsiteRequest generates a request for the DeleteBucketWebsite operation. +func (c *S3) DeleteBucketWebsiteRequest(input *DeleteBucketWebsiteInput) (req *aws.Request, output *DeleteBucketWebsiteOutput) { + op := &aws.Operation{ + Name: "DeleteBucketWebsite", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?website", + } + + if input == nil { + input = &DeleteBucketWebsiteInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketWebsiteOutput{} + req.Data = output + return +} + +// DeleteBucketWebsite This operation removes the website configuration from the bucket. +func (c *S3) DeleteBucketWebsite(input *DeleteBucketWebsiteInput) (*DeleteBucketWebsiteOutput, error) { + req, out := c.DeleteBucketWebsiteRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketWebsiteWithContext(ctx aws.Context, input *DeleteBucketWebsiteInput) (*DeleteBucketWebsiteOutput, error) { + req, out := c.DeleteBucketWebsiteRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// DeleteObjectRequest generates a request for the DeleteObject operation. +func (c *S3) DeleteObjectRequest(input *DeleteObjectInput) (req *aws.Request, output *DeleteObjectOutput) { + op := &aws.Operation{ + Name: "DeleteObject", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &DeleteObjectInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteObjectOutput{} + req.Data = output + return +} + +// DeleteObject Removes the null version (if there is one) of an object and inserts a delete +// marker, which becomes the latest version of the object. If there isn't a +// null version, Amazon S3 does not remove any objects. +func (c *S3) DeleteObject(input *DeleteObjectInput) (*DeleteObjectOutput, error) { + req, out := c.DeleteObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteObjectWithContext(ctx aws.Context, input *DeleteObjectInput) (*DeleteObjectOutput, error) { + req, out := c.DeleteObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// DeleteObjectsRequest generates a request for the DeleteObjects operation. +func (c *S3) DeleteObjectsRequest(input *DeleteObjectsInput) (req *aws.Request, output *DeleteObjectsOutput) { + op := &aws.Operation{ + Name: "DeleteObjects", + HTTPMethod: "POST", + HTTPPath: "/{Bucket}?delete", + } + + if input == nil { + input = &DeleteObjectsInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteObjectsOutput{} + req.Data = output + return +} + +func (c *S3) DeleteObjects(input *DeleteObjectsInput) (*DeleteObjectsOutput, error) { + return c.DeleteObjectsWithContext(context.Background(), input) +} + +func (c *S3) DeleteObjectsWithContext(ctx aws.Context, input *DeleteObjectsInput) (*DeleteObjectsOutput, error) { + var errors []*Error + var okList []*DeletedObject + if input == nil { + input = &DeleteObjectsInput{} + } + for _, t := range input.Delete.Objects { + _, err := c.DeleteObjectWithContext(ctx, &DeleteObjectInput{Bucket: input.Bucket, Key: t.Key}) + if input.IsReTurnResults != nil && *input.IsReTurnResults == true { + if err != nil { + aerr, _ := err.(awserr.Error) + errors = append(errors, &Error{Key: t.Key, Code: aws.String(aerr.Code()), Message: aws.String(aerr.Message())}) + } else { + okList = append(okList, &DeletedObject{Key: t.Key}) + } + } + } + output := &DeleteObjectsOutput{ + Deleted: okList, + Errors: errors, + } + return output, nil +} + +// DeleteBucketPrefix deletes all objects with the specified prefix in the bucket. +func (c *S3) DeleteBucketPrefix(input *DeleteBucketPrefixInput) (*DeleteObjectsOutput, error) { + return c.DeleteBucketPrefixWithContext(context.Background(), input) +} + +func (c *S3) DeleteBucketPrefixWithContext(ctx aws.Context, input *DeleteBucketPrefixInput) (*DeleteObjectsOutput, error) { + var errors []*Error + var okList []*DeletedObject + + var output = &DeleteObjectsOutput{ + Deleted: okList, + Errors: errors, + } + if input == nil { + input = &DeleteBucketPrefixInput{} + } + marker := aws.String("") + maxKeys := aws.Long(100) + if input.MaxKeys != nil { + maxKeys = input.MaxKeys + } + for { + resp, err := c.ListObjectsWithContext(ctx, &ListObjectsInput{ + Bucket: input.Bucket, + Prefix: input.Prefix, + Marker: marker, + MaxKeys: maxKeys, + }) + if err == nil { + for _, t := range resp.Contents { + _, err := c.DeleteObjectWithContext(ctx, &DeleteObjectInput{Bucket: input.Bucket, Key: t.Key}) + if input.IsReTurnResults != nil && *input.IsReTurnResults == true { + if err != nil { + aerr, _ := err.(awserr.Error) + errors = append(errors, &Error{Key: t.Key, Code: aws.String(aerr.Code()), Message: aws.String(aerr.Message())}) + output.Errors = errors + } else { + okList = append(okList, &DeletedObject{Key: t.Key}) + output.Deleted = okList + } + } + } + if *resp.IsTruncated == false { + break + } + marker = resp.Contents[*maxKeys-1].Key + } else { + return output, err + } + } + return output, nil +} + +// TryDeleteBucketPrefix deletes all objects with the specified prefix in the bucket, and retries at most 3 times. +func (c *S3) TryDeleteBucketPrefix(input *DeleteBucketPrefixInput) (*DeleteObjectsOutput, error) { + return c.TryDeleteBucketPrefixWithContext(context.Background(), input) +} + +func (c *S3) TryDeleteBucketPrefixWithContext(ctx aws.Context, input *DeleteBucketPrefixInput) (*DeleteObjectsOutput, error) { + params := input + var output *DeleteObjectsOutput + err := Do(func(attempt int) (bool, error) { + var err error + output, err = c.DeleteBucketPrefixWithContext(ctx, params) + return attempt < 3, err // 重试3次 + }) + return output, err +} + +// GetBucketACLRequest generates a request for the GetBucketACL operation. +func (c *S3) GetBucketACLRequest(input *GetBucketACLInput) (req *aws.Request, output *GetBucketACLOutput) { + op := &aws.Operation{ + Name: "GetBucketAcl", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?acl", + } + + if input == nil { + input = &GetBucketACLInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketACLOutput{} + req.Data = output + return +} + +// GetBucketACL Gets the access control policy for the bucket. +func (c *S3) GetBucketACL(input *GetBucketACLInput) (*GetBucketACLOutput, error) { + req, out := c.GetBucketACLRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketACLWithContext(ctx aws.Context, input *GetBucketACLInput) (*GetBucketACLOutput, error) { + req, out := c.GetBucketACLRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetBucketLocationRequest generates a request for the GetBucketLocation operation. +func (c *S3) GetBucketLocationRequest(input *GetBucketLocationInput) (req *aws.Request, output *GetBucketLocationOutput) { + op := &aws.Operation{ + Name: "GetBucketLocation", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?location", + } + + if input == nil { + input = &GetBucketLocationInput{} + } + + req = c.newRequest(op, input, output) + req.Handlers.Unmarshal.PushFront(buildGetBucketLocation) + output = &GetBucketLocationOutput{} + req.Data = output + return +} + +// GetBucketLocation Returns the region the bucket resides in. +func (c *S3) GetBucketLocation(input *GetBucketLocationInput) (*GetBucketLocationOutput, error) { + req, out := c.GetBucketLocationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketLocationWithContext(ctx aws.Context, input *GetBucketLocationInput) (*GetBucketLocationOutput, error) { + req, out := c.GetBucketLocationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetBucketLoggingRequest generates a request for the GetBucketLogging operation. +func (c *S3) GetBucketLoggingRequest(input *GetBucketLoggingInput) (req *aws.Request, output *GetBucketLoggingOutput) { + op := &aws.Operation{ + Name: "GetBucketLogging", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?logging", + } + + if input == nil { + input = &GetBucketLoggingInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketLoggingOutput{} + req.Data = output + return +} + +// GetBucketLogging Returns the logging status of a bucket and the permissions users have to +// view and modify that status. To use GET, you must be the bucket owner. +func (c *S3) GetBucketLogging(input *GetBucketLoggingInput) (*GetBucketLoggingOutput, error) { + req, out := c.GetBucketLoggingRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketLoggingWithContext(ctx aws.Context, input *GetBucketLoggingInput) (*GetBucketLoggingOutput, error) { + req, out := c.GetBucketLoggingRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetBucketNotificationRequest generates a request for the GetBucketNotification operation. +func (c *S3) GetBucketNotificationRequest(input *GetBucketNotificationConfigurationRequest) (req *aws.Request, output *NotificationConfigurationDeprecated) { + op := &aws.Operation{ + Name: "GetBucketNotification", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?notification", + } + + if input == nil { + input = &GetBucketNotificationConfigurationRequest{} + } + + req = c.newRequest(op, input, output) + output = &NotificationConfigurationDeprecated{} + req.Data = output + return +} + +// GetBucketNotification Deprecated, see the GetBucketNotificationConfiguration operation. +func (c *S3) GetBucketNotification(input *GetBucketNotificationConfigurationRequest) (*NotificationConfigurationDeprecated, error) { + req, out := c.GetBucketNotificationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketNotificationWithContext(ctx aws.Context, input *GetBucketNotificationConfigurationRequest) (*NotificationConfigurationDeprecated, error) { + req, out := c.GetBucketNotificationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetBucketNotificationConfigurationRequest generates a request for the GetBucketNotificationConfiguration operation. +func (c *S3) GetBucketNotificationConfigurationRequest(input *GetBucketNotificationConfigurationRequest) (req *aws.Request, output *NotificationConfiguration) { + op := &aws.Operation{ + Name: "GetBucketNotificationConfiguration", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?notification", + } + + if input == nil { + input = &GetBucketNotificationConfigurationRequest{} + } + + req = c.newRequest(op, input, output) + output = &NotificationConfiguration{} + req.Data = output + return +} + +// GetBucketNotificationConfiguration Returns the notification configuration of a bucket. +func (c *S3) GetBucketNotificationConfiguration(input *GetBucketNotificationConfigurationRequest) (*NotificationConfiguration, error) { + req, out := c.GetBucketNotificationConfigurationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketNotificationConfigurationWithContext(ctx aws.Context, input *GetBucketNotificationConfigurationRequest) (*NotificationConfiguration, error) { + req, out := c.GetBucketNotificationConfigurationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetBucketPolicyRequest generates a request for the GetBucketPolicy operation. +func (c *S3) GetBucketPolicyRequest(input *GetBucketPolicyInput) (req *aws.Request, output *GetBucketPolicyOutput) { + op := &aws.Operation{ + Name: "GetBucketPolicy", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?policy", + } + + if input == nil { + input = &GetBucketPolicyInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketPolicyOutput{} + req.Data = output + return +} + +// GetBucketPolicy Returns the policy of a specified bucket. +func (c *S3) GetBucketPolicy(input *GetBucketPolicyInput) (*GetBucketPolicyOutput, error) { + req, out := c.GetBucketPolicyRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketPolicyWithContext(ctx aws.Context, input *GetBucketPolicyInput) (*GetBucketPolicyOutput, error) { + req, out := c.GetBucketPolicyRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetBucketRequestPaymentRequest generates a request for the GetBucketRequestPayment operation. +func (c *S3) GetBucketRequestPaymentRequest(input *GetBucketRequestPaymentInput) (req *aws.Request, output *GetBucketRequestPaymentOutput) { + op := &aws.Operation{ + Name: "GetBucketRequestPayment", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?requestPayment", + } + + if input == nil { + input = &GetBucketRequestPaymentInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketRequestPaymentOutput{} + req.Data = output + return +} + +// GetBucketRequestPayment Returns the request payment configuration of a bucket. +func (c *S3) GetBucketRequestPayment(input *GetBucketRequestPaymentInput) (*GetBucketRequestPaymentOutput, error) { + req, out := c.GetBucketRequestPaymentRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketRequestPaymentWithContext(ctx aws.Context, input *GetBucketRequestPaymentInput) (*GetBucketRequestPaymentOutput, error) { + req, out := c.GetBucketRequestPaymentRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetBucketVersioningRequest generates a request for the GetBucketVersioning operation. +func (c *S3) GetBucketVersioningRequest(input *GetBucketVersioningInput) (req *aws.Request, output *GetBucketVersioningOutput) { + op := &aws.Operation{ + Name: "GetBucketVersioning", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?versioning", + } + + if input == nil { + input = &GetBucketVersioningInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketVersioningOutput{} + req.Data = output + return +} + +// GetBucketVersioning Returns the versioning state of a bucket. +func (c *S3) GetBucketVersioning(input *GetBucketVersioningInput) (*GetBucketVersioningOutput, error) { + req, out := c.GetBucketVersioningRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketVersioningWithContext(ctx aws.Context, input *GetBucketVersioningInput) (*GetBucketVersioningOutput, error) { + req, out := c.GetBucketVersioningRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetBucketWebsiteRequest generates a request for the GetBucketWebsite operation. +func (c *S3) GetBucketWebsiteRequest(input *GetBucketWebsiteInput) (req *aws.Request, output *GetBucketWebsiteOutput) { + op := &aws.Operation{ + Name: "GetBucketWebsite", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?website", + } + + if input == nil { + input = &GetBucketWebsiteInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketWebsiteOutput{} + req.Data = output + return +} + +// GetBucketWebsite Returns the website configuration for a bucket. +func (c *S3) GetBucketWebsite(input *GetBucketWebsiteInput) (*GetBucketWebsiteOutput, error) { + req, out := c.GetBucketWebsiteRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketWebsiteWithContext(ctx aws.Context, input *GetBucketWebsiteInput) (*GetBucketWebsiteOutput, error) { + req, out := c.GetBucketWebsiteRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetObjectRequest generates a request for the GetObject operation. +func (c *S3) GetObjectRequest(input *GetObjectInput) (req *aws.Request, output *GetObjectOutput) { + op := &aws.Operation{ + Name: "GetObject", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &GetObjectInput{} + } + + req = c.newRequest(op, input, output) + output = &GetObjectOutput{} + req.Data = output + return +} + +// GetObject Retrieves objects from KS3. +func (c *S3) GetObject(input *GetObjectInput) (*GetObjectOutput, error) { + req, out := c.GetObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetObjectWithContext(ctx aws.Context, input *GetObjectInput) (*GetObjectOutput, error) { + req, out := c.GetObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +func (c *S3) GetObjectToFile(input *GetObjectInput, filePath string) error { + // Calls the API to actually download the object. Returns the result instance. + res, err := c.GetObject(input) + if err != nil { + return err + } + return c.SaveObjectToFile(filePath, input, res) +} + +func (c *S3) GetObjectToFileWithContext(ctx aws.Context, input *GetObjectInput, filePath string) error { + // Calls the API to actually download the object. Returns the result instance. + res, err := c.GetObjectWithContext(ctx, input) + if err != nil { + return err + } + return c.SaveObjectToFile(filePath, input, res) +} + +func (c *S3) SaveObjectToFile(filePath string, input *GetObjectInput, res *GetObjectOutput) error { + tempFilePath := filePath + TempFileSuffix + // If the local file does not exist, create a new one. If it exists, overwrite it. + fd, err := os.OpenFile(tempFilePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(0664)) + if err != nil { + return err + } + + var crc64 hash.Hash64 + if c.Config.CrcCheckEnabled { + crc64 = crc.NewCRC(crc.CrcTable(), 0) + } + + if c.Config.CrcCheckEnabled || input.ProgressFn != nil { + var contentLength int64 + if res.ContentLength != nil { + contentLength = *res.ContentLength + } + res.Body = aws.TeeReader(res.Body, crc64, contentLength, input.ProgressFn) + } + + // Copy the data to the local file path. + _, err = io.Copy(fd, res.Body) + fd.Close() + if err != nil { + return err + } + + if c.Config.CrcCheckEnabled { + err = CheckDownloadCrc64(c, res, crc64) + if err != nil { + return err + } + } + + return os.Rename(tempFilePath, filePath) +} + +// GetObjectACLRequest generates a request for the GetObjectACL operation. +func (c *S3) GetObjectACLRequest(input *GetObjectACLInput) (req *aws.Request, output *GetObjectACLOutput) { + op := &aws.Operation{ + Name: "GetObjectAcl", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}/{Key+}?acl", + } + + if input == nil { + input = &GetObjectACLInput{} + } + + req = c.newRequest(op, input, output) + output = &GetObjectACLOutput{} + req.Data = output + return +} + +// GetObjectACL Returns the access control list (ACL) of an object. +func (c *S3) GetObjectACL(input *GetObjectACLInput) (*GetObjectACLOutput, error) { + req, out := c.GetObjectACLRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetObjectACLWithContext(ctx aws.Context, input *GetObjectACLInput) (*GetObjectACLOutput, error) { + req, out := c.GetObjectACLRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GetObjectTorrentRequest generates a request for the GetObjectTorrent operation. +func (c *S3) GetObjectTorrentRequest(input *GetObjectTorrentInput) (req *aws.Request, output *GetObjectTorrentOutput) { + op := &aws.Operation{ + Name: "GetObjectTorrent", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}/{Key+}?torrent", + } + + if input == nil { + input = &GetObjectTorrentInput{} + } + + req = c.newRequest(op, input, output) + output = &GetObjectTorrentOutput{} + req.Data = output + return +} + +// GetObjectTorrent Return torrent files from a bucket. +func (c *S3) GetObjectTorrent(input *GetObjectTorrentInput) (*GetObjectTorrentOutput, error) { + req, out := c.GetObjectTorrentRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetObjectTorrentWithContext(ctx aws.Context, input *GetObjectTorrentInput) (*GetObjectTorrentOutput, error) { + req, out := c.GetObjectTorrentRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// HeadBucketRequest generates a request for the HeadBucket operation. +func (c *S3) HeadBucketRequest(input *HeadBucketInput) (req *aws.Request, output *HeadBucketOutput) { + op := &aws.Operation{ + Name: "HeadBucket", + HTTPMethod: "HEAD", + HTTPPath: "/{Bucket}", + } + + if input == nil { + input = &HeadBucketInput{} + } + + req = c.newRequest(op, input, output) + output = &HeadBucketOutput{} + req.Data = output + return +} + +// HeadBucket This operation is useful to determine if a bucket exists and you have permission +// to access it. +func (c *S3) HeadBucket(input *HeadBucketInput) (*HeadBucketOutput, error) { + req, out := c.HeadBucketRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) HeadBucketWithContext(ctx aws.Context, input *HeadBucketInput) (*HeadBucketOutput, error) { + req, out := c.HeadBucketRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// HeadBucketExist checks if the bucket exists. +func (c *S3) HeadBucketExist(bucket string) (bool, error) { + var err error + req, _ := c.HeadBucketRequest(&HeadBucketInput{ + Bucket: aws.String(bucket), + }) + err = req.Send() + if err == nil && req.HTTPResponse.StatusCode == 200 { + return true, nil + } + return false, err +} + +func (c *S3) HeadBucketExistWithContext(ctx aws.Context, bucket string) (bool, error) { + var err error + req, _ := c.HeadBucketRequest(&HeadBucketInput{ + Bucket: aws.String(bucket), + }) + req.SetContext(ctx) + err = req.Send() + if err == nil && req.HTTPResponse.StatusCode == 200 { + return true, nil + } + return false, err +} + +// HeadObjectRequest generates a request for the HeadObject operation. +func (c *S3) HeadObjectRequest(input *HeadObjectInput) (req *aws.Request, output *HeadObjectOutput) { + op := &aws.Operation{ + Name: "HeadObject", + HTTPMethod: "HEAD", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &HeadObjectInput{} + } + + req = c.newRequest(op, input, output) + output = &HeadObjectOutput{} + req.Data = output + return +} + +// HeadObject The HEAD operation retrieves metadata from an object without returning the +// object itself. This operation is useful if you're only interested in an object's +// metadata. To use HEAD, you must have READ access to the object. +func (c *S3) HeadObject(input *HeadObjectInput) (*HeadObjectOutput, error) { + req, out := c.HeadObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) HeadObjectWithContext(ctx aws.Context, input *HeadObjectInput) (*HeadObjectOutput, error) { + req, out := c.HeadObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// ListBucketsRequest generates a request for the ListBuckets operation. +func (c *S3) ListBucketsRequest(input *ListBucketsInput) (req *aws.Request, output *ListBucketsOutput) { + op := &aws.Operation{ + Name: "ListBuckets", + HTTPMethod: "GET", + HTTPPath: "/", + } + + if input == nil { + input = &ListBucketsInput{} + } + + req = c.newRequest(op, input, output) + output = &ListBucketsOutput{} + req.Data = output + return +} + +// ListBuckets Returns a list of all buckets owned by the authenticated sender of the request. +func (c *S3) ListBuckets(input *ListBucketsInput) (*ListBucketsOutput, error) { + req, out := c.ListBucketsRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ListBucketsWithContext(ctx aws.Context, input *ListBucketsInput) (*ListBucketsOutput, error) { + req, out := c.ListBucketsRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// ListMultipartUploadsRequest generates a request for the ListMultipartUploads operation. +func (c *S3) ListMultipartUploadsRequest(input *ListMultipartUploadsInput) (req *aws.Request, output *ListMultipartUploadsOutput) { + op := &aws.Operation{ + Name: "ListMultipartUploads", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?uploads", + Paginator: &aws.Paginator{ + InputTokens: []string{"KeyMarker", "UploadIdMarker"}, + OutputTokens: []string{"NextKeyMarker", "NextUploadIdMarker"}, + LimitToken: "MaxUploads", + TruncationToken: "IsTruncated", + }, + } + + if input == nil { + input = &ListMultipartUploadsInput{} + } + + req = c.newRequest(op, input, output) + output = &ListMultipartUploadsOutput{} + req.Data = output + return +} + +// ListMultipartUploads This operation lists in-progress multipart uploads. +func (c *S3) ListMultipartUploads(input *ListMultipartUploadsInput) (*ListMultipartUploadsOutput, error) { + req, out := c.ListMultipartUploadsRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ListMultipartUploadsWithContext(ctx aws.Context, input *ListMultipartUploadsInput) (*ListMultipartUploadsOutput, error) { + req, out := c.ListMultipartUploadsRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +func (c *S3) ListMultipartUploadsPages(input *ListMultipartUploadsInput, fn func(p *ListMultipartUploadsOutput, lastPage bool) (shouldContinue bool)) error { + page, _ := c.ListMultipartUploadsRequest(input) + return page.EachPage(func(p interface{}, lastPage bool) bool { + return fn(p.(*ListMultipartUploadsOutput), lastPage) + }) +} + +// ListObjectVersionsRequest generates a request for the ListObjectVersions operation. +func (c *S3) ListObjectVersionsRequest(input *ListObjectVersionsInput) (req *aws.Request, output *ListObjectVersionsOutput) { + op := &aws.Operation{ + Name: "ListObjectVersions", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?versions", + Paginator: &aws.Paginator{ + InputTokens: []string{"KeyMarker", "VersionIdMarker"}, + OutputTokens: []string{"NextKeyMarker", "NextVersionIdMarker"}, + LimitToken: "MaxKeys", + TruncationToken: "IsTruncated", + }, + } + + if input == nil { + input = &ListObjectVersionsInput{} + } + + req = c.newRequest(op, input, output) + output = &ListObjectVersionsOutput{} + req.Data = output + return +} + +// ListObjectVersions Returns metadata about all the versions of objects in a bucket. +func (c *S3) ListObjectVersions(input *ListObjectVersionsInput) (*ListObjectVersionsOutput, error) { + req, out := c.ListObjectVersionsRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ListObjectVersionsWithContext(ctx aws.Context, input *ListObjectVersionsInput) (*ListObjectVersionsOutput, error) { + req, out := c.ListObjectVersionsRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +func (c *S3) ListObjectVersionsPages(input *ListObjectVersionsInput, fn func(p *ListObjectVersionsOutput, lastPage bool) (shouldContinue bool)) error { + page, _ := c.ListObjectVersionsRequest(input) + return page.EachPage(func(p interface{}, lastPage bool) bool { + return fn(p.(*ListObjectVersionsOutput), lastPage) + }) +} + +// ListObjectsRequest generates a request for the ListObjects operation. +func (c *S3) ListObjectsRequest(input *ListObjectsInput) (req *aws.Request, output *ListObjectsOutput) { + op := &aws.Operation{ + Name: "ListObjects", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}", + Paginator: &aws.Paginator{ + InputTokens: []string{"Marker"}, + OutputTokens: []string{"NextMarker || Contents[-1].Key"}, + LimitToken: "MaxKeys", + TruncationToken: "IsTruncated", + }, + } + + if input == nil { + input = &ListObjectsInput{} + } + + req = c.newRequest(op, input, output) + output = &ListObjectsOutput{} + req.Data = output + return +} + +// ListObjects Returns some or all (up to 1000) of the objects in a bucket. You can use +// the request parameters as selection criteria to return a subset of the objects +// in a bucket. +func (c *S3) ListObjects(input *ListObjectsInput) (*ListObjectsOutput, error) { + req, out := c.ListObjectsRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ListObjectsWithContext(ctx aws.Context, input *ListObjectsInput) (*ListObjectsOutput, error) { + req, out := c.ListObjectsRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +func (c *S3) ListObjectsPages(input *ListObjectsInput, fn func(p *ListObjectsOutput, lastPage bool) (shouldContinue bool)) error { + page, _ := c.ListObjectsRequest(input) + return page.EachPage(func(p interface{}, lastPage bool) bool { + return fn(p.(*ListObjectsOutput), lastPage) + }) +} + +// ListPartsRequest generates a request for the ListParts operation. +func (c *S3) ListPartsRequest(input *ListPartsInput) (req *aws.Request, output *ListPartsOutput) { + op := &aws.Operation{ + Name: "ListParts", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}/{Key+}", + Paginator: &aws.Paginator{ + InputTokens: []string{"PartNumberMarker"}, + OutputTokens: []string{"NextPartNumberMarker"}, + LimitToken: "MaxParts", + TruncationToken: "IsTruncated", + }, + } + + if input == nil { + input = &ListPartsInput{} + } + + req = c.newRequest(op, input, output) + output = &ListPartsOutput{} + req.Data = output + return +} + +// ListParts Lists the parts that have been uploaded for a specific multipart upload. +func (c *S3) ListParts(input *ListPartsInput) (*ListPartsOutput, error) { + req, out := c.ListPartsRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ListPartsWithContext(ctx aws.Context, input *ListPartsInput) (*ListPartsOutput, error) { + req, out := c.ListPartsRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +func (c *S3) ListPartsPages(input *ListPartsInput, fn func(p *ListPartsOutput, lastPage bool) (shouldContinue bool)) error { + page, _ := c.ListPartsRequest(input) + return page.EachPage(func(p interface{}, lastPage bool) bool { + return fn(p.(*ListPartsOutput), lastPage) + }) +} + +// PutBucketACLRequest generates a request for the PutBucketACL operation. +func (c *S3) PutBucketACLRequest(input *PutBucketACLInput) (req *aws.Request, output *PutBucketACLOutput) { + op := &aws.Operation{ + Name: "PutBucketAcl", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?acl", + } + + if input == nil { + input = &PutBucketACLInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketACLOutput{} + req.Data = output + return +} + +// PutBucketACL Sets the permissions on a bucket using access control lists (ACL). +func (c *S3) PutBucketACL(input *PutBucketACLInput) (*PutBucketACLOutput, error) { + req, out := c.PutBucketACLRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketACLWithContext(ctx aws.Context, input *PutBucketACLInput) (*PutBucketACLOutput, error) { + req, out := c.PutBucketACLRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// PutBucketLoggingRequest generates a request for the PutBucketLogging operation. +func (c *S3) PutBucketLoggingRequest(input *PutBucketLoggingInput) (req *aws.Request, output *PutBucketLoggingOutput) { + op := &aws.Operation{ + Name: "PutBucketLogging", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?logging", + } + + if input == nil { + input = &PutBucketLoggingInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketLoggingOutput{} + req.Data = output + return +} + +// PutBucketLogging Set the logging parameters for a bucket and to specify permissions for who +// can view and modify the logging parameters. To set the logging status of +// a bucket, you must be the bucket owner. +func (c *S3) PutBucketLogging(input *PutBucketLoggingInput) (*PutBucketLoggingOutput, error) { + req, out := c.PutBucketLoggingRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketLoggingWithContext(ctx aws.Context, input *PutBucketLoggingInput) (*PutBucketLoggingOutput, error) { + req, out := c.PutBucketLoggingRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// PutBucketNotificationRequest generates a request for the PutBucketNotification operation. +func (c *S3) PutBucketNotificationRequest(input *PutBucketNotificationInput) (req *aws.Request, output *PutBucketNotificationOutput) { + op := &aws.Operation{ + Name: "PutBucketNotification", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?notification", + } + + if input == nil { + input = &PutBucketNotificationInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketNotificationOutput{} + req.Data = output + return +} + +// PutBucketNotification Deprecated, see the PutBucketNotificationConfiguraiton operation. +func (c *S3) PutBucketNotification(input *PutBucketNotificationInput) (*PutBucketNotificationOutput, error) { + req, out := c.PutBucketNotificationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketNotificationWithContext(ctx aws.Context, input *PutBucketNotificationInput) (*PutBucketNotificationOutput, error) { + req, out := c.PutBucketNotificationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// PutBucketNotificationConfigurationRequest generates a request for the PutBucketNotificationConfiguration operation. +func (c *S3) PutBucketNotificationConfigurationRequest(input *PutBucketNotificationConfigurationInput) (req *aws.Request, output *PutBucketNotificationConfigurationOutput) { + op := &aws.Operation{ + Name: "PutBucketNotificationConfiguration", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?notification", + } + + if input == nil { + input = &PutBucketNotificationConfigurationInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketNotificationConfigurationOutput{} + req.Data = output + return +} + +// PutBucketNotificationConfiguration Enables notifications of specified events for a bucket. +func (c *S3) PutBucketNotificationConfiguration(input *PutBucketNotificationConfigurationInput) (*PutBucketNotificationConfigurationOutput, error) { + req, out := c.PutBucketNotificationConfigurationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketNotificationConfigurationWithContext(ctx aws.Context, input *PutBucketNotificationConfigurationInput) (*PutBucketNotificationConfigurationOutput, error) { + req, out := c.PutBucketNotificationConfigurationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// PutBucketPolicyRequest generates a request for the PutBucketPolicy operation. +func (c *S3) PutBucketPolicyRequest(input *PutBucketPolicyInput) (req *aws.Request, output *PutBucketPolicyOutput) { + op := &aws.Operation{ + Name: "PutBucketPolicy", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?policy", + } + + if input == nil { + input = &PutBucketPolicyInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketPolicyOutput{} + req.Data = output + return +} + +// PutBucketPolicy Replaces a policy on a bucket. If the bucket already has a policy, the one +// in this request completely replaces it. +func (c *S3) PutBucketPolicy(input *PutBucketPolicyInput) (*PutBucketPolicyOutput, error) { + req, out := c.PutBucketPolicyRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketPolicyWithContext(ctx aws.Context, input *PutBucketPolicyInput) (*PutBucketPolicyOutput, error) { + req, out := c.PutBucketPolicyRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// PutBucketRequestPaymentRequest generates a request for the PutBucketRequestPayment operation. +func (c *S3) PutBucketRequestPaymentRequest(input *PutBucketRequestPaymentInput) (req *aws.Request, output *PutBucketRequestPaymentOutput) { + op := &aws.Operation{ + Name: "PutBucketRequestPayment", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?requestPayment", + } + + if input == nil { + input = &PutBucketRequestPaymentInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketRequestPaymentOutput{} + req.Data = output + return +} + +// PutBucketRequestPayment Sets the request payment configuration for a bucket. By default, the bucket +// owner pays for downloads from the bucket. This configuration parameter enables +// the bucket owner (only) to specify that the person requesting the download +// will be charged for the download. +func (c *S3) PutBucketRequestPayment(input *PutBucketRequestPaymentInput) (*PutBucketRequestPaymentOutput, error) { + req, out := c.PutBucketRequestPaymentRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketRequestPaymentWithContext(ctx aws.Context, input *PutBucketRequestPaymentInput) (*PutBucketRequestPaymentOutput, error) { + req, out := c.PutBucketRequestPaymentRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// PutBucketVersioningRequest generates a request for the PutBucketVersioning operation. +func (c *S3) PutBucketVersioningRequest(input *PutBucketVersioningInput) (req *aws.Request, output *PutBucketVersioningOutput) { + op := &aws.Operation{ + Name: "PutBucketVersioning", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?versioning", + } + + if input == nil { + input = &PutBucketVersioningInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketVersioningOutput{} + req.Data = output + return +} + +// PutBucketVersioning Sets the versioning state of an existing bucket. To set the versioning state, +// you must be the bucket owner. +func (c *S3) PutBucketVersioning(input *PutBucketVersioningInput) (*PutBucketVersioningOutput, error) { + req, out := c.PutBucketVersioningRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketVersioningWithContext(ctx aws.Context, input *PutBucketVersioningInput) (*PutBucketVersioningOutput, error) { + req, out := c.PutBucketVersioningRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// PutBucketWebsiteRequest generates a request for the PutBucketWebsite operation. +func (c *S3) PutBucketWebsiteRequest(input *PutBucketWebsiteInput) (req *aws.Request, output *PutBucketWebsiteOutput) { + op := &aws.Operation{ + Name: "PutBucketWebsite", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?website", + } + + if input == nil { + input = &PutBucketWebsiteInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketWebsiteOutput{} + req.Data = output + return +} + +type metadataGeneratePresignedUrlInput struct { + SDKShapeTraits bool `type:"structure" payload:"GeneratePresignedUrlInput"` +} +type GeneratePresignedUrlInput struct { + + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + ContentMd5 *string `location:"header" locationName:"Content-Md5" type:"string"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // A map of metadata to store with the object in S3. + Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` + + TrafficLimit *int64 `location:"querystring" locationName:"x-kss-traffic-limit" type:"integer"` + + HTTPMethod HTTPMethod `locationName:"HTTPMethod" type:"string" required:"true"` + + // The date and time at which the object is no longer cacheable. + Expires int64 `locationName:"Expires" type:"integer" required:"true"` + + // Sets the Cache-Control header of the response. + ResponseCacheControl *string `location:"querystring" locationName:"response-cache-control" type:"string"` + + // Sets the Content-Disposition header of the response + ResponseContentDisposition *string `location:"querystring" locationName:"response-content-disposition" type:"string"` + + // Sets the Content-Encoding header of the response. + ResponseContentEncoding *string `location:"querystring" locationName:"response-content-encoding" type:"string"` + + // Sets the Content-Language header of the response. + ResponseContentLanguage *string `location:"querystring" locationName:"response-content-language" type:"string"` + + // Sets the Content-Type header of the response. + ResponseContentType *string `location:"querystring" locationName:"response-content-type" type:"string"` + + // Sets the Expires header of the response. + ResponseExpires *time.Time `location:"querystring" locationName:"response-expires" type:"timestamp" timestampFormat:"iso8601"` + + // Deprecated, use ExtendHeaders instead. + Headers map[string]*string `location:"extendHeaders" type:"map"` + + // Deprecated, use ExtendQueryParams instead. + Parameters map[string]*string `location:"extendQueryParams" type:"map"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGeneratePresignedUrlInput `json:"-" xml:"-"` +} +type GeneratePresignedUrlOutput struct { + url *string +} + +// PutBucketWebsite Set the website configuration for a bucket. +func (c *S3) PutBucketWebsite(input *PutBucketWebsiteInput) (*PutBucketWebsiteOutput, error) { + req, out := c.PutBucketWebsiteRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketWebsiteWithContext(ctx aws.Context, input *PutBucketWebsiteInput) (*PutBucketWebsiteOutput, error) { + req, out := c.PutBucketWebsiteRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// PutObjectRequest generates a request for the PutObject operation. +func (c *S3) PutObjectRequest(input *PutObjectInput) (req *aws.Request, output *PutObjectOutput) { + op := &aws.Operation{ + Name: "PutObject", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &PutObjectInput{} + } + + req = c.newRequest(op, input, output) + if c.Config.CrcCheckEnabled { + req.Handlers.CheckCrc64.PushBack(CheckUploadCrc64) + } + if input.ProgressFn != nil { + req.ProgressFn = input.ProgressFn + } + output = &PutObjectOutput{} + req.Data = output + return +} + +// PutObject Adds an object to a bucket. +func (c *S3) PutObject(input *PutObjectInput) (*PutObjectOutput, error) { + req, out := c.PutObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutObjectWithContext(ctx aws.Context, input *PutObjectInput) (*PutObjectOutput, error) { + req, out := c.PutObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +func (c *S3) PutReaderRequest(input *PutReaderInput) (req *aws.Request, output *PutObjectOutput) { + op := &aws.Operation{ + Name: "PutObject", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &PutReaderInput{} + } + input.Body = &Body{io.NopCloser(input.Body)} + + req = c.newRequest(op, input, output) + if c.Config.CrcCheckEnabled { + req.Handlers.CheckCrc64.PushBack(CheckUploadCrc64) + } + if input.ProgressFn != nil { + req.ProgressFn = input.ProgressFn + } + output = &PutObjectOutput{} + req.Data = output + return +} + +// PutReader Adds an object to a bucket. +func (c *S3) PutReader(input *PutReaderInput) (*PutObjectOutput, error) { + req, out := c.PutReaderRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutReaderWithContext(ctx aws.Context, input *PutReaderInput) (*PutObjectOutput, error) { + req, out := c.PutReaderRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// GeneratePresignedUrl generates a presigned url for the object +func (c *S3) GeneratePresignedUrl(input *GeneratePresignedUrlInput) (url string, err error) { + op := &aws.Operation{ + HTTPMethod: string(input.HTTPMethod), + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &GeneratePresignedUrlInput{} + } + + if input.ExtendHeaders == nil { + input.ExtendHeaders = input.Headers + } + + if input.ExtendQueryParams == nil { + input.ExtendQueryParams = input.Parameters + } + + output := &GeneratePresignedUrlOutput{} + req := c.newRequest(op, input, output) + + aws.ValidateParameters(req) + if req.Error != nil { + return "", req.Error + } + + if string(input.HTTPMethod) == "" { + return "", apierr.New("InvalidParameter", "missing required parameter: HTTPMethod", nil) + } + + if input.Expires <= 0 { + return "", apierr.New("InvalidParameter", "Expires is required and must bigger than 0", nil) + } + + if IsV4Signature(c.Config.SignerVersion) { + if input.Expires > 604800 { + return "", apierr.New("InvalidParameter", "Expires must between 1 and 604800 in V4 signature", nil) + } + req.ExpireTime = input.Expires + } else { + req.ExpireTime = input.Expires + time.Now().Unix() + } + req.Sign() + + return req.HTTPRequest.URL.String(), nil +} + +// GeneratePresignedUrlInput generates a presigned url for the object +// Deprecated: Use GeneratePresignedUrl instead. +func (c *S3) GeneratePresignedUrlInput(input *GeneratePresignedUrlInput) (url string) { + opGeneratePresigned := &aws.Operation{ + HTTPMethod: string(input.HTTPMethod), + HTTPPath: "/{Bucket}/{Key+}", + } + output := &GeneratePresignedUrlOutput{} + req := c.newRequest(opGeneratePresigned, input, output) + now := time.Now().Unix() + if c.Config.SignerVersion == "V4" || c.Config.SignerVersion == "V4_UNSIGNED_PAYLOAD_SIGNER" { + req.ExpireTime = input.Expires + } else { + req.ExpireTime = input.Expires + now + } + req.Sign() + return req.HTTPRequest.URL.String() +} + +func (c *S3) PutObjectPreassignedInput(input *PutObjectInput) (*http.Request, error) { + req, _ := c.PutObjectRequest(input) + err := req.Sign() + return req.HTTPRequest, err +} + +// PutObjectACLRequest generates a request for the PutObjectACL operation. +func (c *S3) PutObjectACLRequest(input *PutObjectACLInput) (req *aws.Request, output *PutObjectACLOutput) { + op := &aws.Operation{ + Name: "PutObjectAcl", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}/{Key+}?acl", + } + + if input == nil { + input = &PutObjectACLInput{} + } + + req = c.newRequest(op, input, output) + output = &PutObjectACLOutput{} + req.Data = output + return +} + +// PutObjectACL uses the acl subresource to set the access control list (ACL) permissions +// for an object that already exists in a bucket +func (c *S3) PutObjectACL(input *PutObjectACLInput) (*PutObjectACLOutput, error) { + req, out := c.PutObjectACLRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutObjectACLWithContext(ctx aws.Context, input *PutObjectACLInput) (*PutObjectACLOutput, error) { + req, out := c.PutObjectACLRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// RestoreObjectRequest generates a request for the RestoreObject operation. +func (c *S3) RestoreObjectRequest(input *RestoreObjectInput) (req *aws.Request, output *RestoreObjectOutput) { + op := &aws.Operation{ + Name: "RestoreObject", + HTTPMethod: "POST", + HTTPPath: "/{Bucket}/{Key+}?restore", + } + + if input == nil { + input = &RestoreObjectInput{} + } + + req = c.newRequest(op, input, output) + output = &RestoreObjectOutput{} + req.Data = output + return +} + +func (c *S3) RestoreObject(input *RestoreObjectInput) (*RestoreObjectOutput, error) { + req, out := c.RestoreObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) RestoreObjectWithContext(ctx aws.Context, input *RestoreObjectInput) (*RestoreObjectOutput, error) { + req, out := c.RestoreObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// UploadPartRequest generates a request for the UploadPart operation. +func (c *S3) UploadPartRequest(input *UploadPartInput) (req *aws.Request, output *UploadPartOutput) { + op := &aws.Operation{ + Name: "UploadPart", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &UploadPartInput{} + } + + req = c.newRequest(op, input, output) + if c.Config.CrcCheckEnabled { + req.Handlers.CheckCrc64.PushBack(CheckUploadCrc64) + } + if input.ProgressFn != nil { + req.ProgressFn = input.ProgressFn + } + output = &UploadPartOutput{} + req.Data = output + return +} + +// UploadPart Uploads a part in a multipart upload. +// +// Note: After you initiate multipart upload and upload one or more parts, +// you must either complete or abort multipart upload in order to stop getting +// charged for storage of the uploaded parts. Only after you either complete +// or abort multipart upload, Amazon S3 frees up the parts storage and stops +// charging you for the parts storage. +func (c *S3) UploadPart(input *UploadPartInput) (*UploadPartOutput, error) { + req, out := c.UploadPartRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) UploadPartWithContext(ctx aws.Context, input *UploadPartInput) (*UploadPartOutput, error) { + req, out := c.UploadPartRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +// UploadPartCopyRequest generates a request for the UploadPartCopy operation. +func (c *S3) UploadPartCopyRequest(input *UploadPartCopyInput) (req *aws.Request, output *UploadPartCopyOutput) { + op := &aws.Operation{ + Name: "UploadPartCopy", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}/{Key+}", + } + + if input == nil { + input = &UploadPartCopyInput{} + } + + // URL encode the copy source + if input.CopySource == nil { + input.CopySource = aws.String(BuildCopySource(input.SourceBucket, input.SourceKey)) + } + req = c.newRequest(op, input, output) + output = &UploadPartCopyOutput{} + req.Data = output + return +} + +// UploadPartCopy Uploads a part by copying data from an existing object as data source. +func (c *S3) UploadPartCopy(input *UploadPartCopyInput) (*UploadPartCopyOutput, error) { + req, out := c.UploadPartCopyRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) UploadPartCopyWithContext(ctx aws.Context, input *UploadPartCopyInput) (*UploadPartCopyOutput, error) { + req, out := c.UploadPartCopyRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type AbortMultipartUploadInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + UploadID *string `location:"querystring" locationName:"uploadId" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataAbortMultipartUploadInput `json:"-" xml:"-"` +} + +type metadataAbortMultipartUploadInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type AbortMultipartUploadOutput struct { + // If present, indicates that the requester was successfully charged for the request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + metadataAbortMultipartUploadOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataAbortMultipartUploadOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type AccessControlPolicy struct { + // A list of grants. + Grants []*Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"` + + Owner *Owner `type:"structure"` + + metadataAccessControlPolicy `json:"-" xml:"-"` +} + +type metadataAccessControlPolicy struct { + SDKShapeTraits bool `type:"structure"` +} + +type Bucket struct { + // Date the bucket was created. + CreationDate *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // The name of the bucket. + Name *string `type:"string"` + + Region *string `type:"string"` + + Type *string `type:"string"` + + // Bucket visit type, NORMAL is the standard type, FREQUENTLIST is the high-frequency enumeration type. + VisitType *string `type:"string"` + + // The bucket data redundancy type. + DataRedundancyType *string `type:"string"` + + metadataBucket `json:"-" xml:"-"` +} + +type metadataBucket struct { + SDKShapeTraits bool `type:"structure"` +} + +type BucketLoggingStatus struct { + LoggingEnabled *LoggingEnabled `type:"structure"` + + metadataBucketLoggingStatus `json:"-" xml:"-"` +} + +type metadataBucketLoggingStatus struct { + SDKShapeTraits bool `type:"structure"` +} + +type CloudFunctionConfiguration struct { + CloudFunction *string `type:"string"` + + // Bucket event for which to send notifications. + Event *string `type:"string"` + + Events []*string `locationName:"Event" type:"list" flattened:"true"` + + // Optional unique identifier for configurations in a notification configuration. + // If you don't provide one, Amazon S3 will assign an ID. + ID *string `locationName:"Id" type:"string"` + + InvocationRole *string `type:"string"` + + metadataCloudFunctionConfiguration `json:"-" xml:"-"` +} + +type metadataCloudFunctionConfiguration struct { + SDKShapeTraits bool `type:"structure"` +} + +type CommonPrefix struct { + Prefix *string `type:"string"` + + metadataCommonPrefix `json:"-" xml:"-"` +} + +type metadataCommonPrefix struct { + SDKShapeTraits bool `type:"structure"` +} + +type CompleteMultipartUploadInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + MultipartUpload *CompletedMultipartUpload `locationName:"CompleteMultipartUpload" type:"structure"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + UploadID *string `location:"querystring" locationName:"uploadId" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Specifies whether the object is forbidden to overwrite. + ForbidOverwrite *bool `location:"header" locationName:"x-amz-forbid-overwrite" type:"boolean"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataCompleteMultipartUploadInput `json:"-" xml:"-"` +} + +type metadataCompleteMultipartUploadInput struct { + SDKShapeTraits bool `type:"structure" payload:"MultipartUpload"` +} + +type CompleteMultipartUploadOutput struct { + Bucket *string `type:"string"` + + // Entity tag of the object. + ETag *string `type:"string"` + + // If the object expiration is configured, this will contain the expiration + // date (expiry-date) and rule ID (rule-id). The value of rule-id is URL encoded. + Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` + + Key *string `type:"string"` + + Location *string `type:"string"` + + ChecksumCRC64ECMA *string `type:"string"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // If present, specifies the ID of the AWS Key Management Service (KMS) master + // encryption key that was used for the object. + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in KS3, eg: AES256. + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Version of the object. + VersionID *string `location:"header" locationName:"x-amz-version-id" type:"string"` + + metadataCompleteMultipartUploadOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataCompleteMultipartUploadOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type CompletedMultipartUpload struct { + Parts []*CompletedPart `locationName:"Part" type:"list" flattened:"true"` + + metadataCompletedMultipartUpload `json:"-" xml:"-"` +} + +type metadataCompletedMultipartUpload struct { + SDKShapeTraits bool `type:"structure"` +} + +type CompletedPart struct { + // Entity tag returned when the part was uploaded. + ETag *string `type:"string"` + + // Part number that identifies the part. + PartNumber *int64 `type:"integer"` + + // CRC64 value of a single part + ChecksumCRC64ECMA *string `type:"string"` + + metadataCompletedPart `json:"-" xml:"-"` +} + +type metadataCompletedPart struct { + SDKShapeTraits bool `type:"structure"` +} + +type Condition struct { + // The HTTP error code when the redirect is applied. In the event of an error, + // if the error code equals this value, then the specified redirect is applied. + // Required when parent element Condition is specified and sibling KeyPrefixEquals + // is not specified. If both are specified, then both must be true for the redirect + // to be applied. + HTTPErrorCodeReturnedEquals *string `locationName:"HttpErrorCodeReturnedEquals" type:"string"` + + // The object key name prefix when the redirect is applied. For example, to + // redirect requests for ExamplePage.html, the key prefix will be ExamplePage.html. + // To redirect request for all pages with the prefix docs/, the key prefix will + // be /docs, which identifies all objects in the docs/ folder. Required when + // the parent element Condition is specified and sibling HttpErrorCodeReturnedEquals + // is not specified. If both conditions are specified, both must be true for + // the redirect to be applied. + KeyPrefixEquals *string `type:"string"` + + metadataCondition `json:"-" xml:"-"` +} + +type metadataCondition struct { + SDKShapeTraits bool `type:"structure"` +} + +type CopyObjectInput struct { + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + SourceBucket *string `location:"uri" locationName:"sourceBucket" type:"string"` + + SourceKey *string `location:"uri" locationName:"sourceKey" type:"string"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // The language the content is in. + ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // The name of the source bucket and key name of the source object, separated + // by a slash (/). Must be URL-encoded. + CopySource *string `location:"header" locationName:"x-amz-copy-source" type:"string" required:"true"` + + // Copies the object if its entity tag (ETag) matches the specified tag. + CopySourceIfMatch *string `location:"header" locationName:"x-amz-copy-source-if-match" type:"string"` + + // Copies the object if it has been modified since the specified time. + CopySourceIfModifiedSince *time.Time `location:"header" locationName:"x-amz-copy-source-if-modified-since" type:"timestamp" timestampFormat:"rfc822"` + + // Copies the object if its entity tag (ETag) is different from the specified ETag. + CopySourceIfNoneMatch *string `location:"header" locationName:"x-amz-copy-source-if-none-match" type:"string"` + + // Copies the object if it hasn't been modified since the specified time. + CopySourceIfUnmodifiedSince *time.Time `location:"header" locationName:"x-amz-copy-source-if-unmodified-since" type:"timestamp" timestampFormat:"rfc822"` + + // Specifies the algorithm to use when decrypting the source object (e.g., AES256). + CopySourceSSECustomerAlgorithm *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use to decrypt + // the source object. The encryption key provided in this header must be one + // that was used when the source object was created. + CopySourceSSECustomerKey *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + CopySourceSSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key-MD5" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // Allows grantee to read the object data and its metadata. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Allows grantee to read the object ACL. + GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"` + + // Allows grantee to write the ACL for the applicable object. + GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + + // A map of metadata to store with the object in S3. + Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` + + // Specifies whether the metadata is copied from the source object or replaced + // with metadata provided in the request. + MetadataDirective *string `location:"header" locationName:"x-amz-metadata-directive" type:"string"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Specifies the algorithm to use to when encrypting the object (e.g., AES256, + // aws:kms). + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting + // data. This value is used to store the object and then it is discarded; Amazon + // does not store the encryption key. The key must be appropriate for use with + // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // header. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT + // requests for an object protected by AWS KMS will fail if not made via SSL + // or using SigV4. Documentation on configuring any of the officially supported + // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // The type of storage to use for the object. Defaults to 'STANDARD'. + StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"` + + // If the bucket is configured as a website, redirects requests for this object + // to another object in the same bucket or to an external URL. Amazon S3 stores + // the value of this header in the object metadata. + WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` + + // Specifies the object tag of the object. Multiple tags can be set at the same time, such as: TagA=A&TagB=B. + // Note: Key and Value need to be URL-encoded first. If an item does not have "=", the Value is considered to be an empty string. + Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` + + // Specifies how to set the object tag of the target object. + // Default value: COPY + // Valid values: + // COPY (default value): Copies the object tag of the source object to the target object. + // REPLACE: Ignores the object tag of the source object and directly uses the object tag specified in the request. + TaggingDirective *string `location:"header" locationName:"x-amz-tagging-directive" type:"string"` + + // Specifies whether the object is forbidden to overwrite. + ForbidOverwrite *bool `location:"header" locationName:"x-amz-forbid-overwrite" type:"boolean"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataCopyObjectInput `json:"-" xml:"-"` +} + +type metadataCopyObjectInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type CopyObjectOutput struct { + CopyObjectResult *CopyObjectResult `type:"structure"` + + CopySourceVersionID *string `location:"header" locationName:"x-amz-copy-source-version-id" type:"string"` + + // If the object expiration is configured, the response includes this header. + Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm + // used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // If present, specifies the ID of the AWS Key Management Service (KMS) master + // encryption key that was used for the object. + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + metadataCopyObjectOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataCopyObjectOutput struct { + SDKShapeTraits bool `type:"structure" payload:"CopyObjectResult"` +} + +type CopyObjectResult struct { + ETag *string `type:"string"` + + LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // CRC64 value of a single part + ChecksumCRC64ECMA *string `type:"string"` + + metadataCopyObjectResult `json:"-" xml:"-"` +} + +type metadataCopyObjectResult struct { + SDKShapeTraits bool `type:"structure"` +} + +type CopyPartResult struct { + // Entity tag of the object. + ETag *string `type:"string"` + + // Date and time at which the object was uploaded. + LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // CRC64 value of a single part + ChecksumCRC64ECMA *string `type:"string"` + + metadataCopyPartResult `json:"-" xml:"-"` +} + +type metadataCopyPartResult struct { + SDKShapeTraits bool `type:"structure"` +} + +type CreateBucketConfiguration struct { + // Specifies the region where the bucket will be created. If you don't specify + // a region, the bucket will be created in US Standard. + LocationConstraint *string `type:"string"` + + // The bucket data redundancy type. + // Default value: LRS + // Valid value: LRS丨ZRS + // LRS: local redundancy storage + // ZRS: zone redundancy storage + DataRedundancyType *string `type:"string"` + + metadataCreateBucketConfiguration `json:"-" xml:"-"` +} + +type metadataCreateBucketConfiguration struct { + SDKShapeTraits bool `type:"structure"` +} + +type CreateBucketInput struct { + // The canned ACL to apply to the bucket. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + ProjectId *string `location:"uri" locationName:"ProjectId" type:"string"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + CreateBucketConfiguration *CreateBucketConfiguration `locationName:"CreateBucketConfiguration" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` + + // Allows grantee the read, write, read ACP, and write ACP permissions on the + // bucket. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // Allows grantee to list the objects in the bucket. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Allows grantee to read the bucket ACL. + GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"` + + // Allows grantee to create, overwrite, and delete any object in the bucket. + GrantWrite *string `location:"header" locationName:"x-amz-grant-write" type:"string"` + + // Allows grantee to write the ACL for the applicable bucket. + GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + BucketType *string `location:"header" locationName:"x-amz-bucket-type" type:"string"` + + // Specify the bucket visit type. + BucketVisitType *string `location:"header" locationName:"x-amz-bucket-visit-type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + MetadataCreateBucketInput `json:"-" xml:"-"` +} + +type MetadataCreateBucketInput struct { + SDKShapeTraits bool `type:"structure" payload:"CreateBucketConfiguration"` +} + +type CreateBucketOutput struct { + Location *string `location:"header" locationName:"Location" type:"string"` + + metadataCreateBucketOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataCreateBucketOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type CreateMultipartUploadInput struct { + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + // Specifies the object tag of the object. Multiple tags can be set at the same time, such as: TagA=A&TagB=B. + // Note: Key and Value need to be URL-encoded first. If an item does not have "=", the Value is considered to be an empty string. + Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // The language the content is in. + ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // Allows grantee to read the object data and its metadata. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Allows grantee to read the object ACL. + GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"` + + // Allows grantee to write the ACL for the applicable object. + GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // A map of metadata to store with the object in S3. + Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Specifies the algorithm to use to when encrypting the object (e.g., AES256, + // aws:kms). + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting + // data. This value is used to store the object and then it is discarded; Amazon + // does not store the encryption key. The key must be appropriate for use with + // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // header. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT + // requests for an object protected by AWS KMS will fail if not made via SSL + // or using SigV4. Documentation on configuring any of the officially supported + // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // The type of storage to use for the object. Defaults to 'STANDARD'. + StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"` + + // If the bucket is configured as a website, redirects requests for this object + // to another object in the same bucket or to an external URL. Amazon S3 stores + // the value of this header in the object metadata. + WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` + + // Specifies whether the object is forbidden to overwrite. + ForbidOverwrite *bool `location:"header" locationName:"x-amz-forbid-overwrite" type:"boolean"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataCreateMultipartUploadInput `json:"-" xml:"-"` +} + +type metadataCreateMultipartUploadInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type CreateMultipartUploadOutput struct { + // Name of the bucket to which the multipart upload was initiated. + Bucket *string `locationName:"Bucket" type:"string"` + + // Object key for which the multipart upload was initiated. + Key *string `type:"string"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm + // used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // If present, specifies the ID of the AWS Key Management Service (KMS) master + // encryption key that was used for the object. + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // ID for the initiated multipart upload. + UploadID *string `locationName:"UploadId" type:"string"` + + Metadata map[string]*string `location:"headers" type:"map"` + + metadataCreateMultipartUploadOutput `json:"-" xml:"-"` +} + +type metadataCreateMultipartUploadOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type Delete struct { + Objects []*ObjectIdentifier `locationName:"Object" type:"list" flattened:"true" required:"true"` + + // Element to enable quiet mode for the request. When you add this element, + // you must set its value to true. + Quiet *bool `type:"boolean"` + + metadataDelete `json:"-" xml:"-"` +} + +type metadataDelete struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteBucketInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataDeleteBucketInput `json:"-" xml:"-"` +} + +type metadataDeleteBucketInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteBucketOutput struct { + metadataDeleteBucketOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataDeleteBucketOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteBucketPolicyInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataDeleteBucketPolicyInput `json:"-" xml:"-"` +} + +type metadataDeleteBucketPolicyInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteBucketPolicyOutput struct { + metadataDeleteBucketPolicyOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataDeleteBucketPolicyOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteBucketWebsiteInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataDeleteBucketWebsiteInput `json:"-" xml:"-"` +} + +type metadataDeleteBucketWebsiteInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteBucketWebsiteOutput struct { + metadataDeleteBucketWebsiteOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataDeleteBucketWebsiteOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteMarkerEntry struct { + // Specifies whether the object is (true) or is not (false) the latest version + // of an object. + IsLatest *bool `type:"boolean"` + + // The object key. + Key *string `type:"string"` + + // Date and time the object was last modified. + LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + Owner *Owner `type:"structure"` + + // Version ID of an object. + VersionID *string `locationName:"VersionId" type:"string"` + + metadataDeleteMarkerEntry `json:"-" xml:"-"` +} + +type metadataDeleteMarkerEntry struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteObjectInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // The concatenation of the authentication device's serial number, a space, + // and the value that is displayed on your authentication device. + MFA *string `location:"header" locationName:"x-amz-mfa" type:"string"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // VersionId used to reference a specific version of the object. + VersionID *string `location:"querystring" locationName:"versionId" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataDeleteObjectInput `json:"-" xml:"-"` +} + +type DeleteBucketPrefixInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Prefix of the object. + Prefix *string `type:"string" required:"true"` + // The max number of each list. + MaxKeys *int64 `type:"integer" required:"true"` + // Is return the deletion result. + IsReTurnResults *bool `type:"boolean" required:"true"` +} + +type metadataDeleteObjectInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteObjectOutput struct { + // Specifies whether the versioned object that was permanently deleted was (true) + // or was not (false) a delete marker. + DeleteMarker *bool `location:"header" locationName:"x-amz-delete-marker" type:"boolean"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // Returns the version ID of the delete marker created as a result of the DELETE + // operation. + VersionID *string `location:"header" locationName:"x-amz-version-id" type:"string"` + + metadataDeleteObjectOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataDeleteObjectOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteObjectsInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Delete *Delete `locationName:"Delete" type:"structure" required:"true"` + + IsReTurnResults *bool `type:"boolean" required:"true"` + + // The concatenation of the authentication device's serial number, a space, + // and the value that is displayed on your authentication device. + MFA *string `location:"header" locationName:"x-amz-mfa" type:"string"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataDeleteObjectsInput `json:"-" xml:"-"` +} + +type metadataDeleteObjectsInput struct { + SDKShapeTraits bool `type:"structure" payload:"Delete"` +} + +type DeleteObjectsOutput struct { + Deleted []*DeletedObject `type:"list" flattened:"true"` + + Errors []*Error `locationName:"Error" type:"list" flattened:"true"` + + ErrorsCount *int64 `locationName:"ErrorsCount" type:"integer"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + metadataDeleteObjectsOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataDeleteObjectsOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeletedObject struct { + DeleteMarker *bool `type:"boolean"` + + DeleteMarkerVersionID *string `locationName:"DeleteMarkerVersionId" type:"string"` + + Key *string `type:"string"` + + VersionID *string `locationName:"VersionId" type:"string"` + + metadataDeletedObject `json:"-" xml:"-"` +} + +type metadataDeletedObject struct { + SDKShapeTraits bool `type:"structure"` +} + +type Error struct { + Code *string `type:"string"` + + Key *string `type:"string"` + + Message *string `type:"string"` + + VersionID *string `locationName:"VersionId" type:"string"` + + metadataError `json:"-" xml:"-"` +} + +type metadataError struct { + SDKShapeTraits bool `type:"structure"` +} + +type ErrorDocument struct { + // The object key name to use when a 4XX class error occurs. + Key *string `type:"string" required:"true"` + + metadataErrorDocument `json:"-" xml:"-"` +} + +type metadataErrorDocument struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketACLInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetBucketACLInput `json:"-" xml:"-"` +} + +type metadataGetBucketACLInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketACLOutput struct { + // A list of grants. + Grants []*Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"` + + Owner *Owner `type:"structure"` + + metadataGetBucketACLOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetBucketACLOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketLocationInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetBucketLocationInput `json:"-" xml:"-"` +} + +type metadataGetBucketLocationInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketLocationOutput struct { + LocationConstraint *string `type:"string"` + + metadataGetBucketLocationOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetBucketLocationOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketLoggingInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetBucketLoggingInput `json:"-" xml:"-"` +} + +type metadataGetBucketLoggingInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketLoggingOutput struct { + LoggingEnabled *LoggingEnabled `type:"structure"` + + metadataGetBucketLoggingOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetBucketLoggingOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketNotificationConfigurationRequest struct { + // Name of the buket to get the notification configuration for. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + metadataGetBucketNotificationConfigurationRequest `json:"-" xml:"-"` +} + +type metadataGetBucketNotificationConfigurationRequest struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketPolicyInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetBucketPolicyInput `json:"-" xml:"-"` +} + +type metadataGetBucketPolicyInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketPolicyOutput struct { + // The bucket policy as a JSON document. + Policy *string `type:"string"` + + metadataGetBucketPolicyOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetBucketPolicyOutput struct { + SDKShapeTraits bool `type:"structure" payload:"Policy"` +} + +type GetBucketRequestPaymentInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetBucketRequestPaymentInput `json:"-" xml:"-"` +} + +type metadataGetBucketRequestPaymentInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketRequestPaymentOutput struct { + // Specifies who pays for the download and request fees. + Payer *string `type:"string"` + + metadataGetBucketRequestPaymentOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetBucketRequestPaymentOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketVersioningInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetBucketVersioningInput `json:"-" xml:"-"` +} + +type metadataGetBucketVersioningInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketVersioningOutput struct { + // Specifies whether MFA delete is enabled in the bucket versioning configuration. + // This element is only returned if the bucket has been configured with MFA + // delete. If the bucket has never been so configured, this element is not returned. + MFADelete *string `locationName:"MfaDelete" type:"string"` + + // The versioning state of the bucket. + Status *string `type:"string"` + + metadataGetBucketVersioningOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetBucketVersioningOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketWebsiteInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetBucketWebsiteInput `json:"-" xml:"-"` +} + +type metadataGetBucketWebsiteInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetBucketWebsiteOutput struct { + ErrorDocument *ErrorDocument `type:"structure"` + + IndexDocument *IndexDocument `type:"structure"` + + RedirectAllRequestsTo *RedirectAllRequestsTo `type:"structure"` + + RoutingRules []*RoutingRule `locationNameList:"RoutingRule" type:"list"` + + metadataGetBucketWebsiteOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetBucketWebsiteOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetObjectACLInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // VersionId used to reference a specific version of the object. + VersionID *string `location:"querystring" locationName:"versionId" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetObjectACLInput `json:"-" xml:"-"` +} + +type metadataGetObjectACLInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetObjectACLOutput struct { + // A list of grants. + Grants []*Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"` + + Owner *Owner `type:"structure"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + metadataGetObjectACLOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetObjectACLOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetObjectInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Return the object only if its entity tag (ETag) is the same as the one specified, + // otherwise return a 412 (precondition failed). + IfMatch *string `location:"header" locationName:"If-Match" type:"string"` + + // Return the object only if it has been modified since the specified time, + // otherwise return a 304 (not modified). + IfModifiedSince *time.Time `location:"header" locationName:"If-Modified-Since" type:"timestamp" timestampFormat:"rfc822"` + + // Return the object only if its entity tag (ETag) is different from the one + // specified, otherwise return a 304 (not modified). + IfNoneMatch *string `location:"header" locationName:"If-None-Match" type:"string"` + + // Return the object only if it has not been modified since the specified time, + // otherwise return a 412 (precondition failed). + IfUnmodifiedSince *time.Time `location:"header" locationName:"If-Unmodified-Since" type:"timestamp" timestampFormat:"rfc822"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Downloads the specified range bytes of an object. For more information about + // the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. + Range *string `location:"header" locationName:"Range" type:"string"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Sets the Cache-Control header of the response. + ResponseCacheControl *string `location:"querystring" locationName:"response-cache-control" type:"string"` + + // Sets the Content-Disposition header of the response + ResponseContentDisposition *string `location:"querystring" locationName:"response-content-disposition" type:"string"` + + // Sets the Content-Encoding header of the response. + ResponseContentEncoding *string `location:"querystring" locationName:"response-content-encoding" type:"string"` + + // Sets the Content-Language header of the response. + ResponseContentLanguage *string `location:"querystring" locationName:"response-content-language" type:"string"` + + // Sets the Content-Type header of the response. + ResponseContentType *string `location:"querystring" locationName:"response-content-type" type:"string"` + + // Sets the Expires header of the response. + ResponseExpires *time.Time `location:"querystring" locationName:"response-expires" type:"timestamp" timestampFormat:"iso8601"` + + // Specify the encoding type of the client. + // If you want to compress and transmit the returned content using gzip, + // you need to add a request header: Accept-Encoding:gzip。 + // KS3 will determine whether to return gzip compressed data based on the + // Content-Type and Object size (not less than 1 KB) of the object. + // Value: gzip、br、deflate + AcceptEncoding *string `location:"header" locationName:"Accept-Encoding" type:"string"` + + // Specifies the algorithm to use to when encrypting the object (e.g., AES256, + // aws:kms). + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting + // data. This value is used to store the object and then it is discarded; Amazon + // does not store the encryption key. The key must be appropriate for use with + // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // header. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // VersionId used to reference a specific version of the object. + VersionID *string `location:"querystring" locationName:"versionId" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + TrafficLimit *int64 `location:"header" locationName:"x-kss-traffic-limit" type:"string"` + + // Progress callback function + ProgressFn aws.ProgressFunc `location:"function"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetObjectInput `json:"-" xml:"-"` +} + +type metadataGetObjectInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetObjectOutput struct { + AcceptRanges *string `location:"header" locationName:"accept-ranges" type:"string"` + + // Object data. + Body io.ReadCloser `type:"blob"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // The language the content is in. + ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` + + // Size of the body in bytes. + ContentLength *int64 `location:"header" locationName:"Content-Length" type:"integer"` + + // The portion of the object returned in the response. + ContentRange *string `location:"header" locationName:"Content-Range" type:"string"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Specifies whether the object retrieved was (true) or was not (false) a Delete + // Marker. If false, this response header does not appear in the response. + DeleteMarker *bool `location:"header" locationName:"x-amz-delete-marker" type:"boolean"` + + // An ETag is an opaque identifier assigned by a web server to a specific version + // of a resource found at a URL + ETag *string `location:"header" locationName:"ETag" type:"string"` + + // If the object expiration is configured (see PUT Bucket lifecycle), the response + // includes this header. It includes the expiry-date and rule-id key value pairs + // providing object expiration information. The value of the rule-id is URL + // encoded. + Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // Last modified date of the object + LastModified *time.Time `location:"header" locationName:"Last-Modified" type:"timestamp" timestampFormat:"rfc822"` + + // This is set to the number of metadata entries not returned in x-amz-meta + // headers. This can happen if you create metadata using an API like SOAP that + // supports more flexible metadata than the REST API. For example, using SOAP, + // you can create metadata whose values are not legal HTTP headers. + MissingMeta *int64 `location:"header" locationName:"x-amz-missing-meta" type:"integer"` + + ReplicationStatus *string `location:"header" locationName:"x-amz-replication-status" type:"string"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // Provides information about object restoration operation and expiration time + // of the restored object copy. + Restore *string `location:"header" locationName:"x-amz-restore" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm + // used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // If present, specifies the ID of the AWS Key Management Service (KMS) master + // encryption key that was used for the object. + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // Version of the object. + VersionID *string `location:"header" locationName:"x-amz-version-id" type:"string"` + + // If the bucket is configured as a website, redirects requests for this object + // to another object in the same bucket or to an external URL. Amazon S3 stores + // the value of this header in the object metadata. + WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` + + metadataGetObjectOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetObjectOutput struct { + SDKShapeTraits bool `type:"structure" payload:"Body"` +} + +type GetObjectTorrentInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetObjectTorrentInput `json:"-" xml:"-"` +} + +type metadataGetObjectTorrentInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetObjectTorrentOutput struct { + Body io.ReadCloser `type:"blob"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + metadataGetObjectTorrentOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetObjectTorrentOutput struct { + SDKShapeTraits bool `type:"structure" payload:"Body"` +} + +type Grant struct { + Grantee *Grantee `type:"structure"` + + // Specifies the permission given to the grantee. + Permission *string `type:"string"` + + metadataGrant `json:"-" xml:"-"` +} + +type metadataGrant struct { + SDKShapeTraits bool `type:"structure"` +} + +type Grantee struct { + // Screen name of the grantee. + DisplayName *string `type:"string"` + + // Email address of the grantee. + EmailAddress *string `type:"string"` + + // The canonical user ID of the grantee. + ID *string `type:"string"` + + // Type of grantee + Type *string `locationName:"xsi:type" type:"string" xmlAttribute:"true" required:"true"` + + // URI of the grantee group. + URI *string `type:"string"` + + metadataGrantee `json:"-" xml:"-"` +} + +type metadataGrantee struct { + SDKShapeTraits bool `type:"structure" xmlPrefix:"xsi" xmlURI:"http://www.w3.org/2001/XMLSchema-instance"` +} + +type HeadBucketInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataHeadBucketInput `json:"-" xml:"-"` +} + +type metadataHeadBucketInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type HeadBucketOutput struct { + metadataHeadBucketOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataHeadBucketOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type HeadObjectInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Return the object only if its entity tag (ETag) is the same as the one specified, + // otherwise return a 412 (precondition failed). + IfMatch *string `location:"header" locationName:"If-Match" type:"string"` + + // Return the object only if it has been modified since the specified time, + // otherwise return a 304 (not modified). + IfModifiedSince *time.Time `location:"header" locationName:"If-Modified-Since" type:"timestamp" timestampFormat:"rfc822"` + + // Return the object only if its entity tag (ETag) is different from the one + // specified, otherwise return a 304 (not modified). + IfNoneMatch *string `location:"header" locationName:"If-None-Match" type:"string"` + + // Return the object only if it has not been modified since the specified time, + // otherwise return a 412 (precondition failed). + IfUnmodifiedSince *time.Time `location:"header" locationName:"If-Unmodified-Since" type:"timestamp" timestampFormat:"rfc822"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Downloads the specified range bytes of an object. For more information about + // the HTTP Range header, go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. + Range *string `location:"header" locationName:"Range" type:"string"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Specifies the algorithm to use to when encrypting the object (e.g., AES256, + // aws:kms). + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting + // data. This value is used to store the object and then it is discarded; Amazon + // does not store the encryption key. The key must be appropriate for use with + // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // header. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // VersionId used to reference a specific version of the object. + VersionID *string `location:"querystring" locationName:"versionId" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataHeadObjectInput `json:"-" xml:"-"` +} + +type metadataHeadObjectInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type HeadObjectOutput struct { + AcceptRanges *string `location:"header" locationName:"accept-ranges" type:"string"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // The language the content is in. + ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` + + // Size of the body in bytes. + ContentLength *int64 `location:"header" locationName:"Content-Length" type:"integer"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Specifies whether the object retrieved was (true) or was not (false) a Delete + // Marker. If false, this response header does not appear in the response. + DeleteMarker *bool `location:"header" locationName:"x-amz-delete-marker" type:"boolean"` + + // An ETag is an opaque identifier assigned by a web server to a specific version + // of a resource found at a URL + ETag *string `location:"header" locationName:"ETag" type:"string"` + + // If the object expiration is configured (see PUT Bucket lifecycle), the response + // includes this header. It includes the expiry-date and rule-id key value pairs + // providing object expiration information. The value of the rule-id is URL + // encoded. + Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // Last modified date of the object + LastModified *time.Time `location:"header" locationName:"Last-Modified" type:"timestamp" timestampFormat:"rfc822"` + + // This is set to the number of metadata entries not returned in x-amz-meta + // headers. This can happen if you create metadata using an API like SOAP that + // supports more flexible metadata than the REST API. For example, using SOAP, + // you can create metadata whose values are not legal HTTP headers. + MissingMeta *int64 `location:"header" locationName:"x-amz-missing-meta" type:"integer"` + + ReplicationStatus *string `location:"header" locationName:"x-amz-replication-status" type:"string"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // Provides information about object restoration operation and expiration time + // of the restored object copy. + Restore *string `location:"header" locationName:"x-amz-restore" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm + // used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // If present, specifies the ID of the AWS Key Management Service (KMS) master + // encryption key that was used for the object. + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // Version of the object. + VersionID *string `location:"header" locationName:"x-amz-version-id" type:"string"` + + // If the bucket is configured as a website, redirects requests for this object + // to another object in the same bucket or to an external URL. Amazon S3 stores + // the value of this header in the object metadata. + WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` + + metadataHeadObjectOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataHeadObjectOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type IndexDocument struct { + // A suffix that is appended to a request that is for a directory on the website + // endpoint (e.g. if the suffix is index.html and you make a request to samplebucket/images/ + // the data that is returned will be for the object with the key name images/index.html) + // The suffix must not be empty and must not include a slash character. + Suffix *string `type:"string" required:"true"` + + metadataIndexDocument `json:"-" xml:"-"` +} + +type metadataIndexDocument struct { + SDKShapeTraits bool `type:"structure"` +} + +type Initiator struct { + // Name of the Principal. + DisplayName *string `type:"string"` + + // If the principal is an AWS account, it provides the Canonical User ID. If + // the principal is an IAM User, it provides a user ARN value. + ID *string `type:"string"` + + metadataInitiator `json:"-" xml:"-"` +} + +type metadataInitiator struct { + SDKShapeTraits bool `type:"structure"` +} + +// LambdaFunctionConfiguration Container for specifying the AWS Lambda notification configuration. +type LambdaFunctionConfiguration struct { + Events []*string `locationName:"Event" type:"list" flattened:"true" required:"true"` + + // Optional unique identifier for configurations in a notification configuration. + // If you don't provide one, Amazon S3 will assign an ID. + ID *string `locationName:"Id" type:"string"` + + // Lambda cloud function ARN that Amazon S3 can invoke when it detects events + // of the specified type. + LambdaFunctionARN *string `locationName:"CloudFunction" type:"string" required:"true"` + + metadataLambdaFunctionConfiguration `json:"-" xml:"-"` +} + +type metadataLambdaFunctionConfiguration struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListBucketsInput struct { + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataListBucketsInput `json:"-" xml:"-"` +} + +type metadataListBucketsInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListBucketsOutput struct { + Buckets []*Bucket `locationNameList:"Bucket" type:"list"` + + Owner *Owner `type:"structure"` + + metadataListBucketsOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataListBucketsOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListMultipartUploadsInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Character you use to group keys. + Delimiter *string `location:"querystring" locationName:"delimiter" type:"string"` + + // Requests Amazon S3 to encode the object keys in the response and specifies + // the encoding method to use. An object key may contain any Unicode character; + // however, XML 1.0 parser cannot parse some characters, such as characters + // with an ASCII value from 0 to 10. For characters that are not supported in + // XML 1.0, you can add this parameter to request that Amazon S3 encode the + // keys in the response. + EncodingType *string `location:"querystring" locationName:"encoding-type" type:"string"` + + // Together with upload-id-marker, this parameter specifies the multipart upload + // after which listing should begin. + KeyMarker *string `location:"querystring" locationName:"key-marker" type:"string"` + + // Sets the maximum number of multipart uploads, from 1 to 1,000, to return + // in the response body. 1,000 is the maximum number of uploads that can be + // returned in a response. + MaxUploads *int64 `location:"querystring" locationName:"max-uploads" type:"integer"` + + // Lists in-progress uploads only for those keys that begin with the specified + // prefix. + Prefix *string `location:"querystring" locationName:"prefix" type:"string"` + + // Together with key-marker, specifies the multipart upload after which listing + // should begin. If key-marker is not specified, the upload-id-marker parameter + // is ignored. + UploadIDMarker *string `location:"querystring" locationName:"upload-id-marker" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataListMultipartUploadsInput `json:"-" xml:"-"` +} + +type metadataListMultipartUploadsInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListMultipartUploadsOutput struct { + // Name of the bucket to which the multipart upload was initiated. + Bucket *string `type:"string"` + + CommonPrefixes []*CommonPrefix `type:"list" flattened:"true"` + + Delimiter *string `type:"string"` + + // Encoding type used by Amazon S3 to encode object keys in the response. + EncodingType *string `type:"string"` + + // Indicates whether the returned list of multipart uploads is truncated. A + // value of true indicates that the list was truncated. The list can be truncated + // if the number of multipart uploads exceeds the limit allowed or specified + // by max uploads. + IsTruncated *bool `type:"boolean"` + + // The key at or after which the listing began. + KeyMarker *string `type:"string"` + + // Maximum number of multipart uploads that could have been included in the + // response. + MaxUploads *int64 `type:"integer"` + + // When a list is truncated, this element specifies the value that should be + // used for the key-marker request parameter in a subsequent request. + NextKeyMarker *string `type:"string"` + + // When a list is truncated, this element specifies the value that should be + // used for the upload-id-marker request parameter in a subsequent request. + NextUploadIDMarker *string `locationName:"NextUploadIdMarker" type:"string"` + + // When a prefix is provided in the request, this field contains the specified + // prefix. The result contains only keys starting with the specified prefix. + Prefix *string `type:"string"` + + // Upload ID after which listing began. + UploadIDMarker *string `locationName:"UploadIdMarker" type:"string"` + + Uploads []*MultipartUpload `locationName:"Upload" type:"list" flattened:"true"` + + metadataListMultipartUploadsOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataListMultipartUploadsOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListObjectVersionsInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // A delimiter is a character you use to group keys. + Delimiter *string `location:"querystring" locationName:"delimiter" type:"string"` + + // Requests Amazon S3 to encode the object keys in the response and specifies + // the encoding method to use. An object key may contain any Unicode character; + // however, XML 1.0 parser cannot parse some characters, such as characters + // with an ASCII value from 0 to 10. For characters that are not supported in + // XML 1.0, you can add this parameter to request that Amazon S3 encode the + // keys in the response. + EncodingType *string `location:"querystring" locationName:"encoding-type" type:"string"` + + // Specifies the key to start with when listing objects in a bucket. + KeyMarker *string `location:"querystring" locationName:"key-marker" type:"string"` + + // Sets the maximum number of keys returned in the response. The response might + // contain fewer keys but will never contain more. + MaxKeys *int64 `location:"querystring" locationName:"max-keys" type:"integer"` + + // Limits the response to keys that begin with the specified prefix. + Prefix *string `location:"querystring" locationName:"prefix" type:"string"` + + // Specifies the object version you want to start listing from. + VersionIDMarker *string `location:"querystring" locationName:"version-id-marker" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataListObjectVersionsInput `json:"-" xml:"-"` +} + +type metadataListObjectVersionsInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListObjectVersionsOutput struct { + CommonPrefixes []*CommonPrefix `type:"list" flattened:"true"` + + DeleteMarkers []*DeleteMarkerEntry `locationName:"DeleteMarker" type:"list" flattened:"true"` + + Delimiter *string `type:"string"` + + // Encoding type used by Amazon S3 to encode object keys in the response. + EncodingType *string `type:"string"` + + // A flag that indicates whether or not Amazon S3 returned all of the results + // that satisfied the search criteria. If your results were truncated, you can + // make a follow-up paginated request using the NextKeyMarker and NextVersionIdMarker + // response parameters as a starting place in another request to return the + // rest of the results. + IsTruncated *bool `type:"boolean"` + + // Marks the last Key returned in a truncated response. + KeyMarker *string `type:"string"` + + MaxKeys *int64 `type:"integer"` + + Name *string `type:"string"` + + // Use this value for the key marker request parameter in a subsequent request. + NextKeyMarker *string `type:"string"` + + // Use this value for the next version id marker parameter in a subsequent request. + NextVersionIDMarker *string `locationName:"NextVersionIdMarker" type:"string"` + + Prefix *string `type:"string"` + + VersionIDMarker *string `locationName:"VersionIdMarker" type:"string"` + + Versions []*ObjectVersion `locationName:"Version" type:"list" flattened:"true"` + + metadataListObjectVersionsOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataListObjectVersionsOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListObjectsInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // A delimiter is a character you use to group keys. + Delimiter *string `location:"querystring" locationName:"delimiter" type:"string"` + + // Requests Amazon S3 to encode the object keys in the response and specifies + // the encoding method to use. An object key may contain any Unicode character; + // however, XML 1.0 parser cannot parse some characters, such as characters + // with an ASCII value from 0 to 10. For characters that are not supported in + // XML 1.0, you can add this parameter to request that Amazon S3 encode the + // keys in the response. + EncodingType *string `location:"querystring" locationName:"encoding-type" type:"string"` + + // Specifies the key to start with when listing objects in a bucket. + Marker *string `location:"querystring" locationName:"marker" type:"string"` + + // Sets the maximum number of keys returned in the response. The response might + // contain fewer keys but will never contain more. + MaxKeys *int64 `location:"querystring" locationName:"max-keys" type:"integer"` + + // Limits the response to keys that begin with the specified prefix. + Prefix *string `location:"querystring" locationName:"prefix" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataListObjectsInput `json:"-" xml:"-"` +} + +type metadataListObjectsInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListObjectsOutput struct { + CommonPrefixes []*CommonPrefix `type:"list" flattened:"true"` + + Contents []*Object `type:"list" flattened:"true"` + + Delimiter *string `type:"string"` + + // Encoding type used by Amazon S3 to encode object keys in the response. + EncodingType *string `type:"string"` + + // A flag that indicates whether or not Amazon S3 returned all of the results + // that satisfied the search criteria. + IsTruncated *bool `type:"boolean"` + + Marker *string `type:"string"` + + MaxKeys *int64 `type:"integer"` + + Name *string `type:"string"` + + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. Amazon S3 lists objects in alphabetical + // order Note: This element is returned only if you have delimiter request parameter + // specified. If response does not include the NextMaker and it is truncated, + // you can use the value of the last Key in the response as the marker in the + // subsequent request to get the next set of object keys. + NextMarker *string `type:"string"` + + Prefix *string `type:"string"` + + metadataListObjectsOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataListObjectsOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListPartsInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Sets the maximum number of parts to return. + MaxParts *int64 `location:"querystring" locationName:"max-parts" type:"integer"` + + // Specifies the part after which listing should begin. Only parts with higher + // part numbers will be listed. + PartNumberMarker *int64 `location:"querystring" locationName:"part-number-marker" type:"integer"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Upload ID identifying the multipart upload whose parts are being listed. + UploadID *string `location:"querystring" locationName:"uploadId" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataListPartsInput `json:"-" xml:"-"` +} + +type metadataListPartsInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type ListPartsOutput struct { + // Name of the bucket to which the multipart upload was initiated. + Bucket *string `type:"string"` + + // Identifies who initiated the multipart upload. + Initiator *Initiator `type:"structure"` + + // Indicates whether the returned list of parts is truncated. + IsTruncated *bool `type:"boolean"` + + // Object key for which the multipart upload was initiated. + Key *string `type:"string"` + + // Maximum number of parts that were allowed in the response. + MaxParts *int64 `type:"integer"` + + // When a list is truncated, this element specifies the last part in the list, + // as well as the value to use for the part-number-marker request parameter + // in a subsequent request. + NextPartNumberMarker *int64 `type:"integer"` + + Owner *Owner `type:"structure"` + + // Part number after which listing begins. + PartNumberMarker *int64 `type:"integer"` + + Parts []*Part `locationName:"Part" type:"list" flattened:"true"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // The class of storage used to store the object. + StorageClass *string `type:"string"` + + // Upload ID identifying the multipart upload whose parts are being listed. + UploadID *string `locationName:"UploadId" type:"string"` + + metadataListPartsOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataListPartsOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type LoggingEnabled struct { + // Specifies the bucket where you want Amazon S3 to store server access logs. + // You can have your logs delivered to any bucket that you own, including the + // same bucket that is being logged. You can also configure multiple buckets + // to deliver their logs to the same target bucket. In this case you should + // choose a different TargetPrefix for each source bucket so that the delivered + // log files can be distinguished by key. + TargetBucket *string `type:"string"` + + TargetGrants []*TargetGrant `locationNameList:"Grant" type:"list"` + + // This element lets you specify a prefix for the keys that the log files will + // be stored under. + TargetPrefix *string `type:"string"` + + metadataLoggingEnabled `json:"-" xml:"-"` +} + +type metadataLoggingEnabled struct { + SDKShapeTraits bool `type:"structure"` +} + +type MultipartUpload struct { + // Date and time at which the multipart upload was initiated. + Initiated *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // Identifies who initiated the multipart upload. + Initiator *Initiator `type:"structure"` + + // Key of the object for which the multipart upload was initiated. + Key *string `type:"string"` + + Owner *Owner `type:"structure"` + + // The class of storage used to store the object. + StorageClass *string `type:"string"` + + // Upload ID that identifies the multipart upload. + UploadID *string `locationName:"UploadId" type:"string"` + + metadataMultipartUpload `json:"-" xml:"-"` +} + +type metadataMultipartUpload struct { + SDKShapeTraits bool `type:"structure"` +} + +// NotificationConfiguration Container for specifying the notification configuration of the bucket. If +// this element is empty, notifications are turned off on the bucket. +type NotificationConfiguration struct { + LambdaFunctionConfigurations []*LambdaFunctionConfiguration `locationName:"CloudFunctionConfiguration" type:"list" flattened:"true"` + + QueueConfigurations []*QueueConfiguration `locationName:"QueueConfiguration" type:"list" flattened:"true"` + + TopicConfigurations []*TopicConfiguration `locationName:"TopicConfiguration" type:"list" flattened:"true"` + + metadataNotificationConfiguration `json:"-" xml:"-"` +} + +type metadataNotificationConfiguration struct { + SDKShapeTraits bool `type:"structure"` +} + +type NotificationConfigurationDeprecated struct { + CloudFunctionConfiguration *CloudFunctionConfiguration `type:"structure"` + + QueueConfiguration *QueueConfigurationDeprecated `type:"structure"` + + TopicConfiguration *TopicConfigurationDeprecated `type:"structure"` + + metadataNotificationConfigurationDeprecated `json:"-" xml:"-"` +} + +type metadataNotificationConfigurationDeprecated struct { + SDKShapeTraits bool `type:"structure"` +} + +type Object struct { + ETag *string `type:"string"` + + Key *string `type:"string"` + + LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + Owner *Owner `type:"structure"` + + Size *int64 `type:"integer"` + + // The class of storage used to store the object. + StorageClass *string `type:"string"` + + // The redundancy type of the object. + Redundancy *string `type:"string"` + + metadataObject `json:"-" xml:"-"` +} + +type metadataObject struct { + SDKShapeTraits bool `type:"structure"` +} + +type ObjectIdentifier struct { + // Key name of the object to delete. + Key *string `type:"string" required:"true"` + + // VersionId for the specific version of the object to delete. + VersionID *string `locationName:"VersionId" type:"string"` + + metadataObjectIdentifier `json:"-" xml:"-"` +} + +type metadataObjectIdentifier struct { + SDKShapeTraits bool `type:"structure"` +} + +type ObjectVersion struct { + ETag *string `type:"string"` + + // Specifies whether the object is (true) or is not (false) the latest version + // of an object. + IsLatest *bool `type:"boolean"` + + // The object key. + Key *string `type:"string"` + + // Date and time the object was last modified. + LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + Owner *Owner `type:"structure"` + + // Size in bytes of the object. + Size *int64 `type:"integer"` + + // The class of storage used to store the object. + StorageClass *string `type:"string"` + + // Version ID of an object. + VersionID *string `locationName:"VersionId" type:"string"` + + metadataObjectVersion `json:"-" xml:"-"` +} + +type metadataObjectVersion struct { + SDKShapeTraits bool `type:"structure"` +} + +type Owner struct { + DisplayName *string `type:"string"` + + ID *string `type:"string"` + + metadataOwner `json:"-" xml:"-"` +} + +type metadataOwner struct { + SDKShapeTraits bool `type:"structure"` +} + +type Part struct { + // Entity tag returned when the part was uploaded. + ETag *string `type:"string"` + + // Date and time at which the part was uploaded. + LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // Part number identifying the part. + PartNumber *int64 `type:"integer"` + + // Size of the uploaded part data. + Size *int64 `type:"integer"` + + metadataPart `json:"-" xml:"-"` +} + +type metadataPart struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutBucketACLInput struct { + // The canned ACL to apply to the bucket. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + AccessControlPolicy *AccessControlPolicy `locationName:"AccessControlPolicy" type:"structure"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Allows grantee the read, write, read ACP, and write ACP permissions on the + // bucket. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // Allows grantee to list the objects in the bucket. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Allows grantee to read the bucket ACL. + GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"` + + // Allows grantee to create, overwrite, and delete any object in the bucket. + GrantWrite *string `location:"header" locationName:"x-amz-grant-write" type:"string"` + + // Allows grantee to write the ACL for the applicable bucket. + GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketACLInput `json:"-" xml:"-"` +} + +type metadataPutBucketACLInput struct { + SDKShapeTraits bool `type:"structure" payload:"AccessControlPolicy"` +} + +type PutBucketACLOutput struct { + metadataPutBucketACLOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutBucketACLOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutBucketLoggingInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + BucketLoggingStatus *BucketLoggingStatus `locationName:"BucketLoggingStatus" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketLoggingInput `json:"-" xml:"-"` +} + +type metadataPutBucketLoggingInput struct { + SDKShapeTraits bool `type:"structure" payload:"BucketLoggingStatus"` +} + +type PutBucketLoggingOutput struct { + metadataPutBucketLoggingOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutBucketLoggingOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutBucketNotificationConfigurationInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Container for specifying the notification configuration of the bucket. If + // this element is empty, notifications are turned off on the bucket. + NotificationConfiguration *NotificationConfiguration `locationName:"NotificationConfiguration" type:"structure" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketNotificationConfigurationInput `json:"-" xml:"-"` +} + +type metadataPutBucketNotificationConfigurationInput struct { + SDKShapeTraits bool `type:"structure" payload:"NotificationConfiguration"` +} + +type PutBucketNotificationConfigurationOutput struct { + metadataPutBucketNotificationConfigurationOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutBucketNotificationConfigurationOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutBucketNotificationInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + NotificationConfiguration *NotificationConfigurationDeprecated `locationName:"NotificationConfiguration" type:"structure" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketNotificationInput `json:"-" xml:"-"` +} + +type metadataPutBucketNotificationInput struct { + SDKShapeTraits bool `type:"structure" payload:"NotificationConfiguration"` +} + +type PutBucketNotificationOutput struct { + metadataPutBucketNotificationOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutBucketNotificationOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutBucketPolicyInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // The bucket policy as a JSON document. + Policy *string `type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketPolicyInput `json:"-" xml:"-"` +} + +type metadataPutBucketPolicyInput struct { + SDKShapeTraits bool `type:"structure" payload:"Policy"` +} + +type PutBucketPolicyOutput struct { + metadataPutBucketPolicyOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutBucketPolicyOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutBucketRequestPaymentInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + RequestPaymentConfiguration *RequestPaymentConfiguration `locationName:"RequestPaymentConfiguration" type:"structure" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketRequestPaymentInput `json:"-" xml:"-"` +} + +type metadataPutBucketRequestPaymentInput struct { + SDKShapeTraits bool `type:"structure" payload:"RequestPaymentConfiguration"` +} + +type PutBucketRequestPaymentOutput struct { + metadataPutBucketRequestPaymentOutput `json:"-" xml:"-"` +} + +type metadataPutBucketRequestPaymentOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutBucketVersioningInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // The concatenation of the authentication device's serial number, a space, + // and the value that is displayed on your authentication device. + MFA *string `location:"header" locationName:"x-amz-mfa" type:"string"` + + VersioningConfiguration *VersioningConfiguration `locationName:"VersioningConfiguration" type:"structure" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketVersioningInput `json:"-" xml:"-"` +} + +type metadataPutBucketVersioningInput struct { + SDKShapeTraits bool `type:"structure" payload:"VersioningConfiguration"` +} + +type PutBucketVersioningOutput struct { + metadataPutBucketVersioningOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutBucketVersioningOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutBucketWebsiteInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + WebsiteConfiguration *WebsiteConfiguration `locationName:"WebsiteConfiguration" type:"structure" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketWebsiteInput `json:"-" xml:"-"` +} + +type metadataPutBucketWebsiteInput struct { + SDKShapeTraits bool `type:"structure" payload:"WebsiteConfiguration"` +} + +type PutBucketWebsiteOutput struct { + metadataPutBucketWebsiteOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutBucketWebsiteOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutObjectACLInput struct { + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + AccessControlPolicy *AccessControlPolicy `locationName:"AccessControlPolicy" type:"structure"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Allows grantee the read, write, read ACP, and write ACP permissions on the + // bucket. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // Allows grantee to list the objects in the bucket. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Allows grantee to read the bucket ACL. + GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"` + + // Allows grantee to create, overwrite, and delete any object in the bucket. + GrantWrite *string `location:"header" locationName:"x-amz-grant-write" type:"string"` + + // Allows grantee to write the ACL for the applicable bucket. + GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutObjectACLInput `json:"-" xml:"-"` +} + +type metadataPutObjectACLInput struct { + SDKShapeTraits bool `type:"structure" payload:"AccessControlPolicy"` +} + +type PutObjectACLOutput struct { + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + metadataPutObjectACLOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutObjectACLOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type PutObjectInput struct { + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + // Object data. + Body io.ReadSeeker `type:"blob"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // The language the content is in. + ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` + + // Size of the body in bytes. This parameter is useful when the size of the + // body cannot be determined automatically. + ContentLength *int64 `location:"header" locationName:"Content-Length" type:"integer"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // Allows grantee to read the object data and its metadata. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Allows grantee to read the object ACL. + GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"` + + // Allows grantee to write the ACL for the applicable object. + GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // A map of metadata to store with the object in S3. + Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` + + // Specifies the object tag of the object. Multiple tags can be set at the same time, such as: TagA=A&TagB=B. + // Note: Key and Value need to be URL-encoded first. If an item does not have "=", the Value is considered to be an empty string. + Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` + + // Specifies whether the object is forbidden to overwrite. + ForbidOverwrite *bool `location:"header" locationName:"x-amz-forbid-overwrite" type:"boolean"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Specifies the algorithm to use to when encrypting the object (e.g., AES256, + // aws:kms). + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting + // data. This value is used to store the object and then it is discarded; Amazon + // does not store the encryption key. The key must be appropriate for use with + // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // header. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT + // requests for an object protected by AWS KMS will fail if not made via SSL + // or using SigV4. Documentation on configuring any of the officially supported + // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // The type of storage to use for the object. Defaults to 'STANDARD'. + StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"` + + // If the bucket is configured as a website, redirects requests for this object + // to another object in the same bucket or to an external URL. Amazon S3 stores + // the value of this header in the object metadata. + WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` + + ContentMaxLength *int64 `location:"header" locationName:"x-amz-content-maxlength" type:"integer"` + + CallbackUrl *string `location:"header" locationName:"x-kss-callbackurl" type:"string"` + + CallbackBody *string `location:"header" locationName:"x-kss-callbackbody" type:"string"` + + TrafficLimit *int64 `location:"header" locationName:"x-kss-traffic-limit" type:"integer"` + + ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"` + + // Progress callback function + ProgressFn aws.ProgressFunc `location:"function"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutObjectInput `json:"-" xml:"-"` +} + +type PutReaderInput struct { + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + // Object data. + Body io.Reader `type:"blob"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // The language the content is in. + ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` + + // Size of the body in bytes. This parameter is useful when the size of the + // body cannot be determined automatically. + ContentLength *int64 `location:"header" locationName:"Content-Length" type:"integer"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // Allows grantee to read the object data and its metadata. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Allows grantee to read the object ACL. + GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"` + + // Allows grantee to write the ACL for the applicable object. + GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // A map of metadata to store with the object in S3. + Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` + + // Specifies the object tag of the object. Multiple tags can be set at the same time, such as: TagA=A&TagB=B. + // Note: Key and Value need to be URL-encoded first. If an item does not have "=", the Value is considered to be an empty string. + Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` + + // Specifies whether the object is forbidden to overwrite. + ForbidOverwrite *bool `location:"header" locationName:"x-amz-forbid-overwrite" type:"boolean"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Specifies the algorithm to use to when encrypting the object (e.g., AES256, + // aws:kms). + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting + // data. This value is used to store the object and then it is discarded; Amazon + // does not store the encryption key. The key must be appropriate for use with + // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // header. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT + // requests for an object protected by AWS KMS will fail if not made via SSL + // or using SigV4. Documentation on configuring any of the officially supported + // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // The type of storage to use for the object. Defaults to 'STANDARD'. + StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"` + + // If the bucket is configured as a website, redirects requests for this object + // to another object in the same bucket or to an external URL. Amazon S3 stores + // the value of this header in the object metadata. + WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` + + ContentMaxLength *int64 `location:"header" locationName:"x-amz-content-maxlength" type:"integer"` + + CallbackUrl *string `location:"header" locationName:"x-kss-callbackurl" type:"string"` + + CallbackBody *string `location:"header" locationName:"x-kss-callbackbody" type:"string"` + + TrafficLimit *int64 `location:"header" locationName:"x-kss-traffic-limit" type:"integer"` + + ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"` + + // Progress callback function + ProgressFn aws.ProgressFunc `location:"function"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutObjectInput `json:"-" xml:"-"` +} + +type metadataPutObjectInput struct { + SDKShapeTraits bool `type:"structure" payload:"Body"` +} + +type PutObjectOutput struct { + // Entity tag for the uploaded object. + ETag *string `location:"header" locationName:"ETag" type:"string"` + + // If the object expiration is configured, this will contain the expiration + // date (expiry-date) and rule ID (rule-id). The value of rule-id is URL encoded. + Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm + // used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // If present, specifies the ID of the AWS Key Management Service (KMS) master + // encryption key that was used for the object. + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // Version of the object. + VersionID *string `location:"header" locationName:"x-amz-version-id" type:"string"` + + NewFileName *string `location:"header" locationName:"newfilename" type:"string"` + + metadataPutObjectOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutObjectOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +// QueueConfiguration Container for specifying an configuration when you want Amazon S3 to publish +// events to an Amazon Simple Queue Service (Amazon SQS) queue. +type QueueConfiguration struct { + Events []*string `locationName:"Event" type:"list" flattened:"true" required:"true"` + + // Optional unique identifier for configurations in a notification configuration. + // If you don't provide one, Amazon S3 will assign an ID. + ID *string `locationName:"Id" type:"string"` + + // Amazon SQS queue ARN to which Amazon S3 will publish a message when it detects + // events of specified type. + QueueARN *string `locationName:"Queue" type:"string" required:"true"` + + metadataQueueConfiguration `json:"-" xml:"-"` +} + +type metadataQueueConfiguration struct { + SDKShapeTraits bool `type:"structure"` +} + +type QueueConfigurationDeprecated struct { + // Bucket event for which to send notifications. + Event *string `type:"string"` + + Events []*string `locationName:"Event" type:"list" flattened:"true"` + + // Optional unique identifier for configurations in a notification configuration. + // If you don't provide one, Amazon S3 will assign an ID. + ID *string `locationName:"Id" type:"string"` + + Queue *string `type:"string"` + + metadataQueueConfigurationDeprecated `json:"-" xml:"-"` +} + +type metadataQueueConfigurationDeprecated struct { + SDKShapeTraits bool `type:"structure"` +} + +type Redirect struct { + // The HTTP redirect code to use on the response. Not required if one of the + // siblings is present. + HTTPRedirectCode *string `locationName:"HttpRedirectCode" type:"string"` + + // The host name to use in the redirect request. + HostName *string `type:"string"` + + // Protocol to use (http, https) when redirecting requests. The default is the + // protocol that is used in the original request. + Protocol *string `type:"string"` + + // The object key prefix to use in the redirect request. For example, to redirect + // requests for all pages with prefix docs/ (objects in the docs/ folder) to + // documents/, you can set a condition block with KeyPrefixEquals set to docs/ + // and in the Redirect set ReplaceKeyPrefixWith to /documents. Not required + // if one of the siblings is present. Can be present only if ReplaceKeyWith + // is not provided. + ReplaceKeyPrefixWith *string `type:"string"` + + // The specific object key to use in the redirect request. For example, redirect + // request to error.html. Not required if one of the sibling is present. Can + // be present only if ReplaceKeyPrefixWith is not provided. + ReplaceKeyWith *string `type:"string"` + + metadataRedirect `json:"-" xml:"-"` +} + +type metadataRedirect struct { + SDKShapeTraits bool `type:"structure"` +} + +type RedirectAllRequestsTo struct { + // Name of the host where requests will be redirected. + HostName *string `type:"string" required:"true"` + + // Protocol to use (http, https) when redirecting requests. The default is the + // protocol that is used in the original request. + Protocol *string `type:"string"` + + metadataRedirectAllRequestsTo `json:"-" xml:"-"` +} + +type metadataRedirectAllRequestsTo struct { + SDKShapeTraits bool `type:"structure"` +} + +type RequestPaymentConfiguration struct { + // Specifies who pays for the download and request fees. + Payer *string `type:"string" required:"true"` + + metadataRequestPaymentConfiguration `json:"-" xml:"-"` +} + +type metadataRequestPaymentConfiguration struct { + SDKShapeTraits bool `type:"structure"` +} + +type RestoreObjectInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + RestoreRequest *RestoreRequest `locationName:"RestoreRequest" type:"structure"` + + VersionID *string `location:"querystring" locationName:"versionId" type:"string"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataRestoreObjectInput `json:"-" xml:"-"` +} + +type metadataRestoreObjectInput struct { + SDKShapeTraits bool `type:"structure" payload:"RestoreRequest"` +} + +type RestoreObjectOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type RestoreRequest struct { + // Lifetime of the active copy in days + Days *int64 `type:"integer" required:"true"` + + metadataRestoreRequest `json:"-" xml:"-"` +} + +type metadataRestoreRequest struct { + SDKShapeTraits bool `type:"structure"` +} + +type RoutingRule struct { + // A container for describing a condition that must be met for the specified + // redirect to apply. For example, 1. If request is for pages in the /docs folder, + // redirect to the /documents folder. 2. If request results in HTTP error 4xx, + // redirect request to another host where you might process the error. + Condition *Condition `type:"structure"` + + // Container for redirect information. You can redirect requests to another + // host, to another page, or with another protocol. In the event of an error, + // you can can specify a different error code to return. + Redirect *Redirect `type:"structure" required:"true"` + + metadataRoutingRule `json:"-" xml:"-"` +} + +type metadataRoutingRule struct { + SDKShapeTraits bool `type:"structure"` +} + +type Tag struct { + // Name of the tag. + Key *string `type:"string" required:"true"` + + // Value of the tag. + Value *string `type:"string" required:"true"` + + metadataTag `json:"-" xml:"-"` +} + +type metadataTag struct { + SDKShapeTraits bool `type:"structure"` +} + +type Tagging struct { + TagSet []*Tag `locationNameList:"Tag" type:"list" required:"true"` + + metadataTagging `json:"-" xml:"-"` +} + +func (t *Tagging) ToString() string { + if t == nil || t.TagSet == nil || len(t.TagSet) == 0 { + return "" + } + + var builder strings.Builder + for i := 0; i < len(t.TagSet); i++ { + if i > 0 { + builder.WriteString("&") + } + tag := t.TagSet[i] + encodedKey := url.QueryEscape(aws.ToString(tag.Key)) + encodedValue := url.QueryEscape(aws.ToString(tag.Value)) + builder.WriteString(encodedKey) + builder.WriteString("=") + builder.WriteString(encodedValue) + } + + return builder.String() +} + +type metadataTagging struct { + SDKShapeTraits bool `type:"structure"` +} + +type TargetGrant struct { + Grantee *Grantee `type:"structure"` + + // Logging permissions assigned to the Grantee for the bucket. + Permission *string `type:"string"` + + metadataTargetGrant `json:"-" xml:"-"` +} + +type metadataTargetGrant struct { + SDKShapeTraits bool `type:"structure"` +} + +// TopicConfiguration Container for specifying the configuration when you want Amazon S3 to publish +// events to an Amazon Simple Notification Service (Amazon SNS) topic. +type TopicConfiguration struct { + Events []*string `locationName:"Event" type:"list" flattened:"true" required:"true"` + + // Optional unique identifier for configurations in a notification configuration. + // If you don't provide one, Amazon S3 will assign an ID. + ID *string `locationName:"Id" type:"string"` + + // Amazon SNS topic ARN to which Amazon S3 will publish a message when it detects + // events of specified type. + TopicARN *string `locationName:"Topic" type:"string" required:"true"` + + metadataTopicConfiguration `json:"-" xml:"-"` +} + +type metadataTopicConfiguration struct { + SDKShapeTraits bool `type:"structure"` +} + +type TopicConfigurationDeprecated struct { + // Bucket event for which to send notifications. + Event *string `type:"string"` + + Events []*string `locationName:"Event" type:"list" flattened:"true"` + + // Optional unique identifier for configurations in a notification configuration. + // If you don't provide one, Amazon S3 will assign an ID. + ID *string `locationName:"Id" type:"string"` + + // Amazon SNS topic to which Amazon S3 will publish a message to report the + // specified events for the bucket. + Topic *string `type:"string"` + + metadataTopicConfigurationDeprecated `json:"-" xml:"-"` +} + +type metadataTopicConfigurationDeprecated struct { + SDKShapeTraits bool `type:"structure"` +} + +type UploadPartCopyInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + SourceBucket *string `location:"uri" locationName:"sourceBucket" type:"string"` + + SourceKey *string `location:"uri" locationName:"sourceKey" type:"string"` + + // The name of the source bucket and key name of the source object, separated + // by a slash (/). Must be URL-encoded. + CopySource *string `location:"header" locationName:"x-amz-copy-source" type:"string" required:"true"` + + // Copies the object if its entity tag (ETag) matches the specified tag. + CopySourceIfMatch *string `location:"header" locationName:"x-amz-copy-source-if-match" type:"string"` + + // Copies the object if it has been modified since the specified time. + CopySourceIfModifiedSince *time.Time `location:"header" locationName:"x-amz-copy-source-if-modified-since" type:"timestamp" timestampFormat:"rfc822"` + + // Copies the object if its entity tag (ETag) is different than the specified + // ETag. + CopySourceIfNoneMatch *string `location:"header" locationName:"x-amz-copy-source-if-none-match" type:"string"` + + // Copies the object if it hasn't been modified since the specified time. + CopySourceIfUnmodifiedSince *time.Time `location:"header" locationName:"x-amz-copy-source-if-unmodified-since" type:"timestamp" timestampFormat:"rfc822"` + + // The range of bytes to copy from the source object. The range value must use + // the form bytes=first-last, where the first and last are the zero-based byte + // offsets to copy. For example, bytes=0-9 indicates that you want to copy the + // first ten bytes of the source. You can copy a range only if the source object + // is greater than 5 GB. + CopySourceRange *string `location:"header" locationName:"x-amz-copy-source-range" type:"string"` + + // Specifies the algorithm to use when decrypting the source object (e.g., AES256). + CopySourceSSECustomerAlgorithm *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use to decrypt + // the source object. The encryption key provided in this header must be one + // that was used when the source object was created. + CopySourceSSECustomerKey *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + CopySourceSSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key-MD5" type:"string"` + + // Part number of part being copied. + PartNumber *int64 `location:"querystring" locationName:"partNumber" type:"integer" required:"true"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Specifies the algorithm to use to when encrypting the object (e.g., AES256, + // aws:kms). + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting + // data. This value is used to store the object and then it is discarded; Amazon + // does not store the encryption key. The key must be appropriate for use with + // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // header. This must be the same encryption key specified in the initiate multipart + // upload request. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Upload ID identifying the multipart upload whose part is being copied. + UploadID *string `location:"querystring" locationName:"uploadId" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataUploadPartCopyInput `json:"-" xml:"-"` +} + +type metadataUploadPartCopyInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type UploadPartCopyOutput struct { + CopyPartResult *CopyPartResult `type:"structure"` + + // The version of the source object that was copied, if you have enabled versioning + // on the source bucket. + CopySourceVersionID *string `location:"header" locationName:"x-amz-copy-source-version-id" type:"string"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm + // used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // If present, specifies the ID of the AWS Key Management Service (KMS) master + // encryption key that was used for the object. + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + metadataUploadPartCopyOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataUploadPartCopyOutput struct { + SDKShapeTraits bool `type:"structure" payload:"CopyPartResult"` +} + +type UploadPartInput struct { + Body io.ReadSeeker `type:"blob"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Size of the body in bytes. This parameter is useful when the size of the + // body cannot be determined automatically. + ContentLength *int64 `location:"header" locationName:"Content-Length" type:"integer"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Part number of part being uploaded. + PartNumber *int64 `location:"querystring" locationName:"partNumber" type:"integer" required:"true"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Specifies the algorithm to use to when encrypting the object (e.g., AES256, + // aws:kms). + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting + // data. This value is used to store the object and then it is discarded; Amazon + // does not store the encryption key. The key must be appropriate for use with + // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // header. This must be the same encryption key specified in the initiate multipart + // upload request. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Upload ID identifying the multipart upload whose part is being uploaded. + UploadID *string `location:"querystring" locationName:"uploadId" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + TrafficLimit *int64 `location:"header" locationName:"x-kss-traffic-limit" type:"string"` + + ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"` + + // Progress callback function + ProgressFn aws.ProgressFunc `location:"function"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataUploadPartInput `json:"-" xml:"-"` +} + +type metadataUploadPartInput struct { + SDKShapeTraits bool `type:"structure" payload:"Body"` +} + +type UploadPartOutput struct { + // Entity tag for the uploaded object. + ETag *string `location:"header" locationName:"ETag" type:"string"` + + // CRC64 value of a single part. + ChecksumCRC64ECMA *string `location:"header" locationName:"x-amz-checksum-crc64ecma" type:"string"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm + // used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // If present, specifies the ID of the AWS Key Management Service (KMS) master + // encryption key that was used for the object. + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + metadataUploadPartOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataUploadPartOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +type VersioningConfiguration struct { + // Specifies whether MFA delete is enabled in the bucket versioning configuration. + // This element is only returned if the bucket has been configured with MFA + // delete. If the bucket has never been so configured, this element is not returned. + MFADelete *string `locationName:"MfaDelete" type:"string"` + + // The versioning state of the bucket. + Status *string `type:"string"` + + metadataVersioningConfiguration `json:"-" xml:"-"` +} + +type metadataVersioningConfiguration struct { + SDKShapeTraits bool `type:"structure"` +} + +type WebsiteConfiguration struct { + ErrorDocument *ErrorDocument `type:"structure"` + + IndexDocument *IndexDocument `type:"structure"` + + RedirectAllRequestsTo *RedirectAllRequestsTo `type:"structure"` + + RoutingRules []*RoutingRule `locationNameList:"RoutingRule" type:"list"` + + metadataWebsiteConfiguration `json:"-" xml:"-"` +} + +type metadataWebsiteConfiguration struct { + SDKShapeTraits bool `type:"structure"` +} + +func (c *S3) DeleteObjectTaggingRequest(input *DeleteObjectTaggingInput) (req *aws.Request, output *DeleteObjectTaggingOutput) { + op := &aws.Operation{ + Name: "DeleteObjectTagging", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}/{Key+}?tagging", + } + + if input == nil { + input = &DeleteObjectTaggingInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteObjectTaggingOutput{} + req.Data = output + return +} + +func (c *S3) DeleteObjectTagging(input *DeleteObjectTaggingInput) (*DeleteObjectTaggingOutput, error) { + req, out := c.DeleteObjectTaggingRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteObjectTaggingWithContext(ctx aws.Context, input *DeleteObjectTaggingInput) (*DeleteObjectTaggingOutput, error) { + req, out := c.DeleteObjectTaggingRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteObjectTaggingInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataDeleteObjectTaggingInput `json:"-" xml:"-"` +} + +type metadataDeleteObjectTaggingInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type DeleteObjectTaggingOutput struct { + metadataDeleteObjectTaggingOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataDeleteObjectTaggingOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +// GetObjectTaggingRequest generates a request for the GetObjectTagging operation. +func (c *S3) GetObjectTaggingRequest(input *GetObjectTaggingInput) (req *aws.Request, output *GetObjectTaggingOutput) { + op := &aws.Operation{ + Name: "GetObjectTagging", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}/{Key+}?tagging", + } + + if input == nil { + input = &GetObjectTaggingInput{} + } + + req = c.newRequest(op, input, output) + output = &GetObjectTaggingOutput{} + req.Data = output + return +} + +// GetObjectTagging gets the tagging configuration for an object. +func (c *S3) GetObjectTagging(input *GetObjectTaggingInput) (*GetObjectTaggingOutput, error) { + req, out := c.GetObjectTaggingRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetObjectTaggingWithContext(ctx aws.Context, input *GetObjectTaggingInput) (*GetObjectTaggingOutput, error) { + req, out := c.GetObjectTaggingRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetObjectTaggingInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataGetObjectTaggingInput `json:"-" xml:"-"` +} + +type metadataGetObjectTaggingInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type GetObjectTaggingOutput struct { + Tagging *Tagging `locationName:"Tagging" type:"structure"` + + metadataGetObjectTaggingOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetObjectTaggingOutput struct { + SDKShapeTraits bool `type:"structure" payload:"Tagging"` +} + +// PutObjectTaggingRequest generates a request for the PutObjectTagging operation. +func (c *S3) PutObjectTaggingRequest(input *PutObjectTaggingInput) (req *aws.Request, output *PutObjectTaggingOutput) { + op := &aws.Operation{ + Name: "PutObjectTagging", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}/{Key+}?tagging", + } + + if input == nil { + input = &PutObjectTaggingInput{} + } + input.ContentType = aws.String("application/xml") + req = c.newRequest(op, input, output) + output = &PutObjectTaggingOutput{} + req.Data = output + return +} + +// PutObjectTagging sets the tagging configuration for an object. +func (c *S3) PutObjectTagging(input *PutObjectTaggingInput) (*PutObjectTaggingOutput, error) { + req, out := c.PutObjectTaggingRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutObjectTaggingWithContext(ctx aws.Context, input *PutObjectTaggingInput) (*PutObjectTaggingOutput, error) { + req, out := c.PutObjectTaggingRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type PutObjectTaggingInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + Tagging *Tagging `locationName:"Tagging" type:"structure"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutObjectTaggingInput `json:"-" xml:"-"` +} + +type metadataPutObjectTaggingInput struct { + SDKShapeTraits bool `type:"structure" payload:"Tagging"` +} + +type PutObjectTaggingOutput struct { + metadataPutObjectTaggingOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataPutObjectTaggingOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +// FetchObjectRequest generates a request for the FetchObject operation. +func (c *S3) FetchObjectRequest(input *FetchObjectInput) (req *aws.Request, output *FetchObjectOutput) { + op := &aws.Operation{ + Name: "FetchObject", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}/{Key+}?fetch", + } + + if input == nil { + input = &FetchObjectInput{} + } + + if input.SourceUrl != nil { + input.SourceUrl = aws.String(url.QueryEscape(*input.SourceUrl)) + } + req = c.newRequest(op, input, output) + output = &FetchObjectOutput{} + req.Data = output + return +} + +func (c *S3) FetchObject(input *FetchObjectInput) (*FetchObjectOutput, error) { + req, out := c.FetchObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) FetchObjectWithContext(ctx aws.Context, input *FetchObjectInput) (*FetchObjectOutput, error) { + req, out := c.FetchObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type FetchObjectInput struct { + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-kss-acl" type:"string"` + + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // The language the content is in. + ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // The name of the source bucket and key name of the source object, separated + // by a slash (/). Must be URL-encoded. + SourceUrl *string `location:"header" locationName:"x-kss-sourceurl" type:"string" required:"true"` + + CallbackUrl *string `location:"header" locationName:"x-kss-callbackurl" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // Allows grantee to read the object data and its metadata. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Allows grantee to read the object ACL. + GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"` + + // Allows grantee to write the ACL for the applicable object. + GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` + + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // A map of metadata to store with the object in S3. + Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` + + // Specifies whether the metadata is copied from the source object or replaced + // with metadata provided in the request. + MetadataDirective *string `location:"header" locationName:"x-amz-metadata-directive" type:"string"` + + // Confirms that the requester knows that she or he will be charged for the + // request. Bucket owners need not specify this parameter in their requests. + // Documentation on downloading objects from requester pays buckets can be found + // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html + RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string"` + + // Specifies the algorithm to use to when encrypting the object (e.g., AES256, + // aws:kms). + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting + // data. This value is used to store the object and then it is discarded; Amazon + // does not store the encryption key. The key must be appropriate for use with + // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm + // header. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // Amazon S3 uses this header for a message integrity check to ensure the encryption + // key was transmitted without error. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT + // requests for an object protected by AWS KMS will fail if not made via SSL + // or using SigV4. Documentation on configuring any of the officially supported + // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // The type of storage to use for the object. Defaults to 'STANDARD'. + StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"` + + // If the bucket is configured as a website, redirects requests for this object + // to another object in the same bucket or to an external URL. Amazon S3 stores + // the value of this header in the object metadata. + WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` + + // Specifies the object tag of the object. Multiple tags can be set at the same time, such as: TagA=A&TagB=B. + // Note: Key and Value need to be URL-encoded first. If an item does not have "=", the Value is considered to be an empty string. + Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataFetchObjectInput `json:"-" xml:"-"` +} + +type metadataFetchObjectInput struct { + SDKShapeTraits bool `type:"structure"` +} + +type FetchObjectOutput struct { + + // If the object expiration is configured, the response includes this header. + Expiration *string `location:"header" locationName:"x-amz-expiration" type:"string"` + + // If present, indicates that the requester was successfully charged for the + // request. + RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm + // used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // If present, specifies the ID of the AWS Key Management Service (KMS) master + // encryption key that was used for the object. + SSEKMSKeyID *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + + // The Server-side encryption algorithm used when storing this object in S3 + // (e.g., AES256, aws:kms). + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataFetchObjectOutput `json:"-" xml:"-"` +} + +type metadataFetchObjectOutput struct { + SDKShapeTraits bool `type:"structure"` +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/api_object.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/api_object.go new file mode 100644 index 0000000000..d528a6380d --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/api_object.go @@ -0,0 +1,169 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "io" + "time" +) + +type AppendObjectInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // The name of the object. + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // The starting position of the AppendObject operation. + // When the AppendObject operation is successful, the x-kss-next-append-position header describes the starting position of the next operation. + Position *int64 `location:"querystring" locationName:"position" type:"integer" required:"true"` + + // The readable body payload to send to KS3. + Body io.Reader `type:"blob"` + + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // Size of the body in bytes. This parameter is useful when the size of the + // body cannot be determined automatically. + ContentLength *int64 `location:"header" locationName:"Content-Length" type:"integer"` + + // Calculate MD5 value for message content (excluding header) + ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // When using Expect: 100-continue, the client will only send the request body after receiving confirmation from the server. + // If the information in the request header is rejected, the client will not send the request body. + Expect *string `location:"header" locationName:"Expect" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // A map of metadata to store with the object in S3. + Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` + + // The type of storage to use for the object. Defaults to 'STANDARD'. + StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"` + + // Set the maximum allowed size for a single addition of content + ContentMaxLength *int64 `location:"header" locationName:"x-amz-content-maxlength" type:"integer"` + + // Specifies the object tag of the object. Multiple tags can be set at the same time, such as: TagA=A&TagB=B. + // Note: Key and Value need to be URL-encoded first. If an item does not have "=", the Value is considered to be an empty string. + Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` + + // Allows grantee to read the object data and its metadata. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // The Server-side encryption algorithm used when storing this object in KS3, eg: AES256. + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // Specifies the algorithm to use to when encrypting the object, eg: AES256. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for KS3 to use in encrypting data. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Progress callback function + ProgressFn aws.ProgressFunc `location:"function"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataAppendObjectInput `json:"-" xml:"-"` +} + +type metadataAppendObjectInput struct { + SDKShapeTraits bool `type:"structure" payload:"Body"` +} + +type AppendObjectOutput struct { + // Entity tag for the uploaded object. + ETag *string `location:"header" locationName:"ETag" type:"string"` + + // The position that should be provided for the next request, which is the size of the current object. + NextAppendPosition *int64 `location:"header" locationName:"x-amz-next-append-position" type:"integer"` + + // The type of Object. + ObjectType *string `location:"header" locationName:"x-amz-object-type" type:"string"` + + // The Server-side encryption algorithm used when storing this object in KS3, eg: AES256. + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header confirming the encryption algorithm used. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // If server-side encryption with a customer-provided encryption key was requested, + // the response will include this header to provide round trip message integrity + // verification of the customer-provided encryption key. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataAppendObjectOutput `json:"-" xml:"-"` +} + +type metadataAppendObjectOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +// AppendObjectRequest generates a request for the AppendObject operation. +func (c *S3) AppendObjectRequest(input *AppendObjectInput) (req *aws.Request, output *AppendObjectOutput) { + op := &aws.Operation{ + Name: "AppendObject", + HTTPMethod: "POST", + HTTPPath: "/{Bucket}/{Key+}?append", + } + + if input == nil { + input = &AppendObjectInput{} + } + + req = c.newRequest(op, input, output) + + if input.ProgressFn != nil { + req.ProgressFn = input.ProgressFn + } + + output = &AppendObjectOutput{} + req.Data = output + return +} + +// AppendObject is used to append data to an Appendable object. +func (c *S3) AppendObject(input *AppendObjectInput) (*AppendObjectOutput, error) { + req, out := c.AppendObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) AppendObjectWithContext(ctx aws.Context, input *AppendObjectInput) (*AppendObjectOutput, error) { + req, out := c.AppendObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_accessmonitor.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_accessmonitor.go new file mode 100644 index 0000000000..baf6053df4 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_accessmonitor.go @@ -0,0 +1,134 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +type PutBucketAccessMonitorInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Bucket access monitor configuration. + AccessMonitorConfiguration *AccessMonitorConfiguration `locationName:"AccessMonitorConfiguration" type:"structure" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketAccessMonitorInput `json:"-" xml:"-"` +} + +type metadataPutBucketAccessMonitorInput struct { + SDKShapeTraits bool `type:"structure" payload:"AccessMonitorConfiguration"` +} + +type AccessMonitorConfiguration struct { + // Specifies whether to enable access tracking for the bucket. The value range is as follows: + // Enabled: After the bucket enables access tracking, the access tracking enable time is used as the default + // last access time for all objects in the bucket. + // Disabled: The access tracking status of the bucket can be changed to Disabled only when the bucket does not + // have a lifecycle rule based on the last access time matching rule. + Status *string `locationName:"Status" type:"string" required:"true"` +} + +type PutBucketAccessMonitorOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutBucketAccessMonitorRequest generates a request for the PutBucketAccessMonitor operation. +func (c *S3) PutBucketAccessMonitorRequest(input *PutBucketAccessMonitorInput) (req *aws.Request, output *PutBucketAccessMonitorOutput) { + op := &aws.Operation{ + Name: "PutBucketAccessMonitor", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?accessmonitor", + } + + if input == nil { + input = &PutBucketAccessMonitorInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketAccessMonitorOutput{} + req.Data = output + return +} + +// PutBucketAccessMonitor sets bucket access monitor configuration. +func (c *S3) PutBucketAccessMonitor(input *PutBucketAccessMonitorInput) (*PutBucketAccessMonitorOutput, error) { + req, out := c.PutBucketAccessMonitorRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketAccessMonitorWithContext(ctx aws.Context, input *PutBucketAccessMonitorInput) (*PutBucketAccessMonitorOutput, error) { + req, out := c.PutBucketAccessMonitorRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketAccessMonitorInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketAccessMonitorOutput struct { + // Bucket access monitor configuration. + AccessMonitorConfiguration *AccessMonitorConfiguration `locationName:"AccessMonitorConfiguration" type:"structure"` + + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketAccessMonitorOutput `json:"-" xml:"-"` +} + +type metadataGetBucketAccessMonitorOutput struct { + SDKShapeTraits bool `type:"structure" payload:"AccessMonitorConfiguration"` +} + +// GetBucketAccessMonitorRequest generates a request for the GetBucketAccessMonitor operation. +func (c *S3) GetBucketAccessMonitorRequest(input *GetBucketAccessMonitorInput) (req *aws.Request, output *GetBucketAccessMonitorOutput) { + op := &aws.Operation{ + Name: "GetBucketAccessMonitor", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?accessmonitor", + } + + if input == nil { + input = &GetBucketAccessMonitorInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketAccessMonitorOutput{} + req.Data = output + return +} + +// GetBucketAccessMonitor gets bucket access monitor configuration. +func (c *S3) GetBucketAccessMonitor(input *GetBucketAccessMonitorInput) (*GetBucketAccessMonitorOutput, error) { + req, out := c.GetBucketAccessMonitorRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketAccessMonitorWithContext(ctx aws.Context, input *GetBucketAccessMonitorInput) (*GetBucketAccessMonitorOutput, error) { + req, out := c.GetBucketAccessMonitorRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_encryption.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_encryption.go new file mode 100644 index 0000000000..39988da02b --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_encryption.go @@ -0,0 +1,194 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +type PutBucketEncryptionInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Bucket encryption configuration container. + ServerSideEncryptionConfiguration *ServerSideEncryptionConfiguration `locationName:"ServerSideEncryptionConfiguration" type:"structure" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketEncryptionInput `json:"-" xml:"-"` +} + +type metadataPutBucketEncryptionInput struct { + SDKShapeTraits bool `type:"structure" payload:"ServerSideEncryptionConfiguration"` + + AutoFillMD5 bool +} + +type ServerSideEncryptionConfiguration struct { + // Default encryption rule for bucket. + Rule *BucketEncryptionRule `locationName:"Rule" type:"structure" required:"true"` +} + +type BucketEncryptionRule struct { + // The child element of the default encryption configuration for the bucket. + ApplyServerSideEncryptionByDefault *ApplyServerSideEncryptionByDefault `locationName:"ApplyServerSideEncryptionByDefault" type:"structure" required:"true"` +} + +type ApplyServerSideEncryptionByDefault struct { + // The server-side encryption algorithm to be used by the bucket's default encryption configuration. + SSEAlgorithm *string `locationName:"SSEAlgorithm" type:"string" required:"true"` +} + +type PutBucketEncryptionOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutBucketEncryptionRequest generates a request for the PutBucketEncryption operation. +func (c *S3) PutBucketEncryptionRequest(input *PutBucketEncryptionInput) (req *aws.Request, output *PutBucketEncryptionOutput) { + op := &aws.Operation{ + Name: "PutBucketEncryption", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?encryption", + } + + if input == nil { + input = &PutBucketEncryptionInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(op, input, output) + output = &PutBucketEncryptionOutput{} + req.Data = output + return +} + +// PutBucketEncryption sets bucket encryption configuration. +func (c *S3) PutBucketEncryption(input *PutBucketEncryptionInput) (*PutBucketEncryptionOutput, error) { + req, out := c.PutBucketEncryptionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketEncryptionWithContext(ctx aws.Context, input *PutBucketEncryptionInput) (*PutBucketEncryptionOutput, error) { + req, out := c.PutBucketEncryptionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketEncryptionInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketEncryptionOutput struct { + // Bucket encryption configuration container. + ServerSideEncryptionConfiguration *ServerSideEncryptionConfiguration `locationName:"ServerSideEncryptionConfiguration" type:"structure"` + + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketEncryptionOutput `json:"-" xml:"-"` +} + +type metadataGetBucketEncryptionOutput struct { + SDKShapeTraits bool `type:"structure" payload:"ServerSideEncryptionConfiguration"` +} + +// GetBucketEncryptionRequest generates a request for the GetBucketEncryption operation. +func (c *S3) GetBucketEncryptionRequest(input *GetBucketEncryptionInput) (req *aws.Request, output *GetBucketEncryptionOutput) { + op := &aws.Operation{ + Name: "GetBucketEncryption", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?encryption", + } + + if input == nil { + input = &GetBucketEncryptionInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketEncryptionOutput{} + req.Data = output + return +} + +// GetBucketEncryption gets bucket encryption configuration. +func (c *S3) GetBucketEncryption(input *GetBucketEncryptionInput) (*GetBucketEncryptionOutput, error) { + req, out := c.GetBucketEncryptionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketEncryptionWithContext(ctx aws.Context, input *GetBucketEncryptionInput) (*GetBucketEncryptionOutput, error) { + req, out := c.GetBucketEncryptionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteBucketEncryptionInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type DeleteBucketEncryptionOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// DeleteBucketEncryptionRequest generates a request for the DeleteBucketEncryption operation. +func (c *S3) DeleteBucketEncryptionRequest(input *DeleteBucketEncryptionInput) (req *aws.Request, output *DeleteBucketEncryptionOutput) { + op := &aws.Operation{ + Name: "DeleteBucketEncryption", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?encryption", + } + + if input == nil { + input = &DeleteBucketEncryptionInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketEncryptionOutput{} + req.Data = output + return +} + +// DeleteBucketEncryption deletes bucket encryption configuration. +func (c *S3) DeleteBucketEncryption(input *DeleteBucketEncryptionInput) (*DeleteBucketEncryptionOutput, error) { + req, out := c.DeleteBucketEncryptionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketEncryptionWithContext(ctx aws.Context, input *DeleteBucketEncryptionInput) (*DeleteBucketEncryptionOutput, error) { + req, out := c.DeleteBucketEncryptionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_location.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_location.go new file mode 100644 index 0000000000..174991be58 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_location.go @@ -0,0 +1,27 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/aws/awserr" + "io" + "regexp" +) + +var reBucketLocation = regexp.MustCompile(`>([^<>]+)<\/LocationConstraint`) + +func buildGetBucketLocation(r *aws.Request) { + if r.DataFilled() { + out := r.Data.(*GetBucketLocationOutput) + b, err := io.ReadAll(r.HTTPResponse.Body) + if err != nil { + r.Error = awserr.New("Unmarshal", + "failed reading response body", err) + return + } + match := reBucketLocation.FindSubmatch(b) + if len(match) > 1 { + loc := string(match[1]) + out.LocationConstraint = aws.String(loc) + } + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_qos.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_qos.go new file mode 100644 index 0000000000..004f9aab6b --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_qos.go @@ -0,0 +1,200 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +type PutBucketQosInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Bucket flow control configuration container. + BucketQosConfiguration *BucketQosConfiguration `locationName:"BucketQosConfiguration" type:"structure" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketQosInput `json:"-" xml:"-"` +} + +type metadataPutBucketQosInput struct { + SDKShapeTraits bool `type:"structure" payload:"BucketQosConfiguration"` +} + +type BucketQosConfiguration struct { + // Set the bucket flow control quota. + Quotas []*BucketQosQuota `locationName:"Quota" type:"list" flattened:"true" required:"true"` +} + +type BucketQosQuota struct { + // Specify the storage medium type that needs flow control. Options: Extreme/Normal (default) + // Extreme: SSD type storage medium + // Normal (default): HDD type storage medium + StorageMedium *string `locationName:"StorageMedium" type:"string"` + + // External network upload bandwidth, in Gbps, the value must be a positive integer. + ExtranetUploadBandwidth *int64 `locationName:"ExtranetUploadBandwidth" type:"integer"` + + // Intranet network upload bandwidth, in Gbps, the value must be a positive integer. + IntranetUploadBandwidth *int64 `locationName:"IntranetUploadBandwidth" type:"integer"` + + // External network download bandwidth, in Gbps, the value must be a positive integer. + ExtranetDownloadBandwidth *int64 `locationName:"ExtranetDownloadBandwidth" type:"integer"` + + // Intranet network download bandwidth, in Gbps, the value must be a positive integer. + IntranetDownloadBandwidth *int64 `locationName:"IntranetDownloadBandwidth" type:"integer"` +} + +type PutBucketQosOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutBucketQosRequest generates a request for the PutBucketQos operation. +func (c *S3) PutBucketQosRequest(input *PutBucketQosInput) (req *aws.Request, output *PutBucketQosOutput) { + op := &aws.Operation{ + Name: "PutBucketQos", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?bucketqos", + } + + if input == nil { + input = &PutBucketQosInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketQosOutput{} + req.Data = output + return +} + +// PutBucketQos sets bucket flow control configuration. +func (c *S3) PutBucketQos(input *PutBucketQosInput) (*PutBucketQosOutput, error) { + req, out := c.PutBucketQosRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketQosWithContext(ctx aws.Context, input *PutBucketQosInput) (*PutBucketQosOutput, error) { + req, out := c.PutBucketQosRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketQosInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketQosOutput struct { + // Bucket flow control configuration container. + BucketQosConfiguration *BucketQosConfiguration `locationName:"BucketQosConfiguration" type:"structure"` + + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketQosOutput `json:"-" xml:"-"` +} + +type metadataGetBucketQosOutput struct { + SDKShapeTraits bool `type:"structure" payload:"BucketQosConfiguration"` +} + +// GetBucketQosRequest generates a request for the GetBucketQos operation. +func (c *S3) GetBucketQosRequest(input *GetBucketQosInput) (req *aws.Request, output *GetBucketQosOutput) { + op := &aws.Operation{ + Name: "GetBucketQos", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?bucketqos", + } + + if input == nil { + input = &GetBucketQosInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketQosOutput{} + req.Data = output + return +} + +// GetBucketQos gets bucket flow control configuration. +func (c *S3) GetBucketQos(input *GetBucketQosInput) (*GetBucketQosOutput, error) { + req, out := c.GetBucketQosRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketQosWithContext(ctx aws.Context, input *GetBucketQosInput) (*GetBucketQosOutput, error) { + req, out := c.GetBucketQosRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteBucketQosInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type DeleteBucketQosOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// DeleteBucketQosRequest generates a request for the DeleteBucketQos operation. +func (c *S3) DeleteBucketQosRequest(input *DeleteBucketQosInput) (req *aws.Request, output *DeleteBucketQosOutput) { + op := &aws.Operation{ + Name: "DeleteBucketQos", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?bucketqos", + } + + if input == nil { + input = &DeleteBucketQosInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketQosOutput{} + req.Data = output + return +} + +// DeleteBucketQos deletes bucket flow control configuration. +func (c *S3) DeleteBucketQos(input *DeleteBucketQosInput) (*DeleteBucketQosOutput, error) { + req, out := c.DeleteBucketQosRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketQosWithContext(ctx aws.Context, input *DeleteBucketQosInput) (*DeleteBucketQosOutput, error) { + req, out := c.DeleteBucketQosRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_tagging.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_tagging.go new file mode 100644 index 0000000000..ebcfde7587 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_tagging.go @@ -0,0 +1,179 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +type PutBucketTaggingInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Bucket tagging configuration container. + Tagging *Tagging `locationName:"Tagging" type:"structure" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketTaggingInput `json:"-" xml:"-"` +} + +type metadataPutBucketTaggingInput struct { + SDKShapeTraits bool `type:"structure" payload:"Tagging"` + + AutoFillMD5 bool +} + +type PutBucketTaggingOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutBucketTaggingRequest generates a request for the PutBucketTagging operation. +func (c *S3) PutBucketTaggingRequest(input *PutBucketTaggingInput) (req *aws.Request, output *PutBucketTaggingOutput) { + op := &aws.Operation{ + Name: "PutBucketTagging", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?tagging", + } + + if input == nil { + input = &PutBucketTaggingInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(op, input, output) + output = &PutBucketTaggingOutput{} + req.Data = output + return +} + +// PutBucketTagging sets bucket tagging configuration. +func (c *S3) PutBucketTagging(input *PutBucketTaggingInput) (*PutBucketTaggingOutput, error) { + req, out := c.PutBucketTaggingRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketTaggingWithContext(ctx aws.Context, input *PutBucketTaggingInput) (*PutBucketTaggingOutput, error) { + req, out := c.PutBucketTaggingRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketTaggingInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketTaggingOutput struct { + // Bucket tagging configuration container. + Tagging *Tagging `locationName:"Tagging" type:"structure"` + + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketTaggingOutput `json:"-" xml:"-"` +} + +type metadataGetBucketTaggingOutput struct { + SDKShapeTraits bool `type:"structure" payload:"Tagging"` +} + +// GetBucketTaggingRequest generates a request for the GetBucketTagging operation. +func (c *S3) GetBucketTaggingRequest(input *GetBucketTaggingInput) (req *aws.Request, output *GetBucketTaggingOutput) { + op := &aws.Operation{ + Name: "GetBucketTagging", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?tagging", + } + + if input == nil { + input = &GetBucketTaggingInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketTaggingOutput{} + req.Data = output + return +} + +// GetBucketTagging gets bucket tagging configuration. +func (c *S3) GetBucketTagging(input *GetBucketTaggingInput) (*GetBucketTaggingOutput, error) { + req, out := c.GetBucketTaggingRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketTaggingWithContext(ctx aws.Context, input *GetBucketTaggingInput) (*GetBucketTaggingOutput, error) { + req, out := c.GetBucketTaggingRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteBucketTaggingInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type DeleteBucketTaggingOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// DeleteBucketTaggingRequest generates a request for the DeleteBucketTagging operation. +func (c *S3) DeleteBucketTaggingRequest(input *DeleteBucketTaggingInput) (req *aws.Request, output *DeleteBucketTaggingOutput) { + op := &aws.Operation{ + Name: "DeleteBucketTagging", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?tagging", + } + + if input == nil { + input = &DeleteBucketTaggingInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketTaggingOutput{} + req.Data = output + return +} + +// DeleteBucketTagging deletes bucket tagging configuration. +func (c *S3) DeleteBucketTagging(input *DeleteBucketTaggingInput) (*DeleteBucketTaggingOutput, error) { + req, out := c.DeleteBucketTaggingRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketTaggingWithContext(ctx aws.Context, input *DeleteBucketTaggingInput) (*DeleteBucketTaggingOutput, error) { + req, out := c.DeleteBucketTaggingRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_transfer_acceleration.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_transfer_acceleration.go new file mode 100644 index 0000000000..5635eaca0d --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/bucket_transfer_acceleration.go @@ -0,0 +1,133 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +type PutBucketTransferAccelerationInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Bucket transfer acceleration configuration container. + TransferAccelerationConfiguration *TransferAccelerationConfiguration `locationName:"TransferAccelerationConfiguration" type:"structure" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketTransferAccelerationInput `json:"-" xml:"-"` +} + +type metadataPutBucketTransferAccelerationInput struct { + SDKShapeTraits bool `type:"structure" payload:"TransferAccelerationConfiguration"` + + AutoFillMD5 bool +} + +type TransferAccelerationConfiguration struct { + // Whether the target bucket has enabled transfer acceleration. + Enabled *bool `locationName:"Enabled" type:"boolean" required:"true"` +} + +type PutBucketTransferAccelerationOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutBucketTransferAccelerationRequest generates a request for the PutBucketTransferAcceleration operation. +func (c *S3) PutBucketTransferAccelerationRequest(input *PutBucketTransferAccelerationInput) (req *aws.Request, output *PutBucketTransferAccelerationOutput) { + op := &aws.Operation{ + Name: "PutBucketTransferAcceleration", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?transferAcceleration", + } + + if input == nil { + input = &PutBucketTransferAccelerationInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(op, input, output) + output = &PutBucketTransferAccelerationOutput{} + req.Data = output + return +} + +// PutBucketTransferAcceleration sets bucket transfer acceleration configuration. +func (c *S3) PutBucketTransferAcceleration(input *PutBucketTransferAccelerationInput) (*PutBucketTransferAccelerationOutput, error) { + req, out := c.PutBucketTransferAccelerationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketTransferAccelerationWithContext(ctx aws.Context, input *PutBucketTransferAccelerationInput) (*PutBucketTransferAccelerationOutput, error) { + req, out := c.PutBucketTransferAccelerationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketTransferAccelerationInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketTransferAccelerationOutput struct { + // Bucket transfer acceleration configuration container. + TransferAccelerationConfiguration *TransferAccelerationConfiguration `locationName:"TransferAccelerationConfiguration" type:"structure"` + + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketTransferAccelerationOutput `json:"-" xml:"-"` +} + +type metadataGetBucketTransferAccelerationOutput struct { + SDKShapeTraits bool `type:"structure" payload:"TransferAccelerationConfiguration"` +} + +// GetBucketTransferAccelerationRequest generates a request for the GetBucketTransferAcceleration operation. +func (c *S3) GetBucketTransferAccelerationRequest(input *GetBucketTransferAccelerationInput) (req *aws.Request, output *GetBucketTransferAccelerationOutput) { + op := &aws.Operation{ + Name: "GetBucketTransferAcceleration", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?transferAcceleration", + } + + if input == nil { + input = &GetBucketTransferAccelerationInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketTransferAccelerationOutput{} + req.Data = output + return +} + +// GetBucketTransferAcceleration gets bucket transfer acceleration configuration. +func (c *S3) GetBucketTransferAcceleration(input *GetBucketTransferAccelerationInput) (*GetBucketTransferAccelerationOutput, error) { + req, out := c.GetBucketTransferAccelerationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketTransferAccelerationWithContext(ctx aws.Context, input *GetBucketTransferAccelerationInput) (*GetBucketTransferAccelerationOutput, error) { + req, out := c.GetBucketTransferAccelerationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/checkpoint.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/checkpoint.go new file mode 100644 index 0000000000..16ecba26c0 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/checkpoint.go @@ -0,0 +1,520 @@ +package s3 + +import ( + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest" + "os" + "path/filepath" + "strconv" +) + +const ( + DefaultTaskNum int64 = 3 + + MaxPartNum int64 = 10000 + + MaxPartSize int64 = 5 * 1024 * 1024 * 1024 + + MinPartSize int64 = 100 * 1024 + + DefaultPartSize int64 = 5 * 1024 * 1024 + + FilePermMode = os.FileMode(0664) + + DirPermMode = os.FileMode(0755) + + CheckpointFileSuffixUploader = ".ucp" + + CheckpointFileSuffixDownloader = ".dcp" + + CheckpointFileSuffixCopier = ".ccp" + + TempFileSuffix = ".temp" + + CheckpointMagic = "B62CAE41-F268-4EC5-839D-FBE475E3FA02" +) + +// ------------------------------------ UploadCheckpoint ------------------------------------ + +type UploadCheckpoint struct { + Magic string + MD5 string + CpFilePath string // checkpoint file full path + UploadFilePath string // Local file path + UploadFileSize int64 // Local file size + UploadFileLastModified string // Local file last modified time + BucketName string // Bucket name + ObjectKey string // Object key + PartSize int64 // Part size + UploadId string // Upload ID + PartETagList []*CompletedPart // Completed parts +} + +func newUploadCheckpoint(u *Uploader) (*UploadCheckpoint, error) { + request := u.uploadFileRequest + fileSize := aws.ToLong(request.FileSize) + partSize := u.getPartSize(fileSize, aws.ToLong(request.PartSize)) + cp := &UploadCheckpoint{ + Magic: CheckpointMagic, + UploadFileSize: fileSize, + BucketName: aws.ToString(request.Bucket), + ObjectKey: aws.ToString(request.Key), + PartSize: partSize, + PartETagList: make([]*CompletedPart, 0), + } + + filePath := aws.ToString(request.UploadFile) + if filePath != "" { + fileInfo, err := os.Stat(filePath) + if err != nil { + return nil, err + } + cp.UploadFilePath = filePath + cp.UploadFileLastModified = fileInfo.ModTime().String() + } else { + if request.ObjectMeta != nil { + cp.UploadFileLastModified = aws.ToString(request.ObjectMeta[HTTPHeaderLastModified]) + } + } + + return cp, nil +} + +func generateUploadCpFilePath(request *UploadFileInput) (string, error) { + name := fmt.Sprintf("%s/%s", *request.Bucket, *request.Key) + md5Hash := md5.New() + md5Hash.Write([]byte("ks3://" + rest.EscapePath(name, false))) + destHash := hex.EncodeToString(md5Hash.Sum(nil)) + + filePath := aws.ToString(request.UploadFile) + absPath, _ := filepath.Abs(filePath) + md5Hash.Reset() + md5Hash.Write([]byte(absPath)) + srcHash := hex.EncodeToString(md5Hash.Sum(nil)) + + var dir string + baseDir := aws.ToString(request.CheckpointDir) + if baseDir == "" { + dir = os.TempDir() + } else { + dir = filepath.Dir(baseDir) + } + + cpFilePath := filepath.Join(dir, fmt.Sprintf("%v-%v%v", srcHash, destHash, CheckpointFileSuffixUploader)) + + return cpFilePath, nil +} + +// load checkpoint from local file +func (cp *UploadCheckpoint) load() error { + if cp.CpFilePath == "" { + return nil + } + + if !FileExists(cp.CpFilePath) { + return nil + } + + // 读取断点文件 + contents, err := os.ReadFile(cp.CpFilePath) + if err != nil { + return err + } + + ucp := UploadCheckpoint{} + if err = json.Unmarshal(contents, &ucp); err != nil { + return err + } + + // 判断断点文件是否有效 + if !cp.isValid(ucp) { + err := cp.remove() + if err != nil { + return err + } + } + + // 读取断点文件成功，将断点文件中的信息赋值给当前对象 + cp.UploadId = ucp.UploadId + cp.PartETagList = ucp.PartETagList + + return nil +} + +func (cp *UploadCheckpoint) isValid(ucp UploadCheckpoint) bool { + md5sum := ucp.checksum() + if CheckpointMagic != ucp.Magic || md5sum != ucp.MD5 { + return false + } + + if cp.BucketName != ucp.BucketName || + cp.ObjectKey != ucp.ObjectKey || + cp.PartSize != ucp.PartSize || + cp.UploadFilePath != ucp.UploadFilePath || + cp.UploadFileSize != ucp.UploadFileSize || + cp.UploadFileLastModified != ucp.UploadFileLastModified { + return false + } + + if len(ucp.UploadId) == 0 { + return false + } + + return true +} + +func (cp *UploadCheckpoint) dump() error { + if cp.CpFilePath == "" { + return nil + } + + dir := filepath.Dir(cp.CpFilePath) + if !DirExists(dir) { + err := os.MkdirAll(dir, DirPermMode) + if err != nil { + return err + } + } + + cp.MD5 = cp.checksum() + str, err := json.Marshal(cp) + if err != nil { + return err + } + + return os.WriteFile(cp.CpFilePath, str, FilePermMode) +} + +func (cp *UploadCheckpoint) checksum() string { + str := cp.MD5 + cp.MD5 = "" + json, _ := json.Marshal(cp) + sum := md5.Sum(json) + md5sum := hex.EncodeToString(sum[:]) + cp.MD5 = str + return md5sum +} + +func (cp *UploadCheckpoint) remove() error { + if cp.CpFilePath == "" { + return nil + } + + return os.Remove(cp.CpFilePath) +} + +// ------------------------------------ DownloadCheckpoint ------------------------------------ + +type DownloadCheckpoint struct { + Magic string + MD5 string + CpFilePath string // checkpoint file full path + DownloadFilePath string // Local file path + BucketName string // Bucket name + ObjectKey string // Object key + ObjectSize int64 // Object size + ObjectLastModified string // Object last modified + PartSize int64 // Part size + PartETagList []*CompletedPart // Completed parts +} + +func newDownloadCheckpoint(d *Downloader) (*DownloadCheckpoint, error) { + request := d.downloadFileRequest + meta := d.downloadFileMeta + objectSize, _ := strconv.ParseInt(aws.ToString(meta[HTTPHeaderContentLength]), 10, 64) + lastModified := aws.ToString(meta[HTTPHeaderLastModified]) + cp := &DownloadCheckpoint{ + Magic: CheckpointMagic, + BucketName: aws.ToString(request.Bucket), + ObjectKey: aws.ToString(request.Key), + DownloadFilePath: aws.ToString(request.DownloadFile), + ObjectSize: objectSize, + ObjectLastModified: lastModified, + PartSize: aws.ToLong(request.PartSize), + PartETagList: make([]*CompletedPart, 0), + } + + return cp, nil +} + +func generateDownloadCpFilePath(request *DownloadFileInput) (string, error) { + name := fmt.Sprintf("%v/%v", *request.Bucket, *request.Key) + md5Hash := md5.New() + md5Hash.Write([]byte("ks3://" + rest.EscapePath(name, false))) + destHash := hex.EncodeToString(md5Hash.Sum(nil)) + + filePath := aws.ToString(request.DownloadFile) + absPath, _ := filepath.Abs(filePath) + md5Hash.Reset() + md5Hash.Write([]byte(absPath)) + srcHash := hex.EncodeToString(md5Hash.Sum(nil)) + + var dir string + baseDir := aws.ToString(request.CheckpointDir) + if baseDir == "" { + dir = os.TempDir() + } else { + dir = filepath.Dir(baseDir) + } + + cpFilePath := filepath.Join(dir, fmt.Sprintf("%v-%v%v", srcHash, destHash, CheckpointFileSuffixDownloader)) + + return cpFilePath, nil +} + +// load checkpoint from local file +func (cp *DownloadCheckpoint) load() error { + if cp.CpFilePath == "" { + return nil + } + + if !FileExists(cp.CpFilePath) { + return nil + } + + // 读取断点文件 + contents, err := os.ReadFile(cp.CpFilePath) + if err != nil { + return err + } + + dcp := DownloadCheckpoint{} + if err = json.Unmarshal(contents, &dcp); err != nil { + return err + } + + // 判断断点文件是否有效 + if !cp.isValid(dcp) { + cp.remove() + return nil + } + + // 读取断点文件成功，将断点文件中的信息赋值给当前对象 + cp.PartETagList = dcp.PartETagList + + return nil +} + +func (cp *DownloadCheckpoint) isValid(dcp DownloadCheckpoint) bool { + md5sum := dcp.checksum() + if CheckpointMagic != dcp.Magic || md5sum != dcp.MD5 { + return false + } + + if cp.BucketName != dcp.BucketName || + cp.ObjectKey != dcp.ObjectKey || + cp.PartSize != dcp.PartSize || + cp.DownloadFilePath != dcp.DownloadFilePath || + cp.ObjectSize != dcp.ObjectSize || + cp.ObjectLastModified != dcp.ObjectLastModified { + return false + } + + return true +} + +func (cp *DownloadCheckpoint) dump() error { + if cp.CpFilePath == "" { + return nil + } + + dir := filepath.Dir(cp.CpFilePath) + if !DirExists(dir) { + err := os.MkdirAll(dir, DirPermMode) + if err != nil { + return err + } + } + + cp.MD5 = cp.checksum() + str, err := json.Marshal(cp) + if err != nil { + return err + } + + return os.WriteFile(cp.CpFilePath, str, FilePermMode) +} + +func (cp *DownloadCheckpoint) checksum() string { + str := cp.MD5 + cp.MD5 = "" + json, _ := json.Marshal(cp) + sum := md5.Sum(json) + md5sum := hex.EncodeToString(sum[:]) + cp.MD5 = str + return md5sum +} + +func (cp *DownloadCheckpoint) remove() error { + if cp.CpFilePath == "" { + return nil + } + + return os.Remove(cp.CpFilePath) +} + +// ------------------------------------ CopyCheckpoint ------------------------------------ + +type CopyCheckpoint struct { + Magic string + MD5 string + CpFilePath string // checkpoint file full path + BucketName string // Bucket name + ObjectKey string // Object key + SrcBucketName string // Source bucket name + SrcObjectKey string // Source object key + SrcObjectSize int64 // Source object size + SrcObjectLastModified string // Source object last modified time + PartSize int64 // Part size + UploadId string // Upload ID + PartETagList []*CompletedPart // Completed parts +} + +func newCopyCheckpoint(c *Copier) (*CopyCheckpoint, error) { + request := c.copyFileRequest + meta := c.copyObjectMeta + + objectSize, _ := strconv.ParseInt(aws.ToString(meta[HTTPHeaderContentLength]), 10, 64) + lastModified := aws.ToString(meta[HTTPHeaderLastModified]) + partSize := c.getPartSize(objectSize, aws.ToLong(request.PartSize)) + cp := &CopyCheckpoint{ + Magic: CheckpointMagic, + BucketName: aws.ToString(request.Bucket), + ObjectKey: aws.ToString(request.Key), + SrcBucketName: aws.ToString(request.SourceBucket), + SrcObjectKey: aws.ToString(request.SourceKey), + SrcObjectSize: objectSize, + SrcObjectLastModified: lastModified, + PartSize: partSize, + PartETagList: make([]*CompletedPart, 0), + } + + return cp, nil +} + +func generateCopyCpFilePath(request *CopyFileInput) (string, error) { + dstName := fmt.Sprintf("%s/%s", *request.Bucket, *request.Key) + md5Hash := md5.New() + md5Hash.Write([]byte("ks3://" + rest.EscapePath(dstName, false))) + destHash := hex.EncodeToString(md5Hash.Sum(nil)) + + srcName := fmt.Sprintf("%s/%s", *request.SourceBucket, *request.SourceKey) + md5Hash.Reset() + md5Hash.Write([]byte(srcName)) + srcHash := hex.EncodeToString(md5Hash.Sum(nil)) + + var dir string + baseDir := aws.ToString(request.CheckpointDir) + if baseDir == "" { + dir = os.TempDir() + } else { + dir = filepath.Dir(baseDir) + } + + cpFilePath := filepath.Join(dir, fmt.Sprintf("%v-%v%v", srcHash, destHash, CheckpointFileSuffixCopier)) + + return cpFilePath, nil +} + +// load checkpoint from local file +func (cp *CopyCheckpoint) load() error { + if cp.CpFilePath == "" { + return nil + } + + if !FileExists(cp.CpFilePath) { + return nil + } + + // 读取断点文件 + contents, err := os.ReadFile(cp.CpFilePath) + if err != nil { + return err + } + + ccp := CopyCheckpoint{} + if err = json.Unmarshal(contents, &ccp); err != nil { + return err + } + + // 判断断点文件是否有效 + if !cp.isValid(ccp) { + err := cp.remove() + if err != nil { + return err + } + } + + // 读取断点文件成功，将断点文件中的信息赋值给当前对象 + cp.UploadId = ccp.UploadId + cp.PartETagList = ccp.PartETagList + + return nil +} + +func (cp *CopyCheckpoint) isValid(ccp CopyCheckpoint) bool { + md5sum := ccp.checksum() + if CheckpointMagic != ccp.Magic || md5sum != ccp.MD5 { + return false + } + + if cp.BucketName != ccp.BucketName || + cp.ObjectKey != ccp.ObjectKey || + cp.SrcBucketName != ccp.SrcBucketName || + cp.SrcObjectKey != ccp.SrcObjectKey || + cp.SrcObjectSize != ccp.SrcObjectSize || + cp.SrcObjectLastModified != ccp.SrcObjectLastModified || + cp.PartSize != ccp.PartSize { + return false + } + + if len(ccp.UploadId) == 0 { + return false + } + + return true +} + +func (cp *CopyCheckpoint) dump() error { + if cp.CpFilePath == "" { + return nil + } + + dir := filepath.Dir(cp.CpFilePath) + if !DirExists(dir) { + err := os.MkdirAll(dir, DirPermMode) + if err != nil { + return err + } + } + + cp.MD5 = cp.checksum() + str, err := json.Marshal(cp) + if err != nil { + return err + } + + return os.WriteFile(cp.CpFilePath, str, FilePermMode) +} + +func (cp *CopyCheckpoint) checksum() string { + str := cp.MD5 + cp.MD5 = "" + json, _ := json.Marshal(cp) + sum := md5.Sum(json) + md5sum := hex.EncodeToString(sum[:]) + cp.MD5 = str + return md5sum +} + +func (cp *CopyCheckpoint) remove() error { + if cp.CpFilePath == "" { + return nil + } + + return os.Remove(cp.CpFilePath) +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/const.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/const.go new file mode 100644 index 0000000000..f35477ec6e --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/const.go @@ -0,0 +1,97 @@ +package s3 + +// HTTP headers +const ( + HTTPHeaderAcceptEncoding string = "Accept-Encoding" + HTTPHeaderAuthorization = "Authorization" + HTTPHeaderCacheControl = "Cache-Control" + HTTPHeaderContentDisposition = "Content-Disposition" + HTTPHeaderContentEncoding = "Content-Encoding" + HTTPHeaderContentLength = "Content-Length" + HTTPHeaderContentMD5 = "Content-MD5" + HTTPHeaderContentType = "Content-Type" + HTTPHeaderContentLanguage = "Content-Language" + HTTPHeaderLastModified = "Last-Modified" + HTTPHeaderDate = "Date" + HTTPHeaderEtag = "Etag" + HTTPHeaderExpires = "Expires" + HTTPHeaderHost = "Host" + HTTPHeaderAmzACL = "X-Amz-Acl" + HTTPHeaderAmzChecksumCrc64ecma = "X-Amz-Checksum-Crc64ecma" + HTTPHeaderAmzStorageClass = "X-Amz-Storage-Class" + HTTPHeaderAmzDataRedundancyType = "X-Amz-Data-Redundancy-Type" + HTTPHeaderAmzZRSSwitchEnable = "X-Amz-Zrs-Switch-Enable" + HTTPHeaderAmzAllowSameActionOverlap = "X-Amz-Allow-Same-Action-Overlap" + HTTPHeaderAmzBucketType = "X-Amz-Bucket-Type" + HTTPHeaderAmzBucketVisitType = "X-Amz-Bucket-Visit-Type" +) + +// ACL +const ( + ACLPrivate string = "private" + ACLPublicRead string = "public-read" + ACLPublicReadWrite string = "public-read-write" +) + +// StorageClass +const ( + StorageClassExtremePL3 string = "EXTREME_PL3" + StorageClassExtremePL2 string = "EXTREME_PL2" + StorageClassExtremePL1 string = "EXTREME_PL1" + StorageClassStandard string = "STANDARD" + StorageClassIA string = "STANDARD_IA" + StorageClassDeepIA string = "DEEP_IA" + StorageClassArchive string = "ARCHIVE" + StorageClassDeepColdArchive string = "DEEP_COLD_ARCHIVE" +) + +// BucketType +const ( + BucketTypeExtremePL3 string = "EXTREME_PL3" + BucketTypeExtremePL2 string = "EXTREME_PL2" + BucketTypeExtremePL1 string = "EXTREME_PL1" + BucketTypeNormal string = "NORMAL" + BucketTypeIA string = "IA" + BucketTypeDeepIA string = "DEEP_IA" + BucketTypeArchive string = "ARCHIVE" +) + +const ( + BucketVisitTypeNormal string = "NORMAL" + BucketVisitTypeFrequentList string = "FREQUENTLIST" +) + +type HTTPMethod string + +const ( + PUT HTTPMethod = "PUT" + GET HTTPMethod = "GET" + DELETE HTTPMethod = "DELETE" + HEAD HTTPMethod = "HEAD" + POST HTTPMethod = "POST" +) + +const ( + AllUsersUri = "http://acs.amazonaws.com/groups/global/AllUsers" + MetaPrefix = "x-amz-meta-" +) + +const ( + DataRedundancyTypeLRS string = "LRS" + DataRedundancyTypeZRS string = "ZRS" +) + +const ( + StorageMediumNormal string = "Normal" + StorageMediumExtreme string = "Extreme" +) + +const ( + AlgorithmAES256 string = "AES256" + AlgorithmSM4 string = "SM4" +) + +const ( + StatusEnabled string = "Enabled" + StatusDisabled string = "Disabled" +) diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/copier.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/copier.go new file mode 100644 index 0000000000..944072b12a --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/copier.go @@ -0,0 +1,804 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "github.com/ks3sdklib/aws-sdk-go/aws" + "io" + "os" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +type CopyFileInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Object key of the object. + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // The name of the source bucket. + SourceBucket *string `location:"uri" locationName:"SourceBucket" type:"string" required:"true"` + + // Object key of the source object. + SourceKey *string `location:"uri" locationName:"SourceKey" type:"string" required:"true"` + + // The size of each part. + PartSize *int64 `type:"integer"` + + // The number of tasks to upload the file. + TaskNum *int64 `type:"integer"` + + // Whether to enable checkpoint. + EnableCheckpoint *bool `type:"boolean"` + + // The directory to store the checkpoint file. + CheckpointDir *string `type:"string"` + + // The checkpoint file path. + CheckpointFile *string `type:"string"` + + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // A map of metadata to store with the object in S3. + Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` + + // Specifies whether the metadata is copied from the source object or replaced + // with metadata provided in the request. + MetadataDirective *string `location:"header" locationName:"x-amz-metadata-directive" type:"string"` + + // The type of storage to use for the object. Defaults to 'STANDARD'. + StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"` + + // Specifies the object tag of the object. Multiple tags can be set at the same time, such as: TagA=A&TagB=B. + // Note: Key and Value need to be URL-encoded first. If an item does not have "=", the Value is considered to be an empty string. + Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` + + // Specifies how to set the object tag of the target object. + // Default value: COPY + // Valid values: + // COPY (default value): Copies the object tag of the source object to the target object. + // REPLACE: Ignores the object tag of the source object and directly uses the object tag specified in the request. + TaggingDirective *string `location:"header" locationName:"x-amz-tagging-directive" type:"string"` + + // Specifies whether the object is forbidden to overwrite. + ForbidOverwrite *bool `location:"header" locationName:"x-amz-forbid-overwrite" type:"boolean"` + + // Allows grantee to read the object data and its metadata. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // Copies the object if its entity tag (ETag) matches the specified tag. + CopySourceIfMatch *string `location:"header" locationName:"x-amz-copy-source-if-match" type:"string"` + + // Copies the object if it has been modified since the specified time. + CopySourceIfModifiedSince *time.Time `location:"header" locationName:"x-amz-copy-source-if-modified-since" type:"timestamp" timestampFormat:"rfc822"` + + // Copies the object if its entity tag (ETag) is different from the specified ETag. + CopySourceIfNoneMatch *string `location:"header" locationName:"x-amz-copy-source-if-none-match" type:"string"` + + // Copies the object if it hasn't been modified since the specified time. + CopySourceIfUnmodifiedSince *time.Time `location:"header" locationName:"x-amz-copy-source-if-unmodified-since" type:"timestamp" timestampFormat:"rfc822"` + + // Specifies the decryption algorithm used to decrypt the data source object. Valid value: AES256. + CopySourceSSECustomerAlgorithm *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-algorithm" type:"string"` + + // The base64-encoded encryption key used for KS3 decryption specified by the user. + // Its value must be the same as the key used when the data source object was created. + CopySourceSSECustomerKey *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + // If the server encrypted with a user-provided encryption key, when decryption is requested, + // the response will include this header to provide data consistency verification information + // for the user-provided encryption key. + CopySourceSSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key-MD5" type:"string"` + + // The Server-side encryption algorithm used when storing this object in KS3, eg: AES256. + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // Specifies the algorithm to use to when encrypting the object, eg: AES256. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for KS3 to use in encrypting data. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Progress callback function + ProgressFn aws.ProgressFunc `location:"function"` +} + +type CopyFileOutput struct { + Bucket *string + + Key *string + + ETag *string + + ChecksumCRC64ECMA *string +} + +func (c *S3) CopyFile(request *CopyFileInput) (*CopyFileOutput, error) { + return c.CopyFileWithContext(context.Background(), request) +} + +func (c *S3) CopyFileWithContext(ctx context.Context, request *CopyFileInput) (*CopyFileOutput, error) { + return newCopier(c, ctx, request).copyFile() +} + +func (c *S3) CopyFileAcrossRegion(request *CopyFileInput, dstClient *S3) (*UploadFileOutput, error) { + return c.CopyFileAcrossRegionWithContext(context.Background(), request, dstClient) +} + +func (c *S3) CopyFileAcrossRegionWithContext(ctx context.Context, request *CopyFileInput, dstClient *S3) (*UploadFileOutput, error) { + uploadFileRequest, err := c.buildUploadFileRequest(ctx, request) + if err != nil { + return nil, err + } + + return dstClient.UploadFileWithContext(ctx, uploadFileRequest) +} + +func (c *S3) buildUploadFileRequest(ctx context.Context, request *CopyFileInput) (*UploadFileInput, error) { + if request == nil { + return nil, errors.New("copyFileRequest is required") + } + + if aws.ToString(request.Bucket) == "" { + return nil, errors.New("bucket is required") + } + + if aws.ToString(request.Key) == "" { + return nil, errors.New("key is required") + } + + if aws.ToString(request.SourceBucket) == "" { + return nil, errors.New("source bucket is required") + } + + if aws.ToString(request.SourceKey) == "" { + return nil, errors.New("source key is required") + } + + input := &UploadFileInput{ + Bucket: request.Bucket, + Key: request.Key, + PartSize: request.PartSize, + TaskNum: request.TaskNum, + EnableCheckpoint: request.EnableCheckpoint, + CheckpointDir: request.CheckpointDir, + CheckpointFile: request.CheckpointFile, + ACL: request.ACL, + CacheControl: request.CacheControl, + ContentDisposition: request.ContentDisposition, + ContentEncoding: request.ContentEncoding, + ContentType: request.ContentType, + Expires: request.Expires, + Metadata: request.Metadata, + StorageClass: request.StorageClass, + Tagging: request.Tagging, + ForbidOverwrite: request.ForbidOverwrite, + GrantRead: request.GrantRead, + GrantFullControl: request.GrantFullControl, + ServerSideEncryption: request.ServerSideEncryption, + SSECustomerAlgorithm: request.SSECustomerAlgorithm, + SSECustomerKey: request.SSECustomerKey, + SSECustomerKeyMD5: request.SSECustomerKeyMD5, + ProgressFn: request.ProgressFn, + } + + fetcher := &Fetcher{ + client: c, + request: request, + } + + var filePartFetcher FilePartFetcher = fetcher + input.FilePartFetcher = &filePartFetcher + + resp, err := c.HeadObject(&HeadObjectInput{ + Bucket: request.SourceBucket, + Key: request.SourceKey, + IfModifiedSince: request.CopySourceIfModifiedSince, + IfUnmodifiedSince: request.CopySourceIfUnmodifiedSince, + IfMatch: request.CopySourceIfMatch, + IfNoneMatch: request.CopySourceIfNoneMatch, + SSECustomerAlgorithm: request.CopySourceSSECustomerAlgorithm, + SSECustomerKey: request.CopySourceSSECustomerKey, + SSECustomerKeyMD5: request.CopySourceSSECustomerKeyMD5, + }) + if err != nil { + return nil, err + } + input.ObjectMeta = resp.Metadata + + if !strings.EqualFold(aws.ToString(request.MetadataDirective), "REPLACE") { + input.CacheControl = resp.Metadata[HTTPHeaderCacheControl] + input.ContentDisposition = resp.Metadata[HTTPHeaderContentDisposition] + input.ContentEncoding = resp.Metadata[HTTPHeaderContentEncoding] + input.ContentType = resp.Metadata[HTTPHeaderContentType] + expires, err := time.Parse("Mon, 02 Jan 2006 15:04:05 GMT", aws.ToString(resp.Metadata[HTTPHeaderExpires])) + if err == nil { + input.Expires = aws.Time(expires) + } + + metaData := map[string]*string{} + for k, v := range resp.Metadata { + if strings.HasPrefix(strings.ToLower(k), MetaPrefix) { + metaData[k] = v + } + } + input.Metadata = metaData + } + if !strings.EqualFold(aws.ToString(request.TaggingDirective), "REPLACE") { + taggingResp, err := c.GetObjectTaggingWithContext(ctx, &GetObjectTaggingInput{ + Bucket: request.SourceBucket, + Key: request.SourceKey, + }) + if err != nil { + return nil, err + } + + tagStr := taggingResp.Tagging.ToString() + if tagStr != "" { + input.Tagging = aws.String(tagStr) + } + } + if input.ACL == nil { + aclResp, err := c.GetObjectACLWithContext(ctx, &GetObjectACLInput{ + Bucket: request.SourceBucket, + Key: request.SourceKey, + }) + if err != nil { + return nil, err + } + + input.ACL = aws.String(GetCannedACL(aclResp.Grants)) + } + if input.StorageClass == nil { + input.StorageClass = resp.Metadata[HTTPHeaderAmzStorageClass] + } + + return input, nil +} + +type Fetcher struct { + client *S3 + request *CopyFileInput +} + +type Body struct { + io.ReadCloser +} + +func (b *Body) Seek(offset int64, whence int) (int64, error) { + return 0, os.ErrInvalid +} + +func (f *Fetcher) Fetch(objectRange []int64) (io.ReadSeeker, error) { + resp, err := f.client.GetObject(&GetObjectInput{ + Bucket: f.request.SourceBucket, + Key: f.request.SourceKey, + Range: aws.String(fmt.Sprintf("bytes=%d-%d", objectRange[0], objectRange[1])), + SSECustomerAlgorithm: f.request.CopySourceSSECustomerAlgorithm, + SSECustomerKey: f.request.CopySourceSSECustomerKey, + SSECustomerKeyMD5: f.request.CopySourceSSECustomerKeyMD5, + }) + if err != nil { + return nil, err + } + + body := &Body{resp.Body} + + return body, nil +} + +type Copier struct { + client *S3 + + context context.Context + + copyFileRequest *CopyFileInput + + copyCheckpoint *CopyCheckpoint + + CompletedSize int64 + + copyObjectMeta map[string]*string + + mu sync.Mutex + + error error +} + +func newCopier(s3 *S3, ctx context.Context, request *CopyFileInput) *Copier { + return &Copier{ + client: s3, + context: ctx, + copyFileRequest: request, + } +} + +func (c *Copier) copyFile() (*CopyFileOutput, error) { + err := c.validate() + if err != nil { + return nil, err + } + + c.copyObjectMeta, err = c.headObject() + if err != nil { + return nil, err + } + + fileSize, _ := strconv.ParseInt(aws.ToString(c.copyObjectMeta[HTTPHeaderContentLength]), 10, 64) + + var resp *CopyFileOutput + if fileSize <= aws.ToLong(c.copyFileRequest.PartSize) { + resp, err = c.copyObject() + } else { + resp, err = c.multipartCopy() + } + if err != nil { + return nil, err + } + + if c.client.Config.CrcCheckEnabled { + clientCrc64, _ := strconv.ParseUint(aws.ToString(c.copyObjectMeta[HTTPHeaderAmzChecksumCrc64ecma]), 10, 64) + serverCrc64, _ := strconv.ParseUint(aws.ToString(resp.ChecksumCRC64ECMA), 10, 64) + c.client.Config.LogDebug("check file crc64, client crc64:%d, server crc64:%d", clientCrc64, serverCrc64) + if serverCrc64 != 0 && clientCrc64 != serverCrc64 { + return nil, errors.New(fmt.Sprintf("crc64 check failed, client crc64:%d, server crc64:%d", clientCrc64, serverCrc64)) + } + } + + return resp, err +} + +func (c *Copier) validate() error { + request := c.copyFileRequest + if request == nil { + return errors.New("copyFileRequest is required") + } + + if aws.ToString(request.Bucket) == "" { + return errors.New("bucket is required") + } + + if aws.ToString(request.Key) == "" { + return errors.New("key is required") + } + + if aws.ToString(request.SourceBucket) == "" { + return errors.New("source bucket is required") + } + + if aws.ToString(request.SourceKey) == "" { + return errors.New("source key is required") + } + + if request.PartSize == nil { + request.PartSize = aws.Long(DefaultPartSize) + } else if aws.ToLong(request.PartSize) < MinPartSize { + request.PartSize = aws.Long(MinPartSize) + } else if aws.ToLong(request.PartSize) > MaxPartSize { + request.PartSize = aws.Long(MaxPartSize) + } + + if aws.ToLong(request.TaskNum) <= 0 { + request.TaskNum = aws.Long(DefaultTaskNum) + } + + return nil +} + +func (c *Copier) copyObject() (*CopyFileOutput, error) { + request := c.copyFileRequest + input := &CopyObjectInput{ + Bucket: request.Bucket, + Key: request.Key, + SourceBucket: request.SourceBucket, + SourceKey: request.SourceKey, + ACL: request.ACL, + CacheControl: request.CacheControl, + ContentDisposition: request.ContentDisposition, + ContentEncoding: request.ContentEncoding, + ContentType: request.ContentType, + Expires: request.Expires, + Metadata: request.Metadata, + MetadataDirective: request.MetadataDirective, + StorageClass: request.StorageClass, + Tagging: request.Tagging, + TaggingDirective: request.TaggingDirective, + ForbidOverwrite: request.ForbidOverwrite, + GrantRead: request.GrantRead, + GrantFullControl: request.GrantFullControl, + CopySourceIfMatch: request.CopySourceIfMatch, + CopySourceIfNoneMatch: request.CopySourceIfNoneMatch, + CopySourceIfModifiedSince: request.CopySourceIfModifiedSince, + CopySourceIfUnmodifiedSince: request.CopySourceIfUnmodifiedSince, + CopySourceSSECustomerAlgorithm: request.CopySourceSSECustomerAlgorithm, + CopySourceSSECustomerKey: request.CopySourceSSECustomerKey, + CopySourceSSECustomerKeyMD5: request.CopySourceSSECustomerKeyMD5, + ServerSideEncryption: request.ServerSideEncryption, + SSECustomerAlgorithm: request.SSECustomerAlgorithm, + SSECustomerKey: request.SSECustomerKey, + SSECustomerKeyMD5: request.SSECustomerKeyMD5, + } + if c.copyFileRequest.StorageClass == nil { + input.StorageClass = c.copyObjectMeta[HTTPHeaderAmzStorageClass] + } + + resp, err := c.client.CopyObjectWithContext(c.context, input) + if err != nil { + return nil, err + } + + return &CopyFileOutput{ + Bucket: request.Bucket, + Key: request.Key, + ETag: resp.CopyObjectResult.ETag, + ChecksumCRC64ECMA: resp.CopyObjectResult.ChecksumCRC64ECMA, + }, nil +} + +func (c *Copier) multipartCopy() (*CopyFileOutput, error) { + ccp, err := newCopyCheckpoint(c) + if err != nil { + return nil, err + } + c.copyCheckpoint = ccp + + if aws.ToBoolean(c.copyFileRequest.EnableCheckpoint) { + cpFilePath := aws.ToString(c.copyFileRequest.CheckpointFile) + if cpFilePath == "" { + cpFilePath, err = generateCopyCpFilePath(c.copyFileRequest) + if err != nil { + return nil, err + } + } + ccp.CpFilePath = cpFilePath + + err = c.copyCheckpoint.load() + if err != nil { + return nil, err + } + + if ccp.UploadId != "" && !c.isUploadIdValid() { + ccp.UploadId = "" + ccp.PartETagList = make([]*CompletedPart, 0) + ccp.remove() + } + } + + if ccp.UploadId == "" { + ccp.UploadId, err = c.initUploadId() + if err != nil { + return nil, err + } + ccp.dump() + } + + fileSize := ccp.SrcObjectSize + partSize := ccp.PartSize + totalPartNum := (fileSize-1)/partSize + 1 + tasks := make(chan CopyPartTask, totalPartNum) + + var i int64 + for i = 0; i < totalPartNum; i++ { + partNum := i + 1 + offset := (partNum - 1) * partSize + actualPartSize := c.getActualPartSize(fileSize, partSize, partNum) + partETag := c.getPartETag(partNum) + if partETag != nil { + c.publishProgress(actualPartSize) + } else { + uploadPartTask := CopyPartTask{ + partNumber: partNum, + offset: offset, + actualPartSize: actualPartSize, + } + tasks <- uploadPartTask + } + } + close(tasks) + + var wg sync.WaitGroup + for i = 0; i < aws.ToLong(c.copyFileRequest.TaskNum); i++ { + wg.Add(1) + go c.runTask(tasks, &wg) + } + wg.Wait() + + if c.error != nil { + return nil, c.error + } + + completedMultipartUpload := c.getMultipartUploadParts() + resp, err := c.completeMultipartUpload(completedMultipartUpload) + if err != nil { + return nil, err + } + + return c.getCopyFileOutput(resp), nil +} + +func (c *Copier) getCopyFileOutput(resp *CompleteMultipartUploadOutput) *CopyFileOutput { + return &CopyFileOutput{ + Bucket: resp.Bucket, + Key: resp.Key, + ETag: resp.ETag, + ChecksumCRC64ECMA: resp.ChecksumCRC64ECMA, + } +} + +func (c *Copier) getPartSize(fileSize int64, originPartSize int64) int64 { + partSize := originPartSize + totalPartNum := (fileSize-1)/partSize + 1 + for totalPartNum > MaxPartNum { + partSize += originPartSize + totalPartNum = (fileSize-1)/partSize + 1 + } + return partSize +} + +func (c *Copier) getActualPartSize(fileSize int64, partSize int64, partNum int64) int64 { + offset := (partNum - 1) * partSize + actualPartSize := partSize + if offset+partSize >= fileSize { + actualPartSize = fileSize - offset + } + return actualPartSize +} + +func (c *Copier) getPartETag(partNumber int64) *CompletedPart { + for _, partETag := range c.copyCheckpoint.PartETagList { + if *partETag.PartNumber == partNumber { + return partETag + } + } + return nil +} + +type CopyPartTask struct { + partNumber int64 + + offset int64 + + actualPartSize int64 +} + +func (c *Copier) runTask(tasks <-chan CopyPartTask, wg *sync.WaitGroup) { + defer wg.Done() + for task := range tasks { + if c.error != nil { + return + } + + partETag, err := c.copyPart(task) + if err != nil { + c.setError(err) + return + } + + c.updatePart(partETag) + } +} + +func (c *Copier) copyPart(task CopyPartTask) (CompletedPart, error) { + request := c.copyFileRequest + ccp := c.copyCheckpoint + var partETag CompletedPart + + start := task.offset + end := task.offset + task.actualPartSize - 1 + resp, err := c.client.UploadPartCopyWithContext(c.context, &UploadPartCopyInput{ + Bucket: aws.String(ccp.BucketName), + Key: aws.String(ccp.ObjectKey), + SourceBucket: aws.String(ccp.SrcBucketName), + SourceKey: aws.String(ccp.SrcObjectKey), + UploadID: aws.String(ccp.UploadId), + PartNumber: aws.Long(task.partNumber), + CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)), + CopySourceIfMatch: request.CopySourceIfMatch, + CopySourceIfNoneMatch: request.CopySourceIfNoneMatch, + CopySourceIfModifiedSince: request.CopySourceIfModifiedSince, + CopySourceIfUnmodifiedSince: request.CopySourceIfUnmodifiedSince, + CopySourceSSECustomerAlgorithm: request.CopySourceSSECustomerAlgorithm, + CopySourceSSECustomerKey: request.CopySourceSSECustomerKey, + CopySourceSSECustomerKeyMD5: request.CopySourceSSECustomerKeyMD5, + SSECustomerAlgorithm: request.SSECustomerAlgorithm, + SSECustomerKey: request.SSECustomerKey, + SSECustomerKeyMD5: request.SSECustomerKeyMD5, + }) + + if err != nil { + return partETag, err + } + + partETag.PartNumber = aws.Long(task.partNumber) + partETag.ETag = resp.CopyPartResult.ETag + partETag.ChecksumCRC64ECMA = resp.CopyPartResult.ChecksumCRC64ECMA + c.publishProgress(task.actualPartSize) + + return partETag, nil +} + +func (c *Copier) updatePart(partETag CompletedPart) { + c.mu.Lock() + defer c.mu.Unlock() + + c.copyCheckpoint.PartETagList = append(c.copyCheckpoint.PartETagList, &partETag) + c.copyCheckpoint.dump() +} + +func (c *Copier) setError(err error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.error == nil { + c.error = err + } +} + +func (c *Copier) getMultipartUploadParts() *CompletedMultipartUpload { + partETags := c.copyCheckpoint.PartETagList + // 按照PartNumber排序 + sort.Sort(CompletedParts(partETags)) + return &CompletedMultipartUpload{ + Parts: partETags, + } +} + +func (c *Copier) completeMultipartUpload(completedMultipartUpload *CompletedMultipartUpload) (*CompleteMultipartUploadOutput, error) { + resp, err := c.client.CompleteMultipartUploadWithContext(c.context, &CompleteMultipartUploadInput{ + Bucket: c.copyFileRequest.Bucket, + Key: c.copyFileRequest.Key, + UploadID: aws.String(c.copyCheckpoint.UploadId), + MultipartUpload: completedMultipartUpload, + ForbidOverwrite: c.copyFileRequest.ForbidOverwrite, + }) + if err != nil { + return nil, err + } + c.copyCheckpoint.remove() + return resp, err +} + +func (c *Copier) publishProgress(actualPartSize int64) { + if c.copyFileRequest.ProgressFn != nil { + atomic.AddInt64(&c.CompletedSize, actualPartSize) + c.copyFileRequest.ProgressFn(actualPartSize, c.CompletedSize, c.copyCheckpoint.SrcObjectSize) + } +} + +func (c *Copier) headObject() (map[string]*string, error) { + request := c.copyFileRequest + resp, err := c.client.HeadObjectWithContext(c.context, &HeadObjectInput{ + Bucket: request.SourceBucket, + Key: request.SourceKey, + IfModifiedSince: request.CopySourceIfModifiedSince, + IfUnmodifiedSince: request.CopySourceIfUnmodifiedSince, + IfMatch: request.CopySourceIfMatch, + IfNoneMatch: request.CopySourceIfNoneMatch, + SSECustomerAlgorithm: request.CopySourceSSECustomerAlgorithm, + SSECustomerKey: request.CopySourceSSECustomerKey, + SSECustomerKeyMD5: request.CopySourceSSECustomerKeyMD5, + }) + if err != nil { + return nil, err + } + return resp.Metadata, err +} + +func (c *Copier) initUploadId() (string, error) { + request := c.copyFileRequest + input := &CreateMultipartUploadInput{ + Bucket: request.Bucket, + Key: request.Key, + ACL: request.ACL, + CacheControl: request.CacheControl, + ContentDisposition: request.ContentDisposition, + ContentEncoding: request.ContentEncoding, + ContentType: request.ContentType, + Expires: request.Expires, + Metadata: request.Metadata, + StorageClass: request.StorageClass, + Tagging: request.Tagging, + ForbidOverwrite: request.ForbidOverwrite, + GrantRead: request.GrantRead, + GrantFullControl: request.GrantFullControl, + ServerSideEncryption: request.ServerSideEncryption, + SSECustomerAlgorithm: request.SSECustomerAlgorithm, + SSECustomerKey: request.SSECustomerKey, + SSECustomerKeyMD5: request.SSECustomerKeyMD5, + } + if !strings.EqualFold(aws.ToString(request.MetadataDirective), "REPLACE") { + input.CacheControl = c.copyObjectMeta[HTTPHeaderCacheControl] + input.ContentDisposition = c.copyObjectMeta[HTTPHeaderContentDisposition] + input.ContentEncoding = c.copyObjectMeta[HTTPHeaderContentEncoding] + input.ContentType = c.copyObjectMeta[HTTPHeaderContentType] + expires, err := time.Parse("Mon, 02 Jan 2006 15:04:05 GMT", aws.ToString(c.copyObjectMeta[HTTPHeaderExpires])) + if err == nil { + input.Expires = aws.Time(expires) + } + + metaData := map[string]*string{} + for k, v := range c.copyObjectMeta { + if strings.HasPrefix(strings.ToLower(k), MetaPrefix) { + metaData[k] = v + } + } + input.Metadata = metaData + } + if !strings.EqualFold(aws.ToString(request.TaggingDirective), "REPLACE") { + taggingResp, err := c.client.GetObjectTaggingWithContext(c.context, &GetObjectTaggingInput{ + Bucket: request.SourceBucket, + Key: request.SourceKey, + }) + if err != nil { + return "", err + } + + tagStr := taggingResp.Tagging.ToString() + if tagStr != "" { + input.Tagging = aws.String(tagStr) + } + } + if c.copyFileRequest.ACL == nil { + aclResp, err := c.client.GetObjectACLWithContext(c.context, &GetObjectACLInput{ + Bucket: request.SourceBucket, + Key: request.SourceKey, + }) + if err != nil { + return "", err + } + + input.ACL = aws.String(GetCannedACL(aclResp.Grants)) + } + if c.copyFileRequest.StorageClass == nil { + input.StorageClass = c.copyObjectMeta[HTTPHeaderAmzStorageClass] + } + + resp, err := c.client.CreateMultipartUploadWithContext(c.context, input) + if err != nil { + return "", err + } + return aws.ToString(resp.UploadID), nil +} + +func (c *Copier) isUploadIdValid() bool { + _, err := c.client.ListPartsWithContext(c.context, &ListPartsInput{ + Bucket: c.copyFileRequest.Bucket, + Key: c.copyFileRequest.Key, + UploadID: aws.String(c.copyCheckpoint.UploadId), + }) + if err != nil && strings.Contains(err.Error(), "NoSuchUpload") { + return false + } + return true +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/cors.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/cors.go new file mode 100644 index 0000000000..911b16c804 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/cors.go @@ -0,0 +1,196 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +type PutBucketCORSInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + CORSConfiguration *CORSConfiguration `locationName:"CORSConfiguration" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/" ` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketCORSInput `json:"-" xml:"-"` +} + +type metadataPutBucketCORSInput struct { + SDKShapeTraits bool `type:"structure" payload:"CORSConfiguration"` + + AutoFillMD5 bool +} + +type CORSConfiguration struct { + Rules []*CORSRule `locationName:"CORSRule" type:"list" flattened:"true"` + + // Whether non-cross-domain requests return the Vary: Origin header. + NonCrossOriginResponseVary *bool `locationName:"NonCrossOriginResponseVary" type:"boolean" xml:"NonCrossOriginResponseVary"` +} + +type CORSRule struct { + AllowedHeaders []string `locationName:"AllowedHeader" type:"list" flattened:"true"` + AllowedMethods []string `locationName:"AllowedMethod" type:"list" flattened:"true"` + AllowedOrigins []string `locationName:"AllowedOrigin" type:"list" flattened:"true"` + ExposeHeaders []string `locationName:"ExposeHeader" type:"list" flattened:"true"` + MaxAgeSeconds *int64 `locationName:"MaxAgeSeconds"` +} + +type PutBucketCORSOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutBucketCORSRequest generates a request for the PutBucketCORS operation. +func (c *S3) PutBucketCORSRequest(input *PutBucketCORSInput) (req *aws.Request, output *PutBucketCORSOutput) { + op := &aws.Operation{ + Name: "PutBucketCors", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?cors", + } + + if input == nil { + input = &PutBucketCORSInput{} + } + + //目前默认为true + input.AutoFillMD5 = true + req = c.newRequest(op, input, output) + output = &PutBucketCORSOutput{} + req.Data = output + return +} + +// PutBucketCORS Sets the cors configuration for a bucket. +func (c *S3) PutBucketCORS(input *PutBucketCORSInput) (*PutBucketCORSOutput, error) { + req, out := c.PutBucketCORSRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketCORSWithContext(ctx aws.Context, input *PutBucketCORSInput) (*PutBucketCORSOutput, error) { + req, out := c.PutBucketCORSRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketCORSInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketCORSOutput struct { + CORSConfiguration *CORSConfiguration `locationName:"CORSConfiguration" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketCORSOutput `json:"-" xml:"-"` +} + +type metadataGetBucketCORSOutput struct { + SDKShapeTraits bool `type:"structure" payload:"CORSConfiguration"` +} + +// GetBucketCORSRequest generates a request for the GetBucketCORS operation. +func (c *S3) GetBucketCORSRequest(input *GetBucketCORSInput) (req *aws.Request, output *GetBucketCORSOutput) { + op := &aws.Operation{ + Name: "GetBucketCors", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?cors", + } + + if input == nil { + input = &GetBucketCORSInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketCORSOutput{} + req.Data = output + return +} + +// GetBucketCORS Returns the cors configuration for the bucket. +func (c *S3) GetBucketCORS(input *GetBucketCORSInput) (*GetBucketCORSOutput, error) { + req, out := c.GetBucketCORSRequest(input) + err := req.Send() + if req.Data != nil { + out = req.Data.(*GetBucketCORSOutput) + } + return out, err +} + +func (c *S3) GetBucketCORSWithContext(ctx aws.Context, input *GetBucketCORSInput) (*GetBucketCORSOutput, error) { + req, out := c.GetBucketCORSRequest(input) + req.SetContext(ctx) + err := req.Send() + if req.Data != nil { + out = req.Data.(*GetBucketCORSOutput) + } + return out, err +} + +type DeleteBucketCORSInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type DeleteBucketCORSOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// DeleteBucketCORSRequest generates a request for the DeleteBucketCORS operation. +func (c *S3) DeleteBucketCORSRequest(input *DeleteBucketCORSInput) (req *aws.Request, output *DeleteBucketCORSOutput) { + op := &aws.Operation{ + Name: "DeleteBucketCors", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?cors", + } + + if input == nil { + input = &DeleteBucketCORSInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketCORSOutput{} + req.Data = output + return +} + +// DeleteBucketCORS Deletes the cors configuration information set for the bucket. +func (c *S3) DeleteBucketCORS(input *DeleteBucketCORSInput) (*DeleteBucketCORSOutput, error) { + req, out := c.DeleteBucketCORSRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketCORSWithContext(ctx aws.Context, input *DeleteBucketCORSInput) (*DeleteBucketCORSOutput, error) { + req, out := c.DeleteBucketCORSRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/crc_check.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/crc_check.go new file mode 100644 index 0000000000..072acd7c26 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/crc_check.go @@ -0,0 +1,42 @@ +package s3 + +import ( + "fmt" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/apierr" + "hash" + "strconv" +) + +func CheckUploadCrc64(r *aws.Request) { + clientCrc := r.Crc64.Sum64() + serverCrc := uint64(0) + if r.HTTPResponse.Header.Get("X-Amz-Checksum-Crc64ecma") != "" { + serverCrc, _ = strconv.ParseUint(r.HTTPResponse.Header.Get("X-Amz-Checksum-Crc64ecma"), 10, 64) + } + + r.Config.LogInfo("client crc:%d, server crc:%d", clientCrc, serverCrc) + + if serverCrc != 0 && clientCrc != serverCrc { + r.Error = apierr.New("CRCCheckError", fmt.Sprintf("client crc and server crc do not match, request id:[%s]", r.HTTPResponse.Header.Get("X-Kss-Request-Id")), nil) + r.Config.LogError("%s", r.Error.Error()) + } +} + +func CheckDownloadCrc64(s3 *S3, res *GetObjectOutput, crc hash.Hash64) error { + var err error + clientCrc := crc.Sum64() + serverCrc := uint64(0) + if res.Metadata["X-Amz-Checksum-Crc64ecma"] != nil { + serverCrc, _ = strconv.ParseUint(*res.Metadata["X-Amz-Checksum-Crc64ecma"], 10, 64) + } + + s3.Config.LogInfo("client crc:%d, server crc:%d", clientCrc, serverCrc) + + if serverCrc != 0 && clientCrc != serverCrc { + err = apierr.New("CRCCheckError", fmt.Sprintf("client crc and server crc do not match, request id:[%s]", *res.Metadata["X-Kss-Request-Id"]), nil) + s3.Config.LogError("%s", err.Error()) + } + + return err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/data_redundancy_switch.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/data_redundancy_switch.go new file mode 100644 index 0000000000..1bcd5a8f34 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/data_redundancy_switch.go @@ -0,0 +1,126 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "time" +) + +type PutBucketDataRedundancySwitchInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // The bucket data redundancy type. + // Valid value: LRS丨ZRS + // LRS: local redundancy storage + // ZRS: zone redundancy storage + DataRedundancyType *string `location:"header" locationName:"x-amz-data-redundancy-type" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type PutBucketDataRedundancySwitchOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutBucketDataRedundancySwitchRequest generates a request for the PutBucketDataRedundancySwitch operation. +func (c *S3) PutBucketDataRedundancySwitchRequest(input *PutBucketDataRedundancySwitchInput) (req *aws.Request, output *PutBucketDataRedundancySwitchOutput) { + op := &aws.Operation{ + Name: "PutBucketDataRedundancySwitch", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?dataRedundancySwitch", + } + + if input == nil { + input = &PutBucketDataRedundancySwitchInput{} + } + + req = c.newRequest(op, input, output) + output = &PutBucketDataRedundancySwitchOutput{} + req.Data = output + return +} + +// PutBucketDataRedundancySwitch sets the data redundancy type for the bucket. +func (c *S3) PutBucketDataRedundancySwitch(input *PutBucketDataRedundancySwitchInput) (*PutBucketDataRedundancySwitchOutput, error) { + req, out := c.PutBucketDataRedundancySwitchRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketDataRedundancySwitchWithContext(ctx aws.Context, input *PutBucketDataRedundancySwitchInput) (*PutBucketDataRedundancySwitchOutput, error) { + req, out := c.PutBucketDataRedundancySwitchRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketDataRedundancySwitchInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketDataRedundancySwitchOutput struct { + // The bucket data redundancy switch configuration. + DataRedundancySwitch *DataRedundancySwitch `locationName:"DataRedundancySwitch" type:"structure"` + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketDataRedundancySwitchOutput `json:"-" xml:"-"` +} + +type DataRedundancySwitch struct { + // The bucket data redundancy type. + DataRedundancyType *string `locationName:"DataRedundancyType" type:"string"` + // Time when zone redundancy is enabled. + SwitchTime *time.Time `locationName:"SwitchTime" type:"timestamp" timestampFormat:"iso8601"` +} + +type metadataGetBucketDataRedundancySwitchOutput struct { + SDKShapeTraits bool `type:"structure" payload:"DataRedundancySwitch"` +} + +// GetBucketDataRedundancySwitchRequest generates a request for the GetBucketDataRedundancySwitch operation. +func (c *S3) GetBucketDataRedundancySwitchRequest(input *GetBucketDataRedundancySwitchInput) (req *aws.Request, output *GetBucketDataRedundancySwitchOutput) { + op := &aws.Operation{ + Name: "GetBucketDataRedundancySwitch", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?dataRedundancySwitch", + } + + if input == nil { + input = &GetBucketDataRedundancySwitchInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketDataRedundancySwitchOutput{} + req.Data = output + return +} + +// GetBucketDataRedundancySwitch gets the data redundancy switch configuration for the bucket. +func (c *S3) GetBucketDataRedundancySwitch(input *GetBucketDataRedundancySwitchInput) (*GetBucketDataRedundancySwitchOutput, error) { + req, out := c.GetBucketDataRedundancySwitchRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketDataRedundancySwitchWithContext(ctx aws.Context, input *GetBucketDataRedundancySwitchInput) (*GetBucketDataRedundancySwitchOutput, error) { + req, out := c.GetBucketDataRedundancySwitchRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/decompresspolicy.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/decompresspolicy.go new file mode 100644 index 0000000000..250d9ace06 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/decompresspolicy.go @@ -0,0 +1,219 @@ +package s3 + +import "github.com/ks3sdklib/aws-sdk-go/aws" + +// PutBucketDecompressPolicyRequest generates a request for the PutBucketDecompressPolicy operation. +func (c *S3) PutBucketDecompressPolicyRequest(input *PutBucketDecompressPolicyInput) (req *aws.Request, output *PutBucketDecompressPolicyOutput) { + op := &aws.Operation{ + Name: "PutBucketDecompressPolicy", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?decompresspolicy", + } + + if input == nil { + input = &PutBucketDecompressPolicyInput{} + } + + req = c.newRequest(op, input, output) + req.ContentType = "application/json" + output = &PutBucketDecompressPolicyOutput{} + req.Data = output + return +} + +// PutBucketDecompressPolicy sets the decompression policy for the bucket. +func (c *S3) PutBucketDecompressPolicy(input *PutBucketDecompressPolicyInput) (*PutBucketDecompressPolicyOutput, error) { + req, out := c.PutBucketDecompressPolicyRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketDecompressPolicyWithContext(ctx aws.Context, input *PutBucketDecompressPolicyInput) (*PutBucketDecompressPolicyOutput, error) { + req, out := c.PutBucketDecompressPolicyRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type PutBucketDecompressPolicyInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + BucketDecompressPolicy *BucketDecompressPolicy `locationName:"BucketDecompressPolicy" type:"structure"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketDecompressPolicyInput `json:"-" xml:"-"` +} + +type metadataPutBucketDecompressPolicyInput struct { + SDKShapeTraits bool `type:"structure" payload:"BucketDecompressPolicy"` +} + +type BucketDecompressPolicy struct { + Rules []*DecompressPolicyRule `json:"rules,omitempty" type:"list" locationName:"rules" required:"true"` +} + +type DecompressPolicyRule struct { + // The name of the decompression strategy and the unique identifier of the decompression rule configured by the bucket. Value range: [1, 256]. + // Description: The length is 1-256 characters and can only be composed of uppercase or lowercase English letters, numbers, underscores (_), and hyphens (-). + Id *string `json:"id,omitempty" type:"string" locationName:"id" required:"true"` + + // ZIP online decompression trigger event, currently supports the following operations: + // "ObjectCreated:*": represents all operations for creating objects, including Put, Post, Copy objects, and merging segmentation tasks; + // "ObjectCreated:Put": Use the Put method to upload a ZIP package; + // "ObjectCreated:Post": Use the Post method to upload a ZIP package; + // "ObjectCreated:Copy": Use the Copy method to copy a ZIP package; + // "ObjectCreated:CompleteMultipartUpload": Use merge to upload ZIP packages in chunks. + Events *string `json:"events,omitempty" type:"string" locationName:"events" required:"true"` + + // Match rule prefix (ZIP package that matches the prefix). + // If no prefix is specified, all ZIP packages uploaded will be matched by default. + Prefix *string `json:"prefix,omitempty" type:"string" locationName:"prefix"` + + // Match rule suffix. + // The default is. zip, and currently only supports ZIP package format. + Suffix []*string `json:"suffix,omitempty" type:"list" locationName:"suffix" required:"true"` + + // The processing method for files with the same name after decompression is not to overwrite them by default + // The parameter values are as follows: + // 0 (default value): Do not overwrite skip, keep existing objects in the bucket, skip objects with the same name, do not decompress; + // 1: Overwrite, preserve the extracted object, and delete any existing objects with the same name in the bucket. + Overwrite *int64 `json:"overwrite,omitempty" type:"integer" locationName:"overwrite" required:"true"` + + // The address for task callback, URL address. + Callback *string `json:"callback,omitempty" type:"string" locationName:"callback"` + + // Task callback format, JSON format (required if callback address is set). + CallbackFormat *string `json:"callback_format,omitempty" type:"string" locationName:"callback_format"` + + // Specify the prefix of the output file in the target bucket after decompression. If it is not empty, it must end with a '/'. + // If left blank, it will be saved by default in the root path of the storage bucket. + PathPrefix *string `json:"path_prefix,omitempty" type:"string" locationName:"path_prefix"` + + // Specify whether the compressed file path requires a compressed file name, with the following parameter values: + // 0 (default): Keep compressed file name + // 1: Extract directly to the target directory + PathPrefixReplaced *int64 `json:"path_prefix_replaced,omitempty" type:"integer" locationName:"path_prefix_replaced"` + + // Type of file decompression strategy. + // Fixed value: decompress. + PolicyType *string `json:"policy_type,omitempty" type:"string" locationName:"policy_type" required:"true"` +} + +type PutBucketDecompressPolicyOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// GetBucketDecompressPolicyRequest generates a request for the GetBucketDecompressPolicy operation. +func (c *S3) GetBucketDecompressPolicyRequest(input *GetBucketDecompressPolicyInput) (req *aws.Request, output *GetBucketDecompressPolicyOutput) { + op := &aws.Operation{ + Name: "GetBucketDecompressPolicy", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?decompresspolicy", + } + + if input == nil { + input = &GetBucketDecompressPolicyInput{} + } + + req = c.newRequest(op, input, output) + req.ContentType = "application/json" + output = &GetBucketDecompressPolicyOutput{ + BucketDecompressPolicy: &BucketDecompressPolicy{}, + } + req.Data = output + return +} + +// GetBucketDecompressPolicy gets the decompression policy for the bucket. +func (c *S3) GetBucketDecompressPolicy(input *GetBucketDecompressPolicyInput) (*GetBucketDecompressPolicyOutput, error) { + req, out := c.GetBucketDecompressPolicyRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketDecompressPolicyWithContext(ctx aws.Context, input *GetBucketDecompressPolicyInput) (*GetBucketDecompressPolicyOutput, error) { + req, out := c.GetBucketDecompressPolicyRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketDecompressPolicyInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketDecompressPolicyOutput struct { + BucketDecompressPolicy *BucketDecompressPolicy `locationName:"BucketDecompressPolicy" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketDecompressPolicyOutput `json:"-" xml:"-"` +} + +type metadataGetBucketDecompressPolicyOutput struct { + SDKShapeTraits bool `type:"structure" payload:"BucketDecompressPolicy"` +} + +// DeleteBucketDecompressPolicyRequest generates a request for the DeleteBucketDecompressPolicy operation. +func (c *S3) DeleteBucketDecompressPolicyRequest(input *DeleteBucketDecompressPolicyInput) (req *aws.Request, output *DeleteBucketDecompressPolicyOutput) { + op := &aws.Operation{ + Name: "DeleteBucketDecompressPolicy", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?decompresspolicy", + } + + if input == nil { + input = &DeleteBucketDecompressPolicyInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketDecompressPolicyOutput{} + req.Data = output + return +} + +// DeleteBucketDecompressPolicy deletes the decompression policy for the bucket. +func (c *S3) DeleteBucketDecompressPolicy(input *DeleteBucketDecompressPolicyInput) (*DeleteBucketDecompressPolicyOutput, error) { + req, out := c.DeleteBucketDecompressPolicyRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketDecompressPolicyWithContext(ctx aws.Context, input *DeleteBucketDecompressPolicyInput) (*DeleteBucketDecompressPolicyOutput, error) { + req, out := c.DeleteBucketDecompressPolicyRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteBucketDecompressPolicyInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} +type DeleteBucketDecompressPolicyOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/downloader.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/downloader.go new file mode 100644 index 0000000000..624c6323b2 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/downloader.go @@ -0,0 +1,542 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/crc" + "hash" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + "sync/atomic" + "time" +) + +type DownloadFileInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Object key of the object. + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // The path of the file to be downloaded. + DownloadFile *string `type:"string" locationName:"DownloadFile" required:"true"` + + // The size of each part. + PartSize *int64 `type:"integer" locationName:"PartSize"` + + // The number of tasks to download the file. + TaskNum *int64 `type:"integer" locationName:"TaskNum"` + + // Whether to enable checkpoint. + EnableCheckpoint *bool `type:"boolean" locationName:"EnableCheckpoint"` + + // The directory to store the checkpoint file. + CheckpointDir *string `type:"string" locationName:"CheckpointDir"` + + // The checkpoint file path. + CheckpointFile *string `type:"string" locationName:"CheckpointFile"` + + // 下载的范围，range[0]为开始位置，range[1]为结束位置 + // range[0] 小于 0 时表示从文件头开始下载 + // range[1] 小于 0 时表示下载到文件末尾 + // range[0] 和 range[1] 都小于 0 时表示下载整个文件 + // range[0] 和 range[1] 都大于等于 0 时表示下载指定范围的文件 + // range[0] 大于 range[1] 且 range[1] 非负时表示下载整个文件 + // 例如： + // range=[0, 99] 表示下载文件的前100个字节 + // range=[100, 199] 表示下载文件的第101个字节至第200个字节 + // range=[100, -1] 表示下载文件的第101个字节至文件末尾 + // range=[-1, 100] 表示下载文件的后100个字节 + // Downloads the specified range bytes of an object. + Range []int64 `locationName:"Range" type:"list"` + + // Sets the Content-Type header of the response. + ResponseContentType *string `location:"querystring" locationName:"response-content-type" type:"string"` + + // Sets the Content-Language header of the response. + ResponseContentLanguage *string `location:"querystring" locationName:"response-content-language" type:"string"` + + // Sets the Expires header of the response. + ResponseExpires *time.Time `location:"querystring" locationName:"response-expires" type:"timestamp" timestampFormat:"iso8601"` + + // Sets the Cache-Control header of the response. + ResponseCacheControl *string `location:"querystring" locationName:"response-cache-control" type:"string"` + + // Sets the Content-Disposition header of the response + ResponseContentDisposition *string `location:"querystring" locationName:"response-content-disposition" type:"string"` + + // Sets the Content-Encoding header of the response. + ResponseContentEncoding *string `location:"querystring" locationName:"response-content-encoding" type:"string"` + + // Return the object only if it has been modified since the specified time, + // otherwise return a 304 (not modified). + IfModifiedSince *time.Time `location:"header" locationName:"If-Modified-Since" type:"timestamp" timestampFormat:"rfc822"` + + // Return the object only if it has not been modified since the specified time, + // otherwise return a 412 (precondition failed). + IfUnmodifiedSince *time.Time `location:"header" locationName:"If-Unmodified-Since" type:"timestamp" timestampFormat:"rfc822"` + + // Return the object only if its entity tag (ETag) is the same as the one specified, + // otherwise return a 412 (precondition failed). + IfMatch *string `location:"header" locationName:"If-Match" type:"string"` + + // Return the object only if its entity tag (ETag) is different from the one + // specified, otherwise return a 304 (not modified). + IfNoneMatch *string `location:"header" locationName:"If-None-Match" type:"string"` + + // Specify the encoding type of the client. + // If you want to compress and transmit the returned content using gzip, + // you need to add a request header: Accept-Encoding:gzip。 + // KS3 will determine whether to return gzip compressed data based on the + // Content-Type and Object size (not less than 1 KB) of the object. + // Value: gzip、br、deflate + AcceptEncoding *string `location:"header" locationName:"Accept-Encoding" type:"string"` + + // Specifies the algorithm to use to when encrypting the object, eg: AES256. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for KS3 to use in encrypting data. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Progress callback function + ProgressFn aws.ProgressFunc `location:"function"` +} + +type DownloadFileOutput struct { + Bucket *string + + Key *string + + ETag *string + + ChecksumCRC64ECMA *string + + ObjectMeta map[string]*string +} + +func (c *S3) DownloadFile(request *DownloadFileInput) (*DownloadFileOutput, error) { + return c.DownloadFileWithContext(context.Background(), request) +} + +func (c *S3) DownloadFileWithContext(ctx context.Context, request *DownloadFileInput) (*DownloadFileOutput, error) { + return newDownloader(c, ctx, request).downloadFile() +} + +type Downloader struct { + client *S3 + + context context.Context + + downloadFileRequest *DownloadFileInput + + downloadCheckpoint *DownloadCheckpoint + + CompletedSize int64 + + downloadFileSize int64 + + downloadFileMeta map[string]*string + + mu sync.Mutex + + error error +} + +func newDownloader(s3 *S3, ctx context.Context, request *DownloadFileInput) *Downloader { + return &Downloader{ + client: s3, + context: ctx, + downloadFileRequest: request, + } +} + +func (d *Downloader) downloadFile() (*DownloadFileOutput, error) { + err := d.validate() + if err != nil { + return nil, err + } + + d.downloadFileMeta, err = d.headObject() + if err != nil { + return nil, err + } + + dcp, err := newDownloadCheckpoint(d) + if err != nil { + return nil, err + } + d.downloadCheckpoint = dcp + + if aws.ToBoolean(d.downloadFileRequest.EnableCheckpoint) { + cpFilePath := aws.ToString(d.downloadFileRequest.CheckpointFile) + if cpFilePath == "" { + cpFilePath, err = generateDownloadCpFilePath(d.downloadFileRequest) + if err != nil { + return nil, err + } + } + dcp.CpFilePath = cpFilePath + + err = dcp.load() + if err != nil { + return nil, err + } + + if !FileExists(dcp.DownloadFilePath + TempFileSuffix) { + dcp.PartETagList = make([]*CompletedPart, 0) + dcp.remove() + } + } + + err = d.createDownloadDir(dcp.DownloadFilePath + TempFileSuffix) + if err != nil { + return nil, err + } + + objectRange := d.getObjectRange() + d.downloadFileSize = objectRange[1] - objectRange[0] + 1 + partSize := aws.ToLong(d.downloadFileRequest.PartSize) + totalPartNum := (d.downloadFileSize-1)/partSize + 1 + tasks := make(chan DownloadPartTask, totalPartNum) + + var i int64 + for i = 0; i < totalPartNum; i++ { + partNum := i + 1 + start := objectRange[0] + i*partSize + end := Min(start+partSize-1, objectRange[1]) + actualPartSize := end - start + 1 + if d.getPartETag(partNum) != nil { + d.publishProgress(actualPartSize) + } else { + downloadPartTask := DownloadPartTask{ + partNumber: partNum, + start: start, + end: end, + actualPartSize: actualPartSize, + } + tasks <- downloadPartTask + } + } + close(tasks) + + var wg sync.WaitGroup + for i = 0; i < aws.ToLong(d.downloadFileRequest.TaskNum); i++ { + wg.Add(1) + go d.runTask(tasks, &wg) + } + wg.Wait() + + if d.error != nil { + return nil, d.error + } + + if d.downloadFileRequest.Range == nil && d.client.Config.CrcCheckEnabled { + clientCrc64 := d.getCrc64Ecma(dcp.PartETagList) + serverCrc64, _ := strconv.ParseUint(aws.ToString(d.downloadFileMeta[HTTPHeaderAmzChecksumCrc64ecma]), 10, 64) + d.client.Config.LogDebug("check file crc64, client crc64:%d, server crc64:%d", clientCrc64, serverCrc64) + if serverCrc64 != 0 && clientCrc64 != serverCrc64 { + return nil, errors.New(fmt.Sprintf("crc64 check failed, client crc64:%d, server crc64:%d", clientCrc64, serverCrc64)) + } + } + + err = d.complete() + if err != nil { + return nil, err + } + + return d.getDownloadFileOutput(), nil +} + +func (d *Downloader) validate() error { + request := d.downloadFileRequest + if request == nil { + return errors.New("download file request is required") + } + + if aws.ToString(request.Bucket) == "" { + return errors.New("bucket is required") + } + + if aws.ToString(request.Key) == "" { + return errors.New("key is required") + } + + err := d.normalizeDownloadPath() + if err != nil { + return err + } + + if request.PartSize == nil { + request.PartSize = aws.Long(DefaultPartSize) + } else if aws.ToLong(request.PartSize) < MinPartSize { + request.PartSize = aws.Long(MinPartSize) + } else if aws.ToLong(request.PartSize) > MaxPartSize { + request.PartSize = aws.Long(MaxPartSize) + } + + if aws.ToLong(request.TaskNum) <= 0 { + request.TaskNum = aws.Long(DefaultTaskNum) + } + + return nil +} + +func (d *Downloader) getDownloadFileOutput() *DownloadFileOutput { + return &DownloadFileOutput{ + Bucket: d.downloadFileRequest.Bucket, + Key: d.downloadFileRequest.Key, + ETag: d.downloadFileMeta[HTTPHeaderEtag], + ChecksumCRC64ECMA: d.downloadFileMeta[HTTPHeaderAmzChecksumCrc64ecma], + ObjectMeta: d.downloadFileMeta, + } +} + +func (d *Downloader) getActualPartSize(fileSize int64, partSize int64, partNum int64) int64 { + offset := (partNum - 1) * partSize + actualPartSize := partSize + if offset+partSize >= fileSize { + actualPartSize = fileSize - offset + } + return actualPartSize +} + +func (d *Downloader) getPartETag(partNumber int64) *CompletedPart { + for _, partETag := range d.downloadCheckpoint.PartETagList { + if *partETag.PartNumber == partNumber { + return partETag + } + } + return nil +} + +type DownloadPartTask struct { + partNumber int64 + + actualPartSize int64 + + start int64 + + end int64 +} + +func (d *Downloader) runTask(tasks <-chan DownloadPartTask, wg *sync.WaitGroup) { + defer wg.Done() + for task := range tasks { + if d.error != nil { + return + } + + partETag, err := d.downloadPart(task) + if err != nil { + d.setError(err) + return + } + + d.updatePart(partETag) + } +} + +func (d *Downloader) downloadPart(task DownloadPartTask) (CompletedPart, error) { + request := d.downloadFileRequest + dcp := d.downloadCheckpoint + tempFilePath := dcp.DownloadFilePath + TempFileSuffix + var completedPart CompletedPart + resp, err := d.client.GetObjectWithContext(d.context, &GetObjectInput{ + Bucket: aws.String(dcp.BucketName), + Key: aws.String(dcp.ObjectKey), + Range: aws.String(fmt.Sprintf("bytes=%d-%d", task.start, task.end)), + ResponseContentType: request.ResponseContentType, + ResponseContentLanguage: request.ResponseContentLanguage, + ResponseExpires: request.ResponseExpires, + ResponseCacheControl: request.ResponseCacheControl, + ResponseContentDisposition: request.ResponseContentDisposition, + ResponseContentEncoding: request.ResponseContentEncoding, + IfModifiedSince: request.IfModifiedSince, + IfUnmodifiedSince: request.IfUnmodifiedSince, + IfMatch: request.IfMatch, + IfNoneMatch: request.IfNoneMatch, + AcceptEncoding: request.AcceptEncoding, + SSECustomerAlgorithm: request.SSECustomerAlgorithm, + SSECustomerKey: request.SSECustomerKey, + SSECustomerKeyMD5: request.SSECustomerKeyMD5, + }) + + if err != nil { + return completedPart, err + } + defer resp.Body.Close() + + var crc64 hash.Hash64 + crc64 = crc.NewCRC(crc.CrcTable(), 0) + resp.Body = aws.TeeReader(resp.Body, crc64, task.actualPartSize, nil) + + fd, err := os.OpenFile(tempFilePath, os.O_WRONLY|os.O_CREATE, FilePermMode) + if err != nil { + return completedPart, err + } + defer fd.Close() + + _, err = fd.Seek((task.partNumber-1)*dcp.PartSize, io.SeekStart) + if err != nil { + return completedPart, err + } + + _, err = io.Copy(fd, resp.Body) + if err != nil { + return completedPart, err + } + + completedPart.PartNumber = aws.Long(task.partNumber) + completedPart.ChecksumCRC64ECMA = aws.String(strconv.FormatUint(crc64.Sum64(), 10)) + d.publishProgress(task.actualPartSize) + + return completedPart, nil +} + +func (d *Downloader) updatePart(partETag CompletedPart) { + d.mu.Lock() + defer d.mu.Unlock() + + d.downloadCheckpoint.PartETagList = append(d.downloadCheckpoint.PartETagList, &partETag) + d.downloadCheckpoint.dump() +} + +func (d *Downloader) setError(err error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.error == nil { + d.error = err + } +} + +func (d *Downloader) publishProgress(actualPartSize int64) { + if d.downloadFileRequest.ProgressFn != nil { + atomic.AddInt64(&d.CompletedSize, actualPartSize) + d.downloadFileRequest.ProgressFn(actualPartSize, d.CompletedSize, d.downloadFileSize) + } +} + +func (d *Downloader) getCrc64Ecma(parts []*CompletedPart) uint64 { + if parts == nil || len(parts) == 0 { + return 0 + } + + sort.Sort(CompletedParts(d.downloadCheckpoint.PartETagList)) + + crcTemp, _ := strconv.ParseUint(*parts[0].ChecksumCRC64ECMA, 10, 64) + for i := 1; i < len(parts); i++ { + crc2, _ := strconv.ParseUint(*parts[i].ChecksumCRC64ECMA, 10, 64) + partSize := d.getActualPartSize(d.downloadFileSize, aws.ToLong(d.downloadFileRequest.PartSize), *parts[i].PartNumber) + crcTemp = crc.CRC64Combine(crcTemp, crc2, (uint64)(partSize)) + } + + return crcTemp +} + +func (d *Downloader) complete() error { + fileName := aws.ToString(d.downloadFileRequest.DownloadFile) + tempFileName := fileName + TempFileSuffix + err := os.Rename(tempFileName, fileName) + if err != nil { + return err + } + d.downloadCheckpoint.remove() + return nil +} + +func (d *Downloader) headObject() (map[string]*string, error) { + request := d.downloadFileRequest + resp, err := d.client.HeadObjectWithContext(d.context, &HeadObjectInput{ + Bucket: request.Bucket, + Key: request.Key, + IfModifiedSince: request.IfModifiedSince, + IfUnmodifiedSince: request.IfUnmodifiedSince, + IfMatch: request.IfMatch, + IfNoneMatch: request.IfNoneMatch, + SSECustomerAlgorithm: request.SSECustomerAlgorithm, + SSECustomerKey: request.SSECustomerKey, + SSECustomerKeyMD5: request.SSECustomerKeyMD5, + }) + if err != nil { + return nil, err + } + return resp.Metadata, err +} + +func (d *Downloader) createDownloadDir(filePath string) error { + dir := filepath.Dir(filePath) + if !DirExists(dir) { + err := os.MkdirAll(dir, DirPermMode) + if err != nil { + return err + } + } + return nil +} + +func (d *Downloader) getObjectRange() []int64 { + objectRange := d.downloadFileRequest.Range + objectSize := d.downloadCheckpoint.ObjectSize + if objectRange == nil { + return []int64{0, objectSize - 1} + } + + if !d.isValidRange(objectRange, objectSize) { + d.client.Config.LogWarn("Invalid range value: %v, ignore it and request for entire object", objectRange) + return []int64{0, objectSize - 1} + } + objectStart := objectRange[0] + objectEnd := objectRange[1] + + if objectStart < 0 { + return []int64{objectSize - objectEnd, objectSize - 1} + } + + if objectEnd < 0 { + return []int64{objectStart, objectSize - 1} + } + + return []int64{objectStart, Min(objectEnd, objectSize-1)} +} + +func (d *Downloader) isValidRange(objectRange []int64, objectSize int64) bool { + if len(objectRange) != 2 { + return false + } + + objectStart := objectRange[0] + objectEnd := objectRange[1] + + if objectStart < 0 && objectEnd < 0 || objectEnd >= 0 && objectStart > objectEnd { + return false + } + + return objectStart < objectSize +} + +func (d *Downloader) normalizeDownloadPath() error { + downloadPath := aws.ToString(d.downloadFileRequest.DownloadFile) + if downloadPath == "" { + downloadPath = aws.ToString(d.downloadFileRequest.Key) + } + // 规范化路径 + normalizedPath := filepath.Clean(downloadPath) + // 获取绝对路径 + absPath, err := filepath.Abs(normalizedPath) + if err != nil { + return err + } + d.downloadFileRequest.DownloadFile = aws.String(absPath) + + return nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/errors.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/errors.go new file mode 100644 index 0000000000..6d3e726cf5 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/errors.go @@ -0,0 +1,60 @@ +// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. + +package s3 + +const ( + + // ErrCodeBucketAlreadyExists for service response error code + // "BucketAlreadyExists". + // + // The requested bucket name is not available. The bucket namespace is shared + // by all users of the system. Select a different name and try again. + ErrCodeBucketAlreadyExists = "BucketAlreadyExists" + + // ErrCodeBucketAlreadyOwnedByYou for service response error code + // "BucketAlreadyOwnedByYou". + // + // The bucket you tried to create already exists, and you own it. Amazon S3 + // returns this error in all AWS Regions except in the North Virginia Region. + // For legacy compatibility, if you re-create an existing bucket that you already + // own in the North Virginia Region, Amazon S3 returns 200 OK and resets the + // bucket access control lists (ACLs). + ErrCodeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou" + + // ErrCodeInvalidObjectState for service response error code + // "InvalidObjectState". + // + // Object is archived and inaccessible until restored. + ErrCodeInvalidObjectState = "InvalidObjectState" + + // ErrCodeNoSuchBucket for service response error code + // "NoSuchBucket". + // + // The specified bucket does not exist. + ErrCodeNoSuchBucket = "NoSuchBucket" + + // ErrCodeNoSuchKey for service response error code + // "NoSuchKey". + // + // The specified key does not exist. + ErrCodeNoSuchKey = "NoSuchKey" + + // ErrCodeNoSuchUpload for service response error code + // "NoSuchUpload". + // + // The specified multipart upload does not exist. + ErrCodeNoSuchUpload = "NoSuchUpload" + + // ErrCodeObjectAlreadyInActiveTierError for service response error code + // "ObjectAlreadyInActiveTierError". + // + // This action is not allowed against this storage tier. + ErrCodeObjectAlreadyInActiveTierError = "ObjectAlreadyInActiveTierError" + + // ErrCodeObjectNotInActiveTierError for service response error code + // "ObjectNotInActiveTierError". + // + // The source object of the COPY action is not in the active tier and is only + // stored in Amazon S3 Glacier. + ErrCodeObjectNotInActiveTierError = "ObjectNotInActiveTierError" +) diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/host_style_bucket.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/host_style_bucket.go new file mode 100644 index 0000000000..27fce0b8e6 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/host_style_bucket.go @@ -0,0 +1,53 @@ +package s3 + +import ( + "regexp" + "strings" + + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/aws/awsutil" +) + +var reDomain = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-\_]{1,61}[a-z0-9]$`) +var reIPAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`) + +// dnsCompatibleBucketName returns true if the bucket name is DNS compatible. +// Buckets created outside the classic region MUST be DNS compatible. +func dnsCompatibleBucketName(bucket string) bool { + return reDomain.MatchString(bucket) && + !reIPAddress.MatchString(bucket) && + !strings.Contains(bucket, "..") +} + +// hostStyleBucketName returns true if the request should put the bucket in +// the host. This is false if S3ForcePathStyle is explicitly set or if the +// bucket is not DNS compatible. +func hostStyleBucketName(r *aws.Request, bucket string) bool { + if r.Config.S3ForcePathStyle { + return false + } + + // Bucket might be DNS compatible but dots in the hostname will fail + // certificate validation, so do not use host-style. + if r.HTTPRequest.URL.Scheme == "https" && strings.Contains(bucket, ".") { + return false + } + + // Use host-style if the bucket is DNS compatible + return dnsCompatibleBucketName(bucket) +} + +func updateHostWithBucket(r *aws.Request) { + b := awsutil.ValuesAtPath(r.Params, "Bucket") + if len(b) == 0 { + return + } + + if bucket := b[0].(string); bucket != "" && hostStyleBucketName(r, bucket) { + r.HTTPRequest.URL.Host = bucket + "." + r.HTTPRequest.URL.Host + r.HTTPRequest.URL.Path = strings.Replace(r.HTTPRequest.URL.Path, "/{Bucket}", "", -1) + if r.HTTPRequest.URL.Path == "" { + r.HTTPRequest.URL.Path = "/" + } + } +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/inventory.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/inventory.go new file mode 100644 index 0000000000..ef3cfc6464 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/inventory.go @@ -0,0 +1,306 @@ +package s3 + +import "github.com/ks3sdklib/aws-sdk-go/aws" + +// PutBucketInventoryRequest generates a request for the PutBucketInventory operation. +func (c *S3) PutBucketInventoryRequest(input *PutBucketInventoryInput) (req *aws.Request, output *PutBucketInventoryOutput) { + op := &aws.Operation{ + Name: "PutBucketInventory", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?inventory", + } + + if input == nil { + input = &PutBucketInventoryInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(op, input, output) + output = &PutBucketInventoryOutput{} + req.Data = output + return +} + +// PutBucketInventory creates a new inventory configuration. +func (c *S3) PutBucketInventory(input *PutBucketInventoryInput) (*PutBucketInventoryOutput, error) { + req, out := c.PutBucketInventoryRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketInventoryWithContext(ctx aws.Context, input *PutBucketInventoryInput) (*PutBucketInventoryOutput, error) { + req, out := c.PutBucketInventoryRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type PutBucketInventoryInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Id *string `location:"querystring" locationName:"id" type:"string" required:"true"` + + InventoryConfiguration *InventoryConfiguration `locationName:"InventoryConfiguration" type:"structure" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketInventoryInput `json:"-" xml:"-"` +} + +type metadataPutBucketInventoryInput struct { + SDKShapeTraits bool `type:"structure" payload:"InventoryConfiguration"` + + AutoFillMD5 bool +} + +type PutBucketInventoryOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type InventoryConfiguration struct { + // The list name specified by the user is unique within a single bucket. + Id *string `locationName:"Id" type:"string" required:"true"` + + // Is the inventory function enabled. + IsEnabled *bool `locationName:"IsEnabled" type:"boolean" required:"true"` + + // Specify scanning prefix information. + Filter *InventoryFilter `locationName:"Filter" type:"structure"` + + // Storage inventory results. + Destination *Destination `locationName:"Destination" type:"structure" required:"true"` + + // Container for storing inventory export cycle information. + Schedule *Schedule `locationName:"Schedule" type:"structure" required:"true"` + + // Set the configuration items included in the inventory results. + OptionalFields *OptionalFields `locationName:"OptionalFields" type:"structure" required:"true"` +} + +type InventoryFilter struct { + // The storage path prefix of the inventory file. + Prefix *string `locationName:"Prefix" type:"string" required:"true"` + + // The starting timestamp of the last modification time of the filtered file, in seconds. + LastModifyBeginTimeStamp *string `locationName:"LastModifyBeginTimeStamp" type:"string"` + + // End timestamp of the last modification time of the filtered file, in seconds. + LastModifyEndTimeStamp *string `locationName:"LastModifyEndTimeStamp" type:"string"` +} + +type Destination struct { + // Bucket information stored after exporting the inventory results. + KS3BucketDestination *KS3BucketDestination `locationName:"KS3BucketDestination" type:"structure" required:"true"` +} + +type KS3BucketDestination struct { + // The file format of the inventory file is a CSV file compressed using GZIP after exporting the manifest file. + Format *string `locationName:"Format" type:"string" required:"true"` + + // Bucket owner's account ID. + AccountId *string `locationName:"AccountId" type:"string"` + + // Bucket for storing exported inventory files. + Bucket *string `locationName:"Bucket" type:"string" required:"true"` + + // The storage path prefix of the inventory file. + Prefix *string `locationName:"Prefix" type:"string"` +} + +type Schedule struct { + // Cycle of exporting inventory files. + Frequency *string `locationName:"Frequency" type:"string" required:"true"` +} + +type OptionalFields struct { + // Configuration items included in the inventory results. + // Valid values: + // Size: The size of the object. + // LastModifiedDate: The last modified time of an object. + // ETag: The ETag value of an object, used to identify its contents. + // StorageClass: The storage type of Object. + // IsMultipartUploaded: Is it an object uploaded through shard upload method. + // EncryptionStatus: Whether the object is encrypted. If the object is encrypted, the value of this field is True; otherwise, it is False. + Field []*string `locationName:"Field" type:"list" flattened:"true"` +} + +// GetBucketInventoryRequest generates a request for the GetBucketInventory operation. +func (c *S3) GetBucketInventoryRequest(input *GetBucketInventoryInput) (req *aws.Request, output *GetBucketInventoryOutput) { + op := &aws.Operation{ + Name: "GetBucketInventory", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?inventory", + } + + if input == nil { + input = &GetBucketInventoryInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketInventoryOutput{} + req.Data = output + return +} + +// GetBucketInventory gets the inventory configuration for the bucket. +func (c *S3) GetBucketInventory(input *GetBucketInventoryInput) (*GetBucketInventoryOutput, error) { + req, out := c.GetBucketInventoryRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketInventoryWithContext(ctx aws.Context, input *GetBucketInventoryInput) (*GetBucketInventoryOutput, error) { + req, out := c.GetBucketInventoryRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketInventoryInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Id *string `location:"querystring" locationName:"id" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketInventoryOutput struct { + InventoryConfiguration *InventoryConfiguration `locationName:"Inventory" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketInventoryOutput `json:"-" xml:"-"` +} + +type metadataGetBucketInventoryOutput struct { + SDKShapeTraits bool `type:"structure" payload:"InventoryConfiguration"` +} + +// DeleteBucketInventoryRequest generates a request for the DeleteBucketInventory operation. +func (c *S3) DeleteBucketInventoryRequest(input *DeleteBucketInventoryInput) (req *aws.Request, output *DeleteBucketInventoryOutput) { + op := &aws.Operation{ + Name: "DeleteBucketInventory", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?inventory", + } + + if input == nil { + input = &DeleteBucketInventoryInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketInventoryOutput{} + req.Data = output + return +} + +// DeleteBucketInventory deletes the inventory configuration for the bucket. +func (c *S3) DeleteBucketInventory(input *DeleteBucketInventoryInput) (*DeleteBucketInventoryOutput, error) { + req, out := c.DeleteBucketInventoryRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketInventoryWithContext(ctx aws.Context, input *DeleteBucketInventoryInput) (*DeleteBucketInventoryOutput, error) { + req, out := c.DeleteBucketInventoryRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteBucketInventoryInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + Id *string `location:"querystring" locationName:"id" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} +type DeleteBucketInventoryOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// ListBucketInventoryRequest generates a request for the ListBucketInventory operation. +func (c *S3) ListBucketInventoryRequest(input *ListBucketInventoryInput) (req *aws.Request, output *ListBucketInventoryOutput) { + op := &aws.Operation{ + Name: "ListBucketInventory", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?inventory", + } + + if input == nil { + input = &ListBucketInventoryInput{} + } + + req = c.newRequest(op, input, output) + output = &ListBucketInventoryOutput{} + req.Data = output + return +} + +// ListBucketInventory lists the inventory configurations for the bucket. +func (c *S3) ListBucketInventory(input *ListBucketInventoryInput) (*ListBucketInventoryOutput, error) { + req, out := c.ListBucketInventoryRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ListBucketInventoryWithContext(ctx aws.Context, input *ListBucketInventoryInput) (*ListBucketInventoryOutput, error) { + req, out := c.ListBucketInventoryRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type ListBucketInventoryInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContinuationToken *string `location:"querystring" locationName:"continuation-token" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type ListInventoryConfigurationsResult struct { + InventoryConfigurations []*InventoryConfiguration `locationName:"InventoryConfiguration" type:"list" flattened:"true"` + + IsTruncated *bool `locationName:"IsTruncated" type:"boolean"` + + NextContinuationToken *string `locationName:"NextContinuationToken" type:"string"` +} + +type ListBucketInventoryOutput struct { + InventoryConfigurationsResult *ListInventoryConfigurationsResult `locationName:"InventoryConfigurationsResult" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataListBucketInventoryOutput `json:"-" xml:"-"` +} + +type metadataListBucketInventoryOutput struct { + SDKShapeTraits bool `type:"structure" payload:"InventoryConfigurationsResult"` +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/ks3bill.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/ks3bill.go new file mode 100644 index 0000000000..0a6a6bf238 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/ks3bill.go @@ -0,0 +1,443 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/signer/open_api" +) + +type QueryKs3DataInput struct { + // 查询用量开始时间，格式为：yyyyMMdd，如：20250901 表示查询从2025年9月1日0点0分开始的用量 + StartTime *string `location:"querystring" locationName:"StartTime" type:"string" required:"true"` + + // 查询用量结束时间，格式为：yyyyMMdd，如：20250902 表示查询到2025年9月2日23点59分结束的用量 + EndTime *string `location:"querystring" locationName:"EndTime" type:"string" required:"true"` + + // 支持按天粒度查询，固定值：Day + DateType *string `location:"querystring" locationName:"DateType" type:"string"` + + // 存储空间名称，最多支持同时查询5个存储桶的用量明细 + BucketNames []string `location:"querystrings" locationName:"Bucketname" type:"list" required:"true"` + + // 可以查询单个或多个计费项，如果不填，则查询除带宽外的所有计费项 + // DataSize：存储量 + // NetworkFlowUp：外网上行流量 + // NetworkFlow：外网下行流量 + // CDNFlow：CDN回源流量 + // ReplicationFlow：跨区域复制流量 + // RequestsGet：GET类请求次数 + // RequestsPut：PUT类请求次数 + // RestoreSize：数据取回量 + // TagNum：对象标签梳理 + // BandwidthUp：上行带宽（不区分外网和CDN） + // BandwidthDown：下行带宽（不区分外网和CDN） + // NetBandwidthUp：外网上行带宽 + // NetBandwidthDown：外网下行带宽 + // CDNBandwidthDown：CDN回源带宽 + // IntranetBandwidthUp：内网上行带宽 + // IntranetBandwidthDown：内网下行带宽 + // IntranetFlowUp：内网上行流量 + // IntranetFlowDown：内网下行流量 + // ObjectNum：桶内的对象数量 + Ks3Products []string `location:"querystrings" locationName:"Ks3Product" type:"list"` + + // 可以查询单个或多个统计项的流量情况，可选值：Object、Referer、IP、UA，返回TOP200数据 + Transfers []string `location:"querystrings" locationName:"Transfer" type:"list"` + + // 可以查询单个或多个统计项的请求次数情况，可选值：Object、Referer、IP、UA，返回TOP200数据 + Requests []string `location:"querystrings" locationName:"Request" type:"list"` +} + +type QueryKs3DataOutput struct { + // 查询结果 + Ks3DataResult *Ks3DataResult `locationName:"Ks3DataResult" type:"structure"` + + // 响应头 + Metadata map[string]*string `location:"headers" type:"map"` + + // HTTP 状态码 + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataQueryKs3DataOutput `json:"-" xml:"-"` +} + +type metadataQueryKs3DataOutput struct { + SDKShapeTraits bool `type:"structure" payload:"Ks3DataResult"` +} + +type Ks3DataResult struct { + // 响应码 + Code *string `locationName:"Code" type:"string"` + + // 响应信息 + Message *string `locationName:"Message" type:"string"` + + // 包含一个或多个Bucket的容器 + Data *Ks3Data `locationName:"Data" type:"structure"` + + // 请求ID + RequestId *string `locationName:"RequestId" type:"string"` +} + +type Ks3Data struct { + // 包含一个或多个Bucket的列表 + Buckets []*Ks3DataBucket `locationName:"Buckets" type:"list"` +} + +type Ks3DataBucket struct { + // Bucket的名称 + Name *string `locationName:"Name" type:"string"` + + // 数据开始时间 + StartTime *string `locationName:"StartTime" type:"string"` + + // 数据结束时间 + EndTime *string `locationName:"EndTime" type:"string"` + + // Bucket的标准存储量，单位是Bytes + StandardDataSize *string `locationName:"StandardDataSize" type:"string"` + + // Bucket的低频存储量，单位是Bytes + StandardIADataSize *string `locationName:"StandardIADataSize" type:"string"` + + // Bucket的归档存储量，单位是Bytes + ArchiveDataSize *string `locationName:"ArchiveDataSize" type:"string"` + + // 标准存储的PUT请求次数 + StandardPutRequest *string `locationName:"StandardPutRequest" type:"string"` + + // 低频存储的PUT请求次数 + StandardIAPutRequest *string `locationName:"StandardIAPutRequest" type:"string"` + + // 归档存储的PUT请求次数 + ArchivePutRequest *string `locationName:"ArchivePutRequest" type:"string"` + + // 标准存储的GET请求次数 + StandardGetRequest *string `locationName:"StandardGetRequest" type:"string"` + + // 低频存储的GET请求次数 + StandardIAGetRequest *string `locationName:"StandardIAGetRequest" type:"string"` + + // 归档存储的GET请求次数 + ArchiveGetRequest *string `locationName:"ArchiveGetRequest" type:"string"` + + // 外网下行流量，单位是Bytes + NetworkFlow *string `locationName:"NetworkFlow" type:"string"` + + // CDN回源流量，单位是Bytes + CDNFlow *string `locationName:"CDNFlow" type:"string"` + + // 跨区域复制流量，单位是Bytes + ReplicationFlow *string `locationName:"ReplicationFlow" type:"string"` + + // 外网上行带宽，不区分外网上行和CDN上行，单位是bps + BandwidthUp []map[string]*string `locationName:"BandwidthUp" type:"list"` + + // 外网下行带宽，不区分外网下行和CDN下行，单位是bps + BandwidthDown []map[string]*string `locationName:"BandwidthDown" type:"list"` + + // 外网下行带宽，单位是bps + OuterBandwidthDown []map[string]*string `locationName:"OuterBandwidthDown" type:"list"` + + // CDN回源带宽，单位是bps + CDNBandwidthDown []map[string]*string `locationName:"CDNBandwidthDown" type:"list"` + + // 低频存储数据取回量，单位是Bytes + StandardIAData *string `locationName:"StandardIAData" type:"string"` + + // 外网上行带宽 + NetBandwidthUp []map[string]*string `locationName:"NetBandwidthUp" type:"list"` + + // 外网上行流量 + NetworkFlowUp *string `locationName:"NetworkFlowUp" type:"string"` + + // 内网上行带宽 + IntranetBandwidthUp []map[string]*string `locationName:"IntranetBandwidthUp" type:"list"` + + // 内网上行流量 + IntranetFlowUp *string `locationName:"IntranetFlowUp" type:"string"` + + // 内网下行带宽 + IntranetBandwidthDown []map[string]*string `locationName:"IntranetBandwidthDown" type:"list"` + + // 内网下行流量 + IntranetFlowDown *string `locationName:"IntranetFlowDown" type:"string"` + + // 桶内的对象数量 + ObjectNum *string `locationName:"ObjectNum" type:"string"` + + // 归档存储解冻数据量，单位是Bytes + ArchiveData *string `locationName:"ArchiveData" type:"string"` + + // 对象标签的数量 + TagNum *string `locationName:"TagNum" type:"string"` + + // Object、Referer、IP、UA产生的流量 + Transfer *Ks3DataTransfer `locationName:"Transfer" type:"structure"` + + // Object、Referer、IP、UA产生的请求次数 + Request *Ks3DataRequest `locationName:"Request" type:"structure"` +} + +type Ks3DataTransfer struct { + // 指定Object产生的流量 + Objects []*Ks3DataTransferObject `locationName:"Object" type:"list"` + + // 指定Referer产生的流量 + Referers []*Ks3DataTransferReferer `locationName:"Referer" type:"list"` + + // 指定IP产生的流量 + IPs []*Ks3DataTransferIP `locationName:"Ip" type:"list"` + + // 指定UA产生的流量 + UAs []*Ks3DataTransferUA `locationName:"Ua" type:"list"` +} + +type Ks3DataTransferObject struct { + // Object名称 + Object *string `locationName:"object" type:"string"` + + // Object产生的流量 + Traffic *string `locationName:"traffic" type:"string"` +} + +type Ks3DataTransferReferer struct { + // Referer名称 + Referer *string `locationName:"referer" type:"string"` + + // Referer产生的流量 + Traffic *string `locationName:"traffic" type:"string"` +} + +type Ks3DataTransferIP struct { + // IP地址 + IP *string `locationName:"ip" type:"string"` + + // IP产生的流量 + Traffic *string `locationName:"traffic" type:"string"` +} + +type Ks3DataTransferUA struct { + // UA名称 + UA *string `locationName:"ua" type:"string"` + + // UA产生的流量 + Traffic *string `locationName:"traffic" type:"string"` +} + +type Ks3DataRequest struct { + // 指定Object产生的流量 + Objects []*Ks3DataRequestObject `locationName:"Object" type:"list"` + + // 指定Referer产生的流量 + Referers []*Ks3DataRequestReferer `locationName:"Referer" type:"list"` + + // 指定IP产生的流量 + IPs []*Ks3DataRequestIP `locationName:"Ip" type:"list"` + + // 指定UA产生的流量 + UAs []*Ks3DataRequestUA `locationName:"Ua" type:"list"` +} + +type Ks3DataRequestObject struct { + // Object名称 + Object *string `locationName:"object" type:"string"` + + // Object产生的请求次数 + Times *string `locationName:"times" type:"string"` +} + +type Ks3DataRequestReferer struct { + // Referer名称 + Referer *string `locationName:"referer" type:"string"` + + // Referer产生的请求次数 + Times *string `locationName:"times" type:"string"` +} + +type Ks3DataRequestIP struct { + // IP地址 + IP *string `locationName:"ip" type:"string"` + + // IP产生的请求次数 + Times *string `locationName:"times" type:"string"` +} + +type Ks3DataRequestUA struct { + // UA名称 + UA *string `locationName:"ua" type:"string"` + + // UA产生的请求次数 + Times *string `locationName:"times" type:"string"` +} + +// QueryKs3DataRequest generates a request for the QueryKs3DataRequest operation. +func (c *S3) QueryKs3DataRequest(input *QueryKs3DataInput) (req *aws.Request, output *QueryKs3DataOutput) { + op := &aws.Operation{ + Name: "QueryKs3Data", + HTTPMethod: "GET", + HTTPPath: "/?Action=QueryKs3Data", + } + + if input == nil { + input = &QueryKs3DataInput{} + } + + if input.StartTime != nil && aws.ToString(input.StartTime) != "" { + input.StartTime = aws.String(aws.ToString(input.StartTime) + "0000") + } + + if input.EndTime != nil && aws.ToString(input.EndTime) != "" { + input.EndTime = aws.String(aws.ToString(input.EndTime) + "2359") + } + + if input.DateType == nil || aws.ToString(input.DateType) == "" { + input.DateType = aws.String("Day") + } + + req = c.newRequest(op, input, output) + req.RequestType = "ks3bill" + req.ContentType = "application/json" + req.HTTPRequest.URL.Host = c.Config.Ks3BillEndpoint + req.Handlers.Sign.Clear() + req.Handlers.Sign.PushBack(open_api.Sign) + output = &QueryKs3DataOutput{ + Ks3DataResult: &Ks3DataResult{}, + } + req.Data = output + return +} + +// QueryKs3Data 桶用量详情及业务分析查询 +func (c *S3) QueryKs3Data(input *QueryKs3DataInput) (*QueryKs3DataOutput, error) { + req, out := c.QueryKs3DataRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) QueryKs3DataWithContext(ctx aws.Context, input *QueryKs3DataInput) (*QueryKs3DataOutput, error) { + req, out := c.QueryKs3DataRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type QueryBucketRankInput struct { + // 查询用量开始时间，格式为：yyyyMMdd，如：20250901 表示查询从2025年9月1日0点0分开始的用量 + StartTime *string `location:"querystring" locationName:"StartTime" type:"string" required:"true"` + + // 查询用量结束时间，格式为：yyyyMMdd，如：20250902 表示查询到2025年9月2日23点59分结束的用量 + EndTime *string `location:"querystring" locationName:"EndTime" type:"string" required:"true"` + + // 支持按天粒度查询，固定值：Day + DateType *string `location:"querystring" locationName:"DateType" type:"string"` + + // 可以查询单个或多个统计项，如果不填，则查询所有统计项 + // DataSize：存储量 + // Flow：外网下行流量 + // RequestsGet：GET类请求次数 + // RequestsPut：PUT类请求次数 + Ks3Products []string `location:"querystrings" locationName:"Ks3Product" type:"list"` + + // TOP排序的Bucket数量，取值范围为[1-500]，默认值为200 + Number *int64 `location:"querystring" locationName:"Number" type:"integer"` +} + +type QueryBucketRankOutput struct { + // 查询结果 + BucketRankResult *BucketRankResult `locationName:"BucketRankResult" type:"structure"` + + // 响应头 + Metadata map[string]*string `location:"headers" type:"map"` + + // HTTP 状态码 + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataQueryBucketRankOutput `json:"-" xml:"-"` +} + +type metadataQueryBucketRankOutput struct { + SDKShapeTraits bool `type:"structure" payload:"BucketRankResult"` +} + +type BucketRankResult struct { + // 响应码 + Code *string `locationName:"Code" type:"string"` + + // 响应信息 + Message *string `locationName:"Message" type:"string"` + + // 桶用量排序数据 + Data *BucketRankData `locationName:"Data" type:"structure"` + + // 请求ID + RequestId *string `locationName:"RequestId" type:"string"` +} + +type BucketRankData struct { + // Bucket的存储量，单位是Bytes + DataSize []map[string]*string `locationName:"DataSize" type:"list"` + + // Bucket的外网下行流量，单位是Bytes + Flow []map[string]*string `locationName:"Flow" type:"list"` + + // Bucket的GET类请求次数，单位是次 + RequestsGet []map[string]*string `locationName:"RequestsGet" type:"list"` + + // Bucket的PUT类请求次数，单位是次 + RequestsPut []map[string]*string `locationName:"RequestsPut" type:"list"` +} + +// QueryBucketRankRequest generates a request for the QueryBucketRankRequest operation. +func (c *S3) QueryBucketRankRequest(input *QueryBucketRankInput) (req *aws.Request, output *QueryBucketRankOutput) { + op := &aws.Operation{ + Name: "QueryBucketRank", + HTTPMethod: "GET", + HTTPPath: "/?Action=QueryBucketRank", + } + + if input == nil { + input = &QueryBucketRankInput{} + } + + if input.StartTime != nil && aws.ToString(input.StartTime) != "" { + input.StartTime = aws.String(aws.ToString(input.StartTime) + "0000") + } + + if input.EndTime != nil && aws.ToString(input.EndTime) != "" { + input.EndTime = aws.String(aws.ToString(input.EndTime) + "2359") + } + + if input.DateType == nil || aws.ToString(input.DateType) == "" { + input.DateType = aws.String("Day") + } + + if input.Number == nil { + input.Number = aws.Long(200) + } + + req = c.newRequest(op, input, output) + req.RequestType = "ks3bill" + req.ContentType = "application/json" + req.HTTPRequest.URL.Host = c.Config.Ks3BillEndpoint + req.Handlers.Sign.Clear() + req.Handlers.Sign.PushBack(open_api.Sign) + output = &QueryBucketRankOutput{ + BucketRankResult: &BucketRankResult{}, + } + req.Data = output + return +} + +// QueryBucketRank 桶用量排序查询 +func (c *S3) QueryBucketRank(input *QueryBucketRankInput) (*QueryBucketRankOutput, error) { + req, out := c.QueryBucketRankRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) QueryBucketRankWithContext(ctx aws.Context, input *QueryBucketRankInput) (*QueryBucketRankOutput, error) { + req, out := c.QueryBucketRankRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/lifecycle.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/lifecycle.go new file mode 100644 index 0000000000..cfa10452e3 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/lifecycle.go @@ -0,0 +1,261 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "time" +) + +type PutBucketLifecycleInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + LifecycleConfiguration *LifecycleConfiguration `locationName:"LifecycleConfiguration" type:"structure"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Specifies whether lifecycle rules allow prefix overlap. + AllowSameActionOverlap *bool `location:"header" locationName:"x-amz-allow-same-action-overlap" type:"boolean"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketLifecycleInput `json:"-" xml:"-"` +} + +type metadataPutBucketLifecycleInput struct { + SDKShapeTraits bool `type:"structure" payload:"LifecycleConfiguration"` + + AutoFillMD5 bool +} + +type LifecycleConfiguration struct { + Rules []*LifecycleRule `locationName:"Rule" type:"list" flattened:"true" required:"true"` +} + +type LifecycleRule struct { + // Unique identifier for the rule. The value cannot be longer than 255 characters. + ID *string `type:"string"` + + // If 'Enabled', the rule is currently being applied. If 'Disabled', the rule + // is not currently being applied. + Status *string `type:"string" required:"true"` + + // Specifies the prefix, each Rule can only have one Filter, and the prefixes of different + // Rules cannot conflict. + Filter *LifecycleFilter `type:"structure"` + + // Specifies the time when the object is deleted + Expiration *LifecycleExpiration `type:"structure"` + + // Specifies when an object transitions to a specified storage class. + Transitions []*Transition `locationName:"Transition" type:"list" flattened:"true"` + + // Specifies the expiration time for multipart uploads. + AbortIncompleteMultipartUpload *AbortIncompleteMultipartUpload `type:"structure"` +} + +type LifecycleFilter struct { + Prefix *string `type:"string"` + And *And `locationName:"And" type:"structure"` +} + +type And struct { + Prefix *string `type:"string"` + Tags []*Tag `locationName:"Tag" type:"list" flattened:"true"` +} + +type LifecycleExpiration struct { + // Indicates at what date the object is to be moved or deleted. Should be in + // GMT ISO 8601 Format. + Date *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // Indicates the lifetime, in days, of the objects that are subject to the rule. + // The value must be a non-zero positive integer. + Days *int64 `type:"integer"` +} + +type Transition struct { + // Indicates at what date the object is to be moved or deleted. Should be in + // GMT ISO 8601 Format. + Date *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // Specifies the number of days after the object is last modified or accessed that the lifecycle rule takes effect. + // When the value of IsAccessTime in the request is true, this parameter indicates that the lifecycle rule takes + // effect based on the last access time of the object. When IsAccessTime is not set in the request or is set to false, + // this parameter indicates that the lifecycle rule takes effect based on the last modification time of the object. + // This parameter is mutually exclusive with Date. + Days *int64 `type:"integer"` + + // The class of storage used to store the object. + StorageClass *string `type:"string"` + + // Specifies whether to use the last access time matching rule. + // true: indicates that the last access time of the object is used for matching. + // false: indicates that the last modification time of the object is used for matching. + IsAccessTime *bool `type:"boolean"` + + // Specifies whether to convert the object to the source storage type when accessed again after the object is + // converted to another storage type. This is only valid when IsAccessTime is set to true. + // true: Indicates that the object is converted to the source storage type when accessed again. + // false: Indicates that the object is still the target storage type when accessed again. + ReturnToStdWhenVisit *bool `type:"boolean"` +} + +type AbortIncompleteMultipartUpload struct { + // Relative expiration time: The expiration time in days after the last modified time + DaysAfterInitiation *int64 `type:"integer"` + // objects created before the date will be expired + Date *string `type:"string"` +} + +type PutBucketLifecycleOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutBucketLifecycleRequest generates a request for the PutBucketLifecycle operation. +func (c *S3) PutBucketLifecycleRequest(input *PutBucketLifecycleInput) (req *aws.Request, output *PutBucketLifecycleOutput) { + op := &aws.Operation{ + Name: "PutBucketLifecycle", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?lifecycle", + } + + if input == nil { + input = &PutBucketLifecycleInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(op, input, output) + output = &PutBucketLifecycleOutput{} + req.Data = output + return +} + +// PutBucketLifecycle Sets lifecycle configuration for your bucket. If a lifecycle configuration +// exists, it replaces it. +func (c *S3) PutBucketLifecycle(input *PutBucketLifecycleInput) (*PutBucketLifecycleOutput, error) { + req, out := c.PutBucketLifecycleRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketLifecycleWithContext(ctx aws.Context, input *PutBucketLifecycleInput) (*PutBucketLifecycleOutput, error) { + req, out := c.PutBucketLifecycleRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketLifecycleInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketLifecycleOutput struct { + Rules []*LifecycleRule `locationName:"Rule" type:"list" flattened:"true"` + + metadataGetBucketLifecycleOutput `json:"-" xml:"-"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type metadataGetBucketLifecycleOutput struct { + SDKShapeTraits bool `type:"structure"` +} + +// GetBucketLifecycleRequest generates a request for the GetBucketLifecycle operation. +func (c *S3) GetBucketLifecycleRequest(input *GetBucketLifecycleInput) (req *aws.Request, output *GetBucketLifecycleOutput) { + op := &aws.Operation{ + Name: "GetBucketLifecycle", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?lifecycle", + } + + if input == nil { + input = &GetBucketLifecycleInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketLifecycleOutput{} + req.Data = output + return +} + +// GetBucketLifecycle Returns the lifecycle configuration information set on the bucket. +func (c *S3) GetBucketLifecycle(input *GetBucketLifecycleInput) (*GetBucketLifecycleOutput, error) { + req, out := c.GetBucketLifecycleRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketLifecycleWithContext(ctx aws.Context, input *GetBucketLifecycleInput) (*GetBucketLifecycleOutput, error) { + req, out := c.GetBucketLifecycleRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteBucketLifecycleInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type DeleteBucketLifecycleOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// DeleteBucketLifecycleRequest generates a request for the DeleteBucketLifecycle operation. +func (c *S3) DeleteBucketLifecycleRequest(input *DeleteBucketLifecycleInput) (req *aws.Request, output *DeleteBucketLifecycleOutput) { + op := &aws.Operation{ + Name: "DeleteBucketLifecycle", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?lifecycle", + } + + if input == nil { + input = &DeleteBucketLifecycleInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketLifecycleOutput{} + req.Data = output + return +} + +// DeleteBucketLifecycle Deletes the lifecycle configuration from the bucket. +func (c *S3) DeleteBucketLifecycle(input *DeleteBucketLifecycleInput) (*DeleteBucketLifecycleOutput, error) { + req, out := c.DeleteBucketLifecycleRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketLifecycleWithContext(ctx aws.Context, input *DeleteBucketLifecycleInput) (*DeleteBucketLifecycleOutput, error) { + req, out := c.DeleteBucketLifecycleRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/mirror.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/mirror.go new file mode 100644 index 0000000000..b0f48dc968 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/mirror.go @@ -0,0 +1,214 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +type BucketMirror struct { + Version *string `json:"version" type:"string" locationName:"version"` + UseDefaultRobots *bool `json:"use_default_robots" locationName:"use_default_robots"` + AsyncMirrorRule *AsyncMirrorRule `json:"async_mirror_rule,omitempty" type:"structure" locationName:"async_mirror_rule"` + SyncMirrorRules []*SyncMirrorRules `json:"sync_mirror_rules,omitempty" type:"list" locationName:"sync_mirror_rules"` + CreatedTime *string `json:"created_time,omitempty" type:"string" locationName:"created_time"` + LastModifiedTime *string `json:"last_modified_time,omitempty" type:"string" locationName:"last_modified_time"` +} +type SavingSetting struct { + ACL *string `json:"acl,omitempty" required:"true" locationName:"acl"` +} +type AsyncMirrorRule struct { + MirrorUrls []*string `json:"mirror_urls,omitempty" required:"true" locationName:"mirror_urls"` + SavingSetting *SavingSetting `json:"saving_setting,omitempty" required:"true" locationName:"saving_setting"` + MirrorType *string `json:"mirror_type,omitempty" locationName:"mirror_type"` +} +type MatchCondition struct { + HTTPCodes []*string `json:"http_codes" locationName:"http_codes"` + KeyPrefixes []*string `json:"key_prefixes" locationName:"key_prefixes"` +} +type SetHeaders struct { + Key *string `json:"key,omitempty" locationName:"key"` + Value *string `json:"value,omitempty" locationName:"value"` +} +type RemoveHeaders struct { + Key *string `json:"key,omitempty" locationName:"key"` +} +type PassHeaders struct { + Key *string `json:"key,omitempty" locationName:"key"` +} +type HeaderSetting struct { + SetHeaders []*SetHeaders `json:"set_headers,omitempty" locationName:"set_headers"` + RemoveHeaders []*RemoveHeaders `json:"remove_headers,omitempty" locationName:"remove_headers"` + PassAll *bool `json:"pass_all,omitempty" locationName:"pass_all"` + PassHeaders []*PassHeaders `json:"pass_headers,omitempty" locationName:"pass_headers"` +} +type MirrorRequestSetting struct { + PassQueryString *bool `json:"pass_query_string,omitempty" locationName:"pass_query_string"` + Follow3Xx *bool `json:"follow3xx,omitempty" locationName:"follow3xx"` + HeaderSetting *HeaderSetting `json:"header_setting,omitempty" locationName:"header_setting"` +} +type SyncMirrorRules struct { + MatchCondition MatchCondition `json:"match_condition" locationName:"match_condition"` + MirrorURL *string `json:"mirror_url,omitempty" locationName:"mirror_url"` + MirrorRequestSetting *MirrorRequestSetting `json:"mirror_request_setting,omitempty" locationName:"mirror_request_setting"` + SavingSetting *SavingSetting `json:"saving_setting,omitempty" locationName:"saving_setting"` + MirrorType *string `json:"mirror_type,omitempty" locationName:"mirror_type"` +} + +type PutBucketMirrorInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + BucketMirror *BucketMirror `locationName:"BucketMirror" json:"-" type:"structure"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketMirrorInput `json:"-" xml:"-"` +} + +type metadataPutBucketMirrorInput struct { + SDKShapeTraits bool `type:"structure" payload:"BucketMirror"` +} + +type PutBucketMirrorOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type GetBucketMirrorInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} +type GetBucketMirrorOutput struct { + BucketMirror *BucketMirror `locationName:"BucketMirror" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketMirrorInput `json:"-" xml:"-"` +} + +type metadataGetBucketMirrorInput struct { + SDKShapeTraits bool `type:"structure" payload:"BucketMirror"` +} + +type DeleteBucketMirrorInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} +type DeleteBucketMirrorOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +func (c *S3) PutBucketMirrorRequest(input *PutBucketMirrorInput) (req *aws.Request, output *PutBucketMirrorOutput) { + op := &aws.Operation{ + Name: "PutBucketMirror", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?mirror", + } + + if input == nil { + input = &PutBucketMirrorInput{} + } + + req = c.newRequest(op, input, output) + req.ContentType = "application/json" + output = &PutBucketMirrorOutput{} + req.Data = output + return +} + +func (c *S3) PutBucketMirror(input *PutBucketMirrorInput) (*PutBucketMirrorOutput, error) { + req, out := c.PutBucketMirrorRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketMirrorWithContext(ctx aws.Context, input *PutBucketMirrorInput) (*PutBucketMirrorOutput, error) { + req, out := c.PutBucketMirrorRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketMirrorRequest(input *GetBucketMirrorInput) (req *aws.Request, output *GetBucketMirrorOutput) { + op := &aws.Operation{ + Name: "GetBucketMirror", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?mirror", + } + + if input == nil { + input = &GetBucketMirrorInput{} + } + + req = c.newRequest(op, input, output) + req.ContentType = "application/json" + output = &GetBucketMirrorOutput{ + BucketMirror: &BucketMirror{}, + } + req.Data = output + return +} + +func (c *S3) GetBucketMirror(input *GetBucketMirrorInput) (*GetBucketMirrorOutput, error) { + req, out := c.GetBucketMirrorRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketMirrorWithContext(ctx aws.Context, input *GetBucketMirrorInput) (*GetBucketMirrorOutput, error) { + req, out := c.GetBucketMirrorRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketMirrorRequest(input *DeleteBucketMirrorInput) (req *aws.Request, output *DeleteBucketMirrorOutput) { + op := &aws.Operation{ + Name: "DeleteBucketMirror", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?mirror", + } + + if input == nil { + input = &DeleteBucketMirrorInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketMirrorOutput{} + req.Data = output + return +} + +func (c *S3) DeleteBucketMirror(input *DeleteBucketMirrorInput) (*DeleteBucketMirrorOutput, error) { + req, out := c.DeleteBucketMirrorRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketMirrorWithContext(ctx aws.Context, input *DeleteBucketMirrorInput) (*DeleteBucketMirrorOutput, error) { + req, out := c.DeleteBucketMirrorRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/replication.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/replication.go new file mode 100644 index 0000000000..23700f6a66 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/replication.go @@ -0,0 +1,190 @@ +package s3 + +import "github.com/ks3sdklib/aws-sdk-go/aws" + +// PutBucketReplicationRequest generates a request for the PutBucketReplication operation. +func (c *S3) PutBucketReplicationRequest(input *PutBucketReplicationInput) (req *aws.Request, output *PutBucketReplicationOutput) { + op := &aws.Operation{ + Name: "PutBucketReplication", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?crr", + } + + if input == nil { + input = &PutBucketReplicationInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(op, input, output) + output = &PutBucketReplicationOutput{} + req.Data = output + return +} + +// PutBucketReplication creates a new replication configuration. +func (c *S3) PutBucketReplication(input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) { + req, out := c.PutBucketReplicationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketReplicationWithContext(ctx aws.Context, input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) { + req, out := c.PutBucketReplicationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type PutBucketReplicationInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + ReplicationConfiguration *ReplicationConfiguration `locationName:"Replication" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketReplicationInput `json:"-" xml:"-"` +} + +type metadataPutBucketReplicationInput struct { + SDKShapeTraits bool `type:"structure" payload:"ReplicationConfiguration"` + + AutoFillMD5 bool +} + +type PutBucketReplicationOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +type ReplicationConfiguration struct { + // Prefix matching, only objects that match prefix rules will be copied. Each copying rule + // can add up to 10 prefix matching rules, and prefixes cannot overlap with each other. + Prefix []*string `locationName:"prefix" type:"list" flattened:"true"` + + // Indicate whether to enable delete replication. If set to Enabled, it means enabled; if set to + // Disabled or not, it means disabled. If set to delete replication, when the source bucket deletes + // an object, the replica of that object in the target bucket will also be deleted. + DeleteMarkerStatus *string `locationName:"DeleteMarkerStatus" type:"string" required:"true"` + + // Target bucket for copying rules. + TargetBucket *string `locationName:"targetBucket" type:"string" required:"true"` + + // Specify whether to copy historical data. Whether to copy the data from the source bucket + // to the target bucket before enabling data replication. + // Enabled: Copy historical data to the target bucket (default value) + // Disabled: Do not copy historical data, only copy new data after enabling the rule to the target bucket. + HistoricalObjectReplication *string `locationName:"HistoricalObjectReplication" type:"string"` + + // Region of the target bucket. + Region *string `locationName:"region" type:"string"` +} + +// GetBucketReplicationRequest generates a request for the GetBucketReplication operation. +func (c *S3) GetBucketReplicationRequest(input *GetBucketReplicationInput) (req *aws.Request, output *GetBucketReplicationOutput) { + op := &aws.Operation{ + Name: "GetBucketReplication", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?crr", + } + + if input == nil { + input = &GetBucketReplicationInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketReplicationOutput{} + req.Data = output + return +} + +// GetBucketReplication gets the replication configuration for the bucket. +func (c *S3) GetBucketReplication(input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) { + req, out := c.GetBucketReplicationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketReplicationWithContext(ctx aws.Context, input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) { + req, out := c.GetBucketReplicationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketReplicationInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketReplicationOutput struct { + ReplicationConfiguration *ReplicationConfiguration `locationName:"Replication" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketReplicationOutput `json:"-" xml:"-"` +} + +type metadataGetBucketReplicationOutput struct { + SDKShapeTraits bool `type:"structure" payload:"ReplicationConfiguration"` +} + +// DeleteBucketReplicationRequest generates a request for the DeleteBucketReplication operation. +func (c *S3) DeleteBucketReplicationRequest(input *DeleteBucketReplicationInput) (req *aws.Request, output *DeleteBucketReplicationOutput) { + op := &aws.Operation{ + Name: "DeleteBucketReplication", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?crr", + } + + if input == nil { + input = &DeleteBucketReplicationInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteBucketReplicationOutput{} + req.Data = output + return +} + +// DeleteBucketReplication deletes the replication configuration for the bucket. +func (c *S3) DeleteBucketReplication(input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) { + req, out := c.DeleteBucketReplicationRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteBucketReplicationWithContext(ctx aws.Context, input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) { + req, out := c.DeleteBucketReplicationRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteBucketReplicationInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} +type DeleteBucketReplicationOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/requester_qos.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/requester_qos.go new file mode 100644 index 0000000000..3c94273da6 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/requester_qos.go @@ -0,0 +1,194 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" +) + +type PutRequesterQosInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Requester flow control configuration container. + RequesterQosConfiguration *RequesterQosConfiguration `locationName:"RequesterQosConfiguration" type:"structure" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutRequesterQosInput `json:"-" xml:"-"` +} + +type metadataPutRequesterQosInput struct { + SDKShapeTraits bool `type:"structure" payload:"RequesterQosConfiguration"` +} + +type RequesterQosConfiguration struct { + // Set the requester flow control rules. + Rules []*RequesterQosRule `locationName:"Rule" type:"list" flattened:"true" required:"true"` +} + +type RequesterQosRule struct { + // Specify the account type that needs flow control. + // Optional values: User/Role. + UserType *string `locationName:"UserType" type:"string" required:"true"` + + // Specify the account that needs flow control. + // Format: accountId/userName、accountId/roleName. + Krn *string `locationName:"Krn" type:"string" required:"true"` + + // Set access account flow control quota. + Quotas []*BucketQosQuota `locationName:"Quota" type:"list" flattened:"true" required:"true"` +} + +type PutRequesterQosOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutRequesterQosRequest generates a request for the PutRequesterQos operation. +func (c *S3) PutRequesterQosRequest(input *PutRequesterQosInput) (req *aws.Request, output *PutRequesterQosOutput) { + op := &aws.Operation{ + Name: "PutRequesterQos", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?requesterqos", + } + + if input == nil { + input = &PutRequesterQosInput{} + } + + req = c.newRequest(op, input, output) + output = &PutRequesterQosOutput{} + req.Data = output + return +} + +// PutRequesterQos sets requester flow control configuration. +func (c *S3) PutRequesterQos(input *PutRequesterQosInput) (*PutRequesterQosOutput, error) { + req, out := c.PutRequesterQosRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutRequesterQosWithContext(ctx aws.Context, input *PutRequesterQosInput) (*PutRequesterQosOutput, error) { + req, out := c.PutRequesterQosRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetRequesterQosInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetRequesterQosOutput struct { + // Requester flow control configuration container. + RequesterQosConfiguration *RequesterQosConfiguration `locationName:"RequesterQosConfiguration" type:"structure"` + + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetRequesterQosOutput `json:"-" xml:"-"` +} + +type metadataGetRequesterQosOutput struct { + SDKShapeTraits bool `type:"structure" payload:"RequesterQosConfiguration"` +} + +// GetRequesterQosRequest generates a request for the GetRequesterQos operation. +func (c *S3) GetRequesterQosRequest(input *GetRequesterQosInput) (req *aws.Request, output *GetRequesterQosOutput) { + op := &aws.Operation{ + Name: "GetRequesterQos", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?requesterqos", + } + + if input == nil { + input = &GetRequesterQosInput{} + } + + req = c.newRequest(op, input, output) + output = &GetRequesterQosOutput{} + req.Data = output + return +} + +// GetRequesterQos gets requester flow control configuration. +func (c *S3) GetRequesterQos(input *GetRequesterQosInput) (*GetRequesterQosOutput, error) { + req, out := c.GetRequesterQosRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetRequesterQosWithContext(ctx aws.Context, input *GetRequesterQosInput) (*GetRequesterQosOutput, error) { + req, out := c.GetRequesterQosRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteRequesterQosInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type DeleteRequesterQosOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// DeleteRequesterQosRequest generates a request for the DeleteRequesterQos operation. +func (c *S3) DeleteRequesterQosRequest(input *DeleteRequesterQosInput) (req *aws.Request, output *DeleteRequesterQosOutput) { + op := &aws.Operation{ + Name: "DeleteRequesterQos", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}?requesterqos", + } + + if input == nil { + input = &DeleteRequesterQosInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteRequesterQosOutput{} + req.Data = output + return +} + +// DeleteRequesterQos deletes requester flow control configuration. +func (c *S3) DeleteRequesterQos(input *DeleteRequesterQosInput) (*DeleteRequesterQosOutput, error) { + req, out := c.DeleteRequesterQosRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteRequesterQosWithContext(ctx aws.Context, input *DeleteRequesterQosInput) (*DeleteRequesterQosOutput, error) { + req, out := c.DeleteRequesterQosRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/retention.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/retention.go new file mode 100644 index 0000000000..5725d51549 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/retention.go @@ -0,0 +1,386 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "time" +) + +// PutBucketRetentionRequest generates a request for the PutBucketRetention operation. +func (c *S3) PutBucketRetentionRequest(input *PutBucketRetentionInput) (req *aws.Request, output *PutBucketRetentionOutput) { + op := &aws.Operation{ + Name: "PutBucketRetention", + HTTPMethod: "PUT", + HTTPPath: "/{Bucket}?retention", + } + + if input == nil { + input = &PutBucketRetentionInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(op, input, output) + output = &PutBucketRetentionOutput{} + req.Data = output + return +} + +// PutBucketRetention sets the retention configuration on a bucket. +func (c *S3) PutBucketRetention(input *PutBucketRetentionInput) (*PutBucketRetentionOutput, error) { + req, out := c.PutBucketRetentionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutBucketRetentionWithContext(ctx aws.Context, input *PutBucketRetentionInput) (*PutBucketRetentionOutput, error) { + req, out := c.PutBucketRetentionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type PutBucketRetentionInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + RetentionConfiguration *BucketRetentionConfiguration `locationName:"RetentionConfiguration" type:"structure"` + + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutBucketRetentionInput `json:"-" xml:"-"` +} + +type metadataPutBucketRetentionInput struct { + SDKShapeTraits bool `type:"structure" payload:"RetentionConfiguration"` + + AutoFillMD5 bool +} + +type BucketRetentionConfiguration struct { + // Whether to enable multiple versions in the recycle bin. When the request does not carry this parameter, + // multiple versions are enabled by default. + EnableMultipleVersion *bool `locationName:"EnableMultipleVersion" type:"boolean"` + + // A container that contains a specific rule for the recycle bin. + Rule *RetentionRule `locationName:"Rule" type:"structure" required:"true"` +} + +type RetentionRule struct { + // The open status of the recycle bin is not case-sensitive. + // Valid values: Enabled, Disabled. Enabled indicates enabling the recycle bin, Disabled indicates disabling the recycle bin. + Status *string `locationName:"Status" type:"string" required:"true"` + + // Specify how many days after the object enters the recycle bin to be completely deleted. + // When Days is not set, the object will be permanently retained in the recycle bin after deletion. + // Value range: 1-365 + Days *int64 `locationName:"Days" type:"integer"` +} + +type PutBucketRetentionOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// GetBucketRetentionRequest generates a request for the GetBucketRetention operation. +func (c *S3) GetBucketRetentionRequest(input *GetBucketRetentionInput) (req *aws.Request, output *GetBucketRetentionOutput) { + op := &aws.Operation{ + Name: "GetBucketRetention", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?retention", + } + + if input == nil { + input = &GetBucketRetentionInput{} + } + + req = c.newRequest(op, input, output) + output = &GetBucketRetentionOutput{ + RetentionConfiguration: &BucketRetentionConfiguration{}, + } + req.Data = output + return +} + +// GetBucketRetention gets the retention configuration for the bucket. +func (c *S3) GetBucketRetention(input *GetBucketRetentionInput) (*GetBucketRetentionOutput, error) { + req, out := c.GetBucketRetentionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetBucketRetentionWithContext(ctx aws.Context, input *GetBucketRetentionInput) (*GetBucketRetentionOutput, error) { + req, out := c.GetBucketRetentionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetBucketRetentionInput struct { + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetBucketRetentionOutput struct { + RetentionConfiguration *BucketRetentionConfiguration `locationName:"RetentionConfiguration" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetBucketRetentionInput `json:"-" xml:"-"` +} + +type metadataGetBucketRetentionInput struct { + SDKShapeTraits bool `type:"structure" payload:"RetentionConfiguration"` +} + +// ListRetentionRequest generates a request for the ListRetention operation. +func (c *S3) ListRetentionRequest(input *ListRetentionInput) (req *aws.Request, output *ListRetentionOutput) { + op := &aws.Operation{ + Name: "ListRetention", + HTTPMethod: "GET", + HTTPPath: "/{Bucket}?recycle", + } + + if input == nil { + input = &ListRetentionInput{} + } + + req = c.newRequest(op, input, output) + output = &ListRetentionOutput{} + req.Data = output + return +} + +// ListRetention lists the objects in the recycle bin. +func (c *S3) ListRetention(input *ListRetentionInput) (*ListRetentionOutput, error) { + req, out := c.ListRetentionRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ListRetentionWithContext(ctx aws.Context, input *ListRetentionInput) (*ListRetentionOutput, error) { + req, out := c.ListRetentionRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type ListRetentionInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Specifies the key to start with when listing objects in a bucket. + Marker *string `location:"querystring" locationName:"marker" type:"string"` + + // Sets the maximum number of keys returned in the response. The response might + // contain fewer keys but will never contain more. + MaxKeys *int64 `location:"querystring" locationName:"max-keys" type:"integer"` + + // Limits the response to keys that begin with the specified prefix. + Prefix *string `location:"querystring" locationName:"prefix" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type ListRetentionOutput struct { + // A container that lists information about the list of objects in the recycle bin. + ListRetentionResult *ListRetentionResult `locationName:"ListRetentionResult" type:"structure"` + + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataListRetentionOutput `json:"-" xml:"-"` +} + +type metadataListRetentionOutput struct { + SDKShapeTraits bool `type:"structure" payload:"ListRetentionResult"` +} + +type ListRetentionResult struct { + // The name of the bucket. + Name *string `type:"string"` + + // Specify the prefix of the Key when requesting this List. + Prefix *string `type:"string"` + + // The maximum number of objects returned is 1000 by default. + MaxKeys *int64 `type:"integer"` + + // Specify the starting position of the object in the target bucket. + Marker *string `type:"string"` + + // The starting point for the next listed file. Users can use this value as a marker parameter + // for the next List Retention. + NextMarker *string `type:"string"` + + // Whether it has been truncated. If the number of records in the Object list exceeds the set + // maximum value, it will be truncated. + IsTruncated *bool `type:"boolean"` + + // The encoding method for Object names. + EncodingType *string `type:"string"` + + // List of Objects Listed. + Contents []*RetentionObject `type:"list" flattened:"true"` +} + +type RetentionObject struct { + // The key of the object. + Key *string `type:"string"` + + // The size of the object is measured in bytes. + Size *int64 `type:"integer"` + + // The entity label of an object, ETag, is generated when uploading an object to identify its content. + ETag *string `type:"string"` + + // The last time the object was modified. + LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // The owner information of this bucket. + Owner *Owner `type:"structure"` + + // The class of storage used to store the object. + StorageClass *string `type:"string"` + + // The version ID of the object. + RetentionId *string `type:"string"` + + // The time when the object was moved to the recycle bin. + RecycleTime *time.Time `type:"timestamp" timestampFormat:"iso8601"` + + // The time when an object is completely deleted from the recycle bin. + EstimatedClearTime *time.Time `type:"timestamp" timestampFormat:"iso8601"` +} + +// RecoverObjectRequest generates a request for the RecoverObject operation. +func (c *S3) RecoverObjectRequest(input *RecoverObjectInput) (req *aws.Request, output *RecoverObjectOutput) { + op := &aws.Operation{ + Name: "RecoverObject", + HTTPMethod: "POST", + HTTPPath: "/{Bucket}/{Key+}?recover", + } + + if input == nil { + input = &RecoverObjectInput{} + } + + req = c.newRequest(op, input, output) + output = &RecoverObjectOutput{} + req.Data = output + return +} + +// RecoverObject recovers the object from the recycle bin. +func (c *S3) RecoverObject(input *RecoverObjectInput) (*RecoverObjectOutput, error) { + req, out := c.RecoverObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) RecoverObjectWithContext(ctx aws.Context, input *RecoverObjectInput) (*RecoverObjectOutput, error) { + req, out := c.RecoverObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type RecoverObjectInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // The key of the object. + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Does it support overwriting when an object with the same name exists in the bucket after being + // recovered from the recycle bin. When the value is true, it indicates overwriting, and the overwritten + // objects in the bucket will enter the recycle bin. + RetentionOverwrite *bool `location:"header" locationName:"x-kss-retention-overwrite" type:"boolean"` + + // Specify the deletion ID of the recovered object. When the request header is not included, + // only the latest version is restored by default. + RetentionId *string `location:"header" locationName:"x-kss-retention-id" type:"string"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type RecoverObjectOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// ClearObjectRequest generates a request for the ClearObject operation. +func (c *S3) ClearObjectRequest(input *ClearObjectInput) (req *aws.Request, output *ClearObjectOutput) { + op := &aws.Operation{ + Name: "ClearObject", + HTTPMethod: "DELETE", + HTTPPath: "/{Bucket}/{Key+}?clear", + } + + if input == nil { + input = &ClearObjectInput{} + } + + req = c.newRequest(op, input, output) + output = &ClearObjectOutput{} + req.Data = output + return +} + +// ClearObject clears the object from the recycle bin. +func (c *S3) ClearObject(input *ClearObjectInput) (*ClearObjectOutput, error) { + req, out := c.ClearObjectRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) ClearObjectWithContext(ctx aws.Context, input *ClearObjectInput) (*ClearObjectOutput, error) { + req, out := c.ClearObjectRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type ClearObjectInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // The key of the object. + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // Specify the deletion ID of the deleted object. + RetentionId *string `location:"header" locationName:"x-kss-retention-id" type:"string" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type ClearObjectOutput struct { + Metadata map[string]*string `location:"headers" type:"map"` + + StatusCode *int64 `location:"statusCode" type:"integer"` +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/service.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/service.go new file mode 100644 index 0000000000..8263edc241 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/service.go @@ -0,0 +1,70 @@ +// THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT. + +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/protocol/body" + "github.com/ks3sdklib/aws-sdk-go/internal/signer/v2" + "github.com/ks3sdklib/aws-sdk-go/internal/signer/v4" + "strings" +) + +// S3 is a client for Amazon S3. +type S3 struct { + *aws.Service +} + +// Used for custom service initialization logic +var initService func(*aws.Service) + +// Used for custom request initialization logic +var initRequest func(*aws.Request) + +// New returns a new S3 client. +func New(config *aws.Config) *S3 { + service := &aws.Service{ + Config: aws.DefaultConfig.Merge(config), + ServiceName: "s3", + APIVersion: "2006-03-01", + } + service.Initialize() + + // Handlers + if service.Config.SignerVersion == "V4" || service.Config.SignerVersion == "V4_UNSIGNED_PAYLOAD_SIGNER" { + service.Handlers.Sign.PushBack(v4.Sign) + } else { + service.Handlers.Sign.PushBack(v2.Sign) + } + + service.Handlers.Build.PushBack(body.Build) + service.Handlers.Unmarshal.PushBack(body.UnmarshalBody) + service.Handlers.UnmarshalMeta.PushBack(body.UnmarshalMeta) + service.Handlers.UnmarshalError.PushBack(body.UnmarshalError) + + // Run custom service initialization if present + if initService != nil { + initService(service) + } + + return &S3{service} +} + +// newRequest creates a new request for a S3 operation and runs any +// custom request initialization. +func (c *S3) newRequest(op *aws.Operation, params, data interface{}) *aws.Request { + r := aws.NewRequest(c.Service, op, params, data) + if r.Config.DomainMode { + r.HTTPRequest.URL.Path = strings.Replace(r.HTTPRequest.URL.Path, "/{Bucket}", "", -1) + if r.HTTPRequest.URL.Path == "" { + r.HTTPRequest.URL.Path = "/" + } + } else { + updateHostWithBucket(r) + } + // Run custom request initialization if present + if initRequest != nil { + initRequest(r) + } + return r +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/try.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/try.go new file mode 100644 index 0000000000..d2ba553c74 --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/try.go @@ -0,0 +1,35 @@ +package s3 +import "errors" + +// MaxRetries is the maximum number of retries before bailing. +var MaxRetries = 10 + +var errMaxRetriesReached = errors.New("exceeded retry limit") + +// Func represents functions that can be retried. +type Func func(attempt int) (retry bool, err error) + +// Do keeps trying the function until the second argument +// returns false, or no error is returned. +func Do(fn Func) error { + var err error + var cont bool + attempt := 1 + for { + cont, err = fn(attempt) + if !cont || err == nil { + break + } + attempt++ + if attempt > MaxRetries { + return errMaxRetriesReached + } + } + return err +} + +// IsMaxRetries checks whether the error is due to hitting the +// maximum number of retries or not. +func IsMaxRetries(err error) bool { + return err == errMaxRetriesReached +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/uploader.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/uploader.go new file mode 100644 index 0000000000..00f0b8e8ea --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/uploader.go @@ -0,0 +1,603 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/internal/crc" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +type UploadFileInput struct { + // The name of the bucket. + Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + + // Object key of the object. + Key *string `location:"uri" locationName:"Key" type:"string" required:"true"` + + // The path of the file to be uploaded. + UploadFile *string `type:"string" required:"true"` + + // The size of the file to be uploaded. + FileSize *int64 `type:"integer"` + + // The file part fetcher. + FilePartFetcher *FilePartFetcher `type:"structure"` + + // The object metadata. + ObjectMeta map[string]*string `type:"structure"` + + // The size of each part. + PartSize *int64 `type:"integer"` + + // The number of tasks to upload the file. + TaskNum *int64 `type:"integer"` + + // Whether to enable checkpoint. + EnableCheckpoint *bool `type:"boolean"` + + // The directory to store the checkpoint file. + CheckpointDir *string `type:"string"` + + // The checkpoint file path. + CheckpointFile *string `type:"string"` + + // The canned ACL to apply to the object. + ACL *string `location:"header" locationName:"x-amz-acl" type:"string"` + + // Specifies caching behavior along the request/reply chain. + CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` + + // Specifies presentational information for the object. + ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the media-type referenced + // by the Content-Type header field. + ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` + + // A standard MIME type describing the format of the object data. + ContentType *string `location:"header" locationName:"Content-Type" type:"string"` + + // The date and time at which the object is no longer cacheable. + Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp" timestampFormat:"rfc822"` + + // A map of metadata to store with the object in S3. + Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` + + // The type of storage to use for the object. Defaults to 'STANDARD'. + StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string"` + + // Specifies the object tag of the object. Multiple tags can be set at the same time, such as: TagA=A&TagB=B. + // Note: Key and Value need to be URL-encoded first. If an item does not have "=", the Value is considered to be an empty string. + Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` + + // Specifies whether the object is forbidden to overwrite. + ForbidOverwrite *bool `location:"header" locationName:"x-amz-forbid-overwrite" type:"boolean"` + + // Allows grantee to read the object data and its metadata. + GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` + + // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. + GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` + + // The Server-side encryption algorithm used when storing this object in KS3, eg: AES256. + ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string"` + + // Specifies the algorithm to use to when encrypting the object, eg: AES256. + SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` + + // Specifies the customer-provided encryption key for KS3 to use in encrypting data. + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + + // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. + SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` + + // Progress callback function + ProgressFn aws.ProgressFunc `location:"function"` +} + +type UploadFileOutput struct { + Bucket *string + + Key *string + + ETag *string + + ChecksumCRC64ECMA *string +} + +type FilePartFetcher interface { + Fetch(objectRange []int64) (io.ReadSeeker, error) +} + +func (c *S3) UploadFile(request *UploadFileInput) (*UploadFileOutput, error) { + return c.UploadFileWithContext(context.Background(), request) +} + +func (c *S3) UploadFileWithContext(ctx context.Context, request *UploadFileInput) (*UploadFileOutput, error) { + return newUploader(c, ctx, request).uploadFile() +} + +type Uploader struct { + client *S3 + + context context.Context + + uploadFileRequest *UploadFileInput + + uploadCheckpoint *UploadCheckpoint + + CompletedSize int64 + + mu sync.Mutex + + error error +} + +func newUploader(s3 *S3, ctx context.Context, request *UploadFileInput) *Uploader { + return &Uploader{ + client: s3, + context: ctx, + uploadFileRequest: request, + } +} + +func (u *Uploader) uploadFile() (*UploadFileOutput, error) { + err := u.validate() + if err != nil { + return nil, err + } + + if aws.ToString(u.uploadFileRequest.UploadFile) != "" && aws.ToLong(u.uploadFileRequest.FileSize) <= aws.ToLong(u.uploadFileRequest.PartSize) { + return u.putObject() + } + + return u.multipartUpload() +} + +func (u *Uploader) validate() error { + request := u.uploadFileRequest + if request == nil { + return errors.New("upload file request is required") + } + + if aws.ToString(request.Bucket) == "" { + return errors.New("bucket is required") + } + + if aws.ToString(request.Key) == "" { + return errors.New("key is required") + } + + err := u.normalizeUploadPath() + if err != nil { + return err + } + + filePath := aws.ToString(request.UploadFile) + if filePath == "" && request.FilePartFetcher == nil { + return errors.New("upload file or file part fetcher is required") + } + + if filePath != "" { + fileInfo, err := os.Stat(filePath) + if err != nil { + return err + } + if fileInfo.IsDir() { + return errors.New("upload file not a file") + } + request.FileSize = aws.Long(fileInfo.Size()) + } else { + if request.ObjectMeta != nil { + fileSize, _ := strconv.ParseInt(aws.ToString(request.ObjectMeta[HTTPHeaderContentLength]), 10, 64) + request.FileSize = aws.Long(fileSize) + } + } + + if request.FilePartFetcher != nil && request.FileSize == nil { + return errors.New("file size is required") + } + + if request.PartSize == nil { + request.PartSize = aws.Long(DefaultPartSize) + } else if aws.ToLong(request.PartSize) < MinPartSize { + request.PartSize = aws.Long(MinPartSize) + } else if aws.ToLong(request.PartSize) > MaxPartSize { + request.PartSize = aws.Long(MaxPartSize) + } + + if aws.ToLong(request.TaskNum) <= 0 { + request.TaskNum = aws.Long(DefaultTaskNum) + } + + return nil +} + +func (u *Uploader) putObject() (*UploadFileOutput, error) { + request := u.uploadFileRequest + fd, err := os.Open(aws.ToString(request.UploadFile)) + if err != nil { + return nil, err + } + defer fd.Close() + + resp, err := u.client.PutObjectWithContext(u.context, &PutObjectInput{ + Bucket: request.Bucket, + Key: request.Key, + Body: fd, + ACL: request.ACL, + CacheControl: request.CacheControl, + ContentDisposition: request.ContentDisposition, + ContentEncoding: request.ContentEncoding, + ContentType: request.ContentType, + Expires: request.Expires, + Metadata: request.Metadata, + StorageClass: request.StorageClass, + Tagging: request.Tagging, + ForbidOverwrite: request.ForbidOverwrite, + GrantRead: request.GrantRead, + GrantFullControl: request.GrantFullControl, + ServerSideEncryption: request.ServerSideEncryption, + SSECustomerAlgorithm: request.SSECustomerAlgorithm, + SSECustomerKey: request.SSECustomerKey, + SSECustomerKeyMD5: request.SSECustomerKeyMD5, + ProgressFn: request.ProgressFn, + }) + if err != nil { + return nil, err + } + + return &UploadFileOutput{ + Bucket: request.Bucket, + Key: request.Key, + ETag: resp.ETag, + ChecksumCRC64ECMA: resp.Metadata[HTTPHeaderAmzChecksumCrc64ecma], + }, nil +} + +func (u *Uploader) multipartUpload() (*UploadFileOutput, error) { + ucp, err := newUploadCheckpoint(u) + if err != nil { + return nil, err + } + u.uploadCheckpoint = ucp + + if aws.ToBoolean(u.uploadFileRequest.EnableCheckpoint) { + cpFilePath := aws.ToString(u.uploadFileRequest.CheckpointFile) + if cpFilePath == "" { + cpFilePath, err = generateUploadCpFilePath(u.uploadFileRequest) + if err != nil { + return nil, err + } + } + ucp.CpFilePath = cpFilePath + + err = ucp.load() + if err != nil { + return nil, err + } + + if ucp.UploadId != "" && !u.isUploadIdValid() { + ucp.UploadId = "" + ucp.PartETagList = make([]*CompletedPart, 0) + ucp.remove() + } + } + + if ucp.UploadId == "" { + ucp.UploadId, err = u.initUploadId() + if err != nil { + return nil, err + } + ucp.dump() + } + + fileSize := ucp.UploadFileSize + partSize := ucp.PartSize + totalPartNum := (fileSize-1)/partSize + 1 + tasks := make(chan UploadPartTask, totalPartNum) + + var i int64 + for i = 0; i < totalPartNum; i++ { + partNum := i + 1 + offset := i * partSize + actualPartSize := u.getActualPartSize(fileSize, partSize, partNum) + partETag := u.getPartETag(partNum) + if partETag != nil { + u.publishProgress(actualPartSize) + } else { + uploadPartTask := UploadPartTask{ + partNumber: partNum, + offset: offset, + actualPartSize: actualPartSize, + } + tasks <- uploadPartTask + } + } + close(tasks) + + var wg sync.WaitGroup + for i = 0; i < aws.ToLong(u.uploadFileRequest.TaskNum); i++ { + wg.Add(1) + go u.runTask(tasks, &wg) + } + wg.Wait() + + if u.error != nil { + return nil, u.error + } + + completedMultipartUpload := u.getMultipartUploadParts() + resp, err := u.completeMultipartUpload(completedMultipartUpload) + if err != nil { + return nil, err + } + + if u.client.Config.CrcCheckEnabled { + clientCrc64 := u.getCrc64Ecma(completedMultipartUpload.Parts) + serverCrc64, _ := strconv.ParseUint(aws.ToString(resp.ChecksumCRC64ECMA), 10, 64) + u.client.Config.LogDebug("check file crc64, client crc64:%d, server crc64:%d", clientCrc64, serverCrc64) + if serverCrc64 != 0 && clientCrc64 != serverCrc64 { + return nil, errors.New(fmt.Sprintf("crc64 check failed, client crc64:%d, server crc64:%d", clientCrc64, serverCrc64)) + } + } + + return u.getUploadFileOutput(resp), nil +} + +func (u *Uploader) getUploadFileOutput(resp *CompleteMultipartUploadOutput) *UploadFileOutput { + return &UploadFileOutput{ + Bucket: resp.Bucket, + Key: resp.Key, + ETag: resp.ETag, + ChecksumCRC64ECMA: resp.ChecksumCRC64ECMA, + } +} + +func (u *Uploader) getPartSize(fileSize int64, originPartSize int64) int64 { + partSize := originPartSize + totalPartNum := (fileSize-1)/partSize + 1 + for totalPartNum > MaxPartNum { + partSize += originPartSize + totalPartNum = (fileSize-1)/partSize + 1 + } + return partSize +} + +func (u *Uploader) getActualPartSize(fileSize int64, partSize int64, partNum int64) int64 { + offset := (partNum - 1) * partSize + actualPartSize := partSize + if offset+partSize >= fileSize { + actualPartSize = fileSize - offset + } + return actualPartSize +} + +func (u *Uploader) getPartETag(partNumber int64) *CompletedPart { + for _, partETag := range u.uploadCheckpoint.PartETagList { + if *partETag.PartNumber == partNumber { + return partETag + } + } + return nil +} + +type UploadPartTask struct { + partNumber int64 + + offset int64 + + actualPartSize int64 +} + +func (u *Uploader) runTask(tasks <-chan UploadPartTask, wg *sync.WaitGroup) { + defer wg.Done() + for task := range tasks { + if u.error != nil { + return + } + + partETag, err := u.uploadPart(task) + if err != nil { + u.setError(err) + return + } + + u.updatePart(partETag) + } +} + +func (u *Uploader) uploadPart(task UploadPartTask) (CompletedPart, error) { + request := u.uploadFileRequest + ucp := u.uploadCheckpoint + offset := task.offset + actualPartSize := task.actualPartSize + var partETag CompletedPart + var reader io.ReadSeeker + if ucp.UploadFilePath != "" { + fd, err := os.Open(ucp.UploadFilePath) + if err != nil { + return partETag, err + } + defer fd.Close() + + reader = io.NewSectionReader(fd, offset, actualPartSize) + } else { + var err error + reader, err = (*u.uploadFileRequest.FilePartFetcher).Fetch([]int64{offset, offset + actualPartSize - 1}) + if err != nil { + return partETag, err + } + } + + resp, err := u.client.UploadPartWithContext(u.context, &UploadPartInput{ + Bucket: aws.String(ucp.BucketName), + Key: aws.String(ucp.ObjectKey), + UploadID: aws.String(ucp.UploadId), + PartNumber: aws.Long(task.partNumber), + Body: reader, + ContentLength: aws.Long(actualPartSize), + SSECustomerAlgorithm: request.SSECustomerAlgorithm, + SSECustomerKey: request.SSECustomerKey, + SSECustomerKeyMD5: request.SSECustomerKeyMD5, + }) + + if err != nil { + return partETag, err + } + + partETag.PartNumber = aws.Long(task.partNumber) + partETag.ETag = resp.ETag + partETag.ChecksumCRC64ECMA = resp.ChecksumCRC64ECMA + u.publishProgress(actualPartSize) + + return partETag, nil +} + +func (u *Uploader) updatePart(partETag CompletedPart) { + u.mu.Lock() + defer u.mu.Unlock() + + u.uploadCheckpoint.PartETagList = append(u.uploadCheckpoint.PartETagList, &partETag) + u.uploadCheckpoint.dump() +} + +func (u *Uploader) setError(err error) { + u.mu.Lock() + defer u.mu.Unlock() + + if u.error == nil { + u.error = err + } +} + +type CompletedParts []*CompletedPart + +func (cp CompletedParts) Len() int { + return len(cp) +} + +func (cp CompletedParts) Less(i, j int) bool { + return *cp[i].PartNumber < *cp[j].PartNumber +} + +func (cp CompletedParts) Swap(i, j int) { + cp[i], cp[j] = cp[j], cp[i] +} + +func (u *Uploader) getMultipartUploadParts() *CompletedMultipartUpload { + partETags := u.uploadCheckpoint.PartETagList + // 按照PartNumber排序 + sort.Sort(CompletedParts(partETags)) + return &CompletedMultipartUpload{ + Parts: partETags, + } +} + +func (u *Uploader) completeMultipartUpload(completedMultipartUpload *CompletedMultipartUpload) (*CompleteMultipartUploadOutput, error) { + resp, err := u.client.CompleteMultipartUploadWithContext(u.context, &CompleteMultipartUploadInput{ + Bucket: u.uploadFileRequest.Bucket, + Key: u.uploadFileRequest.Key, + UploadID: aws.String(u.uploadCheckpoint.UploadId), + MultipartUpload: completedMultipartUpload, + ForbidOverwrite: u.uploadFileRequest.ForbidOverwrite, + }) + if err != nil { + return nil, err + } + u.uploadCheckpoint.remove() + return resp, err +} + +func (u *Uploader) publishProgress(actualPartSize int64) { + if u.uploadFileRequest.ProgressFn != nil { + atomic.AddInt64(&u.CompletedSize, actualPartSize) + u.uploadFileRequest.ProgressFn(actualPartSize, u.CompletedSize, aws.ToLong(u.uploadFileRequest.FileSize)) + } +} + +func (u *Uploader) getCrc64Ecma(parts []*CompletedPart) uint64 { + if parts == nil || len(parts) == 0 { + return 0 + } + + fileSize := u.uploadCheckpoint.UploadFileSize + partSize := u.uploadCheckpoint.PartSize + + crcTemp, _ := strconv.ParseUint(*parts[0].ChecksumCRC64ECMA, 10, 64) + for i := 1; i < len(parts); i++ { + crc2, _ := strconv.ParseUint(*parts[i].ChecksumCRC64ECMA, 10, 64) + actualPartSize := u.getActualPartSize(fileSize, partSize, *parts[i].PartNumber) + crcTemp = crc.CRC64Combine(crcTemp, crc2, (uint64)(actualPartSize)) + } + + return crcTemp +} + +func (u *Uploader) initUploadId() (string, error) { + request := u.uploadFileRequest + resp, err := u.client.CreateMultipartUploadWithContext(u.context, &CreateMultipartUploadInput{ + Bucket: request.Bucket, + Key: request.Key, + ACL: request.ACL, + CacheControl: request.CacheControl, + ContentDisposition: request.ContentDisposition, + ContentEncoding: request.ContentEncoding, + ContentType: request.ContentType, + Expires: request.Expires, + Metadata: request.Metadata, + StorageClass: request.StorageClass, + Tagging: request.Tagging, + ForbidOverwrite: request.ForbidOverwrite, + GrantRead: request.GrantRead, + GrantFullControl: request.GrantFullControl, + ServerSideEncryption: request.ServerSideEncryption, + SSECustomerAlgorithm: request.SSECustomerAlgorithm, + SSECustomerKey: request.SSECustomerKey, + SSECustomerKeyMD5: request.SSECustomerKeyMD5, + }) + if err != nil { + return "", err + } + return aws.ToString(resp.UploadID), nil +} + +func (u *Uploader) isUploadIdValid() bool { + _, err := u.client.ListPartsWithContext(u.context, &ListPartsInput{ + Bucket: u.uploadFileRequest.Bucket, + Key: u.uploadFileRequest.Key, + UploadID: aws.String(u.uploadCheckpoint.UploadId), + }) + if err != nil && strings.Contains(err.Error(), "NoSuchUpload") { + return false + } + return true +} + +func (u *Uploader) normalizeUploadPath() error { + uploadPath := aws.ToString(u.uploadFileRequest.UploadFile) + if uploadPath == "" { + return nil + } + // 规范化路径 + normalizedPath := filepath.Clean(uploadPath) + // 获取绝对路径 + absPath, err := filepath.Abs(normalizedPath) + if err != nil { + return err + } + u.uploadFileRequest.UploadFile = aws.String(absPath) + + return nil +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/util.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/util.go new file mode 100644 index 0000000000..1a5800df5d --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/util.go @@ -0,0 +1,108 @@ +package s3 + +import ( + "crypto/md5" + "encoding/base64" + "io" + "net/url" + "os" +) + +// GetBase64MD5Str 计算Base64格式字符串的MD5值 +func GetBase64MD5Str(str string) string { + // 创建一个MD5哈希对象 + hash := md5.New() + + // 将字符串转换为字节数组并计算MD5哈希值 + hash.Write([]byte(str)) + md5Hash := hash.Sum(nil) + + // 将MD5哈希值转换为Base64格式 + base64Str := base64.StdEncoding.EncodeToString(md5Hash) + + return base64Str +} + +// GetBase64Str 计算Base64格式字符串 +func GetBase64Str(str string) string { + return base64.StdEncoding.EncodeToString([]byte(str)) +} + +// GetBase64FileMD5Str 计算Base64格式文件的MD5值 +func GetBase64FileMD5Str(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + hash := md5.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + + md5Hash := hash.Sum(nil) + + // 将MD5哈希值转换为Base64格式 + base64Str := base64.StdEncoding.EncodeToString(md5Hash) + + return base64Str, err +} + +// BuildCopySource 构建拷贝源 +func BuildCopySource(bucket *string, key *string) string { + if bucket == nil || key == nil { + return "" + } + return "/" + *bucket + "/" + url.QueryEscape(*key) +} + +// GetCannedACL 获取访问控制权限 +func GetCannedACL(Grants []*Grant) string { + allUsersPermissions := map[string]*string{} + for _, value := range Grants { + if value.Grantee.URI != nil && *value.Grantee.URI == AllUsersUri { + allUsersPermissions[*value.Permission] = value.Permission + } + } + _, read := allUsersPermissions["READ"] + _, write := allUsersPermissions["WRITE"] + if read && write { + return ACLPublicReadWrite + } else if read { + return ACLPublicRead + } else { + return ACLPrivate + } +} + +// FileExists returns whether the given file exists or not +func FileExists(filename string) bool { + info, err := os.Stat(filename) + if os.IsNotExist(err) { + return false + } + return !info.IsDir() +} + +// DirExists returns whether the given directory exists or not +func DirExists(dir string) bool { + info, err := os.Stat(dir) + if os.IsNotExist(err) { + return false + } + return info.IsDir() +} + +// Min returns the smaller of two integers +func Min(a, b int64) int64 { + if a < b { + return a + } + return b +} + +// IsV4Signature checks if the signer is V4 or V4_UNSIGNED_PAYLOAD_SIGNER +func IsV4Signature(signer string) bool { + return signer == "V4" || signer == "V4_UNSIGNED_PAYLOAD_SIGNER" +} diff --git a/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/vpc_access_block.go b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/vpc_access_block.go new file mode 100644 index 0000000000..469608468b --- /dev/null +++ b/vendor/github.com/ks3sdklib/aws-sdk-go/service/s3/vpc_access_block.go @@ -0,0 +1,206 @@ +package s3 + +import ( + "github.com/ks3sdklib/aws-sdk-go/aws" + "time" +) + +type PutVpcAccessBlockInput struct { + // Vpc access block configuration container. + VpcAccessBlockConfiguration *VpcAccessBlockConfiguration `locationName:"VpcAccessBlockConfiguration" type:"structure" required:"true"` + + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` + + metadataPutVpcAccessBlockInput `json:"-" xml:"-"` +} + +type metadataPutVpcAccessBlockInput struct { + SDKShapeTraits bool `type:"structure" payload:"VpcAccessBlockConfiguration"` + + AutoFillMD5 bool +} + +type VpcAccessBlockConfiguration struct { + // Set up VPC to access the KS3 public Region rules container. + Rules []*VpcAccessBlockRule `locationName:"Rule" type:"list" flattened:"true" required:"true"` +} + +type VpcAccessBlockRule struct { + // The unique identifier of a Rule. The ID cannot be repeated in a rule. + RuleID *string `locationName:"RuleID" type:"string" required:"true"` + + // Region to which the VPC belongs. + Region *string `locationName:"Region" type:"string" required:"true"` + + // Set the VPC ID of the container. + VPC *VPC `locationName:"VPC" type:"structure"` + + // Set the bucket's container. + BucketAllowAccess *BucketAllowAccess `locationName:"BucketAllowAccess" type:"structure"` + + // Whether to enable this rule. + Status *string `locationName:"Status" type:"string" required:"true"` + + // Creation time. + CreationDate *time.Time `locationName:"CreationDate" type:"timestamp" timestampFormat:"iso8601"` +} + +type VPC struct { + // List of VPC IDs that are not allowed to access resources in this Region. + IDs []string `locationName:"ID" type:"list" flattened:"true"` +} + +type BucketAllowAccess struct { + // List of Bucket names that are allowed to be accessed. + Names []string `locationName:"Name" type:"list" flattened:"true"` +} + +type PutVpcAccessBlockOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// PutVpcAccessBlockRequest generates a request for the PutVpcAccessBlock operation. +func (c *S3) PutVpcAccessBlockRequest(input *PutVpcAccessBlockInput) (req *aws.Request, output *PutVpcAccessBlockOutput) { + op := &aws.Operation{ + Name: "PutVpcAccessBlock", + HTTPMethod: "PUT", + HTTPPath: "/?VpcAccessBlock", + } + + if input == nil { + input = &PutVpcAccessBlockInput{} + } + + input.AutoFillMD5 = true + req = c.newRequest(op, input, output) + output = &PutVpcAccessBlockOutput{} + req.Data = output + return +} + +// PutVpcAccessBlock sets vpc access block configuration. +func (c *S3) PutVpcAccessBlock(input *PutVpcAccessBlockInput) (*PutVpcAccessBlockOutput, error) { + req, out := c.PutVpcAccessBlockRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) PutVpcAccessBlockWithContext(ctx aws.Context, input *PutVpcAccessBlockInput) (*PutVpcAccessBlockOutput, error) { + req, out := c.PutVpcAccessBlockRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type GetVpcAccessBlockInput struct { + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type GetVpcAccessBlockOutput struct { + // Vpc access block configuration container. + VpcAccessBlockConfiguration *VpcAccessBlockConfiguration `locationName:"VpcAccessBlockConfiguration" type:"structure"` + + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` + + metadataGetVpcAccessBlockOutput `json:"-" xml:"-"` +} + +type metadataGetVpcAccessBlockOutput struct { + SDKShapeTraits bool `type:"structure" payload:"VpcAccessBlockConfiguration"` +} + +// GetVpcAccessBlockRequest generates a request for the GetVpcAccessBlock operation. +func (c *S3) GetVpcAccessBlockRequest(input *GetVpcAccessBlockInput) (req *aws.Request, output *GetVpcAccessBlockOutput) { + op := &aws.Operation{ + Name: "GetVpcAccessBlock", + HTTPMethod: "GET", + HTTPPath: "/?VpcAccessBlock", + } + + if input == nil { + input = &GetVpcAccessBlockInput{} + } + + req = c.newRequest(op, input, output) + output = &GetVpcAccessBlockOutput{} + req.Data = output + return +} + +// GetVpcAccessBlock gets vpc access block configuration. +func (c *S3) GetVpcAccessBlock(input *GetVpcAccessBlockInput) (*GetVpcAccessBlockOutput, error) { + req, out := c.GetVpcAccessBlockRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) GetVpcAccessBlockWithContext(ctx aws.Context, input *GetVpcAccessBlockInput) (*GetVpcAccessBlockOutput, error) { + req, out := c.GetVpcAccessBlockRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} + +type DeleteVpcAccessBlockInput struct { + // Set extend request headers. If the existing fields do not support setting the request header you need, you can set it through this field. + ExtendHeaders map[string]*string `location:"extendHeaders" type:"map"` + + // Set extend query params. If the existing fields do not support setting the query param you need, you can set it through this field. + ExtendQueryParams map[string]*string `location:"extendQueryParams" type:"map"` +} + +type DeleteVpcAccessBlockOutput struct { + // The HTTP headers of the response. + Metadata map[string]*string `location:"headers" type:"map"` + + // The HTTP status code of the response. + StatusCode *int64 `location:"statusCode" type:"integer"` +} + +// DeleteVpcAccessBlockRequest generates a request for the DeleteVpcAccessBlock operation. +func (c *S3) DeleteVpcAccessBlockRequest(input *DeleteVpcAccessBlockInput) (req *aws.Request, output *DeleteVpcAccessBlockOutput) { + op := &aws.Operation{ + Name: "DeleteVpcAccessBlock", + HTTPMethod: "DELETE", + HTTPPath: "/?VpcAccessBlock", + } + + if input == nil { + input = &DeleteVpcAccessBlockInput{} + } + + req = c.newRequest(op, input, output) + output = &DeleteVpcAccessBlockOutput{} + req.Data = output + return +} + +// DeleteVpcAccessBlock deletes vpc access block configuration. +func (c *S3) DeleteVpcAccessBlock(input *DeleteVpcAccessBlockInput) (*DeleteVpcAccessBlockOutput, error) { + req, out := c.DeleteVpcAccessBlockRequest(input) + err := req.Send() + return out, err +} + +func (c *S3) DeleteVpcAccessBlockWithContext(ctx aws.Context, input *DeleteVpcAccessBlockInput) (*DeleteVpcAccessBlockOutput, error) { + req, out := c.DeleteVpcAccessBlockRequest(input) + req.SetContext(ctx) + err := req.Send() + return out, err +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7cb77b7118..9c19805598 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -849,6 +849,29 @@ github.com/koding/websocketproxy # github.com/kr/fs v0.1.0 ## explicit github.com/kr/fs +# github.com/ks3sdklib/aws-sdk-go v1.8.1 +## explicit; go 1.16 +github.com/ks3sdklib/aws-sdk-go/aws +github.com/ks3sdklib/aws-sdk-go/aws/awserr +github.com/ks3sdklib/aws-sdk-go/aws/awsutil +github.com/ks3sdklib/aws-sdk-go/aws/credentials +github.com/ks3sdklib/aws-sdk-go/aws/retry +github.com/ks3sdklib/aws-sdk-go/internal/apierr +github.com/ks3sdklib/aws-sdk-go/internal/crc +github.com/ks3sdklib/aws-sdk-go/internal/endpoints +github.com/ks3sdklib/aws-sdk-go/internal/protocol/body +github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil +github.com/ks3sdklib/aws-sdk-go/internal/protocol/jsonrpc +github.com/ks3sdklib/aws-sdk-go/internal/protocol/query +github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/queryutil +github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest +github.com/ks3sdklib/aws-sdk-go/internal/protocol/restjson +github.com/ks3sdklib/aws-sdk-go/internal/protocol/restxml +github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil +github.com/ks3sdklib/aws-sdk-go/internal/signer/open_api +github.com/ks3sdklib/aws-sdk-go/internal/signer/v2 +github.com/ks3sdklib/aws-sdk-go/internal/signer/v4 +github.com/ks3sdklib/aws-sdk-go/service/s3 # github.com/leodido/go-urn v1.2.0 ## explicit; go 1.13 github.com/leodido/go-urn @@ -1877,7 +1900,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250915054625-7251d9eeceec +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250924031606-a2597eb1f66f ## explicit; go 1.24 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing @@ -2028,7 +2051,7 @@ yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 ## explicit; go 1.12 yunion.io/x/s3cli -# yunion.io/x/sqlchemy v1.1.3-0.20250902120005-3b18bc6371c7 +# yunion.io/x/sqlchemy v1.1.3-0.20250923143347-b98e89dbc372 ## explicit; go 1.17 yunion.io/x/sqlchemy yunion.io/x/sqlchemy/backends diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_cluster.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_cluster.go index fc11184222..7ae25c25b9 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_cluster.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_cluster.go @@ -231,7 +231,16 @@ func (rds *SDBInstanceCluster) GetIDBInstanceAccounts() ([]cloudprovider.ICloudD } func (rds *SDBInstanceCluster) GetIDBInstanceBackups() ([]cloudprovider.ICloudDBInstanceBackup, error) { - return nil, cloudprovider.ErrNotSupported + ret := []cloudprovider.ICloudDBInstanceBackup{} + backups, err := rds.region.DescribeDBClusterSnapshots(rds.DBClusterIdentifier, "") + if err != nil { + return nil, err + } + for i := range backups { + backups[i].region = rds.region + ret = append(ret, &backups[i]) + } + return ret, nil } func (rds *SDBInstanceCluster) ChangeConfig(ctx context.Context, config *cloudprovider.SManagedDBInstanceChangeConfig) error { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_cluster_snapshots.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_cluster_snapshots.go new file mode 100644 index 0000000000..f007a71d80 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_cluster_snapshots.go @@ -0,0 +1,172 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aws + +import ( + "time" + + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" +) + +type SDBInstanceClusterSnapshot struct { + multicloud.SDBInstanceBackupBase + AwsTags + + region *SRegion + + AllocatedStorage int `xml:"AllocatedStorage"` + AvailabilityZones []string `xml:"AvailabilityZones>AvailabilityZone"` + EngineMode string `xml:"EngineMode"` + Engine string `xml:"Engine"` + PercentProgress int `xml:"PercentProgress"` + VpcId string `xml:"VpcId"` + IAMDatabaseAuthenticationEnabled bool `xml:"IAMDatabaseAuthenticationEnabled"` + DBClusterSnapshotIdentifier string `xml:"DBClusterSnapshotIdentifier"` + ClusterCreateTime time.Time `xml:"ClusterCreateTime"` + EngineVersion string `xml:"EngineVersion"` + MasterUsername string `xml:"MasterUsername"` + SnapshotType string `xml:"SnapshotType"` + StorageEncrypted bool `xml:"StorageEncrypted"` + TagList []struct { + Value string `xml:"Value"` + Key string `xml:"Key"` + } `xml:"TagList>Tag"` + Port int `xml:"Port"` + SnapshotCreateTime time.Time `xml:"SnapshotCreateTime"` + LicenseModel string `xml:"LicenseModel"` + KmsKeyId string `xml:"KmsKeyId"` + DBClusterIdentifier string `xml:"DBClusterIdentifier"` + DBClusterSnapshotArn string `xml:"DBClusterSnapshotArn"` + DbClusterResourceId string `xml:"DbClusterResourceId"` + Status string `xml:"Status"` +} + +func (snapshot *SDBInstanceClusterSnapshot) GetId() string { + return snapshot.DBClusterSnapshotArn +} + +func (snapshot *SDBInstanceClusterSnapshot) GetGlobalId() string { + return snapshot.DBClusterSnapshotArn +} + +func (snapshot *SDBInstanceClusterSnapshot) GetName() string { + return snapshot.DBClusterSnapshotIdentifier +} + +func (snapshot *SDBInstanceClusterSnapshot) GetEngine() string { + return snapshot.Engine +} + +func (snapshot *SDBInstanceClusterSnapshot) GetEngineVersion() string { + return snapshot.EngineVersion +} + +func (snapshot *SDBInstanceClusterSnapshot) GetStartTime() time.Time { + return snapshot.SnapshotCreateTime +} + +func (snapshot *SDBInstanceClusterSnapshot) GetEndTime() time.Time { + return snapshot.SnapshotCreateTime +} + +func (snapshot *SDBInstanceClusterSnapshot) GetBackupMode() string { + switch snapshot.SnapshotType { + case "manual": + return api.BACKUP_MODE_MANUAL + default: + return api.BACKUP_MODE_AUTOMATED + } +} + +func (snapshot *SDBInstanceClusterSnapshot) GetStatus() string { + switch snapshot.Status { + case "available": + return api.DBINSTANCE_BACKUP_READY + default: + log.Errorf("unknown dbinstance snapshot status: %s", snapshot.Status) + return api.DBINSTANCE_BACKUP_UNKNOWN + } +} + +func (self *SDBInstanceClusterSnapshot) Refresh() error { + snap, err := self.region.GetDBClusterSnapshot(self.DBClusterSnapshotIdentifier) + if err != nil { + return err + } + return jsonutils.Update(self, snap) +} + +func (snapshot *SDBInstanceClusterSnapshot) GetBackupSizeMb() int { + return snapshot.AllocatedStorage * 1024 +} + +func (snapshot *SDBInstanceClusterSnapshot) GetDBNames() string { + return "" +} + +func (snapshot *SDBInstanceClusterSnapshot) GetDBInstanceId() string { + return snapshot.DBClusterIdentifier +} + +func (self *SDBInstanceClusterSnapshot) Delete() error { + return self.region.DeleteRdsSnapshot(self.DBClusterSnapshotIdentifier) +} + +func (self *SRegion) GetDBClusterSnapshot(id string) (*SDBInstanceClusterSnapshot, error) { + snapshots, err := self.DescribeDBClusterSnapshots("", id) + if err != nil { + return nil, err + } + for i := range snapshots { + if snapshots[i].GetGlobalId() == id { + return &snapshots[i], nil + } + } + return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s", id) +} + +func (region *SRegion) DescribeDBClusterSnapshots(clusterId, snapshotId string) ([]SDBInstanceClusterSnapshot, error) { + params := map[string]string{} + if len(clusterId) > 0 { + params["DBClusterIdentifier"] = clusterId + } + if len(snapshotId) > 0 { + params["DBClusterSnapshotIdentifier"] = snapshotId + } + ret := []SDBInstanceClusterSnapshot{} + for { + part := struct { + DBClusterSnapshots struct { + DBClusterSnapshot []SDBInstanceClusterSnapshot `xml:"DBClusterSnapshot"` + } + Marker string `xml:"Marker"` + }{} + err := region.rdsRequest("DescribeDBClusterSnapshots", params, &part) + if err != nil { + return nil, err + } + ret = append(ret, part.DBClusterSnapshots.DBClusterSnapshot...) + if len(part.DBClusterSnapshots.DBClusterSnapshot) == 0 || len(part.Marker) == 0 { + break + } + params["Marker"] = part.Marker + } + return ret, nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_snapshot.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_snapshot.go index 3f4dcb1008..b29ffe5f42 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_snapshot.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/dbinstance_snapshot.go @@ -190,5 +190,13 @@ func (region *SRegion) GetIDBInstanceBackups() ([]cloudprovider.ICloudDBInstance snapshots[i].region = region isnapshots = append(isnapshots, &snapshots[i]) } + clusterSnapshots, err := region.DescribeDBClusterSnapshots("", "") + if err != nil { + return nil, errors.Wrap(err, "DescribeDBClusterSnapshots") + } + for i := 0; i < len(clusterSnapshots); i++ { + clusterSnapshots[i].region = region + isnapshots = append(isnapshots, &clusterSnapshots[i]) + } return isnapshots, nil } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go index 501eccd63e..2f391846c7 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go @@ -616,7 +616,7 @@ func (region *SRegion) _createVM(zone string, desc *cloudprovider.SManagedVMCrea if err != nil { return nil, errors.Wrap(err, "region.GetNetwork") } - networkInterface["network"] = vpc.SelfLink + networkInterface["subnetwork"] = getGlobalId(vpc.SelfLink) } if len(desc.IpAddr) > 0 { networkInterface["networkIp"] = desc.IpAddr diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/bucket.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/bucket.go new file mode 100644 index 0000000000..7484897188 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/bucket.go @@ -0,0 +1,599 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ksyun + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/ks3sdklib/aws-sdk-go/aws" + "github.com/ks3sdklib/aws-sdk-go/aws/credentials" + "github.com/ks3sdklib/aws-sdk-go/service/s3" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/fileutils" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" +) + +type SBucket struct { + multicloud.SBaseBucket + SKsyunTags + + region *SRegion + + CreationDate time.Time + Name string + Region string + Type string + VisitType string + DataRedundancyType string +} + +func (b *SBucket) GetProjectId() string { + return "" +} + +func (b *SBucket) GetGlobalId() string { + return b.Name +} + +func (b *SBucket) GetName() string { + return b.Name +} + +func (b *SBucket) GetAcl() cloudprovider.TBucketACLType { + return b.region.GetBucketAcl(b.Name) +} + +func (region *SRegion) GetBucketAcl(bucket string) cloudprovider.TBucketACLType { + svc := region.getS3Client() + input := &s3.GetBucketACLInput{ + Bucket: &bucket, + } + resp, err := svc.GetBucketACL(input) + if err != nil { + return cloudprovider.ACLUnknown + } + return cloudprovider.TBucketACLType(s3.GetCannedACL(resp.Grants)) +} + +func (b *SBucket) GetLocation() string { + return b.Region +} + +func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion { + return b.region +} + +func (b *SBucket) GetCreatedAt() time.Time { + return b.CreationDate +} + +func (b *SBucket) GetStorageClass() string { + return b.DataRedundancyType +} + +// https://docs.ksyun.com/documents/6761 +func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { + ret := []cloudprovider.SBucketAccessUrl{ + { + Url: fmt.Sprintf("%s.%s", b.Name, s3RegionEndpointMap[b.region.Region]), + Description: "ExtranetEndpoint", + Primary: true, + }, + { + Url: fmt.Sprintf("%s.%s", b.Name, strings.Replace(s3RegionEndpointMap[b.region.Region], ".ksyuncs.com", "-internal.ksyuncs.com", 1)), + Description: "IntranetEndpoint", + }, + } + return ret +} + +func (b *SBucket) GetStats() cloudprovider.SBucketStats { + stats, err := cloudprovider.GetIBucketStats(b) + if err != nil { + log.Errorf("GetStats fail %s", err) + } + return stats +} + +func (region *SRegion) SetBucketAcl(bucket string, acl cloudprovider.TBucketACLType) error { + svc := region.getS3Client() + aclStr := string(acl) + input := &s3.PutBucketACLInput{ + Bucket: &bucket, + ACL: &aclStr, + } + _, err := svc.PutBucketACL(input) + return err +} + +func (b *SBucket) SetAcl(aclStr cloudprovider.TBucketACLType) error { + return b.region.SetBucketAcl(b.Name, aclStr) +} + +func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + result := cloudprovider.SListObjectResult{} + svc := b.region.getS3Client() + input := &s3.ListObjectsInput{ + Bucket: &b.Name, + } + if len(prefix) > 0 { + input.Prefix = &prefix + } + if len(delimiter) > 0 { + input.Delimiter = &delimiter + } + if len(marker) > 0 { + input.Marker = &marker + } + if maxCount > 0 { + cnt := int64(maxCount) + input.MaxKeys = &cnt + } + resp, err := svc.ListObjects(input) + if err != nil { + return result, err + } + result.Objects = make([]cloudprovider.ICloudObject, 0) + for _, object := range resp.Contents { + ksObj := cloudprovider.SBaseCloudObject{} + if object.StorageClass != nil { + ksObj.StorageClass = *object.StorageClass + } + if object.Key != nil { + ksObj.Key = *object.Key + } + if object.Size != nil { + ksObj.SizeBytes = *object.Size + } + if object.ETag != nil { + ksObj.ETag = *object.ETag + } + if object.LastModified != nil { + ksObj.LastModified = *object.LastModified + } + + obj := &SObject{ + bucket: b, + SBaseCloudObject: ksObj, + } + result.Objects = append(result.Objects, obj) + } + if resp.CommonPrefixes != nil { + result.CommonPrefixes = make([]cloudprovider.ICloudObject, len(resp.CommonPrefixes)) + for i, commonPrefix := range resp.CommonPrefixes { + result.CommonPrefixes[i] = &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{Key: *commonPrefix.Prefix}, + } + } + } + if resp.IsTruncated != nil { + result.IsTruncated = *resp.IsTruncated + } + if resp.NextMarker != nil { + result.NextMarker = *resp.NextMarker + } + return result, nil +} + +func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, sizeBytes int64, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { + svc := b.region.getS3Client() + input := &s3.PutObjectInput{ + Bucket: &b.Name, + Key: &key, + } + if sizeBytes > 0 { + input.ContentLength = &sizeBytes + } + if meta != nil { + input.Metadata = make(map[string]*string) + for k, v := range meta { + input.Metadata[k] = &v[0] + } + } + if len(cannedAcl) == 0 { + cannedAcl = b.GetAcl() + } + acl := string(cannedAcl) + input.ACL = &acl + if len(storageClassStr) > 0 { + storageClass := string(storageClassStr) + input.StorageClass = &storageClass + } + seeker, err := fileutils.NewReadSeeker(reader, sizeBytes) + if err != nil { + return errors.Wrap(err, "newFakeSeeker") + } + defer seeker.Close() + input.Body = seeker + _, err = svc.PutObject(input) + if err != nil { + return errors.Wrap(err, "PutObject") + } + return nil +} + +func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) (string, error) { + svc := b.region.getS3Client() + input := &s3.CreateMultipartUploadInput{ + Bucket: &b.Name, + Key: &key, + } + if meta != nil { + input.Metadata = make(map[string]*string) + for k, v := range meta { + input.Metadata[k] = &v[0] + } + } + if len(cannedAcl) == 0 { + cannedAcl = b.GetAcl() + } + acl := string(cannedAcl) + input.ACL = &acl + if len(storageClassStr) > 0 { + storageClass := string(storageClassStr) + input.StorageClass = &storageClass + } + output, err := svc.CreateMultipartUpload(input) + if err != nil { + return "", errors.Wrap(err, "CreateMultipartUpload") + } + return *output.UploadID, nil +} + +func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, reader io.Reader, partSize int64, offset, totalSize int64) (string, error) { + svc := b.region.getS3Client() + pn := int64(partIndex) + input := &s3.UploadPartInput{ + Bucket: &b.Name, + Key: &key, + UploadID: &uploadId, + PartNumber: &pn, + } + seeker, err := fileutils.NewReadSeeker(reader, partSize) + if err != nil { + return "", errors.Wrap(err, "newFakeSeeker") + } + defer seeker.Close() + input.Body = seeker + output, err := svc.UploadPart(input) + if err != nil { + return "", errors.Wrap(err, "UploadPart") + } + return *output.ETag, nil +} + +func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { + svc := b.region.getS3Client() + input := &s3.CompleteMultipartUploadInput{ + Bucket: &b.Name, + Key: &key, + UploadID: &uploadId, + MultipartUpload: &s3.CompletedMultipartUpload{}, + } + parts := make([]*s3.CompletedPart, len(partEtags)) + for i := range partEtags { + pn := int64(i + 1) + parts[i] = &s3.CompletedPart{ + PartNumber: &pn, + ETag: &partEtags[i], + } + } + input.MultipartUpload.Parts = parts + _, err := svc.CompleteMultipartUpload(input) + return err +} + +func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { + svc := b.region.getS3Client() + input := &s3.AbortMultipartUploadInput{ + Bucket: &b.Name, + Key: &key, + UploadID: &uploadId, + } + _, err := svc.AbortMultipartUpload(input) + return err +} + +func (b *SBucket) DeleteObject(ctx context.Context, key string) error { + svc := b.region.getS3Client() + input := &s3.DeleteObjectInput{ + Bucket: &b.Name, + Key: &key, + } + _, err := svc.DeleteObject(input) + return err +} + +func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + svc := b.region.getS3Client() + input := &s3.GeneratePresignedUrlInput{ + Bucket: &b.Name, + Key: &key, + } + input.HTTPMethod = s3.HTTPMethod(method) + input.Expires = int64(expire / time.Second) + return svc.GeneratePresignedUrl(input) +} + +func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { + svc := b.region.getS3Client() + input := &s3.CopyObjectInput{ + Bucket: &b.Name, + Key: &destKey, + SourceBucket: &srcBucket, + SourceKey: &srcKey, + } + if meta != nil { + input.Metadata = make(map[string]*string) + for k, v := range meta { + input.Metadata[k] = &v[0] + } + } + if cannedAcl != cloudprovider.ACLPrivate { + acl := string(cannedAcl) + input.ACL = &acl + } + if storageClassStr != "" { + storageClass := string(storageClassStr) + input.StorageClass = &storageClass + } + _, err := svc.CopyObject(input) + return err +} + +func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + svc := b.region.getS3Client() + input := &s3.GetObjectInput{ + Bucket: &b.Name, + Key: &key, + } + if rangeOpt != nil { + rangeInput := rangeOpt.String() + input.Range = &rangeInput + } + output, err := svc.GetObject(input) + if err != nil { + return nil, errors.Wrap(err, "GetObject") + } + return output.Body, nil +} + +func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + svc := b.region.getS3Client() + pn := int64(partNumber) + copySourceRange := fmt.Sprintf("bytes=%d-%d", srcOffset, srcOffset+srcLength-1) + input := &s3.UploadPartCopyInput{ + Bucket: &b.Name, + Key: &key, + UploadID: &uploadId, + PartNumber: &pn, + SourceBucket: &srcBucket, + SourceKey: &srcKey, + CopySourceRange: ©SourceRange, + } + output, err := svc.UploadPartCopy(input) + if err != nil { + return "", errors.Wrap(err, "UploadPartCopy") + } + return *output.CopyPartResult.ETag, nil + +} + +func (b *SBucket) GetTags() (map[string]string, error) { + svc := b.region.getS3Client() + input := &s3.GetBucketTaggingInput{ + Bucket: &b.Name, + } + output, err := svc.GetBucketTagging(input) + if err != nil { + return nil, errors.Wrap(err, "GetBucketTagging") + } + result := map[string]string{} + if output.Tagging == nil { + return nil, nil + } + for _, tag := range output.Tagging.TagSet { + result[*tag.Key] = *tag.Value + } + return result, nil +} + +func (b *SBucket) SetTags(tags map[string]string, replace bool) error { + svc := b.region.getS3Client() + input := &s3.PutBucketTaggingInput{ + Bucket: &b.Name, + } + if replace { + input.Tagging = &s3.Tagging{ + TagSet: make([]*s3.Tag, 0), + } + } + for k, v := range tags { + input.Tagging.TagSet = append(input.Tagging.TagSet, &s3.Tag{Key: &k, Value: &v}) + } + _, err := svc.PutBucketTagging(input) + log.Infof("put tagging %s error: %v", jsonutils.Marshal(input).String(), err) + if err != nil { + return errors.Wrapf(err, "PutBucketTagging(%s)", b.Name) + } + return nil +} + +func (b *SBucket) ListMultipartUploads() ([]cloudprovider.SBucketMultipartUploads, error) { + result := []cloudprovider.SBucketMultipartUploads{} + svc := b.region.getS3Client() + keyMarker := "" + uploadIDMarker := "" + for { + input := &s3.ListMultipartUploadsInput{ + Bucket: &b.Name, + KeyMarker: &keyMarker, + UploadIDMarker: &uploadIDMarker, + } + output, err := svc.ListMultipartUploads(input) + if err != nil { + return nil, errors.Wrap(err, "ListMultipartUploads") + } + for _, upload := range output.Uploads { + result = append(result, cloudprovider.SBucketMultipartUploads{ + ObjectName: *upload.Key, + UploadID: *upload.UploadID, + Initiated: *upload.Initiated, + }) + } + keyMarker = *output.NextKeyMarker + uploadIDMarker = *output.NextUploadIDMarker + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + } + return result, nil +} + +func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { + buckets, err := region.client.GetBuckets() + if err != nil { + return nil, err + } + ret := make([]cloudprovider.ICloudBucket, 0) + for i := range buckets { + buckets[i].region = region + if strings.EqualFold(buckets[i].Region, s3RegionMap[region.Region]) { + ret = append(ret, &buckets[i]) + } + } + return ret, nil +} + +func (region *SRegion) getS3Client() *s3.S3 { + return region.client.getS3Client(region.Region) +} + +// https://docs.ksyun.com/documents/6761?type=3 +var s3RegionMap = map[string]string{ + "": "BEIJING", + "ap-singapore-1": "SINGAPORE", + "cn-beijing-6": "BEIJING", + "cn-guangzhou-1": "GUANGZHOU", + "cn-shanghai-2": "SHANGHAI", + "cn-northwest-1": "QINGYANG", +} +var s3RegionEndpointMap = map[string]string{ + "": "ks3-cn-beijing.ksyuncs.com", + "ap-singapore-1": "ks3-sgp.ksyuncs.com", + "cn-beijing-6": "ks3-cn-beijing.ksyuncs.com", + "cn-guangzhou-1": "ks3-cn-guangzhou.ksyuncs.com", + "cn-shanghai-2": "ks3-cn-shanghai.ksyuncs.com", + "cn-northwest-1": "ks3-cn-qingyang.ksyuncs.com", +} + +func (cli *SKsyunClient) getS3Client(regionId string) *s3.S3 { + aksk := credentials.NewStaticCredentials(cli.accessKeyId, cli.accessKeySecret, "") + cfg := aws.Config{ + Region: s3RegionMap[regionId], + Credentials: aksk, + Endpoint: s3RegionEndpointMap[regionId], + HTTPClient: cli.getDefaultClient(), + SignerVersion: "V4_UNSIGNED_PAYLOAD_SIGNER", + MaxRetries: 1, + } + if cli.debug { + cfg.LogLevel = aws.Debug + } + return s3.New(&cfg) +} + +func (cli *SKsyunClient) GetBuckets() ([]SBucket, error) { + svc := cli.getS3Client("") + input := &s3.ListBucketsInput{} + resp, err := svc.ListBuckets(input) + if err != nil { + return nil, err + } + ret := make([]SBucket, 0) + for _, b := range resp.Buckets { + ret = append(ret, SBucket{ + Name: *b.Name, + Region: *b.Region, + Type: *b.Type, + VisitType: *b.VisitType, + DataRedundancyType: *b.DataRedundancyType, + CreationDate: *b.CreationDate, + }) + } + return ret, nil +} + +func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error { + svc := region.getS3Client() + input := &s3.CreateBucketInput{ + Bucket: &name, + } + if aclStr != "" { + acl := string(aclStr) + input.ACL = &acl + } + _, err := svc.CreateBucket(input) + return err +} + +func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { + buckets, err := region.GetIBuckets() + if err != nil { + return nil, errors.Wrap(err, "region.GetBuckets") + } + for _, b := range buckets { + if b.GetName() == name { + return b, nil + } + } + return nil, errors.Wrap(cloudprovider.ErrNotFound, "Bucket Not Found") +} + +func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { + return region.GetIBucketById(name) +} + +func (region *SRegion) DeleteIBucket(name string) error { + svc := region.getS3Client() + input := &s3.DeleteBucketInput{ + Bucket: &name, + } + _, err := svc.DeleteBucket(input) + return err +} + +func (region *SRegion) IBucketExist(name string) (bool, error) { + svc := region.getS3Client() + input := &s3.HeadBucketInput{ + Bucket: &name, + } + _, err := svc.HeadBucket(input) + if err != nil { + return false, err + } + return true, nil +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/ksyun.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/ksyun.go index 841cb73051..76720f46da 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/ksyun.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/ksyun.go @@ -157,6 +157,10 @@ func (cli *SKsyunClient) getDefaultClient() *http.Client { return nil, nil } } + // ks3 + if len(action) == 0 && strings.Contains(req.URL.String(), "ks3-") { + return nil, nil + } if cli.cpcfg.ReadOnly { return nil, errors.Wrapf(cloudprovider.ErrAccountReadOnly, "%s %s", req.Method, req.URL.Path) } @@ -353,6 +357,7 @@ func (cli *SKsyunClient) GetCapabilities() []string { cloudprovider.CLOUD_CAPABILITY_COMPUTE + cloudprovider.READ_ONLY_SUFFIX, cloudprovider.CLOUD_CAPABILITY_PROJECT + cloudprovider.READ_ONLY_SUFFIX, cloudprovider.CLOUD_CAPABILITY_CLOUDID, + cloudprovider.CLOUD_CAPABILITY_OBJECTSTORE, cloudprovider.CLOUD_CAPABILITY_RDS + cloudprovider.READ_ONLY_SUFFIX, } return caps diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/object.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/object.go new file mode 100644 index 0000000000..79253a0542 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ksyun/object.go @@ -0,0 +1,84 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ksyun + +import ( + "context" + "net/http" + + "github.com/ks3sdklib/aws-sdk-go/service/s3" + + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/pkg/errors" +) + +type SObject struct { + bucket *SBucket + + cloudprovider.SBaseCloudObject +} + +func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { + return o.bucket +} + +func (o *SObject) GetAcl() cloudprovider.TBucketACLType { + svc := o.bucket.region.getS3Client() + input := &s3.GetObjectACLInput{ + Bucket: &o.bucket.Name, + Key: &o.Key, + } + output, err := svc.GetObjectACL(input) + if err != nil { + return cloudprovider.ACLPrivate + } + return cloudprovider.TBucketACLType(s3.GetCannedACL(output.Grants)) +} + +func (o *SObject) SetAcl(acl cloudprovider.TBucketACLType) error { + svc := o.bucket.region.getS3Client() + aclStr := string(acl) + input := &s3.PutObjectACLInput{ + Bucket: &o.bucket.Name, + Key: &o.Key, + ACL: &aclStr, + } + _, err := svc.PutObjectACL(input) + if err != nil { + return errors.Wrap(err, "PutObjectACL") + } + return nil +} + +func (o *SObject) GetMeta() http.Header { + svc := o.bucket.region.getS3Client() + input := &s3.GetObjectInput{ + Bucket: &o.bucket.Name, + Key: &o.Key, + } + output, err := svc.GetObject(input) + if err != nil { + return nil + } + meta := http.Header{} + for k, v := range output.Metadata { + meta.Add(k, *v) + } + return meta +} + +func (o *SObject) SetMeta(ctx context.Context, meta http.Header) error { + return cloudprovider.ObjectSetMeta(ctx, o.bucket, o, meta) +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/objectstore.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/objectstore.go index f93475cc53..3307a69128 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/objectstore.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/objectstore.go @@ -440,6 +440,13 @@ func (cli *SObjectStoreClient) CreateIBucket(name string, storageClass string, a return nil } +func (cli *SObjectStoreClient) SetBucketLifecycle(name string, lifecycle string) error { + if err := cli.client.SetBucketLifecycle(name, lifecycle); err != nil { + return errors.Wrap(err, "SetBucketLifecycle") + } + return nil +} + func minioErrCode(err error) int { if srvErr, ok := err.(s3cli.ErrorResponse); ok { return srvErr.StatusCode diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/dbinstance.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/dbinstance.go index 77f013c483..076254b54a 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/dbinstance.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/dbinstance.go @@ -36,6 +36,11 @@ func (self *SRegion) GetIDBInstances() ([]cloudprovider.ICloudDBInstance, error) return nil, errors.Wrapf(err, "GetITDSQLs") } ret = append(ret, tdsqls...) + mssqls, err := self.GetISQLServers() + if err != nil { + return nil, errors.Wrapf(err, "GetISQLServers") + } + ret = append(ret, mssqls...) return ret, nil } @@ -44,6 +49,8 @@ func (self *SRegion) GetIDBInstanceById(id string) (cloudprovider.ICloudDBInstan return self.GetMySQLInstanceById(id) } else if strings.HasPrefix(id, "tdsqlshard") { return self.GetTDSQL(id) + } else if strings.HasPrefix(id, "mssql") { + return self.GetSQLServer(id) } return nil, cloudprovider.ErrNotFound } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/monitor.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/monitor.go index ae5fdd20a8..0f6133c109 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/monitor.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/monitor.go @@ -16,6 +16,7 @@ package qcloud import ( "fmt" + "strconv" "strings" "time" @@ -84,11 +85,14 @@ func (self *SQcloudClient) metricsRequest(action string, params map[string]strin return monitorRequest(cli, action, params, self.cpcfg.UpdatePermission, self.debug) } -func (self *SQcloudClient) GetMonitorData(ns string, name string, since time.Time, until time.Time, regionId string, dimensionName string, resIds []string) ([]SDataPoint, error) { +func (self *SQcloudClient) GetMonitorData(ns string, name string, period int, since time.Time, until time.Time, regionId string, dimensionName string, resIds []string) ([]SDataPoint, error) { params := make(map[string]string) params["Region"] = regionId params["MetricName"] = name params["Namespace"] = ns + if period > 0 { + params["Period"] = strconv.Itoa(period) + } params["StartTime"] = since.Format(timeutils.IsoTimeFormat) params["EndTime"] = until.Format(timeutils.IsoTimeFormat) for idx, resId := range resIds { @@ -164,7 +168,7 @@ func (self *SQcloudClient) GetEcsMetrics(opts *cloudprovider.MetricListOptions) }, } { for metricName, tag := range metricNames { - metrics, err := self.GetMonitorData("QCE/CVM", metricName, opts.StartTime, opts.EndTime, opts.RegionExtId, "InstanceId", opts.ResourceIds) + metrics, err := self.GetMonitorData("QCE/CVM", metricName, 0, opts.StartTime, opts.EndTime, opts.RegionExtId, "InstanceId", opts.ResourceIds) if err != nil { log.Errorf("GetMonitorData error: %v", err) continue @@ -232,7 +236,7 @@ func (self *SQcloudClient) GetRedisMetrics(opts *cloudprovider.MetricListOptions }, } { for metricName, tag := range metricNames { - metrics, err := self.GetMonitorData("QCE/REDIS_MEM", metricName, opts.StartTime, opts.EndTime, opts.RegionExtId, "instanceid", opts.ResourceIds) + metrics, err := self.GetMonitorData("QCE/REDIS_MEM", metricName, 0, opts.StartTime, opts.EndTime, opts.RegionExtId, "instanceid", opts.ResourceIds) if err != nil { log.Errorf("GetMonitorData error: %v", err) continue @@ -265,7 +269,60 @@ func (self *SQcloudClient) GetRedisMetrics(opts *cloudprovider.MetricListOptions return ret, nil } -func (self *SQcloudClient) GetRdsMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { +func (self *SQcloudClient) GetSQLServerMetrics(opts *cloudprovider.MetricListOptions, resourceIds []string) ([]cloudprovider.MetricValues, error) { + ret := []cloudprovider.MetricValues{} + for metricType, metricNames := range map[cloudprovider.TMetricType]map[string]string{ + cloudprovider.RDS_METRIC_TYPE_CPU_USAGE: { + "Cpu": "", + }, + cloudprovider.RDS_METRIC_TYPE_MEM_USAGE: { + "UsageMemory": "", + }, + cloudprovider.RDS_METRIC_TYPE_DISK_USAGE: { + "FreeStorage": "", + }, + cloudprovider.RDS_METRIC_TYPE_CONN_COUNT: { + "Connections": "", + }, + } { + for metricName, tag := range metricNames { + metrics, err := self.GetMonitorData("QCE/SQLSERVER", metricName, 60, opts.StartTime, opts.EndTime, opts.RegionExtId, "resourceId", resourceIds) + if err != nil { + log.Errorf("GetMonitorData error: %v", err) + continue + } + for i := range metrics { + metric := cloudprovider.MetricValues{} + if len(metrics[i].Dimensions) < 1 || metrics[i].Dimensions[0].Name != "resourceId" { + continue + } + metric.Id = metrics[i].Dimensions[0].Value + metric.MetricType = metricType + metricValue := cloudprovider.MetricValue{} + metricValue.Tags = map[string]string{} + idx := strings.Index(tag, ":") + if idx > 0 { + metricValue.Tags[tag[:idx]] = tag[idx+1:] + } + if len(metrics[i].Timestamps) == 0 { + continue + } + for j := range metrics[i].Timestamps { + metricValue.Value = metrics[i].Values[j] + if metricType == cloudprovider.RDS_METRIC_TYPE_DISK_USAGE { + metricValue.Value = 100 - metricValue.Value + } + metricValue.Timestamp = time.Unix(int64(metrics[i].Timestamps[j]), 0) + metric.Values = append(metric.Values, metricValue) + } + ret = append(ret, metric) + } + } + } + return ret, nil +} + +func (self *SQcloudClient) GetMySQLMetrics(opts *cloudprovider.MetricListOptions, resourceIds []string) ([]cloudprovider.MetricValues, error) { ret := []cloudprovider.MetricValues{} for metricType, metricNames := range map[cloudprovider.TMetricType]map[string]string{ cloudprovider.RDS_METRIC_TYPE_CPU_USAGE: { @@ -303,7 +360,7 @@ func (self *SQcloudClient) GetRdsMetrics(opts *cloudprovider.MetricListOptions) }, } { for metricName, tag := range metricNames { - metrics, err := self.GetMonitorData("QCE/CDB", metricName, opts.StartTime, opts.EndTime, opts.RegionExtId, "InstanceId", opts.ResourceIds) + metrics, err := self.GetMonitorData("QCE/CDB", metricName, 60, opts.StartTime, opts.EndTime, opts.RegionExtId, "InstanceId", resourceIds) if err != nil { log.Errorf("GetMonitorData error: %v", err) continue @@ -336,6 +393,33 @@ func (self *SQcloudClient) GetRdsMetrics(opts *cloudprovider.MetricListOptions) return ret, nil } +func (self *SQcloudClient) GetRdsMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + cdbs, mssqls := []string{}, []string{} + for _, resourceId := range opts.ResourceIds { + if strings.HasPrefix(resourceId, "cdb") { + cdbs = append(cdbs, resourceId) + } else if strings.HasPrefix(resourceId, "mssql") { + mssqls = append(mssqls, resourceId) + } + } + ret := []cloudprovider.MetricValues{} + if len(cdbs) > 0 { + part, err := self.GetMySQLMetrics(opts, cdbs) + if err != nil { + return nil, err + } + ret = append(ret, part...) + } + if len(mssqls) > 0 { + part, err := self.GetSQLServerMetrics(opts, mssqls) + if err != nil { + return nil, err + } + ret = append(ret, part...) + } + return ret, nil +} + func (self *SQcloudClient) GetK8sMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { ret := []cloudprovider.MetricValues{} for metricType, metricName := range map[cloudprovider.TMetricType]string{ diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go index 9997425df8..ccb677778a 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go @@ -458,14 +458,18 @@ func (client *SQcloudClient) getDefaultClient(params map[string]string) (*common return client.getSdkClient(regionId) } +func (client *SQcloudClient) isSubAccount() bool { + return len(client.accountId) > 0 && + client.accountId != client.masterAccountId && len(client.masterAccountId) > 0 && + client.accountId != client.masterAppId && len(client.masterAppId) > 0 +} + func (client *SQcloudClient) getSdkClient(regionId string) (*common.Client, error) { cli, err := common.NewClientWithSecretId(client.secretId, client.secretKey, regionId) if err != nil { return nil, err } - if len(client.accountId) > 0 && - client.accountId != client.masterAccountId && len(client.masterAccountId) > 0 && - client.accountId != client.masterAppId && len(client.masterAppId) > 0 { + if client.isSubAccount() { arn := fmt.Sprintf("qcs::cam::uin/%s:roleName/%s", client.accountId, "OrganizationAccessControlRole") sts := common.DefaultRoleArnProvider(client.secretId, client.secretKey, arn) cli, err = cli.WithProvider(sts) @@ -804,8 +808,8 @@ func (self *SQcloudClient) invalidateIBuckets() { } func (self *SQcloudClient) getIBuckets() ([]cloudprovider.ICloudBucket, error) { - if len(self.accountId) > 0 { - return nil, errors.Wrapf(cloudprovider.ErrNotSupported, "organization") + if self.isSubAccount() { + return nil, fmt.Errorf("cos not support sub account sync") } if self.ibuckets == nil { err := self.fetchBuckets() @@ -1122,7 +1126,7 @@ func (self *SQcloudClient) GetISSLCertificates() ([]cloudprovider.ICloudSSLCerti return result, nil } -func (self *SQcloudClient) GetCapabilities() []string { +func (client *SQcloudClient) GetCapabilities() []string { caps := []string{ cloudprovider.CLOUD_CAPABILITY_PROJECT, cloudprovider.CLOUD_CAPABILITY_COMPUTE, @@ -1149,7 +1153,7 @@ func (self *SQcloudClient) GetCapabilities() []string { cloudprovider.CLOUD_CAPABILITY_WAF + cloudprovider.READ_ONLY_SUFFIX, } // 官方cos sdk 未支持sts - if len(self.accountId) == 0 { + if !client.isSubAccount() { caps = append(caps, cloudprovider.CLOUD_CAPABILITY_OBJECTSTORE) } return caps diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/rds_sqlserver.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/rds_sqlserver.go new file mode 100644 index 0000000000..d17338f814 --- /dev/null +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/rds_sqlserver.go @@ -0,0 +1,459 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package qcloud + +import ( + "context" + "fmt" + "strings" + "time" + + billingapi "yunion.io/x/cloudmux/pkg/apis/billing" + api "yunion.io/x/cloudmux/pkg/apis/compute" + "yunion.io/x/cloudmux/pkg/cloudprovider" + "yunion.io/x/cloudmux/pkg/multicloud" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/billing" +) + +/* + { + "Architecture": "SINGLE", + "BackupCycle": + [], + "BackupCycleType": "", + "BackupModel": "", + "BackupSaveDays": 7, + "BackupTime": "", + "Collation": "Chinese_PRC_CI_AS", + "Cpu": 2, + "CreateTime": "2025-09-16 20:27:09", + "CrossBackupEnabled": "disable", + "CrossBackupSaveDays": 0, + "CrossRegions": + [], + "DnsPodDomain": "", + "EndTime": "0000-00-00 00:00:00", + "HAFlag": "SINGLE", + "InstanceId": "mssql-ezqm0pk4", + "InstanceNote": "", + "InstanceType": "SI", + "IsDrZone": false, + "IsolateOperator": "", + "IsolateTime": "0000-00-00 00:00:00", + "Memory": 4, + "Model": 2, + "MultiSlaveZones": + [], + "Name": "11b58192-1ddd-4dc8-8ae9-454ac919927f", + "PayMode": 0, + "Pid": 1003456, + "ProjectId": 0, + "ROFlag": "", + "Region": "ap-beijing", + "RegionId": 8, + "RenewFlag": 0, + "SlaveZones": + { + "SlaveZone": "", + "SlaveZoneName": "" + }, + "StartTime": "2025-09-16 20:27:09", + "Status": 1, + "Storage": 20, + "Style": "EXCLUSIVE", + "SubFlag": "", + "SubnetId": 1344612, + "TgwWanVPort": 0, + "TimeZone": "China Standard Time", + "Type": "CLOUD_BSSD", + "Uid": "", + "UniqSubnetId": "subnet-ej5fo5gd", + "UniqVpcId": "vpc-dvnlj4aq", + "UpdateTime": "2025-09-16 20:27:09", + "UsedStorage": 0, + "Version": "2019", + "VersionName": "SQL Server 2019 Enterprise", + "Vip": "", + "VpcId": 3832347, + "Vport": 0, + "Zone": "ap-beijing-5", + "ZoneId": 800005 + } +*/ + +type SSQLServer struct { + multicloud.SDBInstanceBase + QcloudTags + + Architecture string + BackupCycle []string + BackupCycleType string + BackupModel string + BackupSaveDays int + BackupTime string + Collation string + Cpu int + CreateTime time.Time + CrossBackupEnabled string + CrossBackupSaveDays int + CrossRegions []string + DnsPodDomain string + EndTime time.Time + HAFlag string + InstanceId string + InstanceNote string + InstanceType string + IsDrZone bool + IsolateOperator string + IsolateTime string + Memory int + Model int + MultiSlaveZones []string + Name string + PayMode int + Pid int + ProjectId string + ROFlag string + Region string + RegionId int + RenewFlag int + SlaveZones struct { + SlaveZone string + SlaveZoneName string + } + StartTime string + Status int + Storage int + Style string + SubFlag string + SubnetId int + TgwWanVPort int + TimeZone string + Type string + Uid string + UniqSubnetId string + UniqVpcId string + UpdateTime string + UsedStorage int + Version string + VersionName string + Vip string + VpcId int + Vport int + Zone string + ZoneId int + + region *SRegion +} + +func (mssql *SSQLServer) GetId() string { + return mssql.InstanceId +} + +func (mssql *SSQLServer) GetGlobalId() string { + return mssql.InstanceId +} + +func (mssql *SSQLServer) GetName() string { + if len(mssql.Name) > 0 { + return mssql.Name + } + return mssql.InstanceId +} + +func (mssql *SSQLServer) GetDiskSizeGB() int { + return mssql.Storage +} + +func (mssql *SSQLServer) GetEngine() string { + return api.DBINSTANCE_TYPE_SQLSERVER +} + +func (mssql *SSQLServer) GetEngineVersion() string { + return mssql.Version +} + +func (mssql *SSQLServer) GetIVpcId() string { + return mssql.UniqVpcId +} + +func (mssql *SSQLServer) Refresh() error { + rds, err := mssql.region.GetSQLServer(mssql.InstanceId) + if err != nil { + return errors.Wrapf(err, "GetSQLServer(%s)", mssql.InstanceId) + } + return jsonutils.Update(mssql, rds) +} + +func (mssql *SSQLServer) GetInstanceType() string { + return fmt.Sprintf("%d核%dMB", mssql.Cpu, mssql.Memory) +} + +func (mssql *SSQLServer) GetMaintainTime() string { + return "" +} + +func (mssql *SSQLServer) GetDBNetworks() ([]cloudprovider.SDBInstanceNetwork, error) { + return []cloudprovider.SDBInstanceNetwork{ + {NetworkId: mssql.UniqSubnetId, IP: mssql.Vip}, + }, nil +} + +func (mssql *SSQLServer) GetConnectionStr() string { + return "" +} + +func (mssql *SSQLServer) GetInternalConnectionStr() string { + return fmt.Sprintf("%s:%d", mssql.Vip, mssql.Vport) +} + +func (mssql *SSQLServer) Reboot() error { + return cloudprovider.ErrNotImplemented +} + +func (mssql *SSQLServer) ChangeConfig(ctx context.Context, opts *cloudprovider.SManagedDBInstanceChangeConfig) error { + return cloudprovider.ErrNotImplemented +} + +func (mssql *SSQLServer) GetMasterInstanceId() string { + return "" +} + +func (mssql *SSQLServer) GetSecurityGroupIds() ([]string, error) { + return mssql.region.DescribeSQLServerDBSecurityGroups(mssql.InstanceId) +} + +func (region *SRegion) DescribeSQLServerDBSecurityGroups(id string) ([]string, error) { + params := map[string]string{ + "InstanceId": id, + } + resp, err := region.sqlserverRequest("DescribeDBSecurityGroups", params) + if err != nil { + return []string{}, errors.Wrapf(err, "DescribeDBSecurityGroups") + } + ret := struct { + SecurityGroupSet []struct { + SecurityGroupId string + } + }{} + err = resp.Unmarshal(&ret) + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal") + } + groups := []string{} + for i := range ret.SecurityGroupSet { + groups = append(groups, ret.SecurityGroupSet[i].SecurityGroupId) + } + + return groups, nil +} + +func (mssql *SSQLServer) SetSecurityGroups(ids []string) error { + return mssql.region.ModifyDBInstanceSecurityGroups(mssql.InstanceId, ids) +} + +func (region *SRegion) ModifyDBInstanceSecurityGroups(rdsId string, secIds []string) error { + params := map[string]string{ + "InstanceId": rdsId, + } + for idx, id := range secIds { + params[fmt.Sprintf("SecurityGroupIdSet.%d", idx)] = id + } + _, err := region.sqlserverRequest("ModifyDBInstanceSecurityGroups", params) + return err +} + +func (mssql *SSQLServer) Renew(bc billing.SBillingCycle) error { + return cloudprovider.ErrNotImplemented +} + +func (mssql *SSQLServer) OpenPublicConnection() error { + return cloudprovider.ErrNotImplemented +} + +func (mssql *SSQLServer) ClosePublicConnection() error { + return cloudprovider.ErrNotImplemented +} + +func (mssql *SSQLServer) GetPort() int { + return mssql.Vport +} + +func (mssql *SSQLServer) GetStatus() string { + switch mssql.Status { + case 1: + return api.DBINSTANCE_DEPLOYING + case 2, 3: + return api.DBINSTANCE_RUNNING + case 5, 6, 8: + return api.DBINSTANCE_DELETING + case 7, 9, 13, 14, 15, 16, 17: + return api.DBINSTANCE_UPGRADING + case 4: + return api.DBINSTANCE_ISOLATE + case 12: + return api.DBINSTANCE_REBOOTING + case 10: + return api.DBINSTANCE_MIGRATING + default: + return api.DBINSTANCE_DEPLOYING + } +} + +func (mssql *SSQLServer) GetCategory() string { + return strings.ToLower(mssql.Architecture) +} + +func (mssql *SSQLServer) GetStorageType() string { + return api.QCLOUD_DBINSTANCE_STORAGE_TYPE_CLOUD_SSD +} + +func (mssql *SSQLServer) GetCreatedAt() time.Time { + // 2019-12-25 09:00:43 #非UTC时间 + return mssql.CreateTime.Add(time.Hour * -8) +} + +func (mssql *SSQLServer) GetBillingType() string { + if mssql.PayMode == 1 { + return billingapi.BILLING_TYPE_PREPAID + } + return billingapi.BILLING_TYPE_POSTPAID +} + +func (mssql *SSQLServer) SetAutoRenew(bc billing.SBillingCycle) error { + return cloudprovider.ErrNotImplemented +} + +func (mssql *SSQLServer) IsAutoRenew() bool { + return mssql.RenewFlag == 1 +} + +func (mssql *SSQLServer) GetExpiredAt() time.Time { + return mssql.EndTime +} + +func (mssql *SSQLServer) GetVcpuCount() int { + return mssql.Cpu +} + +func (mssql *SSQLServer) GetVmemSizeMB() int { + return mssql.Memory * 1024 +} + +func (mssql *SSQLServer) GetZone1Id() string { + return mssql.Zone +} + +func (mssql *SSQLServer) GetZone2Id() string { + return mssql.SlaveZones.SlaveZone +} + +func (mssql *SSQLServer) GetZone3Id() string { + return mssql.SlaveZones.SlaveZone +} + +func (mssql *SSQLServer) GetProjectId() string { + return mssql.ProjectId +} + +func (mssql *SSQLServer) Delete() error { + err := mssql.region.DeleteSQLServer(mssql.InstanceId) + if err != nil { + return errors.Wrapf(err, "DeleteSQLServer") + } + return mssql.region.DeleteSQLServerInRecycleBin(mssql.InstanceId) +} + +func (region *SRegion) GetSQLServers(id string) ([]SSQLServer, error) { + params := map[string]string{} + if len(id) > 0 { + params["InstanceIdSet.0"] = id + } + offset := 0 + ret := []SSQLServer{} + for { + resp, err := region.sqlserverRequest("DescribeDBInstances", params) + if err != nil { + return nil, errors.Wrapf(err, "DescribeDBInstances") + } + part := struct { + DBInstances []SSQLServer + TotalCount int + }{} + err = resp.Unmarshal(&part) + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + for i := range part.DBInstances { + if part.DBInstances[i].Status == 4 { // 过滤掉回收站 + continue + } + part.DBInstances[i].region = region + + ret = append(ret, part.DBInstances[i]) + } + if len(ret) >= part.TotalCount || len(part.DBInstances) == 0 { + break + } + offset++ + params["Offset"] = fmt.Sprintf("%d", offset) + } + return ret, nil +} + +func (region *SRegion) GetISQLServers() ([]cloudprovider.ICloudDBInstance, error) { + servers, err := region.GetSQLServers("") + if err != nil { + return nil, errors.Wrapf(err, "GetSQLServers") + } + ret := []cloudprovider.ICloudDBInstance{} + for i := range servers { + servers[i].region = region + ret = append(ret, &servers[i]) + } + return ret, nil +} + +func (region *SRegion) GetSQLServer(id string) (*SSQLServer, error) { + vms, err := region.GetSQLServers(id) + if err != nil { + return nil, errors.Wrapf(err, "GetSQLServers") + } + for i := range vms { + if vms[i].InstanceId == id { + vms[i].region = region + return &vms[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (region *SRegion) DeleteSQLServer(id string) error { + params := map[string]string{ + "InstanceIdSet.0": id, + } + _, err := region.sqlserverRequest("TerminateDBInstance", params) + return err +} + +func (region *SRegion) DeleteSQLServerInRecycleBin(id string) error { + params := map[string]string{ + "InstanceId": id, + } + _, err := region.sqlserverRequest("DeleteDBInstance", params) + return err +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/tag_base.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/tag_base.go index 030fd26646..73e281cb5b 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/tag_base.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/tag_base.go @@ -34,7 +34,7 @@ type QcloudTags struct { Tags []multicloud.STag // Cdn Tag []multicloud.STag - // TDSQL + // TDSQL, SQLServer ResourceTags []multicloud.STag } diff --git a/vendor/yunion.io/x/sqlchemy/backends/dameng/sync.go b/vendor/yunion.io/x/sqlchemy/backends/dameng/sync.go index 08ef053288..a607448e22 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/dameng/sync.go +++ b/vendor/yunion.io/x/sqlchemy/backends/dameng/sync.go @@ -145,6 +145,16 @@ func (mysql *SDamengBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, change sql := fmt.Sprintf(`ALTER TABLE "%s" ALTER COLUMN "%s" DROP DEFAULT`, ts.Name(), cols.NewCol.Name()) alters = append(alters, sql) } + + if cols.NewCol.IsNullable() != cols.OldCol.IsNullable() { + if cols.NewCol.IsNullable() { + sql := fmt.Sprintf(`ALTER TABLE "%s" MODIFY "%s" NULL`, ts.Name(), cols.NewCol.Name()) + alters = append(alters, sql) + } else { + sql := fmt.Sprintf(`ALTER TABLE "%s" ALTER COLUMN "%s" SET NOT NULL`, ts.Name(), cols.NewCol.Name()) + alters = append(alters, sql) + } + } } } for _, col := range changes.AddColumns { diff --git a/vendor/yunion.io/x/sqlchemy/reflect.go b/vendor/yunion.io/x/sqlchemy/reflect.go index 27c84d4509..e7d3ae2044 100644 --- a/vendor/yunion.io/x/sqlchemy/reflect.go +++ b/vendor/yunion.io/x/sqlchemy/reflect.go @@ -758,3 +758,116 @@ func ConvertValueToString(val interface{}) string { } return jsonutils.Marshal(val).String() } + +func getFloatValue(val interface{}) (float64, error) { + switch v := val.(type) { + case float32: + return float64(v), nil + case float64: + return v, nil + case int: + return float64(v), nil + case int8: + return float64(v), nil + case int16: + return float64(v), nil + case int32: + return float64(v), nil + case int64: + return float64(v), nil + case uint: + return float64(v), nil + case uint8: + return float64(v), nil + case uint16: + return float64(v), nil + case uint32: + return float64(v), nil + case uint64: + return float64(v), nil + case *float32: + return float64(*v), nil + case *float64: + return *v, nil + case *int: + return float64(*v), nil + case *int8: + return float64(*v), nil + case *int16: + return float64(*v), nil + case *int32: + return float64(*v), nil + case *int64: + return float64(*v), nil + case *uint: + return float64(*v), nil + case *uint8: + return float64(*v), nil + case *uint16: + return float64(*v), nil + case *uint32: + return float64(*v), nil + case *uint64: + return float64(*v), nil + case string: + return strconv.ParseFloat(v, 64) + case *string: + return strconv.ParseFloat(*v, 64) + } + return 0, errors.ErrInvalidFormat +} + +func getTimeValue(val interface{}) (time.Time, error) { + switch v := val.(type) { + case time.Time: + return v, nil + case *time.Time: + return *v, nil + case string: + return timeutils.ParseTimeStr(v) + case *string: + return timeutils.ParseTimeStr(*v) + } + return time.Time{}, errors.ErrInvalidFormat +} + +const MIN_FLOAT_EQUAL_DIFF = float64(0.000001) + +func floatEqual(of, nf float64) bool { + if of > nf { + return of < MIN_FLOAT_EQUAL_DIFF+nf + } else if of < nf { + return of+MIN_FLOAT_EQUAL_DIFF > nf + } else { + return true + } +} + +const MIN_MICRO_SECOND_EQUAL_DIFF = 1000000 + +func timeEqual(of, nf time.Time) bool { + ofUnix := of.UnixMicro() + nfUnix := nf.UnixMicro() + if ofUnix == nfUnix { + return true + } + if ofUnix > nfUnix { + return ofUnix < MIN_MICRO_SECOND_EQUAL_DIFF+nfUnix + } else { + return ofUnix+MIN_MICRO_SECOND_EQUAL_DIFF > nfUnix + } +} + +func EqualsGrossValue(of, nf interface{}) bool { + ofFloat, ofErr := getFloatValue(of) + nfFloat, nfErr := getFloatValue(nf) + if ofErr == nil && nfErr == nil { + return floatEqual(ofFloat, nfFloat) + } + ofTime, ofErr := getTimeValue(of) + nfTime, nfErr := getTimeValue(nf) + if ofErr == nil && nfErr == nil { + return timeEqual(ofTime, nfTime) + } + return false +} diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index e14db490e3..727fd369f9 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -174,6 +174,9 @@ func (us *SUpdateSession) SaveUpdateSql(dt interface{}) (*SUpdateSQLResult, erro if ofJsonStr == nfJsonStr { continue } + if EqualsGrossValue(of, nf) { + continue + } } if c.IsZero(nf) && c.IsText() { nf = nil