fanhuadesenlinnn/cloudpods
1
0
Fork 0
mirror of https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git synced 2026-09-20 08:03:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

feature: add support for for VolcEngine (#18348)

Browse Source
This commit is contained in:
LyndonKong
2023-10-18 18:50:08 +08:00
committed by GitHub
parent 9dddc1b58d
commit 9fff0bef22
105 changed files with 22016 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/climc/shell/compute/cloudaccounts.go
View File

@@ -41,6 +41,7 @@ func init() {
cmd.CreateWithKeyword("create-hcs", &options.SHCSAccountCreateOptions{})
cmd.CreateWithKeyword("create-hcsop", &options.SHcsOpCloudAccountCreateOptions{})
cmd.CreateWithKeyword("create-ucloud", &options.SUcloudCloudAccountCreateOptions{})
cmd.CreateWithKeyword("create-volcengine", &options.SVolcengineCloudAccountCreateOptions{})
cmd.CreateWithKeyword("create-zstack", &options.SZStackCloudAccountCreateOptions{})
cmd.CreateWithKeyword("create-s3", &options.SS3CloudAccountCreateOptions{})
cmd.CreateWithKeyword("create-ceph", &options.SCephCloudAccountCreateOptions{})
@@ -71,6 +72,7 @@ func init() {
cmd.UpdateWithKeyword("update-hcso", &options.SHCSOAccountUpdateOptions{})
cmd.UpdateWithKeyword("update-hcs", &options.SHCSAccountUpdateOptions{})
cmd.UpdateWithKeyword("update-ucloud", &options.SUcloudCloudAccountUpdateOptions{})
cmd.UpdateWithKeyword("update-volcengine", &options.SVolcengineCloudAccountUpdateOptions{})
cmd.UpdateWithKeyword("update-zstack", &options.SZStackCloudAccountUpdateOptions{})
cmd.UpdateWithKeyword("update-s3", &options.SS3CloudAccountUpdateOptions{})
cmd.UpdateWithKeyword("update-ctyun", &options.SCtyunCloudAccountUpdateOptions{})
@@ -99,6 +101,7 @@ func init() {
cmd.PerformWithKeyword("update-credential-hcso", "update-credential", &options.SHCSOAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("update-credential-hcs", "update-credential", &options.SHCSOAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("update-credential-ucloud", "update-credential", &options.SUcloudCloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("update-credential-volcengine", "update-credential", &options.SVolcengineCloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("update-credential-zstack", "update-credential", &options.SZStackCloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("update-credential-s3", "update-credential", &options.SS3CloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("update-credential-ctyun", "update-credential", &options.SCtyunCloudAccountUpdateCredentialOptions{})
@@ -122,6 +125,7 @@ func init() {
cmd.PerformWithKeyword("test-connectivity-openstack", "test-connectivity", &options.SOpenStackCloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("test-connectivity-huawei", "test-connectivity", &options.SHuaweiCloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("test-connectivity-ucloud", "test-connectivity", &options.SUcloudCloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("test-connectivity-volcengine", "test-connectivity", &options.SVolcengineCloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("test-connectivity-zstack", "test-connectivity", &options.SZStackCloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("test-connectivity-s3", "test-connectivity", &options.SS3CloudAccountUpdateCredentialOptions{})
cmd.PerformWithKeyword("test-connectivity-ctyun", "test-connectivity", &options.SCtyunCloudAccountUpdateCredentialOptions{})

4
cmd/climc/shell/compute/usages.go
View File

@@ -32,8 +32,8 @@ import (
type GeneralUsageOptions struct {
HostType []string `help:"Host types" choices:"hypervisor|baremetal|esxi|xen|kubelet|hyperv|aliyun|azure|aws|huawei|qcloud|openstack|ucloud|zstack|google|ctyun"`
Provider []string `help:"Provider" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|Google|Ctyun"`
Brand []string `help:"Brands" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|ZStack|DStack|Google|Ctyun"`
Provider []string `help:"Provider" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|VolcEngine|ZStack|Google|Ctyun"`
Brand []string `help:"Brands" choices:"OneCloud|VMware|Aliyun|Azure|Aws|Qcloud|Huawei|OpenStack|Ucloud|VolcEngine|ZStack|DStack|Google|Ctyun"`
Project string `help:"show usage of specified project"`
ProjectDomain string `help:"show usage of specified domain"`

1
cmd/climc/shell/misc/feature.go
View File

@@ -66,6 +66,7 @@ func init() {
"baidu",
"cucloud",
"qingcloud",
"volcengine",
}
const (

3
go.mod
View File

@@ -233,6 +233,8 @@ require (
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
github.com/vmware/govmomi v0.20.1 // indirect
github.com/volcengine/ve-tos-golang-sdk/v2 v2.6.2 // indirect
github.com/volcengine/volc-sdk-golang v1.0.23 // indirect
github.com/willf/bitset v1.1.9 // indirect
github.com/willf/bloom v2.0.3+incompatible // indirect
github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect
@@ -253,7 +255,6 @@ require (
google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect

7
go.sum
View File

@@ -144,6 +144,7 @@ github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI=
github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/aws/aws-sdk-go v1.39.0 h1:74BBwkEmiqBbi2CGflEh34l0YNtIibTjZsibGarkNjo=
github.com/aws/aws-sdk-go v1.39.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
@@ -709,6 +710,10 @@ github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaU
github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/vmware/govmomi v0.20.1 h1:7b/SeTUB3tER8ZLGLLLH3xcnB2xeuLULXmfPFqPSRZA=
github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
github.com/volcengine/ve-tos-golang-sdk/v2 v2.6.2 h1:FZ7zrUf70YKVuES50mHO3z2VW0K+Arq4uGlDqb+1VA0=
github.com/volcengine/ve-tos-golang-sdk/v2 v2.6.2/go.mod h1:IrjK84IJJTuOZOTMv/P18Ydjy/x+ow7fF7q11jAxXLM=
github.com/volcengine/volc-sdk-golang v1.0.23 h1:anOslb2Qp6ywnsbyq9jqR0ljuO63kg9PY+4OehIk5R8=
github.com/volcengine/volc-sdk-golang v1.0.23/go.mod h1:AfG/PZRUkHJ9inETvbjNifTDgut25Wbkm2QoYBTbvyU=
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
github.com/willf/bitset v1.1.9 h1:GBtFynGY9ZWZmEC9sWuu41/7VBXPFCOAbCbqTflOg9c=
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
@@ -1174,8 +1179,6 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231012115531-f16047235f44 h1:UWU0ISJM6AL7cvxzGT6cK4EPUX7VIPp+x0V6gWcX9fg=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231012115531-f16047235f44/go.mod h1:2sgCN7nRPQL3woLfdgqLDd92vwAHqtlz3KKiHxC5BAw=
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug=
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=

5
pkg/apis/compute/cloudaccount_const.go
View File

@@ -54,6 +54,7 @@ const (
CLOUD_PROVIDER_HCS = compute.CLOUD_PROVIDER_HCS
CLOUD_PROVIDER_OPENSTACK = compute.CLOUD_PROVIDER_OPENSTACK
CLOUD_PROVIDER_UCLOUD = compute.CLOUD_PROVIDER_UCLOUD
CLOUD_PROVIDER_VOLCENGINE = compute.CLOUD_PROVIDER_VOLCENGINE
CLOUD_PROVIDER_ZSTACK = compute.CLOUD_PROVIDER_ZSTACK
CLOUD_PROVIDER_GOOGLE = compute.CLOUD_PROVIDER_GOOGLE
CLOUD_PROVIDER_CTYUN = compute.CLOUD_PROVIDER_CTYUN
@@ -134,6 +135,7 @@ var (
CLOUD_PROVIDER_HCS,
CLOUD_PROVIDER_OPENSTACK,
CLOUD_PROVIDER_UCLOUD,
CLOUD_PROVIDER_VOLCENGINE,
CLOUD_PROVIDER_ZSTACK,
CLOUD_PROVIDER_GOOGLE,
CLOUD_PROVIDER_CTYUN,
@@ -194,6 +196,9 @@ var (
CLOUD_PROVIDER_UCLOUD: {
HOST_TYPE_UCLOUD,
},
CLOUD_PROVIDER_VOLCENGINE: {
HOST_TYPE_VOLCENGINE,
},
CLOUD_PROVIDER_ZSTACK: {
HOST_TYPE_ZSTACK,
},

5
pkg/apis/compute/guest_const.go
View File

@@ -191,6 +191,7 @@ const (
HYPERVISOR_HCSOP = compute.HYPERVISOR_HCSOP
HYPERVISOR_OPENSTACK = compute.HYPERVISOR_OPENSTACK
HYPERVISOR_UCLOUD = compute.HYPERVISOR_UCLOUD
HYPERVISOR_VOLCENGINE = compute.HYPERVISOR_VOLCENGINE
HYPERVISOR_ZSTACK = compute.HYPERVISOR_ZSTACK
HYPERVISOR_GOOGLE = compute.HYPERVISOR_GOOGLE
HYPERVISOR_CTYUN = compute.HYPERVISOR_CTYUN
@@ -268,6 +269,7 @@ var HYPERVISORS = []string{
HYPERVISOR_HCSOP,
HYPERVISOR_OPENSTACK,
HYPERVISOR_UCLOUD,
HYPERVISOR_VOLCENGINE,
HYPERVISOR_ZSTACK,
HYPERVISOR_GOOGLE,
HYPERVISOR_CTYUN,
@@ -299,6 +301,7 @@ var PUBLIC_CLOUD_HYPERVISORS = []string{
HYPERVISOR_QCLOUD,
HYPERVISOR_HUAWEI,
HYPERVISOR_UCLOUD,
HYPERVISOR_VOLCENGINE,
HYPERVISOR_GOOGLE,
HYPERVISOR_CTYUN,
HYPERVISOR_ECLOUD,
@@ -343,6 +346,7 @@ var HYPERVISOR_HOSTTYPE = map[string]string{
HYPERVISOR_HCS: HOST_TYPE_HCS,
HYPERVISOR_OPENSTACK: HOST_TYPE_OPENSTACK,
HYPERVISOR_UCLOUD: HOST_TYPE_UCLOUD,
HYPERVISOR_VOLCENGINE: HOST_TYPE_VOLCENGINE,
HYPERVISOR_ZSTACK: HOST_TYPE_ZSTACK,
HYPERVISOR_GOOGLE: HOST_TYPE_GOOGLE,
HYPERVISOR_CTYUN: HOST_TYPE_CTYUN,
@@ -377,6 +381,7 @@ var HOSTTYPE_HYPERVISOR = map[string]string{
HOST_TYPE_HCS: HYPERVISOR_HCS,
HOST_TYPE_OPENSTACK: HYPERVISOR_OPENSTACK,
HOST_TYPE_UCLOUD: HYPERVISOR_UCLOUD,
HOST_TYPE_VOLCENGINE: HYPERVISOR_VOLCENGINE,
HOST_TYPE_ZSTACK: HYPERVISOR_ZSTACK,
HOST_TYPE_GOOGLE: HYPERVISOR_GOOGLE,
HOST_TYPE_CTYUN: HYPERVISOR_CTYUN,

2
pkg/apis/compute/host_const.go
View File

@@ -38,6 +38,7 @@ const (
HOST_TYPE_HCS = compute.HOST_TYPE_HCS
HOST_TYPE_OPENSTACK = compute.HOST_TYPE_OPENSTACK
HOST_TYPE_UCLOUD = compute.HOST_TYPE_UCLOUD
HOST_TYPE_VOLCENGINE = compute.HOST_TYPE_VOLCENGINE
HOST_TYPE_ZSTACK = compute.HOST_TYPE_ZSTACK
HOST_TYPE_GOOGLE = compute.HOST_TYPE_GOOGLE
HOST_TYPE_CTYUN = compute.HOST_TYPE_CTYUN
@@ -133,6 +134,7 @@ var HOST_TYPES = []string{
HOST_TYPE_HCSOP,
HOST_TYPE_OPENSTACK,
HOST_TYPE_UCLOUD,
HOST_TYPE_VOLCENGINE,
HOST_TYPE_ZSTACK,
HOST_TYPE_CTYUN,
HOST_TYPE_GOOGLE,

5
pkg/apis/compute/storage_const.go
View File

@@ -90,6 +90,11 @@ const (
STORAGE_UCLOUD_LOCAL_SSD = compute.STORAGE_UCLOUD_LOCAL_SSD // SSD本地盘
STORAGE_UCLOUD_EXCLUSIVE_LOCAL_DISK = compute.STORAGE_UCLOUD_EXCLUSIVE_LOCAL_DISK // 独享本地盘
// VolcEngine storage types
STORAGE_VOLC_CLOUD_FLEXPL = compute.STORAGE_VOLCENGINE_FlexPL // 极速型SSD云盘, FlexPL规格
STORAGE_VOLC_CLOUD_PL0 = compute.STORAGE_VOLCENGINE_PL0 //极速型SSD云盘, PL0规格
STORAGE_VOLC_CLOUD_PTSSD = compute.STORAGE_VOLCENGINE_PTSSD // 性能型SSD, 上一代产品
// Zstack storage type
STORAGE_ZSTACK_LOCAL_STORAGE = compute.STORAGE_ZSTACK_LOCAL_STORAGE
STORAGE_ZSTACK_CEPH = compute.STORAGE_ZSTACK_CEPH

157
pkg/compute/guestdrivers/volcengine.go Normal file
View File

@@ -0,0 +1,157 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package guestdrivers
import (
"fmt"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/pkg/util/billing"
"yunion.io/x/pkg/util/rbacscope"
"yunion.io/x/pkg/utils"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/mcclient"
)
type SVolcengineGuestDriver struct {
SManagedVirtualizedGuestDriver
}
func init() {
driver := SVolcengineGuestDriver{}
models.RegisterGuestDriver(&driver)
}
func (self *SVolcengineGuestDriver) GetHypervisor() string {
return api.HYPERVISOR_VOLCENGINE
}
func (self *SVolcengineGuestDriver) GetProvider() string {
return api.CLOUD_PROVIDER_VOLCENGINE
}
func (self *SVolcengineGuestDriver) GetComputeQuotaKeys(scope rbacscope.TRbacScope, ownerId mcclient.IIdentityProvider, brand string) models.SComputeResourceKeys {
keys := models.SComputeResourceKeys{}
keys.SBaseProjectQuotaKeys = quotas.OwnerIdProjectQuotaKeys(scope, ownerId)
keys.CloudEnv = api.CLOUD_ENV_PUBLIC_CLOUD
keys.Provider = api.CLOUD_PROVIDER_VOLCENGINE
keys.Brand = api.CLOUD_PROVIDER_VOLCENGINE
keys.Hypervisor = api.HYPERVISOR_VOLCENGINE
return keys
}
func (self *SVolcengineGuestDriver) GetDefaultSysDiskBackend() string {
return ""
}
func (self *SVolcengineGuestDriver) GetMinimalSysDiskSizeGb() int {
return 40
}
func (self *SVolcengineGuestDriver) GetStorageTypes() []string {
return []string{}
}
func (self *SVolcengineGuestDriver) ChooseHostStorage(host *models.SHost, guest *models.SGuest, diskConfig *api.DiskConfig, storageIds []string) (*models.SStorage, error) {
return chooseHostStorage(self, host, diskConfig.Backend, storageIds), nil
}
func (self *SVolcengineGuestDriver) GetDetachDiskStatus() ([]string, error) {
return []string{api.VM_READY, api.VM_RUNNING}, nil
}
func (self *SVolcengineGuestDriver) GetAttachDiskStatus() ([]string, error) {
return []string{api.VM_READY, api.VM_RUNNING}, nil
}
func (self *SVolcengineGuestDriver) GetRebuildRootStatus() ([]string, error) {
return []string{api.VM_READY, api.VM_RUNNING}, nil
}
func (self *SVolcengineGuestDriver) GetChangeConfigStatus(guest *models.SGuest) ([]string, error) {
return []string{api.VM_READY}, nil
}
func (self *SVolcengineGuestDriver) GetDeployStatus() ([]string, error) {
return []string{api.VM_READY, api.VM_RUNNING}, nil
}
func (self *SVolcengineGuestDriver) GetGuestInitialStateAfterCreate() string {
return api.VM_RUNNING
}
func (self *SVolcengineGuestDriver) GetGuestInitialStateAfterRebuild() string {
return api.VM_RUNNING
}
func (self *SVolcengineGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
return cloudprovider.SInstanceCapability{
Hypervisor: self.GetHypervisor(),
Provider: self.GetProvider(),
DefaultAccount: cloudprovider.SDefaultAccount{
Linux: cloudprovider.SOsDefaultAccount{
DefaultAccount: api.VM_DEFAULT_LINUX_LOGIN_USER,
Changeable: false,
},
Windows: cloudprovider.SOsDefaultAccount{
DefaultAccount: api.VM_DEFAULT_WINDOWS_LOGIN_USER,
Changeable: false,
},
},
Storages: cloudprovider.Storage{
DataDisk: []cloudprovider.StorageInfo{
cloudprovider.StorageInfo{
{StorageType: api.STORAGE_VOLCENGINE_PTSSD, MaxSizeGb: 8192, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
{StorageType: api.STORAGE_VOLCENGINE_PL0, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
{StorageType: api.STORAGE_VOLCENGINE_FlexPL, MaxSizeGb: 32768, MinSizeGb: 20, StepSizeGb: 1, Resizable: true},
},
},
SysDisk: []cloudprovider.StorageInfo{
cloudprovider.StorageInfo{
{StorageType: api.STORAGE_VOLCENGINE_PTSSD, MaxSizeGb: 500, MinSizeGb: 40, StepSizeGb: 1, Resizable: true},
{StorageType: api.STORAGE_VOLCENGINE_PL0, MaxSizeGb: 2048, MinSizeGb: 40, StepSizeGb: 1, Resizable: true},
{StorageType: api.STORAGE_VOLCENGINE_FlexPL, MaxSizeGb: 2048, MinSizeGb: 40, StepSizeGb: 1, Resizable: true},
},
},
},
}
}
func (self *SVolcengineGuestDriver) IsSupportedBillingCycle(bc billing.SBillingCycle) bool {
months := bc.GetMonths()
if (months >= 1 && months <= 9) || (months == 12) || (months == 24) || (months == 36) {
return true
}
return false
}
func (self *SVolcengineGuestDriver) IsNeedInjectPasswordByCloudInit() bool {
return true
}
func (self *SVolcengineGuestDriver) IsSupportSetAutoRenew() bool {
return false
}
func (self *SVolcengineGuestDriver) ValidateResizeDisk(guest *models.SGuest, disk *models.SDisk, storage *models.SStorage) error {
if !utils.IsInStringArray(guest.Status, []string{api.VM_RUNNING, api.VM_READY}) {
return fmt.Errorf("cannot resize disk when guest in status %s", guest.Status)
}
return nil
}

81
pkg/compute/hostdrivers/volcengine.go Normal file
View File

@@ -0,0 +1,81 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hostdrivers
import (
"context"
"fmt"
"yunion.io/x/pkg/utils"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
type SVolcengineHostDriver struct {
SManagedVirtualizationHostDriver
}
func init() {
driver := SVolcengineHostDriver{}
models.RegisterHostDriver(&driver)
}
func (self *SVolcengineHostDriver) GetHostType() string {
return api.HOST_TYPE_VOLCENGINE
}
func (self *SVolcengineHostDriver) GetHypervisor() string {
return api.HYPERVISOR_VOLCENGINE
}
func (self *SVolcengineHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb int) error {
if sizeGb%10 != 0 {
return fmt.Errorf("The disk size must be a multiple of 10Gb")
}
min, max := 0, 0
switch storage.StorageType {
case api.STORAGE_VOLC_CLOUD_PL0:
min, max = 10, 32768
case api.STORAGE_VOLC_CLOUD_FLEXPL:
min, max = 10, 32768
default:
return fmt.Errorf("Not support create or resize %s disk", storage.StorageType)
}
if sizeGb < min || sizeGb > max {
return fmt.Errorf("The %s disk size must be in the range of %d ~ %dGB", storage.StorageType, min, max)
}
return nil
}
func (self *SVolcengineHostDriver) ValidateResetDisk(ctx context.Context, userCred mcclient.TokenCredential, disk *models.SDisk, snapshot *models.SSnapshot, guests []models.SGuest, input *api.DiskResetInput) (*api.DiskResetInput, error) {
for _, guest := range guests {
if !utils.IsInStringArray(guest.Status, []string{api.VM_RUNNING, api.VM_READY}) {
return nil, httperrors.NewBadGatewayError("Volcengine reset disk required guest status is running or read")
}
}
return input, nil
}
func (self *SVolcengineHostDriver) RequestDeleteSnapshotWithStorage(ctx context.Context, host *models.SHost, snapshot *models.SSnapshot, task taskman.ITask) error {
return httperrors.NewNotImplementedError("not implement")
}
func (driver *SVolcengineHostDriver) GetStoragecacheQuota(host *models.SHost) int {
return 10
}

68
pkg/compute/regiondrivers/volcengine.go Normal file
View File

@@ -0,0 +1,68 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package regiondrivers
import (
"context"
"yunion.io/x/jsonutils"
api "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/cloudcommon/validators"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
)
type SVolcengineRegionDriver struct {
SManagedVirtualizationRegionDriver
}
func init() {
driver := SVolcengineRegionDriver{}
models.RegisterRegionDriver(&driver)
}
func (self *SVolcengineRegionDriver) IsSecurityGroupBelongVpc() bool {
return true
}
func (self *SVolcengineRegionDriver) IsAllowSecurityGroupNameRepeat() bool {
return false
}
func (self *SVolcengineRegionDriver) GenerateSecurityGroupName(name string) string {
return name
}
func (self *SVolcengineRegionDriver) GetProvider() string {
return api.CLOUD_PROVIDER_VOLCENGINE
}
func (self *SVolcengineRegionDriver) ValidateCreateVpcData(ctx context.Context, userCred mcclient.TokenCredential, input api.VpcCreateInput) (api.VpcCreateInput, error) {
var cidrV = validators.NewIPv4PrefixValidator("cidr_block")
if err := cidrV.Validate(jsonutils.Marshal(input).(*jsonutils.JSONDict)); err != nil {
return input, err
}
err := IsInPrivateIpRange(cidrV.Value.ToIPRange())
if err != nil {
return input, err
}
if cidrV.Value.MaskLen > 29 {
return input, httperrors.NewInputParameterError("%s request the mask range should be less than or equal to 29", self.GetProvider())
}
return input, nil
}

2
pkg/mcclient/options/base.go
View File

@@ -245,7 +245,7 @@ type BaseListOptions struct {
Manager []string `help:"List objects belonging to the cloud provider" json:"manager,omitempty"`
Account string `help:"List objects belonging to the cloud account" json:"account,omitempty"`
Provider []string `help:"List objects from the provider" choices:"OneCloud|VMware|Aliyun|Apsara|Qcloud|Azure|Aws|Huawei|OpenStack|Ucloud|ZStack|Google|Ctyun|Cloudpods|Nutanix|BingoCloud|IncloudSphere|JDcloud|Proxmox|Ceph|Ecloud|HCSO|HCS|HCSOP|H3C|S3|RemoteFile|Ksyun|Baidu|QingCloud" json:"provider,omitempty"`
Provider []string `help:"List objects from the provider" choices:"OneCloud|VMware|Aliyun|Apsara|Qcloud|Azure|Aws|Huawei|OpenStack|Ucloud|VolcEngine|ZStack|Google|Ctyun|Cloudpods|Nutanix|BingoCloud|IncloudSphere|JDcloud|Proxmox|Ceph|Ecloud|HCSO|HCS|HCSOP|H3C|S3|RemoteFile|Ksyun|Baidu|QingCloud" json:"provider,omitempty"`
Brand []string `help:"List objects belonging to a special brand"`
CloudEnv string `help:"Cloud environment" choices:"public|private|onpremise|private_or_onpremise" json:"cloud_env,omitempty"`
PublicCloud *bool `help:"List objects belonging to public cloud" json:"public_cloud"`

28
pkg/mcclient/options/cloudaccounts.go
View File

@@ -346,6 +346,17 @@ func (opts *SUcloudCloudAccountCreateOptions) Params() (jsonutils.JSONObject, er
return params, nil
}
type SVolcengineCloudAccountCreateOptions struct {
SCloudAccountCreateBaseOptions
SAccessKeyCredential
}
func (opts *SVolcengineCloudAccountCreateOptions) Params() (jsonutils.JSONObject, error) {
params := jsonutils.Marshal(opts)
params.(*jsonutils.JSONDict).Add(jsonutils.NewString("VolcEngine"), "provider")
return params, nil
}
type SZStackCloudAccountCreateOptions struct {
SCloudAccountCreateBaseOptions
SUserPasswordCredential
@@ -565,6 +576,15 @@ func (opts *SUcloudCloudAccountUpdateCredentialOptions) Params() (jsonutils.JSON
return jsonutils.Marshal(opts), nil
}
type SVolcengineCloudAccountUpdateCredentialOptions struct {
SCloudAccountIdOptions
SUserPasswordCredential
}
func (opts *SVolcengineCloudAccountUpdateCredentialOptions) Params() (jsonutils.JSONObject, error) {
return jsonutils.Marshal(opts), nil
}
type SZStackCloudAccountUpdateCredentialOptions struct {
SCloudAccountIdOptions
SUserPasswordCredential
@@ -940,6 +960,14 @@ func (opts *SUcloudCloudAccountUpdateOptions) Params() (jsonutils.JSONObject, er
return jsonutils.Marshal(opts), nil
}
type SVolcengineCloudAccountUpdateOptions struct {
SCloudAccountUpdateBaseOptions
}
func (opts *SVolcengineCloudAccountUpdateOptions) Params() (jsonutils.JSONObject, error) {
return jsonutils.Marshal(opts), nil
}
type SZStackCloudAccountUpdateOptions struct {
SCloudAccountUpdateBaseOptions
}

4
pkg/mcclient/options/compute/servers.go
View File

@@ -44,7 +44,7 @@ type ServerListOptions struct {
Gpu *bool `help:"Show gpu servers"`
Secgroup string `help:"Secgroup ID or Name"`
AdminSecgroup string `help:"AdminSecgroup ID or Name"`
Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure|aws|huawei|ucloud|zstack|openstack|google|ctyun|incloudsphere|nutanix|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox|ksyun|baidu|cucloud|qingcloud"`
Hypervisor string `help:"Show server of hypervisor" choices:"kvm|esxi|container|baremetal|aliyun|azure|aws|huawei|ucloud|volcengine|zstack|openstack|google|ctyun|incloudsphere|nutanix|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox|ksyun|baidu|cucloud|qingcloud"`
Region string `help:"Show servers in cloudregion"`
WithEip *bool `help:"Show Servers with EIP"`
WithoutEip *bool `help:"Show Servers without EIP"`
@@ -249,7 +249,7 @@ type ServerConfigs struct {
Host string `help:"Preferred host where virtual server should be created" json:"prefer_host"`
BackupHost string `help:"Perfered host where virtual backup server should be created"`
Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei|openstack|ucloud|zstack|google|ctyun|incloudsphere|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox"`
Hypervisor string `help:"Hypervisor type" choices:"kvm|esxi|baremetal|container|aliyun|azure|qcloud|aws|huawei|openstack|ucloud|volcengine|zstack|google|ctyun|incloudsphere|bingocloud|cloudpods|ecloud|jdcloud|remotefile|h3c|hcs|hcso|hcsop|proxmox"`
ResourceType string `help:"Resource type" choices:"shared|prepaid|dedicated"`
Backup bool `help:"Create server with backup server"`
AutoSwitchToBackupOnHostDown bool `help:"Auto switch to backup server on host down"`

490
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,490 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Third-party dependencies listed below are subject to different license terms.
yaml for Go
Apache License
Version 2.0, January 2004
=========================
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction, and
distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by the copyright
owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all other entities
that control, are controlled by, or are under common control with that entity.
For the purposes of this definition, "control" means (i) the power, direct or
indirect, to cause the direction or management of such entity, whether by
contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity exercising permissions
granted by this License.
"Source" form shall mean the preferred form for making modifications, including
but not limited to software source code, documentation source, and configuration
files.
"Object" form shall mean any form resulting from mechanical transformation or
translation of a Source form, including but not limited to compiled object code,
generated documentation, and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or Object form, made
available under the License, as indicated by a copyright notice that is included
in or attached to the work (an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object form, that is
based on (or derived from) the Work and for which the editorial revisions,
annotations, elaborations, or other modifications represent, as a whole, an
original work of authorship. For the purposes of this License, Derivative Works
shall not include works that remain separable from, or merely link (or bind by
name) to the interfaces of, the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including the original version
of the Work and any modifications or additions to that Work or Derivative Works
thereof, that is intentionally submitted to Licensor for inclusion in the Work by
the copyright owner or by an individual or Legal Entity authorized to submit on
behalf of the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent to the
Licensor or its representatives, including but not limited to communication on
electronic mailing lists, source code control systems, and issue tracking systems
that are managed by, or on behalf of, the Licensor for the purpose of discussing
and improving the Work, but excluding communication that is conspicuously marked
or otherwise designated in writing by the copyright owner as "Not a
Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of
whom a Contribution has been received by Licensor and subsequently incorporated
within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of this
License, each Contributor hereby grants to You a perpetual, worldwide,
non-exclusive, no-charge, royalty-free, irrevocable copyright license to
reproduce, prepare Derivative Works of, publicly display, publicly perform,
sublicense, and distribute the Work and such Derivative Works in Source or Object
form.
3. Grant of Patent License. Subject to the terms and conditions of this License,
each Contributor hereby grants to You a perpetual, worldwide, non-exclusive,
no-charge, royalty-free, irrevocable (except as stated in this section) patent
license to make, have made, use, offer to sell, sell, import, and otherwise
transfer the Work, where such license applies only to those patent claims
licensable by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s) with the Work to
which such Contribution(s) was submitted. If You institute patent litigation
against any entity (including a cross-claim or counterclaim in a lawsuit)
alleging that the Work or a Contribution incorporated within the Work constitutes
direct or contributory patent infringement, then any patent licenses granted to
You under this License for that Work shall terminate as of the date such
litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the Work or
Derivative Works thereof in any medium, with or without modifications, and in
Source or Object form, provided that You meet the following conditions:
a. You must give any other recipients of the Work or Derivative Works a copy of
this License; and
b. You must cause any modified files to carry prominent notices stating that
You changed the files; and
c. You must retain, in the Source form of any Derivative Works that You
distribute, all copyright, patent, trademark, and attribution notices from
the Source form of the Work, excluding those notices that do not pertain to
any part of the Derivative Works; and
d. If the Work includes a "NOTICE" text file as part of its distribution, then
any Derivative Works that You distribute must include a readable copy of the
attribution notices contained within such NOTICE file, excluding those
notices that do not pertain to any part of the Derivative Works, in at least
one of the following places: within a NOTICE text file distributed as part of
the Derivative Works; within the Source form or documentation, if provided
along with the Derivative Works; or, within a display generated by the
Derivative Works, if and wherever such third-party notices normally appear.
The contents of the NOTICE file are for informational purposes only and do
not modify the License. You may add Your own attribution notices within
Derivative Works that You distribute, alongside or as an addendum to the
NOTICE text from the Work, provided that such additional attribution notices
cannot be construed as modifying the License.
You may add Your own copyright statement to Your modifications and may provide
additional or different license terms and conditions for use, reproduction, or
distribution of Your modifications, or for any such Derivative Works as a whole,
provided Your use, reproduction, and distribution of the Work otherwise complies
with the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise, any
Contribution intentionally submitted for inclusion in the Work by You to the
Licensor shall be under the terms and conditions of this License, without any
additional terms or conditions. Notwithstanding the above, nothing herein shall
supersede or modify the terms of any separate license agreement you may have
executed with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade names,
trademarks, service marks, or product names of the Licensor, except as required
for reasonable and customary use in describing the origin of the Work and
reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or agreed to in
writing, Licensor provides the Work (and each Contributor provides its
Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
either express or implied, including, without limitation, any warranties or
conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any risks
associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory, whether in
tort (including negligence), contract, or otherwise, unless required by
applicable law (such as deliberate and grossly negligent acts) or agreed to in
writing, shall any Contributor be liable to You for damages, including any
direct, indirect, special, incidental, or consequential damages of any character
arising as a result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill, work stoppage,
computer failure or malfunction, or any and all other commercial damages or
losses), even if such Contributor has been advised of the possibility of such
damages.
9. Accepting Warranty or Additional Liability. While redistributing the Work or
Derivative Works thereof, You may choose to offer, and charge a fee for,
acceptance of support, warranty, indemnity, or other liability obligations and/or
rights consistent with this License. However, in accepting such obligations, You
may act only on Your own behalf and on Your sole responsibility, not on behalf of
any other Contributor, and only if You agree to indemnify, defend, and hold each
Contributor harmless for any liability incurred by, or claims asserted against,
such Contributor by reason of your accepting any such warranty or additional
liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work
To apply the Apache License to your work, attach the following boilerplate
notice, with the fields enclosed by brackets "[]" replaced with your own
identifying information. (Don't include the brackets!) The text should be
enclosed in the appropriate comment syntax for the file format. We also recommend
that a file or class name and description of purpose be included on the same
"printed page" as the copyright notice for easier identification within
third-party archives.
Copyright [yyyy] [name of copyright owner] Licensed under the Apache License,
Version 2.0 (the "License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law
or agreed to in writing, software distributed under the License is
distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the specific language
governing permissions and limitations under the License.
---
go-check-check
BSD 2-clause "Simplified" License
Copyright (c) 2010-2013 Gustavo Niemeyer <gustavo@niemeyer.net>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE
---
golang.org/x/sync
BSD 3-clause "New" or "Revised" License
Copyright (c) <YEAR>, <OWNER>
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the <ORGANIZATION> nor the names of its contributors may
be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
---
pmezard-go-difflib
BSD 3-clause "New" or "Revised" License
Source: https://github.com/pmezard/go-difflib
Files: *
Copyright: 2013 Patrick Mézard
License: BSD-3-clause
Files: debian/*
Copyright: 2016 Dmitry Smirnov <onlyjob@debian.org>
License: BSD-3-clause
License: BSD-3-clause
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution. .
The names of its contributors may not be used to endorse or promote
products derived from this software without specific prior written
permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE
---
go-spew
ISC License
Copyright (c) 2012-2016 Dave Collins <dave@davec.name>
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE
---
stretchr/objx
The MIT License
Copyright (c) 2014 Stretchr, Inc.
Copyright (c) 2017-2018 objx contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE
---
Go Testify
The MIT License
===============
Copyright (c) <year> <copyright holders>
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in the
Software without restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

211
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/acl.go generated vendored Normal file
View File

@@ -0,0 +1,211 @@
package tos
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
)
// PutObjectAcl AclGrant, AclRules can not set both.
//
// Deprecated: ues PutObjectACL of ClientV2 instead
func (bkt *Bucket) PutObjectAcl(ctx context.Context, input *PutObjectAclInput, options ...Option) (*PutObjectAclOutput, error) {
if err := isValidKey(input.Key); err != nil {
return nil, err
}
var content io.Reader
if input.AclRules != nil {
data, err := json.Marshal(input.AclRules)
if err != nil {
return nil, fmt.Errorf("tos: marshal BucketAcl Ruels err: %s", err.Error())
}
content = bytes.NewReader(data)
}
builder := bkt.client.newBuilder(bkt.name, input.Key, options...).
WithQuery("acl", "").
WithQuery("versionId", input.VersionID)
if grant := input.AclGrant; grant != nil {
builder.WithHeader(HeaderACL, grant.ACL).
WithHeader(HeaderGrantFullControl, grant.GrantFullControl).
WithHeader(HeaderGrantRead, grant.GrantRead).
WithHeader(HeaderGrantReadAcp, grant.GrantReadAcp).
WithHeader(HeaderGrantWriteAcp, grant.GrantWriteAcp)
}
res, err := builder.WithRetry(OnRetryFromStart, StatusCodeClassifier{}).Request(ctx, http.MethodPut, content, bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
return &PutObjectAclOutput{RequestInfo: res.RequestInfo()}, nil
}
// PutObjectACL put object ACL
func (cli *ClientV2) PutObjectACL(ctx context.Context, input *PutObjectACLInput) (*PutObjectACLOutput, error) {
if err := isValidKey(input.Key); err != nil {
return nil, err
}
if len(input.ACL) != 0 {
if err := isValidACL(input.ACL); err != nil {
return nil, err
}
}
var content io.Reader
if len(input.Grants) != 0 {
for _, grants := range input.Grants {
if err := isValidGrantee(grants.GranteeV2.Type); len(grants.GranteeV2.Type) != 0 && err != nil {
return nil, err
}
if err := isValidCannedType(grants.GranteeV2.Canned); len(grants.GranteeV2.Canned) != 0 && err != nil {
return nil, err
}
if err := isValidPermission(grants.Permission); len(grants.Permission) != 0 && err != nil {
return nil, err
}
}
data, err := json.Marshal(&accessControlList{
Owner: input.Owner,
Grants: input.Grants,
BucketOwnerEntrusted: input.BucketOwnerEntrusted,
})
if err != nil {
return nil, InvalidMarshal
}
content = bytes.NewReader(data)
}
builder := cli.newBuilder(input.Bucket, input.Key).
WithQuery("acl", "").
WithParams(*input)
res, err := builder.WithRetry(OnRetryFromStart, StatusCodeClassifier{}).Request(ctx, http.MethodPut, content, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
return &PutObjectACLOutput{
PutObjectAclOutput{RequestInfo: res.RequestInfo()},
}, nil
}
// GetObjectAcl get object ACL
// objectKey: the name of object
// Options: WithVersionID the version of the object
//
// Deprecated: use GetObjectACL of ClientV2 instead
func (bkt *Bucket) GetObjectAcl(ctx context.Context, objectKey string, options ...Option) (*GetObjectAclOutput, error) {
if err := isValidKey(objectKey); err != nil {
return nil, err
}
res, err := bkt.client.newBuilder(bkt.name, objectKey, options...).
WithQuery("acl", "").
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
out := GetObjectAclOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(out.RequestID, res.Body, &out); err != nil {
return nil, err
}
out.VersionID = res.Header.Get(HeaderVersionID)
return &out, nil
}
// GetObjectACL get object ACL
func (cli *ClientV2) GetObjectACL(ctx context.Context, input *GetObjectACLInput) (*GetObjectACLOutput, error) {
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
if err := isValidKey(input.Key); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key).
WithQuery("acl", "").
WithParams(*input).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
out := GetObjectACLOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(out.RequestID, res.Body, &out); err != nil {
return nil, err
}
out.VersionID = res.Header.Get(HeaderVersionID)
return &out, nil
}
func (cli *ClientV2) GetBucketACL(ctx context.Context, input *GetBucketACLInput) (*GetBucketACLOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("acl", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketACLOutput{RequestInfo: res.RequestInfo()}
marshalRes := bucketACL{}
if err = marshalOutput(output.RequestID, res.Body, &marshalRes); err != nil {
return nil, err
}
output.Grants = marshalRes.GrantList
output.Owner = marshalRes.Owner
return &output, nil
}
func (cli *ClientV2) PutBucketACL(ctx context.Context, input *PutBucketACLInput) (*PutBucketACLOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
reqBuilder := cli.newBuilder(input.Bucket, "").
WithQuery("acl", "").
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
WithParams(*input)
var reqData io.Reader
if input.Owner.ID != "" && len(input.Grants) != 0 {
data, contentMD5, err := marshalInput("PutBucketACLInput", bucketACL{
Owner: input.Owner,
GrantList: input.Grants,
})
if err != nil {
return nil, err
}
_ = reqBuilder.WithHeader(HeaderContentMD5, contentMD5)
reqData = bytes.NewReader(data)
}
res, err := reqBuilder.Request(ctx, http.MethodPut, reqData, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketACLOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

124
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/base_client.go generated vendored Normal file
View File

@@ -0,0 +1,124 @@
package tos
import (
"bytes"
"context"
"net/http"
)
func newBaseClient(c *Client) *baseClient {
return &baseClient{Client: c}
}
type baseClient struct {
*Client
}
func (cli *baseClient) PutObjectTagging(ctx context.Context, input *PutObjectTaggingInput, option ...Option) (*PutObjectTaggingOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutObjectTaggingInput", putObjectTaggingInput{
TagSet: input.TagSet,
})
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key, option...).
WithQuery("tagging", "").
WithParams(*input).
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutObjectTaggingOutput{RequestInfo: res.RequestInfo()}
output.VersionID = res.Header.Get(HeaderVersionID)
return &output, nil
}
func (cli *baseClient) GetObjectTagging(ctx context.Context, input *GetObjectTaggingInput, option ...Option) (*GetObjectTaggingOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key, option...).
WithQuery("tagging", "").
WithParams(*input).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetObjectTaggingOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
output.VersionID = res.Header.Get(HeaderVersionID)
return &output, nil
}
func (cli *baseClient) DeleteObjectTagging(ctx context.Context, input *DeleteObjectTaggingInput, option ...Option) (*DeleteObjectTaggingOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key, option...).
WithQuery("tagging", "").
WithParams(*input).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteObjectTaggingOutput{RequestInfo: res.RequestInfo()}
output.VersionID = res.Header.Get(HeaderVersionID)
return &output, nil
}
func (cli *baseClient) RestoreObject(ctx context.Context, input *RestoreObjectInput, option ...Option) (*RestoreObjectOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("RestoreObjectInput", restoreObjectInput{
Days: input.Days,
RestoreJobParameters: input.RestoreJobParameters,
})
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key, option...).
WithParams(*input).
WithQuery("restore", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPost, bytes.NewReader(data), cli.roundTripper(http.StatusOK, http.StatusAccepted))
if err != nil {
return nil, err
}
defer res.Close()
output := RestoreObjectOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

264
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/bucket.go generated vendored Normal file
View File

@@ -0,0 +1,264 @@
package tos
import (
"bytes"
"context"
"net/http"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
)
// Bucket create a Bucket handle
//
// Deprecated: request with bucket handle is deprecated, use ClientV2 instead
func (cli *Client) Bucket(bucket string) (*Bucket, error) {
if err := isValidBucketName(bucket, false); err != nil {
return nil, err
}
return &Bucket{name: bucket, client: cli, baseClient: newBaseClient(cli)}, nil
}
// CreateBucket create a bucket
//
// Deprecated: use CreateBucket of ClientV2 instead
func (cli *Client) CreateBucket(ctx context.Context, input *CreateBucketInput) (*CreateBucketOutput, error) {
if err := isValidBucketName(input.Bucket, false); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithHeader(HeaderACL, input.ACL).
WithHeader(HeaderGrantFullControl, input.GrantFullControl).
WithHeader(HeaderGrantRead, input.GrantRead).
WithHeader(HeaderGrantReadAcp, input.GrantReadAcp).
WithHeader(HeaderGrantWrite, input.GrantWrite).
WithHeader(HeaderGrantWriteAcp, input.GrantWriteAcp).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
return &CreateBucketOutput{
RequestInfo: res.RequestInfo(),
Location: res.Header.Get(HeaderLocation),
}, nil
}
// CreateBucketV2 create a bucket
func (cli *ClientV2) CreateBucketV2(ctx context.Context, input *CreateBucketV2Input) (*CreateBucketV2Output, error) {
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
// TODO: ACL和Grant不能同时设置，可以在sdk校验
if err := isValidACL(input.ACL); len(input.ACL) != 0 && err != nil {
return nil, err
}
if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) != 0 && err != nil {
return nil, err
}
if err := isValidAzRedundancy(input.AzRedundancy); len(input.AzRedundancy) != 0 && err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithParams(*input).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
return &CreateBucketV2Output{
CreateBucketOutput: CreateBucketOutput{
RequestInfo: res.RequestInfo(),
Location: res.Header.Get(HeaderLocation)}}, nil
}
// HeadBucket get some info of a bucket
//
// Deprecated: use HeadBucket of ClientV2 instead
func (cli *Client) HeadBucket(ctx context.Context, bucket string) (*HeadBucketOutput, error) {
if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(bucket, "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodHead, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
return &HeadBucketOutput{
RequestInfo: res.RequestInfo(),
Region: res.Header.Get(HeaderBucketRegion),
StorageClass: enum.StorageClassType(res.Header.Get(HeaderStorageClass)),
AzRedundancy: enum.AzRedundancyType(res.Header.Get(HeaderAzRedundancy)),
}, nil
}
// HeadBucket get some info of a bucket
func (cli *ClientV2) HeadBucket(ctx context.Context, input *HeadBucketInput) (*HeadBucketOutput, error) {
return cli.Client.HeadBucket(ctx, input.Bucket)
}
// DeleteBucket delete a bucket
//
// Deprecated: use DeleteBucket of ClientV2 instead
func (cli *Client) DeleteBucket(ctx context.Context, bucket string) (*DeleteBucketOutput, error) {
if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(bucket, "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
return &DeleteBucketOutput{RequestInfo: res.RequestInfo()}, nil
}
// DeleteBucket delete a bucket.Deleting a non-empty bucket is not allowed.
// A bucket is empty only if there is no exist object and uncanceled segmented tasks.
func (cli *ClientV2) DeleteBucket(ctx context.Context, input *DeleteBucketInput) (*DeleteBucketOutput, error) {
return cli.Client.DeleteBucket(ctx, input.Bucket)
}
// ListBuckets list the buckets that the AK can access
//
// Deprecated: use ListBuckets of ClientV2 instead
func (cli *Client) ListBuckets(ctx context.Context, _ *ListBucketsInput) (*ListBucketsOutput, error) {
res, err := cli.newBuilder("", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := ListBucketsOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
// ListBuckets list the buckets that the AK can access
func (cli *ClientV2) ListBuckets(ctx context.Context, _ *ListBucketsInput) (*ListBucketsOutput, error) {
res, err := cli.newBuilder("", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := ListBucketsOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) PutBucketStorageClass(ctx context.Context, input *PutBucketStorageClassInput) (*PutBucketStorageClassOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
if err := isValidStorageClass(input.StorageClass); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("storageClass", "").
WithParams(*input).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketStorageClassOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketLocation(ctx context.Context, input *GetBucketLocationInput) (*GetBucketLocationOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("location", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketLocationOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) PutBucketVersioning(ctx context.Context, input *PutBucketVersioningInput) (*PutBucketVersioningOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutBucketVersioning", putBucketVersioningInput{
Status: input.Status,
})
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("versioning", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketVersioningOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketVersioning(ctx context.Context, input *GetBucketVersioningInput) (*GetBucketVersioningOutputV2, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("versioning", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketVersioningOutputV2{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}

126
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/check.go generated vendored Normal file
View File

@@ -0,0 +1,126 @@
package tos
import (
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
)
func IsValidBucketName(name string) error {
if length := len(name); length < 3 || length > 63 {
return InvalidBucketNameLength
}
for i := range name {
if char := name[i]; !(('a' <= char && char <= 'z') || ('0' <= char && char <= '9') || char == '-') {
return InvalidBucketNameCharacter
}
}
if name[0] == '-' || name[len(name)-1] == '-' {
return InvalidBucketNameStartingOrEnding
}
return nil
}
// isValidBucketName validate bucket name, return TosClientError if failed
func isValidBucketName(name string, isCustomDomain bool) error {
if isCustomDomain {
return nil
}
return IsValidBucketName(name)
}
// isValidNames validate bucket name and keys, return TosClientError if failed
func isValidNames(bucket string, key string, isCustomDomain bool, keys ...string) error {
if err := isValidBucketName(bucket, isCustomDomain); err != nil {
return err
}
if err := isValidKey(key, keys...); err != nil {
return err
}
return nil
}
// validKey validate single key, return TosClientError if failed
func validKey(key string) error {
if len(key) < 1 {
return InvalidObjectNameLength
}
return nil
}
// isValidKey validate keys, return TosClientError if failed
func isValidKey(key string, keys ...string) error {
if err := validKey(key); err != nil {
return err
}
for _, k := range keys {
if err := validKey(k); err != nil {
return err
}
}
return nil
}
// isValidACL validate aclType, return TosClientError if failed
func isValidACL(aclType enum.ACLType) error {
if aclType == enum.ACLPrivate || aclType == enum.ACLPublicRead || aclType == enum.ACLPublicReadWrite ||
aclType == enum.ACLAuthRead || aclType == enum.ACLBucketOwnerRead ||
aclType == enum.ACLBucketOwnerFullControl || aclType == enum.ACLLogDeliveryWrite ||
aclType == enum.ACLBucketOwnerEntrusted {
return nil
}
return InvalidACL
}
// isValidStorageClass validate Storage Class, return TosClientError if failed
func isValidStorageClass(storageClass enum.StorageClassType) error {
if storageClass == enum.StorageClassIa || storageClass == enum.StorageClassStandard || storageClass == enum.StorageClassArchiveFr || storageClass == enum.StorageClassColdArchive || storageClass == enum.StorageClassIntelligentTiering {
return nil
}
return InvalidStorageClass
}
func isValidGrantee(granteeType enum.GranteeType) error {
if granteeType == enum.GranteeUser || granteeType == enum.GranteeGroup {
return nil
}
return InvalidGrantee
}
func isValidCannedType(cannedType enum.CannedType) error {
if cannedType == enum.CannedAllUsers || cannedType == enum.CannedAuthenticatedUsers {
return nil
}
return InvalidCanned
}
func isValidAzRedundancy(redundancyType enum.AzRedundancyType) error {
if redundancyType == enum.AzRedundancySingleAz || redundancyType == enum.AzRedundancyMultiAz {
return nil
}
return InvalidAzRedundancy
}
func isValidMetadataDirective(directiveType enum.MetadataDirectiveType) error {
if directiveType == enum.MetadataDirectiveCopy || directiveType == enum.MetadataDirectiveReplace {
return nil
}
return InvalidMetadataDirective
}
func isValidPermission(permissionType enum.PermissionType) error {
if permissionType == enum.PermissionRead || permissionType == enum.PermissionReadAcp ||
permissionType == enum.PermissionWriteAcp || permissionType == enum.PermissionWrite ||
permissionType == enum.PermissionFullControl {
return nil
}
return InvalidPermission
}
func isValidSSECAlgorithm(algorithm string) error {
if algorithm == enum.SSETosAlg || algorithm == enum.SSEKMS {
return nil
}
return InvalidSSECAlgorithm
}

55
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/checksum.go generated vendored Normal file
View File

@@ -0,0 +1,55 @@
package tos
import (
"crypto/md5"
"encoding/hex"
"errors"
"hash"
"io"
"strings"
)
var (
ErrETagMissMatch = errors.New("tos: ETag miss match")
)
// ETagCheckReadCloser checks ETag on read EOF
type ETagCheckReadCloser struct {
reader io.Reader
closer io.Closer
checksum hash.Hash
eTag string
requestID string
}
func NewETagCheckReadCloser(reader io.ReadCloser, eTag, requestID string) *ETagCheckReadCloser {
checksum := md5.New()
return &ETagCheckReadCloser{
reader: io.TeeReader(reader, checksum),
closer: reader,
checksum: checksum,
eTag: strings.Trim(eTag, `"`),
requestID: requestID,
}
}
func (ec *ETagCheckReadCloser) Read(p []byte) (n int, err error) {
n, err = ec.reader.Read(p)
if err == io.EOF && len(ec.eTag) > 0 {
sum := ec.checksum.Sum(nil)
if hexSum := hex.EncodeToString(sum); hexSum != ec.eTag {
return n, &ChecksumError{
RequestID: ec.requestID,
ExpectedChecksum: ec.eTag,
ActualChecksum: hexSum,
}
}
}
return n, err
}
func (ec *ETagCheckReadCloser) Close() error {
return ec.closer.Close()
}

740
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/client.go generated vendored Normal file
View File

@@ -0,0 +1,740 @@
package tos
import (
"bytes"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"net"
"net/http"
"net/url"
"runtime"
"strconv"
"strings"
"time"
)
const (
signPolicyDate = "x-tos-date"
signPolicyCredential = "x-tos-credential"
signPolicyAlgorithm = "x-tos-algorithm"
signPolicySecurityToken = "x-tos-security-token"
signPolicyExpiration = "expiration"
signConditionContentLengthRange = "content-length-range"
signConditionBucket = "bucket"
signConditionKey = "key"
signConditions = "conditions"
maxPreSignExpires = 604800 // 7 day
defaultSignExpires = 3600 // 1 hour
)
// Client TOS Client
// use NewClient to create a new Client
//
// example:
// client, err := NewClient(endpoint, WithCredentials(credentials), WithRegion(region))
// if err != nil {
// // ...
// }
// // do something
//
// if you only access the public bucket:
// client, err := NewClient(endpoint)
// // do something
//
// Deprecated: use ClientV2 instead
type Client struct {
scheme string
host string
urlMode urlMode
userAgent string
credentials Credentials // nullable
signer Signer // nullable
transport Transport
recognizer ContentTypeRecognizer
config Config
retry *retryer
enableCRC bool
logger Logger
isCustomDomain bool
}
// ClientV2 TOS ClientV2
// use NewClientV2 to create a new ClientV2
//
// example:
// client, err := NewClientV2(endpoint, WithCredentials(credentials), WithRegion(region))
// if err != nil {
// // ...
// }
// // do something
//
// if you only access the public bucket:
// client, err := NewClientV2(endpoint)
// // do something
//
type ClientV2 struct {
Client
baseClient *baseClient
}
func (cli *ClientV2) Close() {
if t, ok := cli.transport.(*DefaultTransport); ok {
if h, ok := t.client.Transport.(*http.Transport); ok {
h.CloseIdleConnections()
}
}
}
func (cli *ClientV2) SetHTTPTransport(transport http.RoundTripper) {
cli.transport = newDefaultTranposrtWithHTTPTransport(transport)
}
type ClientOption func(*Client)
// WithCredentials set Credentials
//
// see StaticCredentials, WithoutSecretKeyCredentials and FederationCredentials
func WithCredentials(credentials Credentials) ClientOption {
return func(client *Client) {
client.credentials = credentials
}
}
// WithEnableVerifySSL set whether a client verifies the server's certificate chain and host name.
func WithEnableVerifySSL(enable bool) ClientOption {
skip := !enable
return func(client *Client) {
client.config.TransportConfig.InsecureSkipVerify = skip
}
}
// WithRequestTimeout set timeout for single http request
func WithRequestTimeout(timeout time.Duration) ClientOption {
return func(client *Client) {
client.config.TransportConfig.ResponseHeaderTimeout = timeout
}
}
//
// WithLogger sets the tos sdk logger
//
func WithLogger(logger Logger) ClientOption {
return func(client *Client) {
client.logger = logger
}
}
// WithConnectionTimeout set timeout for constructing connection
func WithConnectionTimeout(timeout time.Duration) ClientOption {
return func(client *Client) {
client.config.TransportConfig.DialTimeout = timeout
}
}
// WithProxy set http Proxy for tos client
func WithProxy(proxy *Proxy) ClientOption {
return func(client *Client) {
client.config.TransportConfig.Proxy = proxy
}
}
// WithMaxConnections set maximum number of http connections
func WithMaxConnections(max int) ClientOption {
return func(client *Client) {
client.config.TransportConfig.MaxIdleConns = max
client.config.TransportConfig.MaxIdleConnsPerHost = max
client.config.TransportConfig.MaxConnsPerHost = max
}
}
// WithIdleConnTimeout set max idle time of a http connection
func WithIdleConnTimeout(timeout time.Duration) ClientOption {
return func(client *Client) {
client.config.TransportConfig.IdleConnTimeout = timeout
}
}
// WithUserAgentSuffix set suffix of user-agent
func WithUserAgentSuffix(suffix string) ClientOption {
return func(client *Client) {
client.userAgent = strings.Join([]string{client.userAgent, suffix}, " ")
}
}
// WithDNSCacheTime set dnsCacheTime in Minute
func WithDNSCacheTime(dnsCacheTime int) ClientOption {
return func(client *Client) {
client.config.TransportConfig.DNSCacheTime = time.Minute * time.Duration(dnsCacheTime)
}
}
// WithEnableCRC set if check crc after uploading object.
// Checking crc is enabled by default.
func WithEnableCRC(enableCRC bool) ClientOption {
return func(client *Client) {
client.enableCRC = enableCRC
}
}
// // WithMaxRetryCount set MaxRetryCount
func WithMaxRetryCount(retryCount int) ClientOption {
return func(client *Client) {
if client.retry != nil {
client.retry.SetBackoff(exponentialBackoff(retryCount, DefaultRetryBackoffBase))
}
}
}
func WithCustomDomain(isCustomDomain bool) ClientOption {
return func(client *Client) {
client.isCustomDomain = isCustomDomain
}
}
// WithTransport set Transport
//
// Deprecated: this function is Deprecated.
// If you want to set http.Transport use WithHTTPTransport instead
func WithTransport(transport Transport) ClientOption {
return func(client *Client) {
client.transport = transport
}
}
// WithHTTPTransport set Transport of http.Client
func WithHTTPTransport(transport http.RoundTripper) ClientOption {
return func(client *Client) {
client.transport = newDefaultTranposrtWithHTTPTransport(transport)
}
}
// WithTransportConfig set TransportConfig
func WithTransportConfig(config *TransportConfig) ClientOption {
return func(client *Client) {
// client.config never be nil
client.config.TransportConfig = *config
}
}
// WithSocketTimeout set read-write timeout
func WithSocketTimeout(readTimeout, writeTimeout time.Duration) ClientOption {
return func(client *Client) {
client.config.TransportConfig.ReadTimeout = readTimeout
client.config.TransportConfig.WriteTimeout = writeTimeout
}
}
// WithRegion set region
func WithRegion(region string) ClientOption {
return func(client *Client) {
// client.config never be nil
client.config.Region = region
if endpoint, ok := SupportedRegion()[region]; ok {
if len(client.config.Endpoint) == 0 {
client.config.Endpoint = endpoint
}
}
}
}
// WithSigner for self-defined Signer
func WithSigner(signer Signer) ClientOption {
return func(client *Client) {
client.signer = signer
}
}
// WithPathAccessMode url mode is path model or default mode
//
// Deprecated: This option is deprecated. Setting PathAccessMode will be ignored silently.
func WithPathAccessMode(pathAccessMode bool) ClientOption {
return func(client *Client) {
}
}
// WithAutoRecognizeContentType set to recognize Content-Type or not, the default is enabled.
func WithAutoRecognizeContentType(enable bool) ClientOption {
return func(client *Client) {
if enable {
client.recognizer = ExtensionBasedContentTypeRecognizer{}
} else {
client.recognizer = EmptyContentTypeRecognizer{}
}
}
}
// WithContentTypeRecognizer set ContentTypeRecognizer to recognize Content-Type,
// the default is ExtensionBasedContentTypeRecognizer
func WithContentTypeRecognizer(recognizer ContentTypeRecognizer) ClientOption {
return func(client *Client) {
client.recognizer = recognizer
}
}
func schemeHost(endpoint string) (scheme string, host string, urlMode urlMode) {
if strings.HasPrefix(endpoint, "https://") {
scheme = "https"
host = endpoint[len("https://"):]
} else if strings.HasPrefix(endpoint, "http://") {
scheme = "http"
host = endpoint[len("http://"):]
} else {
scheme = "https"
host = endpoint
}
urlMode = urlModeDefault
hostWithoutPort, _, _ := net.SplitHostPort(host)
if net.ParseIP(host) != nil || net.ParseIP(hostWithoutPort) != nil {
urlMode = urlModePath
}
return scheme, host, urlMode
}
func initClient(client *Client, endpoint string, options ...ClientOption) error {
client.config.Endpoint = endpoint
for _, option := range options {
option(client)
}
client.scheme, client.host, client.urlMode = schemeHost(client.config.Endpoint)
if client.transport == nil {
transport := NewDefaultTransport(&client.config.TransportConfig)
transport.WithDefaultTransportLogger(client.logger)
client.transport = transport
}
if cred := client.credentials; cred != nil && client.signer == nil {
if len(client.config.Region) == 0 {
if region, ok := SupportedEndpoint()[client.host]; ok {
client.config.Region = region
} else {
return newTosClientError("tos: missing Region option", nil)
}
}
signer := NewSignV4(cred, client.config.Region)
signer.WithSignLogger(client.logger)
client.signer = signer
}
return nil
}
// NewClient create a new Tos Client
// endpoint: access endpoint
// options: WithCredentials set Credentials
// WithRegion set region, this is required if WithCredentials is used
// WithSocketTimeout set read-write timeout
// WithTransportConfig set TransportConfig
// WithTransport set self-defined Transport
func NewClient(endpoint string, options ...ClientOption) (*Client, error) {
client := Client{
recognizer: ExtensionBasedContentTypeRecognizer{},
config: defaultConfig(),
userAgent: fmt.Sprintf("tos-go-sdk/%s (%s/%s;%s)", Version, runtime.GOOS, runtime.GOARCH, runtime.Version()),
retry: newRetryer([]time.Duration{}),
}
client.retry.SetJitter(0.25)
err := initClient(&client, endpoint, options...)
if err != nil {
return nil, err
}
return &client, nil
}
// NewClientV2 create a new Tos ClientV2
// endpoint: access endpoint
// options: WithCredentials set Credentials
// WithRegion set region, this is required if WithCredentials is used.
// If Region is supported and the Endpoint parameter is not set, the Endpoint will be resolved automatically
// WithSocketTimeout set read-write timeout
// WithTransportConfig set TransportConfig
// WithTransport set self-defined Transport
// WithLogger set self-defined Logger
// WithEnableCRC set CRC switch.
// WithMaxRetryCount set Max Retry Count
func NewClientV2(endpoint string, options ...ClientOption) (*ClientV2, error) {
if strings.Contains(endpoint, "s3") {
return nil, InvalidS3Endpoint
}
client := ClientV2{
Client: Client{
recognizer: ExtensionBasedContentTypeRecognizer{},
config: defaultConfig(),
retry: newRetryer(exponentialBackoff(DefaultRetryTime, DefaultRetryBackoffBase)),
userAgent: fmt.Sprintf("tos-go-sdk/%s (%s/%s;%s)", Version, runtime.GOOS, runtime.GOARCH, runtime.Version()),
enableCRC: true,
},
}
client.retry.SetJitter(0.25)
err := initClient(&client.Client, endpoint, options...)
if err != nil {
return nil, err
}
client.baseClient = newBaseClient(&client.Client)
return &client, nil
}
func (cli *Client) newBuilder(bucket, object string, options ...Option) *requestBuilder {
rb := &requestBuilder{
Signer: cli.signer,
Scheme: cli.scheme,
Host: cli.host,
Bucket: bucket,
Object: object,
URLMode: cli.urlMode,
Query: make(url.Values),
Header: make(http.Header),
OnRetry: func(req *Request) error { return nil },
Classifier: StatusCodeClassifier{},
IsCustomDomain: cli.isCustomDomain,
}
rb.Header.Set(HeaderUserAgent, cli.userAgent)
if typ := cli.recognizer.ContentType(object); len(typ) > 0 {
rb.Header.Set(HeaderContentType, typ)
}
for _, option := range options {
option(rb)
}
rb.Retry = cli.retry
return rb
}
func (cli *Client) roundTrip(ctx context.Context, req *Request, expectedCode int, expectedCodes ...int) (*Response, error) {
res, err := cli.transport.RoundTrip(ctx, req)
if err != nil {
return nil, err
}
readBody := req.Method != http.MethodHead
if err = checkError(res, readBody, expectedCode, expectedCodes...); err != nil {
return nil, err
}
return res, nil
}
func (cli *Client) roundTripper(expectedCode int, expectedCodes ...int) roundTripper {
return func(ctx context.Context, req *Request) (*Response, error) {
start := time.Now()
resp, err := cli.roundTrip(ctx, req, expectedCode, expectedCodes...)
if cli.logger != nil {
if err != nil {
cli.logger.Info(fmt.Sprintf("[tos] http error:%s.", err.Error()))
} else {
cli.logger.Info(fmt.Sprintf("[tos] Response StatusCode:%d, RequestId:%s, Cost:%d ms", resp.StatusCode, resp.RequestInfo().RequestID, time.Since(start).Milliseconds()))
}
}
return resp, err
}
}
// PreSignedURL return pre-signed url
// httpMethod: HTTP method, {
// PutObject: http.MethodPut
// GetObject: http.MethodGet
// HeadObject: http.MethodHead
// DeleteObject: http.MethodDelete
// },
// bucket: the bucket name
// objectKey: the object name
// ttl: the time-to-live of signed URL
// options: WithVersionID the version id of the object
// Deprecated: use PreSignedURL of ClientV2 instead
func (cli *Client) PreSignedURL(httpMethod string, bucket, objectKey string, ttl time.Duration, options ...Option) (string, error) {
return cli.newBuilder(bucket, objectKey, options...).
PreSignedURL(httpMethod, ttl)
}
// PreSignedURL return pre-signed url
func (cli *ClientV2) PreSignedURL(input *PreSignedURLInput) (*PreSignedURLOutput, error) {
rb := cli.newBuilder(input.Bucket, input.Key)
if input.IsCustomDomain != nil {
rb.IsCustomDomain = *input.IsCustomDomain
}
if input.AlternativeEndpoint != "" {
schema, host, _ := schemeHost(input.AlternativeEndpoint)
rb.Host = host
rb.Scheme = schema
}
for k, v := range input.Header {
rb.WithHeader(k, v)
}
for k, v := range input.Query {
rb.WithQuery(k, v)
}
if input.Expires == 0 {
input.Expires = defaultPreSignedURLExpires
}
signedURL, err := rb.PreSignedURL(string(input.HTTPMethod), time.Second*time.Duration(input.Expires))
if err != nil {
return nil, err
}
signed := make(map[string]string)
for k := range rb.Header {
signed[k] = rb.Header.Get(k)
}
output := &PreSignedURLOutput{
SignedUrl: signedURL,
SignedHeader: signed,
}
return output, nil
}
func (cli *ClientV2) PreSignedPostSignature(ctx context.Context, input *PreSingedPostSignatureInput) (*PreSingedPostSignatureOutput, error) {
algorithm := signPrefix
postPolicy := make(map[string]interface{})
cred := cli.credentials.Credential()
region := cli.config.Region
date := UTCNow()
if input.Expires == 0 {
input.Expires = defaultSignExpires
}
postPolicy[signPolicyExpiration] = date.Add(time.Second * time.Duration(input.Expires)).Format(serverTimeFormat)
cond := make([]interface{}, 0)
credential := fmt.Sprintf("%s/%s/%s/tos/request", cred.AccessKeyID, date.Format(yyMMdd), region)
cond = append(cond, map[string]string{signPolicyAlgorithm: algorithm})
cond = append(cond, map[string]string{signPolicyCredential: credential})
cond = append(cond, map[string]string{signPolicyDate: date.Format(iso8601Layout)})
if cred.SecurityToken != "" {
cond = append(cond, map[string]string{signPolicySecurityToken: cred.SecurityToken})
}
if input.Bucket != "" {
cond = append(cond, map[string]string{signConditionBucket: input.Bucket})
}
if input.Key != "" {
cond = append(cond, map[string]string{signConditionKey: input.Key})
}
for _, condition := range input.Conditions {
if condition.Operator != nil {
cond = append(cond, []string{*condition.Operator, "$" + condition.Key, condition.Value})
} else {
cond = append(cond, map[string]string{condition.Key: condition.Value})
}
}
if input.ContentLengthRange != nil {
cond = append(cond, []interface{}{signConditionContentLengthRange, input.ContentLengthRange.RangeStart, input.ContentLengthRange.RangeEnd})
}
postPolicy[signConditions] = cond
originPolicy, err := json.Marshal(postPolicy)
if err != nil {
return nil, InvalidMarshal
}
signK := SigningKey(&SigningKeyInfo{
Date: date.Format(yyMMdd),
Region: region,
Credential: &cred,
})
policy := base64.StdEncoding.EncodeToString(originPolicy)
return &PreSingedPostSignatureOutput{
OriginPolicy: string(originPolicy),
Policy: policy,
Algorithm: signPrefix,
Credential: credential,
Date: date.Format(iso8601Layout),
Signature: hex.EncodeToString(hmacSHA256(signK, []byte(policy))),
}, nil
}
func (cli *ClientV2) FetchObjectV2(ctx context.Context, input *FetchObjectInputV2) (*FetchObjectOutputV2, error) {
if err := isValidKey(input.Key); err != nil {
return nil, err
}
if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) > 0 && err != nil {
return nil, InvalidStorageClass
}
if err := isValidACL(input.ACL); len(input.ACL) > 0 && err != nil {
return nil, InvalidACL
}
data, contentMD5, err := marshalInput("FetchObjectInputV2", &fetchObjectInput{
URL: input.URL,
IgnoreSameKey: input.IgnoreSameKey,
ContentMD5: input.HexMD5,
})
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key).
WithQuery("fetch", "").
WithHeader(HeaderContentMD5, contentMD5).
WithParams(*input).
WithRetry(OnRetryFromStart, ServerErrorClassifier{}).
Request(ctx, http.MethodPost, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := FetchObjectOutputV2{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
output.VersionID = res.Header.Get(HeaderVersionID)
output.SSECAlgorithm = res.Header.Get(HeaderSSECustomerAlgorithm)
output.SSECKeyMD5 = res.Header.Get(HeaderCopySourceSSECKeyMD5)
return &output, nil
}
func (cli *ClientV2) PutFetchTaskV2(ctx context.Context, input *PutFetchTaskInputV2) (*PutFetchTaskOutputV2, error) {
if err := isValidKey(input.Key); err != nil {
return nil, err
}
if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) > 0 && err != nil {
return nil, err
}
if err := isValidACL(input.ACL); len(input.ACL) > 0 && err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutFetchTaskInputV2", putFetchTaskV2Input{
URL: input.URL,
IgnoreSameKey: input.IgnoreSameKey,
HexMD5: input.HexMD5,
Object: input.Key,
})
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("fetchTask", "").
WithHeader(HeaderContentMD5, contentMD5).
WithParams(*input).
WithRetry(OnRetryFromStart, ServerErrorClassifier{}).
Request(ctx, http.MethodPost, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutFetchTaskOutputV2{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) PreSignedPolicyURL(ctx context.Context, input *PreSingedPolicyURLInput) (*PreSingedPolicyURLOutput, error) {
if err := isValidBucketName(input.Bucket, input.IsCustomDomain); err != nil {
return nil, err
}
if input.Expires == 0 {
input.Expires = defaultSignExpires
}
policyConditions := make(map[string]interface{})
cred := cli.credentials.Credential()
region := cli.config.Region
query := make(url.Values)
date := UTCNow()
query.Add(v4Expires, strconv.FormatInt(input.Expires, 10))
// algorithm
algorithm := signPrefix
query.Add(v4Algorithm, algorithm)
// data
dateQuery := date.Format(iso8601Layout)
query.Add(v4Date, dateQuery)
// credential
credential := fmt.Sprintf("%s/%s/%s/tos/request", cred.AccessKeyID, date.Format(yyMMdd), region)
query.Add(v4Credential, credential)
if cred.SecurityToken != "" {
query.Add(v4SecurityToken, cred.SecurityToken)
}
// input conditions
cond := make([]interface{}, 0)
for _, condition := range input.Conditions {
key := condition.Key
operator := condition.Operator
if key != "key" {
return nil, InvalidPreSignedConditions
}
if operator != nil && !(*operator == "eq" || *operator == "starts-with") {
return nil, InvalidPreSignedConditions
}
if operator != nil {
cond = append(cond, []string{*operator, "$" + key, condition.Value})
} else {
cond = append(cond, map[string]string{key: condition.Value})
}
}
cond = append(cond, map[string]string{"bucket": input.Bucket})
policyConditions[signConditions] = cond
originPolicy, err := json.Marshal(policyConditions)
if err != nil {
return nil, InvalidMarshal
}
policy := base64.StdEncoding.EncodeToString(originPolicy)
query.Add("X-Tos-Policy", policy)
// CanonicalRequest
const split = byte('\n')
var buf bytes.Buffer
var req bytes.Buffer
req.Grow(512)
var queryReq = make(KVs, 0, len(query))
for key, values := range query {
queryReq = append(queryReq, KV{Key: key, Values: values})
}
req.Write(encodeQuery(queryReq))
req.WriteByte(split)
req.WriteString(unsignedPayload)
canonicalStr := req.String()
// StringToSign
buf.Grow(len(signPrefix) + 128)
buf.WriteString(signPrefix)
buf.WriteByte(split)
buf.WriteString(date.Format(iso8601Layout))
buf.WriteByte(split)
buf.WriteString(date.Format(yyMMdd)) // yyMMdd + '/' + region + '/' + service + '/' + request
buf.WriteByte('/')
buf.WriteString(region)
buf.WriteString("/tos/request")
buf.WriteByte(split)
sum := sha256.Sum256([]byte(canonicalStr))
buf.WriteString(hex.EncodeToString(sum[:]))
// SigningKey
signK := SigningKey(&SigningKeyInfo{Date: date.Format(yyMMdd), Region: region, Credential: &cred})
// Signature
sign := hmacSHA256(signK, buf.Bytes())
query.Add(v4Signature, hex.EncodeToString(sign))
rawQuery := query.Encode()
// set scheme and host
resScheme := cli.scheme
resHost := cli.host
if input.AlternativeEndpoint != "" {
resScheme, resHost, _ = schemeHost(input.AlternativeEndpoint)
}
return &PreSingedPolicyURLOutput{
bucket: input.Bucket,
SignatureQuery: rawQuery,
scheme: resScheme,
host: resHost,
isCustomDomain: input.IsCustomDomain,
}, nil
}

31
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/config.go generated vendored Normal file
View File

@@ -0,0 +1,31 @@
package tos
import "time"
type Config struct {
Endpoint string
Region string
TransportConfig TransportConfig
}
func defaultConfig() Config {
return Config{
TransportConfig: DefaultTransportConfig(),
}
}
func DefaultTransportConfig() TransportConfig {
return TransportConfig{
MaxIdleConns: 1024,
MaxIdleConnsPerHost: 1024,
MaxConnsPerHost: 1024,
DialTimeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
IdleConnTimeout: 60 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ResponseHeaderTimeout: 60 * time.Second,
ExpectContinueTimeout: 3 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
}
}

149
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/consts.go generated vendored Normal file
View File

@@ -0,0 +1,149 @@
package tos
import (
"hash/crc64"
"os"
)
const (
// Version tos-go-sdk version
Version = "v2.6.2"
)
const TempFileSuffix = ".temp"
const DefaultFilePerm = os.FileMode(0644)
var DefaultCrcTable = func() *crc64.Table {
return crc64.MakeTable(crc64.ECMA)
}
const DefaultTaskBufferSize = 100
const DefaultListMaxKeys = 1000
func SupportedRegion() map[string]string {
return map[string]string{
"cn-beijing": "tos-cn-beijing.volces.com",
"cn-guangzhou": "tos-cn-guangzhou.volces.com",
"cn-shanghai": "tos-cn-shanghai.volces.com",
}
}
func SupportedEndpoint() map[string]string {
supportEndpoint := make(map[string]string)
for key, value := range SupportedRegion() {
supportEndpoint[value] = key
}
return supportEndpoint
}
const (
defaultPreSignedURLExpires = 3600
maxPreSignedURLExpires = 604800
)
const (
MaxPartSize = 5 * 1024 * 1024 * 1024
MinPartSize = 5 * 1024 * 1024
DefaultPartSize = 20 * 1024 * 1024
)
const (
// Deprecated: use enum.ACLPrivate instead
ACLPrivate = "private"
// Deprecated: use enum.ACLPublicRead instead
ACLPublicRead = "public-read"
// Deprecated: use enum.ACLPublicReadWrite instead
ACLPublicReadWrite = "public-read-write"
// Deprecated: use enum.ACLAuthRead instead
ACLAuthRead = "authenticated-read"
// Deprecated: use enum.ACLBucketOwnerRead instead
ACLBucketOwnerRead = "bucket-owner-read"
// Deprecated: use enum.ACLBucketOwnerFullControl instead
ACLBucketOwnerFullControl = "bucket-owner-full-control"
// Deprecated: use enum.ACLLogDeliveryWrite instead
ACLLogDeliveryWrite = "log-delivery-write"
// Deprecated: use enum.PermissionRead instead
PermissionRead = "READ"
// Deprecated: use enum.PermissionWrite instead
PermissionTypeWrite = "WRITE"
// Deprecated: use enum.PermissionReadAcp instead
PermissionTypeReadAcp = "READ_ACP"
// Deprecated: use enum.PermissionWriteAcp instead
PermissionTypeWriteAcp = "WRITE_ACP"
// Deprecated: use enum.PermissionFullControl instead
PermissionFullControl = "FULL_CONTROL"
)
const (
ISO8601TimeFormat = "2006-01-02T15:04:05.000Z07:00"
)
const (
// MetadataDirectiveReplace replace source object metadata when calling CopyObject
MetadataDirectiveReplace = "REPLACE"
// MetadataDirectiveCopy copy source object metadata when calling CopyObject
MetadataDirectiveCopy = "COPY"
)
const (
QueryPartNumber = "partNumber"
)
const (
HeaderUserAgent = "User-Agent"
HeaderContentLength = "Content-Length"
HeaderContentType = "Content-Type"
HeaderContentMD5 = "Content-MD5"
HeaderContentSha256 = "X-Tos-Content-Sha256"
HeaderContentLanguage = "Content-Language"
HeaderContentEncoding = "Content-Encoding"
HeaderContentDisposition = "Content-Disposition"
HeaderLastModified = "Last-Modified"
HeaderCacheControl = "Cache-Control"
HeaderExpires = "Expires"
HeaderETag = "ETag"
HeaderVersionID = "X-Tos-Version-Id"
HeaderDeleteMarker = "X-Tos-Delete-Marker"
HeaderStorageClass = "X-Tos-Storage-Class"
HeaderAzRedundancy = "X-Tos-Az-Redundancy"
HeaderRestore = "X-Tos-Restore"
HeaderTag = "X-Tos-Tag"
HeaderSSECustomerAlgorithm = "X-Tos-Server-Side-Encryption-Customer-Algorithm"
HeaderSSECustomerKeyMD5 = "X-Tos-Server-Side-Encryption-Customer-Key-MD5"
HeaderSSECustomerKey = "X-Tos-Server-Side-Encryption-Customer-Key"
HeaderServerSideEncryption = "X-Tos-Server-Side-Encryption"
HeaderServerSideEncryptionKmsKeyID = "X-Tos-Server-Side-Encryption-Kms-Key-Id"
HeaderCopySourceSSECAlgorithm = "X-Tos-Server-Side-Encryption-Customer-Algorithm"
HeaderCopySourceSSECKeyMD5 = "X-Tos-Server-Side-Encryption-Customer-Key-MD5"
HeaderCopySourceSSECKey = "X-Tos-Server-Side-Encryption-Customer-Key"
HeaderIfModifiedSince = "If-Modified-Since"
HeaderIfUnmodifiedSince = "If-Unmodified-Since"
HeaderIfMatch = "If-Match"
HeaderIfNoneMatch = "If-None-Match"
HeaderRange = "Range"
HeaderContentRange = "Content-Range"
HeaderRequestID = "X-Tos-Request-Id"
HeaderID2 = "X-Tos-Id-2"
HeaderBucketRegion = "X-Tos-Bucket-Region"
HeaderLocation = "Location"
HeaderACL = "X-Tos-Acl"
HeaderGrantFullControl = "X-Tos-Grant-Full-Control"
HeaderGrantRead = "X-Tos-Grant-Read"
HeaderGrantReadAcp = "X-Tos-Grant-Read-Acp"
HeaderGrantWrite = "X-Tos-Grant-Write"
HeaderGrantWriteAcp = "X-Tos-Grant-Write-Acp"
HeaderNextAppendOffset = "X-Tos-Next-Append-Offset"
HeaderObjectType = "X-Tos-Object-Type"
HeaderHashCrc64ecma = "X-Tos-Hash-Crc64ecma"
HeaderMetadataDirective = "X-Tos-Metadata-Directive"
HeaderCopySource = "X-Tos-Copy-Source"
HeaderCopySourceIfMatch = "X-Tos-Copy-Source-If-Match"
HeaderCopySourceIfNoneMatch = "X-Tos-Copy-Source-If-None-Match"
HeaderCopySourceIfModifiedSince = "X-Tos-Copy-Source-If-Modified-Since"
HeaderCopySourceIfUnmodifiedSince = "X-Tos-Copy-Source-If-Unmodified-Since"
HeaderCopySourceRange = "X-Tos-Copy-Source-Range"
HeaderCopySourceVersionID = "X-Tos-Copy-Source-Version-Id"
HeaderWebsiteRedirectLocation = "X-Tos-Website-Redirect-Location"
HeaderCSType = "X-Tos-Cs-Type"
HeaderMetaPrefix = "X-Tos-Meta-"
)

310
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/copy.go generated vendored Normal file
View File

@@ -0,0 +1,310 @@
package tos
import (
"context"
"fmt"
"net/http"
"net/url"
"strconv"
"time"
)
// CopyObject copy an object
// srcObjectKey: the source object name
// dstObjectKey: the destination object name. srcObjectKey and dstObjectKey belongs to the same bucket.
// options: WithVersionID the version id of source object,
// WithMetadataDirective copy source object metadata or replace with new object metadata,
// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl,
// WithCopySourceIfMatch WithCopySourceIfNoneMatch WithCopySourceIfModifiedSince WithCopySourceIfUnmodifiedSince set copy conditions
// if CopyObject called with WithMetadataDirective(tos.MetadataDirectiveReplace), these options can be used:
// WithContentType set Content-Type,
// WithContentDisposition set Content-Disposition,
// WithContentLanguage set Content-Language,
// WithContentEncoding set Content-Encoding,
// WithCacheControl set Cache-Control,
// WithExpires set Expires,
// WithMeta set meta header(s),
//
// Deprecated: use CopyObject of ClientV2 instead
func (bkt *Bucket) CopyObject(ctx context.Context, srcObjectKey, dstObjectKey string, options ...Option) (*CopyObjectOutput, error) {
if err := isValidKey(dstObjectKey, srcObjectKey); err != nil {
return nil, err
}
return bkt.client.copyObject(ctx, bkt.name, dstObjectKey, bkt.name, srcObjectKey, options...)
}
// CopyObjectTo copy an object to target bucket
// dstBucket: the destination bucket
// dstObjectKey: the destination object name
// srcObjectKey: the source object name
// options: WithVersionID the version id of source object,
// WithMetadataDirective copy source object metadata or replace with new object metadata.
// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl,
// WithCopySourceIfMatch WithCopySourceIfNoneMatch WithCopySourceIfModifiedSince WithCopySourceIfUnmodifiedSince set copy conditions
// if CopyObjectTo called with WithMetadataDirective(tos.MetadataDirectiveReplace), these options can be used:
// WithContentType set Content-Type,
// WithContentDisposition set Content-Disposition,
// WithContentLanguage set Content-Language,
// WithContentEncoding set Content-Encoding,
// WithCacheControl set Cache-Control,
// WithExpires set Expires,
// WithMeta set meta header(s),
//
// Deprecated: use CopyObject of ClientV2 instead
func (bkt *Bucket) CopyObjectTo(ctx context.Context, dstBucket, dstObjectKey, srcObjectKey string, options ...Option) (*CopyObjectOutput, error) {
if err := isValidNames(dstBucket, dstObjectKey, false, srcObjectKey); err != nil {
return nil, err
}
return bkt.client.copyObject(ctx, dstBucket, dstObjectKey, bkt.name, srcObjectKey, options...)
}
// CopyObjectFrom copy an object from target bucket
// srcBucket: the srcBucket bucket
// srcObjectKey: the source object name
// dstObjectKey: the destination object name
// options: WithVersionID the version id of source object,
// WithMetadataDirective copy source object metadata or replace with new object metadata
// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl,
// WithCopySourceIfMatch WithCopySourceIfNoneMatch WithCopySourceIfModifiedSince WithCopySourceIfUnmodifiedSince set copy conditions
// if CopyObjectFrom called with WithMetadataDirective(tos.MetadataDirectiveReplace), these options can be used:
// WithContentType set Content-Type,
// WithContentDisposition set Content-Disposition,
// WithContentLanguage set Content-Language,
// WithContentEncoding set Content-Encoding,
// WithCacheControl set Cache-Control,
// WithExpires set Expires,
// WithMeta set meta header(s),
//
// Deprecated: use CopyObject of ClientV2 instead
func (bkt *Bucket) CopyObjectFrom(ctx context.Context, srcBucket, srcObjectKey, dstObjectKey string, options ...Option) (*CopyObjectOutput, error) {
if err := isValidNames(srcBucket, srcObjectKey, false, dstObjectKey); err != nil {
return nil, err
}
return bkt.client.copyObject(ctx, bkt.name, dstObjectKey, srcBucket, srcObjectKey, options...)
}
func (cli *Client) copyObject(ctx context.Context, dstBucket, dstObject string, srcBucket, srcObject string, options ...Option) (*CopyObjectOutput, error) {
res, err := cli.newBuilder(dstBucket, dstObject, options...).
WithCopySource(srcBucket, srcObject).
WithRetry(nil, ServerErrorClassifier{}).
Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
marshalOut := copyObjectOutput{}
if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &marshalOut); err != nil {
return nil, err
}
if marshalOut.ETag == "" {
return nil, &TosServerError{
TosError: TosError{marshalOut.Message},
RequestInfo: res.RequestInfo(),
Code: marshalOut.Code,
HostID: marshalOut.HostID,
Resource: marshalOut.Resource,
}
}
out := CopyObjectOutput{RequestInfo: res.RequestInfo(), ETag: marshalOut.ETag, LastModified: marshalOut.LastModified}
out.VersionID = res.Header.Get(HeaderVersionID)
out.SourceVersionID = res.Header.Get(HeaderCopySourceVersionID)
out.SSECAlgorithm = res.Header.Get(HeaderSSECustomerAlgorithm)
out.SSECKeyMD5 = res.Header.Get(HeaderSSECustomerKeyMD5)
out.ServerSideEncryption = res.Header.Get(HeaderServerSideEncryption)
out.ServerSideEncryptionKeyID = res.Header.Get(HeaderServerSideEncryptionKmsKeyID)
return &out, nil
}
// CopyObject copy an object
func (cli *ClientV2) CopyObject(ctx context.Context, input *CopyObjectInput) (*CopyObjectOutput, error) {
if err := isValidBucketName(input.SrcBucket, false); err != nil {
return nil, err
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
if err := isValidKey(input.Key, input.SrcKey); err != nil {
return nil, err
}
if err := isValidMetadataDirective(input.MetadataDirective); len(input.MetadataDirective) != 0 && err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key).
WithParams(*input).
WithCopySource(input.SrcBucket, input.SrcKey).
WithRetry(nil, ServerErrorClassifier{}).
Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
marshalOut := copyObjectOutput{}
if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &marshalOut); err != nil {
return nil, err
}
// Body 的 Etag 存在复制成功
if marshalOut.ETag == "" {
return nil, &TosServerError{
TosError: TosError{marshalOut.Message},
RequestInfo: res.RequestInfo(),
Code: marshalOut.Code,
HostID: marshalOut.HostID,
Resource: marshalOut.Resource,
}
}
out := CopyObjectOutput{RequestInfo: res.RequestInfo(), ETag: marshalOut.ETag, LastModified: marshalOut.LastModified}
out.VersionID = res.Header.Get(HeaderVersionID)
out.SourceVersionID = res.Header.Get(HeaderCopySourceVersionID)
return &out, nil
}
type uploadPartCopyOutput struct {
ETag string `json:"ETag,omitempty"`
LastModified string `json:"LastModified,omitempty"`
Error
}
func copyRange(startOffset, partSize *int64) string {
cr := ""
if startOffset != nil {
if partSize != nil {
cr = fmt.Sprintf("bytes=%d-%d", *startOffset, *startOffset+*partSize-1)
} else {
cr = fmt.Sprintf("bytes=%d-", *startOffset)
}
} else if partSize != nil {
cr = fmt.Sprintf("bytes=0-%d", *partSize-1)
}
return cr
}
func copySource(bucket, object, versionID string) string {
if len(versionID) == 0 {
return "/" + bucket + "/" + url.QueryEscape(object)
}
return "/" + bucket + "/" + url.QueryEscape(object) + "?versionId=" + versionID
}
func (up *UploadPartCopyOutput) uploadedPart() uploadedPart {
return uploadedPart{PartNumber: up.PartNumber, ETag: up.ETag}
}
// UploadPartCopy copy a part of object as a part of a multipart upload operation
// input: uploadID, DestinationKey, SourceBucket, SourceKey and other parameters,
// options: WithCopySourceIfMatch WithCopySourceIfNoneMatch WithCopySourceIfModifiedSince WithCopySourceIfUnmodifiedSince set copy conditions
//
// Deprecated: use UploadPartCopy of ClientV2 instead
func (bkt *Bucket) UploadPartCopy(ctx context.Context, input *UploadPartCopyInput, options ...Option) (*UploadPartCopyOutput, error) {
if err := isValidNames(input.SourceBucket, input.DestinationKey, false); err != nil {
return nil, err
}
res, err := bkt.client.newBuilder(bkt.name, input.DestinationKey, options...).
WithQuery("partNumber", strconv.Itoa(input.PartNumber)).
WithQuery("uploadId", input.UploadID).
WithQuery("versionId", input.SourceVersionID).
WithHeader(HeaderCopySourceRange, copyRange(input.StartOffset, input.PartSize)).
WithCopySource(input.SourceBucket, input.SourceKey).
WithRetry(nil, ServerErrorClassifier{}).
Request(ctx, http.MethodPut, nil, bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
var out uploadPartCopyOutput
if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &out); err != nil {
return nil, err
}
if out.ETag == "" {
return nil, &TosServerError{
TosError: TosError{out.Message},
RequestInfo: res.RequestInfo(),
Code: out.Code,
HostID: out.HostID,
Resource: out.Resource,
}
}
return &UploadPartCopyOutput{
RequestInfo: res.RequestInfo(),
VersionID: res.Header.Get(HeaderVersionID),
SourceVersionID: res.Header.Get(HeaderCopySourceVersionID),
PartNumber: input.PartNumber,
ETag: out.ETag,
LastModified: out.LastModified,
}, nil
}
func copyRangeV2(start, end int64) string {
cr := ""
if start == 0 && end == 0 {
return cr
}
if start > end {
return cr
}
cr = fmt.Sprintf("bytes=%d-%d", start, end)
return cr
}
// UploadPartCopyV2 copy a part of object as a part of a multipart upload operation
func (cli *ClientV2) UploadPartCopyV2(
ctx context.Context,
input *UploadPartCopyV2Input) (*UploadPartCopyV2Output, error) {
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
if err := isValidBucketName(input.SrcBucket, false); err != nil {
return nil, err
}
if err := isValidKey(input.SrcKey, input.Key); err != nil {
return nil, err
}
req := cli.newBuilder(input.Bucket, input.Key).
WithParams(*input)
if input.CopySourceRange != "" {
req = req.WithHeader(HeaderCopySourceRange, input.CopySourceRange)
} else if input.CopySourceRangeEnd != 0 {
req = req.WithHeader(HeaderCopySourceRange, copyRangeV2(input.CopySourceRangeStart, input.CopySourceRangeEnd))
}
res, err := req.WithCopySource(input.SrcBucket, input.SrcKey).
WithRetry(nil, ServerErrorClassifier{}).
Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
var out uploadPartCopyOutput
if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &out); err != nil {
return nil, err
}
lastModified, _ := time.ParseInLocation(http.TimeFormat, res.Header.Get(HeaderLastModified), time.UTC)
if out.ETag == "" {
return nil, &TosServerError{
TosError: TosError{out.Message},
RequestInfo: res.RequestInfo(),
Code: out.Code,
HostID: out.HostID,
Resource: out.Resource,
}
}
return &UploadPartCopyV2Output{
RequestInfo: res.RequestInfo(),
PartNumber: input.PartNumber,
ETag: out.ETag,
LastModified: lastModified,
CopySourceVersionID: res.Header.Get(HeaderCopySourceVersionID),
ServerSideEncryption: res.Header.Get(HeaderServerSideEncryption),
ServerSideEncryptionKeyID: res.Header.Get(HeaderServerSideEncryptionKmsKeyID),
SSECAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm),
SSECKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5),
}, nil
}

81
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/cors.go generated vendored Normal file
View File

@@ -0,0 +1,81 @@
package tos
import (
"bytes"
"context"
"net/http"
)
// GetBucketCORS get the bucket's CORS settings.
func (cli *ClientV2) GetBucketCORS(ctx context.Context, input *GetBucketCORSInput) (*GetBucketCORSOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("cors", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketCORSOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
// PutBucketCORS upsert the bucket's CORS settings.
func (cli *ClientV2) PutBucketCORS(ctx context.Context, input *PutBucketCORSInput) (*PutBucketCORSOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutBucketCORSInput", input)
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("cors", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketCORSOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
// DeleteBucketCORS delete the bucket's all CORS settings.
func (cli *ClientV2) DeleteBucketCORS(ctx context.Context, input *DeleteBucketCORSInput) (*DeleteBucketCORSOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("cors", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteBucketCORSOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

113
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/crc.go generated vendored Normal file
View File

@@ -0,0 +1,113 @@
package tos
import (
"hash"
"hash/crc64"
)
// digest represents the partial evaluation of a checksum.
type digest struct {
crc uint64
tab *crc64.Table
}
// NewCRC is similar with crc64.New, but you can set the initial value.
// Following methods are copied from package crc64 to implement Hash interface.
func NewCRC(tab *crc64.Table, init uint64) hash.Hash64 { return &digest{init, tab} }
func (d *digest) Size() int { return crc64.Size }
func (d *digest) BlockSize() int { return 1 }
func (d *digest) Reset() { d.crc = 0 }
func (d *digest) Write(p []byte) (n int, err error) {
d.crc = crc64.Update(d.crc, d.tab, p)
return len(p), nil
}
func (d *digest) Sum64() uint64 { return d.crc }
func (d *digest) Sum(in []byte) []byte {
s := d.Sum64()
return append(in, byte(s>>56), byte(s>>48), byte(s>>40), byte(s>>32), byte(s>>24), byte(s>>16), byte(s>>8), byte(s))
}
// gf2Dim dimension of GF(2) vectors (length of CRC)
const gf2Dim int = 64
func gf2MatrixTimes(mat []uint64, vec uint64) uint64 {
var sum uint64
for i := 0; vec != 0; i++ {
if vec&1 != 0 {
sum ^= mat[i]
}
vec >>= 1
}
return sum
}
func gf2MatrixSquare(square []uint64, mat []uint64) {
for n := 0; n < gf2Dim; n++ {
square[n] = gf2MatrixTimes(mat, mat[n])
}
}
// CRC64Combine combines CRC64
func CRC64Combine(crc1 uint64, crc2 uint64, len2 uint64) uint64 {
var even [gf2Dim]uint64 // Even-power-of-two zeros operator
var odd [gf2Dim]uint64 // Odd-power-of-two zeros operator
// Degenerate case
if len2 == 0 {
return crc1
}
// Put operator for one zero bit in odd
odd[0] = crc64.ECMA // CRC64 polynomial
var row uint64 = 1
for n := 1; n < gf2Dim; n++ {
odd[n] = row
row <<= 1
}
// Put operator for two zero bits in even
gf2MatrixSquare(even[:], odd[:])
// Put operator for four zero bits in odd
gf2MatrixSquare(odd[:], even[:])
// Apply len2 zeros to crc1, first square will put the operator for one zero byte, eight zero bits, in even
for {
// Apply zeros operator for this bit of len2
gf2MatrixSquare(even[:], odd[:])
if len2&1 != 0 {
crc1 = gf2MatrixTimes(even[:], crc1)
}
len2 >>= 1
// If no more bits set, then done
if len2 == 0 {
break
}
// Another iteration of the loop with odd and even swapped
gf2MatrixSquare(odd[:], even[:])
if len2&1 != 0 {
crc1 = gf2MatrixTimes(odd[:], crc1)
}
len2 >>= 1
// If no more bits set, then done
if len2 == 0 {
break
}
}
// Return combined CRC
crc1 ^= crc2
return crc1
}

161
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/credential.go generated vendored Normal file
View File

@@ -0,0 +1,161 @@
package tos
import (
"sync/atomic"
"time"
"unsafe"
"golang.org/x/sync/singleflight"
)
type Credential struct {
AccessKeyID string
AccessKeySecret string
SecurityToken string
}
// Credentials provides Credential
type Credentials interface {
Credential() Credential
}
// StaticCredentials Credentials with static access-key and secret-key
type StaticCredentials struct {
accessKey string
secretKey string
securityToken string
}
// NewStaticCredentials Credentials with static access-key and secret-key
// use StaticCredentials.WithSecurityToken to set security-token
//
// you can use it as:
// client, err := tos.NewClient(endpoint, tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
// // do something more
//
// And you can use tos.WithPerRequestSigner set the 'Signer' for each request.
//
func NewStaticCredentials(accessKeyID, accessKeySecret string) *StaticCredentials {
return &StaticCredentials{
accessKey: accessKeyID,
secretKey: accessKeySecret,
}
}
// WithSecurityToken set security-token
func (sc *StaticCredentials) WithSecurityToken(securityToken string) {
sc.securityToken = securityToken
}
func (sc *StaticCredentials) Credential() Credential {
return Credential{
AccessKeyID: sc.accessKey,
AccessKeySecret: sc.secretKey,
SecurityToken: sc.securityToken,
}
}
// WithoutSecretKeyCredentials Credentials with static access-key and no secret-key
//
// If you don't want to use secret-key directly, but use signed-key, you can use it as:
// signer := tos.NewSignV4(tos.NewWithoutSecretKeyCredentials(accessKey), region)
// signer.WithSigningKey(func(*SigningKeyInfo) []byte { return signingKey})
// client, err := tos.NewClient(endpoint, tos.WithSigner(signer))
// // do something more
//
// And you can use tos.WithPerRequestSigner set the 'Signer' for each request.
//
type WithoutSecretKeyCredentials struct {
accessKey string
securityToken string
}
func NewWithoutSecretKeyCredentials(accessKeyID string) *WithoutSecretKeyCredentials {
return &WithoutSecretKeyCredentials{
accessKey: accessKeyID,
securityToken: "",
}
}
// WithSecurityToken set security-token
func (sc *WithoutSecretKeyCredentials) WithSecurityToken(securityToken string) {
sc.securityToken = securityToken
}
func (sc *WithoutSecretKeyCredentials) Credential() Credential {
return Credential{
AccessKeyID: sc.accessKey,
SecurityToken: sc.securityToken,
}
}
// FederationToken contains Credential and Credential's expiration time
type FederationToken struct {
Credential Credential
Expiration time.Time
}
// FederationTokenProvider provides FederationToken
type FederationTokenProvider interface {
FederationToken() (*FederationToken, error)
}
// FederationCredentials implements Credentials interfaces with flushing Credential periodically
type FederationCredentials struct {
cachedToken *FederationToken
refreshing uint32
preFetch time.Duration
tokenProvider FederationTokenProvider
flight singleflight.Group
}
// NewFederationCredentials FederationCredentials implements Credentials interfaces with flushing Credential periodically
//
// use WithPreFetch set prefetch time
func NewFederationCredentials(tokenProvider FederationTokenProvider) (*FederationCredentials, error) {
cred, err := tokenProvider.FederationToken()
if err != nil {
return nil, err
}
return &FederationCredentials{
cachedToken: cred,
preFetch: 5 * time.Minute,
tokenProvider: tokenProvider,
}, nil
}
// WithPreFetch set prefetch time
func (fc *FederationCredentials) WithPreFetch(preFetch time.Duration) {
fc.preFetch = preFetch
}
func (fc *FederationCredentials) token() *FederationToken {
return (*FederationToken)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&fc.cachedToken))))
}
// Credential for Credentials interface
func (fc *FederationCredentials) Credential() Credential {
now := time.Now()
if token := fc.token(); now.After(token.Expiration) { // 已经过期
_, _, _ = fc.flight.Do("flushing", func() (interface{}, error) {
flushed, err := fc.tokenProvider.FederationToken()
if err != nil {
return nil, err
}
atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&fc.cachedToken)), unsafe.Pointer(flushed))
return flushed, nil
})
} else if now.Add(fc.preFetch).After(token.Expiration) &&
atomic.LoadUint32(&fc.refreshing) == 0 {
// 将要过期, prefetch token
if atomic.CompareAndSwapUint32(&fc.refreshing, 0, 1) {
defer atomic.StoreUint32(&fc.refreshing, 0)
if newToken, err := fc.tokenProvider.FederationToken(); err == nil {
atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&fc.cachedToken)), unsafe.Pointer(newToken))
}
}
}
return fc.token().Credential
}

76
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/custom_domain.go generated vendored Normal file
View File

@@ -0,0 +1,76 @@
package tos
import (
"bytes"
"context"
"net/http"
)
func (cli *ClientV2) PutBucketCustomDomain(ctx context.Context, input *PutBucketCustomDomainInput) (*PutBucketCustomDomainOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
body := putBucketCustomDomainInput{
Rule: input.Rule,
}
data, contentMD5, err := marshalInput("PutBucketCustomDomainInput", body)
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("customdomain", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketCustomDomainOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) ListBucketCustomDomain(ctx context.Context, input *ListBucketCustomDomainInput) (*ListBucketCustomDomainOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("customdomain", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := ListBucketCustomDomainOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) DeleteBucketCustomDomain(ctx context.Context, input *DeleteBucketCustomDomainInput) (*DeleteBucketCustomDomainOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("customdomain", input.Domain).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteBucketCustomDomainOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

213
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/dns_cache.go generated vendored Normal file
View File

@@ -0,0 +1,213 @@
package tos
import (
"container/heap"
"context"
"net"
"strings"
"sync"
"time"
)
const (
DefaultCacheCap = 100
VolceHostSuffix = "volces.com"
HostSplitSep = "."
HostSplitLength = 4
)
type cacheItem struct {
host string
ipList []string
expireAt time.Time
heapIndex int
}
type priorityQueue []*cacheItem
func (p priorityQueue) Len() int {
return len(p)
}
func (p priorityQueue) Peek() *cacheItem {
if p.Len() > 0 {
return p[0]
}
return nil
}
func (p priorityQueue) Less(i, j int) bool {
return p[i].expireAt.Before(p[j].expireAt)
}
func (p priorityQueue) Swap(i, j int) {
p[i], p[j] = p[j], p[i]
p[i].heapIndex = i
p[j].heapIndex = j
}
func (p *priorityQueue) Push(x interface{}) {
n := len(*p)
item := x.(*cacheItem)
item.heapIndex = n
*p = append(*p, item)
}
func (p *priorityQueue) Pop() interface{} {
old := *p
n := len(old)
item := old[n-1]
old[n-1] = nil
item.heapIndex = -1
*p = old[0 : n-1]
return item
}
type cache struct {
lock sync.RWMutex
heap *priorityQueue
cleanTime time.Time
data map[string]cacheItem
expiration time.Duration
}
func (c *cache) Remove(key string, removeIp string) {
c.lock.Lock()
defer c.lock.Unlock()
data, ok := c.data[key]
if !ok {
return
}
value := make([]string, 0, len(data.ipList))
for _, ip := range data.ipList {
if ip == removeIp {
continue
}
value = append(value, ip)
}
// 没有有效的 IP 将缓存删除
if len(value) == 0 {
delete(c.data, key)
return
}
data.ipList = value
c.data[key] = data
}
func (c *cache) Get(key string) ([]string, bool) {
c.lock.RLock()
data, ok := c.data[key]
c.lock.RUnlock()
if !ok {
return nil, false
}
if data.expireAt.Before(time.Now()) {
return nil, false
}
return data.ipList, true
}
func (c *cache) cleanCache() {
c.cleanTime = time.Now().Add(c.expiration)
maxCleanCount := 5
for i := 0; i < maxCleanCount; i++ {
item := c.heap.Peek()
if item == nil {
return
}
if item.expireAt.Before(time.Now()) {
heap.Pop(c.heap)
data, ok := c.data[item.host]
if ok && data.expireAt == item.expireAt {
delete(c.data, item.host)
}
} else {
return
}
}
}
func (c *cache) Put(key string, ipList []string) {
c.lock.Lock()
defer c.lock.Unlock()
item := cacheItem{
ipList: ipList,
expireAt: time.Now().Add(c.expiration),
host: key,
}
c.data[key] = item
heap.Push(c.heap, &item)
// 大于 Cap
if c.heap.Len() > DefaultCacheCap {
item := heap.Pop(c.heap).(*cacheItem)
if item == nil {
return
}
data, ok := c.data[item.host]
if ok && data.expireAt == item.expireAt {
delete(c.data, item.host)
}
}
if time.Now().After(c.cleanTime) {
c.cleanCache()
}
}
type resolver struct {
cache *cache
}
func newResolver(expiration time.Duration) *resolver {
pq := make(priorityQueue, 0)
return &resolver{cache: &cache{
heap: &pq,
cleanTime: time.Now().Add(expiration),
data: make(map[string]cacheItem),
expiration: expiration,
}}
}
func ipToStringList(ips []net.IP) []string {
res := make([]string, len(ips))
for i, ip := range ips {
res[i] = ip.String()
}
return res
}
func wrappedHost(host string) string {
if !strings.HasSuffix(host, VolceHostSuffix) {
return host
}
hostSplit := strings.Split(host, HostSplitSep)
if len(hostSplit) != HostSplitLength {
return host
}
return strings.Join(hostSplit[1:], HostSplitSep)
}
func (r *resolver) GetIpList(ctx context.Context, host string) ([]string, error) {
ipList, ok := r.cache.Get(wrappedHost(host))
if ok {
return ipList, nil
}
ips, err := net.LookupIP(host)
if err != nil {
return nil, err
}
ipsStr := ipToStringList(ips)
r.cache.Put(wrappedHost(host), ipsStr)
return ipsStr, nil
}
func (r *resolver) Remove(host string, ip string) {
r.cache.Remove(wrappedHost(host), ip)
}

388
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/download_file.go generated vendored Normal file
View File

@@ -0,0 +1,388 @@
package tos
import (
"context"
"crypto/md5"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
)
func getDownloadCheckpoint(input *DownloadFileInput, init func(input *HeadObjectV2Output) (*downloadCheckpoint, error), output *HeadObjectV2Output) (checkpoint *downloadCheckpoint, err error) {
enabled := input.EnableCheckpoint
checkpointPath := input.CheckpointFile
if !enabled {
return init(output)
}
checkpoint = &downloadCheckpoint{}
loadCheckPoint(checkpointPath, checkpoint)
if checkpoint.Valid(input, output) {
return
}
parentDir := filepath.Dir(checkpointPath)
stat, err := os.Stat(parentDir)
if err != nil {
err = os.MkdirAll(parentDir, os.ModePerm)
if err != nil {
return nil, newTosClientError(err.Error(), err)
}
} else if !stat.IsDir() {
return nil, newTosClientError("Fail to create folder due to a same file exists.", nil)
}
file, err := os.Create(checkpointPath)
if err != nil {
return nil, newTosClientError(err.Error(), err)
}
_ = file.Close()
checkpoint, err = init(output)
if err != nil {
return nil, err
}
err = checkpoint.WriteToFile()
if err != nil {
return nil, err
}
return
}
func (cli *ClientV2) DownloadFile(ctx context.Context, input *DownloadFileInput) (*DownloadFileOutput, error) {
err := validateDownloadInput(input, cli.isCustomDomain)
if err != nil {
return nil, err
}
headOutput, err := cli.HeadObjectV2(ctx, &input.HeadObjectV2Input)
if err != nil {
return nil, err
}
needDownload, err := parseDownloadFilePath(input)
if err != nil {
return nil, err
}
if !needDownload {
return &DownloadFileOutput{*headOutput}, nil
}
event := downloadEvent{input: input}
init := func(output *HeadObjectV2Output) (*downloadCheckpoint, error) {
err := createDownloadTempFile(input, event)
if err != nil {
return nil, err
}
return initDownloadCheckpoint(input, headOutput)
}
checkpoint, err := getDownloadCheckpoint(input, init, headOutput)
if err != nil {
return nil, err
}
cleaner := func() {
_ = os.Remove(input.CheckpointFile)
_ = os.Remove(input.tempFile)
}
bindCancelHookWithCleaner(input.CancelHook, cleaner)
return cli.downloadFile(ctx, headOutput, checkpoint, input, event)
}
// loadCheckPoint load UploadFile checkpoint or DownloadFile checkpoint.
// checkpoint must be a pointer
func loadCheckPoint(path string, checkpoint interface{}) {
contents, err := ioutil.ReadFile(path)
if err != nil && !os.IsNotExist(err) {
return
}
if len(contents) == 0 {
return
}
json.Unmarshal(contents, &checkpoint)
}
func isDir(filePath string) bool {
stat, err := os.Stat(filePath)
if err != nil {
_, fileName := filepath.Split(filePath)
return fileName == ""
}
return stat.IsDir()
}
// if file is a directory, append suffix to it to make a file name
func withSuffixIfDir(filePath string, suffix string) string {
if isDir(filePath) {
return filepath.Clean(filepath.Join(filePath, suffix))
}
return filePath
}
func getDownloadCheckPointPath(checkpointPath, filePath, bucket, key, versionId string) string {
fileName := strings.Join([]string{filepath.Base(filePath), checkpointPathMd5(bucket, key, versionId), "download"}, ".")
if len(checkpointPath) == 0 {
dirName := filepath.Dir(filePath)
return filepath.Clean(filepath.Join(dirName, fileName))
}
return withSuffixIfDir(checkpointPath, fileName)
}
func checkpointPathMd5(bucket string, key string, versionId string) string {
var data []byte
if versionId != "" {
data = []byte(strings.Join([]string{bucket, key, versionId}, "."))
} else {
data = []byte(strings.Join([]string{bucket, key}, "."))
}
r := md5.Sum(data)
return base64.URLEncoding.EncodeToString(r[:])
}
func parseDownloadFilePath(input *DownloadFileInput) (needDownloadFile bool, err error) {
input.filePath = input.FilePath
inputFile := input.filePath
isDirRes := isDir(input.filePath)
if isDirRes {
input.filePath = filepath.Clean(filepath.Join(input.filePath, input.Key))
}
input.tempFile = input.filePath + TempFileSuffix
if input.EnableCheckpoint {
input.CheckpointFile = getDownloadCheckPointPath(input.CheckpointFile, input.filePath, input.Bucket, input.Key, input.VersionID)
}
if isDirRes && strings.HasSuffix(input.Key, "/") {
err := os.MkdirAll(filepath.Join(inputFile, input.Key), os.ModePerm)
if err != nil {
return false, InvalidFilePath.withCause(err)
}
return false, nil
}
return true, nil
}
func validateDownloadInput(input *DownloadFileInput, isCustomDomain bool) error {
if err := isValidNames(input.Bucket, input.Key, isCustomDomain); err != nil {
return err
}
if input.PartSize == 0 {
input.PartSize = DefaultPartSize
}
if input.PartSize < MinPartSize || input.PartSize > MaxPartSize {
return newTosClientError("The input part size is invalid, please set it range from 5MB to 5GB", nil)
}
if input.TaskNum < 1 {
input.TaskNum = 1
}
if input.TaskNum > 1000 {
input.TaskNum = 1000
}
return nil
}
func initDownloadCheckpoint(input *DownloadFileInput, headOutput *HeadObjectV2Output) (*downloadCheckpoint, error) {
partsNum := headOutput.ContentLength / input.PartSize
remainder := headOutput.ContentLength % input.PartSize
if remainder != 0 {
partsNum++
}
parts := make([]downloadPartInfo, partsNum)
for i := int64(0); i < partsNum; i++ {
parts[i] = downloadPartInfo{
PartNumber: int(i + 1),
RangeStart: i * input.PartSize,
RangeEnd: (i+1)*input.PartSize - 1,
}
}
if remainder != 0 {
parts[partsNum-1].RangeEnd = (partsNum-1)*input.PartSize + remainder - 1
}
if len(parts) > 10000 {
return nil, newTosClientError("tos: part count too many", nil)
}
return &downloadCheckpoint{
checkpointPath: input.CheckpointFile,
Bucket: input.Bucket,
Key: input.Key,
VersionID: input.VersionID,
PartSize: input.PartSize,
IfMatch: input.IfMatch,
IfModifiedSince: input.IfModifiedSince,
IfNoneMatch: input.IfNoneMatch,
IfUnmodifiedSince: input.IfUnmodifiedSince,
SSECAlgorithm: input.SSECAlgorithm,
SSECKeyMD5: input.SSECKey,
ObjectInfo: objectInfo{
Etag: headOutput.ETag,
HashCrc64ecma: headOutput.HashCrc64ecma,
LastModified: headOutput.LastModified,
ObjectSize: headOutput.ContentLength,
},
FileInfo: downloadFileInfo{
FilePath: input.filePath,
TempFilePath: input.tempFile,
},
PartsInfo: parts,
}, nil
}
func checkAndCreateDir(filePath string) error {
dir := filepath.Dir(filePath)
stat, err := os.Stat(dir)
if err != nil {
err = os.MkdirAll(dir, os.ModePerm)
if err != nil {
return err
}
} else if !stat.IsDir() {
return fmt.Errorf("dir name same as file name. ")
}
return nil
}
func createDownloadTempFile(input *DownloadFileInput, event downloadEvent) error {
wrapErr := func(err error) error {
event.postDownloadEvent(&DownloadEvent{
Type: enum.DownloadEventCreateTempFileFailed,
Bucket: input.Bucket,
Key: input.Key,
VersionID: input.VersionID,
FilePath: input.filePath,
TempFilePath: &input.tempFile,
CheckpointFile: &input.CheckpointFile,
})
return newTosClientError("tos: create temp file failed.", err)
}
err := checkAndCreateDir(input.tempFile)
if err != nil {
return wrapErr(err)
}
file, err := os.Create(input.tempFile)
if err != nil {
return wrapErr(err)
}
_ = file.Close()
event.postDownloadEvent(&DownloadEvent{
Type: enum.DownloadEventCreateTempFileSucceed,
Bucket: input.Bucket,
Key: input.Key,
VersionID: input.VersionID,
FilePath: input.filePath,
TempFilePath: &input.tempFile,
CheckpointFile: &input.CheckpointFile,
})
return nil
}
func getDownloadTasks(cli *ClientV2, ctx context.Context, headOutput *HeadObjectV2Output,
checkpoint *downloadCheckpoint, input *DownloadFileInput) []task {
tasks := make([]task, 0)
consumed := int64(0)
subtotal := int64(0)
for _, part := range checkpoint.PartsInfo {
if !part.IsCompleted {
tasks = append(tasks, &downloadTask{
cli: cli,
ctx: ctx,
input: input,
partNumber: part.PartNumber,
rangeStart: part.RangeStart,
rangeEnd: part.RangeEnd,
consumed: &consumed,
subtotal: &subtotal,
total: headOutput.ContentLength,
enableCRC64: cli.enableCRC,
})
} else {
consumed += part.RangeEnd - part.RangeStart + 1
}
}
return tasks
}
func (d downloadEvent) newDownloadEvent() *DownloadEvent {
return &DownloadEvent{
Bucket: d.input.Bucket,
Key: d.input.Key,
VersionID: d.input.VersionID,
FilePath: d.input.filePath,
CheckpointFile: &d.input.CheckpointFile,
TempFilePath: &d.input.tempFile,
}
}
func (d downloadEvent) newDownloadPartSucceedEvent(part downloadPartInfo) *DownloadEvent {
event := d.newSucceedEvent(enum.DownloadEventDownloadPartSucceed)
event.DowloadPartInfo = &DownloadPartInfo{
PartNumber: part.PartNumber,
RangeStart: part.RangeStart,
RangeEnd: part.RangeEnd,
}
return event
}
func (d downloadEvent) newSucceedEvent(eventType enum.DownloadEventType) *DownloadEvent {
event := d.newDownloadEvent()
event.Type = eventType
return event
}
func (d downloadEvent) newFailedEvent(err error, eventType enum.DownloadEventType) *DownloadEvent {
event := d.newDownloadEvent()
event.Type = eventType
event.Err = err
return event
}
func (d downloadEvent) postDownloadEvent(event *DownloadEvent) {
if d.input.DownloadEventListener != nil {
d.input.DownloadEventListener.EventChange(event)
}
}
func (cli *ClientV2) downloadFile(ctx context.Context,
headOutput *HeadObjectV2Output, checkpoint *downloadCheckpoint, input *DownloadFileInput, event downloadEvent) (*DownloadFileOutput, error) {
// prepare tasks
tasks := getDownloadTasks(cli, ctx, headOutput, checkpoint, input)
routinesNum := min(input.TaskNum, len(tasks))
tg := newTaskGroup(getCancelHandle(input.CancelHook), routinesNum, checkpoint, event, input.EnableCheckpoint, tasks)
tg.RunWorker()
// start adding tasks
postDataTransferStatus(input.DataTransferListener, &DataTransferStatus{
Type: enum.DataTransferStarted,
})
tg.Scheduler()
success, err := tg.Wait()
if err != nil {
_ = os.Remove(input.tempFile)
}
if success < len(tasks) {
return nil, newTosClientError("tos: some download task failed.", nil)
}
// Check CRC64
if cli.enableCRC && headOutput.HashCrc64ecma != 0 && combineCRCInDownload(checkpoint.PartsInfo) != headOutput.HashCrc64ecma {
return nil, newTosClientError("tos: crc of entire file mismatch.", nil)
}
err = os.Rename(input.tempFile, input.filePath)
if err != nil {
event.postDownloadEvent(event.newFailedEvent(err, enum.DownloadEventRenameTempFileFailed))
return nil, err
}
event.postDownloadEvent(event.newSucceedEvent(enum.DownloadEventRenameTempFileSucceed))
_ = os.Remove(checkpoint.checkpointPath)
return &DownloadFileOutput{*headOutput}, nil
}

181
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum/enum.go generated vendored Normal file
View File

@@ -0,0 +1,181 @@
package enum
type ACLType string
const (
ACLPrivate ACLType = "private"
ACLPublicRead ACLType = "public-read"
ACLPublicReadWrite ACLType = "public-read-write"
ACLAuthRead ACLType = "authenticated-read"
ACLBucketOwnerRead ACLType = "bucket-owner-read"
ACLBucketOwnerFullControl ACLType = "bucket-owner-full-control"
ACLLogDeliveryWrite ACLType = "log-delivery-write"
ACLBucketOwnerEntrusted ACLType = "bucket-owner-entrusted"
)
type StorageClassType string
const (
StorageClassStandard StorageClassType = "STANDARD"
StorageClassIa StorageClassType = "IA"
StorageClassArchiveFr StorageClassType = "ARCHIVE_FR"
StorageClassIntelligentTiering StorageClassType = "INTELLIGENT_TIERING"
StorageClassColdArchive StorageClassType = "COLD_ARCHIVE"
)
type MetadataDirectiveType string
const (
// MetadataDirectiveReplace replace source object metadata when calling CopyObject
MetadataDirectiveReplace MetadataDirectiveType = "REPLACE"
// MetadataDirectiveCopy copy source object metadata when calling CopyObject
MetadataDirectiveCopy MetadataDirectiveType = "COPY"
)
type AzRedundancyType string
const (
AzRedundancySingleAz AzRedundancyType = "single-az"
AzRedundancyMultiAz AzRedundancyType = "multi-az"
)
type PermissionType string
const (
PermissionRead PermissionType = "READ"
PermissionWrite PermissionType = "WRITE"
PermissionReadAcp PermissionType = "READ_ACP"
PermissionWriteAcp PermissionType = "WRITE_ACP"
PermissionFullControl PermissionType = "FULL_CONTROL"
)
type GranteeType string
const (
GranteeGroup GranteeType = "Group"
GranteeUser GranteeType = "CanonicalUser"
)
type CannedType string
const (
CannedAllUsers CannedType = "AllUsers"
CannedAuthenticatedUsers CannedType = "AuthenticatedUsers"
)
type DataTransferType int
const (
DataTransferStarted DataTransferType = 1
DataTransferRW DataTransferType = 2
DataTransferSucceed DataTransferType = 3
DataTransferFailed DataTransferType = 4
)
type HttpMethodType string
const (
HttpMethodGet HttpMethodType = "GET"
HttpMethodPut HttpMethodType = "PUT"
HttpMethodPost HttpMethodType = "POST"
HttpMethodDelete HttpMethodType = "DELETE"
HttpMethodHead HttpMethodType = "HEAD"
)
type UploadEventType int
const (
UploadEventCreateMultipartUploadSucceed UploadEventType = 1
UploadEventCreateMultipartUploadFailed UploadEventType = 2
UploadEventUploadPartSucceed UploadEventType = 3
UploadEventUploadPartFailed UploadEventType = 4
UploadEventUploadPartAborted UploadEventType = 5 // The task needs to be interrupted in case of 403, 404, 405 errors
UploadEventCompleteMultipartUploadSucceed UploadEventType = 6
UploadEventCompleteMultipartUploadFailed UploadEventType = 7
)
type DownloadEventType int
const (
DownloadEventCreateTempFileSucceed DownloadEventType = 1
DownloadEventCreateTempFileFailed DownloadEventType = 2
DownloadEventDownloadPartSucceed DownloadEventType = 3
DownloadEventDownloadPartFailed DownloadEventType = 4
DownloadEventDownloadPartAborted DownloadEventType = 5 // The task needs to be interrupted in case of 403, 404, 405 errors
DownloadEventRenameTempFileSucceed DownloadEventType = 6
DownloadEventRenameTempFileFailed DownloadEventType = 7
)
type CertStatusType string
const (
CertStatusBound CertStatusType = "CertBound"
CertStatusUnbound CertStatusType = "CertUnbound"
CertStatusExpired CertStatusType = "CertExpired"
)
type StorageClassInheritDirectiveType string
const (
StorageClassIDDestinationBucket StorageClassInheritDirectiveType = "DESTINATION_BUCKET"
StorageClassIDSourceObject StorageClassInheritDirectiveType = "SOURCE_OBJECT"
)
type StatusType string
const (
StatusEnabled StatusType = "Enabled"
StatusDisabled StatusType = "Disabled"
)
const (
LifecycleStatusEnabled StatusType = "Enabled"
LifecycleStatusDisabled StatusType = "Disabled"
)
type RedirectType string
const (
RedirectTypeMirror RedirectType = "Mirror"
RedirectTypeAsync RedirectType = "Async"
)
const (
SSETosAlg = "AES256"
SSEKMS = "kms"
)
type VersioningStatusType string
const (
VersioningStatusEnable VersioningStatusType = "Enabled"
VersioningStatusSuspended VersioningStatusType = "Suspended"
)
type ProtocolType string
const (
ProtocolHttp ProtocolType = "http"
ProtocolHttps ProtocolType = "https"
)
type CopyEventType int
const (
CopyEventCreateMultipartUploadSucceed CopyEventType = 1
CopyEventCreateMultipartUploadFailed CopyEventType = 2
CopyEventUploadPartCopySuccess CopyEventType = 3
CopyEventUploadPartCopyFailed CopyEventType = 4
CopyEventUploadPartCopyAborted CopyEventType = 5
CopyEventCompleteMultipartUploadSucceed CopyEventType = 6
CopyEventCompleteMultipartUploadFailed CopyEventType = 7
)
type TierType string
const (
TierStandard TierType = "Standard"
TierExpedited TierType = "Expedited"
TierBulk TierType = "Bulk"
)

323
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/error.go generated vendored Normal file
View File

@@ -0,0 +1,323 @@
package tos
import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
)
var (
InputIsNilClientError = newTosClientError("input is nil. ", nil)
InputInvalidClientError = newTosClientError("input data is invalid. ", nil)
InvalidPartNumber = newTosClientError("input part number is invalid. ", nil)
InvalidUploadID = newTosClientError("input upload id is invalid. ", nil)
InvalidBucketNameLength = newTosClientError("invalid bucket name, the length must be [3, 63]", nil)
InvalidBucketNameCharacter = newTosClientError("invalid bucket name, the character set is illegal", nil)
InvalidBucketNameStartingOrEnding = newTosClientError("invalid bucket name, the bucket name can be neither starting with '-' nor ending with '-'", nil)
InvalidObjectNameLength = newTosClientError("invalid object name, the length must be [1, 696]", nil)
InvalidObjectNameStartingOrEnding = newTosClientError("invalid object name, the object name can not start with '\\'", nil)
InvalidObjectNameCharacterSet = newTosClientError("invalid object name, the character set is illegal", nil)
InvalidACL = newTosClientError("invalid acl type", nil)
InvalidStorageClass = newTosClientError("invalid storage class", nil)
InvalidGrantee = newTosClientError("invalid grantee type", nil)
InvalidCanned = newTosClientError("invalid canned type", nil)
InvalidAzRedundancy = newTosClientError("invalid az redundancy type", nil)
InvalidMetadataDirective = newTosClientError("invalid metadata directive type", nil)
InvalidPermission = newTosClientError("invalid permission type", nil)
InvalidSSECAlgorithm = newTosClientError("invalid encryption-decryption algorithm", nil)
InvalidPartSize = newTosClientError("invalid part size, the size must be [5242880, 5368709120]", nil)
InvalidSrcFilePath = newTosClientError("invalid file path, the file does not exist", nil)
InvalidFilePartNum = newTosClientError("unsupported part number, the maximum is 10000", nil)
InvalidMarshal = newTosClientError("unable to do serialization/deserialization", nil)
InvalidPreSignedURLExpires = newTosClientError("invalid pre signed url expires, the time must be less 604800 seconds.", nil)
InvalidPreSignedConditions = newTosClientError("invalid pre signed url conditions.", nil)
InvalidFilePath = newTosClientError("invalid file path", nil)
InvalidCheckpointFilePath = newTosClientError("invalid checkpoint file path", nil)
CrcCheckFail = newTosClientError("crc check not equal", nil)
InvalidS3Endpoint = newTosClientError("do not support s3 endpoint, please use tos endpoint", nil)
NotSupportSeek = newTosClientError("reader not support seek", nil)
ProxyNotSupportHttps = newTosClientError("proxy not support https", nil)
ProxyUrlInvalid = newTosClientError("proxy url invalid", nil)
NotificationConfigurationsInvalid = newTosClientError("Notification Configurations invalid", nil)
InvalidCompleteAllPartsLength = newTosClientError("Should not specify both complete all and Parts", nil)
InvalidPartsLength = newTosClientError("You must specify at least one part", nil)
InvlidDeleteMultiObjectsLength = newTosClientError("You must specify at least one object", nil)
)
type TosError struct {
Message string
}
func (e *TosError) Error() string {
return e.Message
}
// for simplify code
func newTosClientError(message string, cause error) *TosClientError {
return &TosClientError{
TosError: TosError{
Message: message,
},
Cause: cause,
}
}
type TosClientError struct {
TosError
Cause error
}
func (t *TosClientError) withCause(err error) error {
t.Cause = err
return t
}
// try to unmarshal server error from response
func newTosServerError(res *Response) error {
data, err := ioutil.ReadAll(io.LimitReader(res.Body, 64<<10)) // avoid too large
if err != nil && len(data) <= 0 {
return &TosServerError{
TosError: TosError{"tos: server returned an empty body"},
RequestInfo: res.RequestInfo(),
}
}
se := Error{StatusCode: res.StatusCode}
if err = json.Unmarshal(data, &se); err != nil {
return &TosServerError{
TosError: TosError{"tos: server returned an invalid body"},
RequestInfo: res.RequestInfo(),
}
}
return &TosServerError{
TosError: TosError{se.Message},
RequestInfo: res.RequestInfo(),
Code: se.Code,
HostID: se.HostID,
Resource: se.Resource,
}
}
// 服务端错误定义参考：https://www.volcengine.com/docs/6349/74874
type TosServerError struct {
TosError `json:"TosError"`
RequestInfo `json:"RequestInfo"`
Code string `json:"Code,omitempty"`
HostID string `json:"HostID,omitempty"`
Resource string `json:"Resource,omitempty"`
}
type Error struct {
StatusCode int `json:"-"`
Code string `json:"Code,omitempty"`
Message string `json:"Message,omitempty"`
RequestID string `json:"RequestId,omitempty"`
HostID string `json:"HostId,omitempty"`
Resource string `json:"Resource,omitempty"`
}
func (e *Error) Error() string {
return fmt.Sprintf("tos: request error: StatusCode=%d, Code=%s, Message=%q, RequestID=%s, HostID=%s",
e.StatusCode, e.Code, e.Message, e.RequestID, e.HostID)
}
// Code return error code saved in TosServerError
func Code(err error) string {
if er, ok := err.(*TosServerError); ok {
return er.Code
}
return ""
}
// StatueCode return status code saved in TosServerError or UnexpectedStatusCodeError
//
// Deprecated: use StatusCode instead
func StatueCode(err error) int {
return StatusCode(err)
}
// StatusCode return status code saved in TosServerError or UnexpectedStatusCodeError
func StatusCode(err error) int {
if er, ok := err.(*TosServerError); ok {
return er.StatusCode
}
if er, ok := err.(*UnexpectedStatusCodeError); ok {
return er.StatusCode
}
return 0
}
func RequestID(err error) string {
switch ev := err.(type) {
case *TosServerError:
return ev.RequestID
case *UnexpectedStatusCodeError:
return ev.RequestID
case *ChecksumError:
return ev.RequestID
case *SerializeError:
return ev.RequestID
}
return ""
}
type UnexpectedStatusCodeError struct {
StatusCode int `json:"StatusCode,omitempty"`
ExpectedCodes []int `json:"ExpectedCodes,omitempty"`
RequestID string `json:"RequestId,omitempty"`
expectedCodes [2]int
responseMsg string
err Error
}
func NewUnexpectedStatusCodeError(statusCode int, expectedCode int, expectedCodes ...int) *UnexpectedStatusCodeError {
err := UnexpectedStatusCodeError{
StatusCode: statusCode,
}
err.ExpectedCodes = err.expectedCodes[:0]
err.ExpectedCodes = append(err.ExpectedCodes, expectedCode)
err.ExpectedCodes = append(err.ExpectedCodes, expectedCodes...)
return &err
}
func (us *UnexpectedStatusCodeError) WithRequestBody(res *Response) *UnexpectedStatusCodeError {
data, err := ioutil.ReadAll(io.LimitReader(res.Body, 64<<10))
if err != nil || len(data) <= 0 {
return us
}
us.responseMsg = string(data)
se := Error{StatusCode: res.StatusCode}
err = json.Unmarshal(data, &se)
if err != nil {
return us
}
us.err = se
return us
}
func (us *UnexpectedStatusCodeError) WithRequestID(requestID string) *UnexpectedStatusCodeError {
us.RequestID = requestID
return us
}
func (us *UnexpectedStatusCodeError) GoString() string {
if us.responseMsg != "" {
return fmt.Sprintf("tos.UnexpectedStatusCodeError{StatusCode:%d, ExpectedCodes:%v, RequestID:%s, ResponseErr:%s}",
us.StatusCode, us.ExpectedCodes, us.RequestID, us.responseMsg)
}
return fmt.Sprintf("tos.UnexpectedStatusCodeError{StatusCode:%d, ExpectedCodes:%v, RequestID:%s}",
us.StatusCode, us.ExpectedCodes, us.RequestID)
}
func (us *UnexpectedStatusCodeError) Error() string {
if us.responseMsg != "" {
return fmt.Sprintf("tos: unexpected status code error: StatusCode=%d, ExpectedCodes=%v, RequestID=%s, ResponseErr:%s",
us.StatusCode, us.ExpectedCodes, us.RequestID, us.responseMsg)
}
return fmt.Sprintf("tos: unexpected status code error: StatusCode=%d, ExpectedCodes=%v, RequestID=%s",
us.StatusCode, us.ExpectedCodes, us.RequestID)
}
type ChecksumError struct {
RequestID string `json:"RequestId,omitempty"`
ExpectedChecksum string `json:"ExpectedChecksum,omitempty"`
ActualChecksum string `json:"ActualChecksum,omitempty"`
}
func (ce *ChecksumError) Error() string {
return fmt.Sprintf("tos: checksum error: RequestID=%s, ExpectedChecksum=%s, ActualChecksum=%s",
ce.RequestID, ce.ExpectedChecksum, ce.ActualChecksum)
}
type SerializeError struct {
RequestID string `json:"RequestId,omitempty"`
Message string `json:"Message,omitempty"`
}
func (se *SerializeError) Error() string {
return fmt.Sprintf("tos: serialize error: RequestID=%s, Message=%q", se.RequestID, se.Message)
}
func checkError(res *Response, readBody bool, okCode int, okCodes ...int) error {
if res.StatusCode == okCode {
return nil
}
for _, code := range okCodes {
if res.StatusCode == code {
return nil
}
}
defer res.Close()
if readBody && res.StatusCode >= http.StatusBadRequest && res.Body != nil {
return newTosServerError(res)
// fall through
}
unexpected := NewUnexpectedStatusCodeError(res.StatusCode, okCode, okCodes...).
WithRequestID(res.RequestInfo().RequestID)
if readBody && res.Body != nil {
unexpected = unexpected.WithRequestBody(res)
}
return &TosServerError{
TosError: TosError{unexpected.Error()},
RequestInfo: res.RequestInfo(),
Code: unexpected.err.Code,
HostID: unexpected.err.HostID,
Resource: unexpected.err.Resource,
}
}
// StatusCodeClassifier classifies Errors.
// If the error is nil, it returns NoRetry;
// if the error is TimeoutException or can be interpreted as TosServerError, and the StatusCode is 5xx or 429, it returns Retry;
// otherwise, it returns NoRetry.
type StatusCodeClassifier struct{}
// Classify implements the classifier interface.
func (classifier StatusCodeClassifier) Classify(err error) retryAction {
if err == nil {
return NoRetry
}
e, ok := err.(*TosServerError)
if ok {
if e.StatusCode >= 500 || e.StatusCode == 429 {
return Retry
}
}
cErr, ok := err.(*TosClientError)
if ok {
_, ok = cErr.Cause.(interface{ Timeout() bool })
if ok {
return Retry
}
}
return NoRetry
}
// ServerErrorClassifier classify errors returned by POST method.
// If the error is nil, it returns NoRetry;
// if the error can be interpreted as TosServerError and its StatusCode is 5xx or 429, it returns Retry;
// otherwise, it returns NoRetry.
type ServerErrorClassifier struct{}
// Classify implements the classifier interface.
func (classifier ServerErrorClassifier) Classify(err error) retryAction {
if err == nil {
return NoRetry
}
e, ok := err.(*TosServerError)
if ok {
if e.StatusCode >= 500 || e.StatusCode == 429 {
return Retry
}
}
return NoRetry
}
type NoRetryClassifier struct{}
// Classify implements the classifier interface.
func (classifier NoRetryClassifier) Classify(_ error) retryAction {
return NoRetry
}

149
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/fetch.go generated vendored Normal file
View File

@@ -0,0 +1,149 @@
package tos
import (
"bytes"
"context"
"net/http"
)
const (
FetchTaskStateFailed = "Failed"
FetchTaskStateSucceed = "Succeed"
FetchTaskStateExpired = "Expired"
FetchTaskStateRunning = "Running"
)
type FetchObjectInput struct {
URL string `json:"URL,omitempty"` // required
Key string `json:"Key,omitempty"` // required
IgnoreSameKey bool `json:"IgnoreSameKey,omitempty"` // optional, default value is false
ContentMD5 string `json:"ContentMD5,omitempty"` // hex-encoded md5, optional
}
type FetchObjectOutput struct {
RequestInfo `json:"-"`
VersionID string `json:"VersionId,omitempty"` // may be empty
ETag string `json:"ETag,omitempty"`
}
type fetchObjectInput struct {
URL string `json:"URL,omitempty"` // required
IgnoreSameKey bool `json:"IgnoreSameKey,omitempty"` // optional, default value is false
ContentMD5 string `json:"ContentMD5,omitempty"` // base64-encoded md5, optional
}
// FetchObject fetch an object from specified URL
// options:
// WithMeta set meta header(s)
// WithServerSideEncryptionCustomer set server side encryption options
// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl
// Calling FetchObject will be blocked util fetch operation is finished
func (bkt *Bucket) FetchObject(ctx context.Context, input *FetchObjectInput, options ...Option) (*FetchObjectOutput, error) {
if err := isValidKey(input.Key); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("FetchObjectInput", &fetchObjectInput{
URL: input.URL,
IgnoreSameKey: input.IgnoreSameKey,
ContentMD5: input.ContentMD5,
})
if err != nil {
return nil, err
}
res, err := bkt.client.newBuilder(bkt.name, input.Key, options...).
WithQuery("fetch", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, ServerErrorClassifier{}).
Request(ctx, http.MethodPost, bytes.NewReader(data), bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
out := FetchObjectOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(out.RequestID, res.Body, &out); err != nil {
return nil, err
}
out.VersionID = res.Header.Get(HeaderVersionID)
return &out, nil
}
type PutFetchTaskInput struct {
URL string `json:"URL,omitempty"` // required
Object string `json:"Object,omitempty"` // object key, required
IgnoreSameKey bool `json:"IgnoreSameKey,omitempty"` // optional, default value is false
ContentMD5 string `json:"ContentMD5,omitempty"` // hex-encoded md5, optional
}
type PutFetchTaskOutput struct {
RequestInfo `json:"-"`
TaskID string `json:"TaskId,omitempty"`
}
// PutFetchTask put a fetch task to a bucket
// options:
// WithMeta set meta header(s)
// WithServerSideEncryptionCustomer set server side encryption options
// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl
// Calling PutFetchTask will return immediately after the task created.
func (bkt *Bucket) PutFetchTask(ctx context.Context, input *PutFetchTaskInput, options ...Option) (*PutFetchTaskOutput, error) {
if err := isValidKey(input.Object); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutFetchTaskInput", input)
if err != nil {
return nil, err
}
res, err := bkt.client.newBuilder(bkt.name, "", options...).
WithQuery("fetchTask", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, ServerErrorClassifier{}).
Request(ctx, http.MethodPost, bytes.NewReader(data), bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
out := PutFetchTaskOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(out.RequestID, res.Body, &out); err != nil {
return nil, err
}
return &out, nil
}
type GetFetchTaskInput struct {
TaskID string `json:"taskID,omitempty"`
}
type GetFetchTaskOutput struct {
RequestInfo `json:"-"`
State string `json:"State,omitempty"`
// Cause string `json:"Cause,omitempty"`
}
// GetFetchTask query the task state by the TaskID
// Task state:
// FetchTaskStateFailed = "Failed"
// FetchTaskStateSucceed = "Succeed"
// FetchTaskStateExpired = "Expired"
// FetchTaskStateRunning = "Running"
func (bkt *Bucket) GetFetchTask(ctx context.Context, input *GetFetchTaskInput, options ...Option) (*GetFetchTaskOutput, error) {
res, err := bkt.client.newBuilder(bkt.name, "", options...).
WithQuery("fetchTask", "").
WithQuery("taskId", input.TaskID).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
out := GetFetchTaskOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(out.RequestID, res.Body, &out); err != nil {
return nil, err
}
return &out, nil
}

106
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/http_trace.go generated vendored Normal file
View File

@@ -0,0 +1,106 @@
package tos
import (
"crypto/tls"
"fmt"
"net/http"
"net/http/httptrace"
"sync/atomic"
)
type accessLogRequest struct {
clientDnsCost int64
clientDialCost int64
clientTlsHandShakeCost int64
clientSendHeadersAndBodyCost int64
clientWaitResponseCost int64
clientSendRequestCost int64
actionStartMs int64
}
func newAccessLogRequest(actionStartMs int64) *accessLogRequest {
return &accessLogRequest{
clientDnsCost: -1,
clientDialCost: -1,
clientTlsHandShakeCost: -1,
clientSendHeadersAndBodyCost: -1,
clientWaitResponseCost: -1,
clientSendRequestCost: -1,
actionStartMs: actionStartMs,
}
}
func (r *accessLogRequest) PrintAccessLog(logger Logger, req *http.Request, response *http.Response) {
if logger == nil {
return
}
atomic.CompareAndSwapInt64(&r.clientSendRequestCost, -1, GetUnixTimeMs()-r.actionStartMs)
var requestId *string
if response != nil {
requestId = StringPtr(response.Header.Get(HeaderRequestID))
}
prefix := buildPrefix(requestId)
if req != nil {
logger.Debug(fmt.Sprintf("%s, method: %s, host: %s, request uri: %s, dns cost: %d ms, dial cost: %d ms, tls handshake cost: %d ms, send headers and body cost: %d ms, wait response cost: %d ms, request cost: %d ms",
prefix, req.Method, req.URL.Host, req.URL.EscapedPath(), r.clientDnsCost, r.clientDialCost, r.clientTlsHandShakeCost,
r.clientSendHeadersAndBodyCost, r.clientWaitResponseCost, r.clientSendRequestCost))
} else {
logger.Debug(fmt.Sprintf("%s, dns cost: %d ms, dial cost: %d ms, tls handshake cost: %d ms, send headers and body cost: %d ms, wait response cost: %d ms, request cost: %d ms",
prefix, r.clientDnsCost, r.clientDialCost, r.clientTlsHandShakeCost, r.clientSendHeadersAndBodyCost, r.clientWaitResponseCost, r.clientSendRequestCost))
}
}
func buildPrefix(requestId *string) string {
prefix := ""
if requestId != nil {
prefix = fmt.Sprintf("[requestId: %s] %s", *requestId, prefix)
}
return prefix
}
func getClientTrace(actionStartMs int64) (*httptrace.ClientTrace, *accessLogRequest) {
var dnsStart int64
var dialStart int64
var tlsHandShakeStart int64
var sendHeadersAndBodyStart int64
var waitResponseStart int64
r := newAccessLogRequest(actionStartMs)
trace := &httptrace.ClientTrace{
GotFirstResponseByte: func() {
r.clientWaitResponseCost = GetUnixTimeMs() - waitResponseStart
},
DNSStart: func(info httptrace.DNSStartInfo) {
dnsStart = GetUnixTimeMs()
},
DNSDone: func(info httptrace.DNSDoneInfo) {
r.clientDnsCost = GetUnixTimeMs() - dnsStart
},
ConnectStart: func(network, addr string) {
dialStart = GetUnixTimeMs()
},
ConnectDone: func(network, addr string, err error) {
now := GetUnixTimeMs()
sendHeadersAndBodyStart = now
r.clientDialCost = now - dialStart
},
TLSHandshakeStart: func() {
tlsHandShakeStart = GetUnixTimeMs()
},
TLSHandshakeDone: func(state tls.ConnectionState, err error) {
now := GetUnixTimeMs()
sendHeadersAndBodyStart = now
r.clientTlsHandShakeCost = now - tlsHandShakeStart
},
GotConn: func(httptrace.GotConnInfo) {
sendHeadersAndBodyStart = GetUnixTimeMs()
},
WroteRequest: func(info httptrace.WroteRequestInfo) {
waitResponseStart = GetUnixTimeMs()
r.clientSendHeadersAndBodyCost = waitResponseStart - sendHeadersAndBodyStart
},
}
return trace, r
}

116
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/lifecycle.go generated vendored Normal file
View File

@@ -0,0 +1,116 @@
package tos
import (
"bytes"
"context"
"net/http"
"time"
)
func (cli *ClientV2) parseLifecycleInput(input *PutBucketLifecycleInput) putBucketLifecycleInput {
lifecycleInput := make([]lifecycleRule, 0, len(input.Rules))
for _, lifecycle := range input.Rules {
var exp *expiration
if lifecycle.Expiration != nil {
exp = &expiration{
Days: lifecycle.Expiration.Days,
}
if !lifecycle.Expiration.Date.IsZero() {
exp.Date = lifecycle.Expiration.Date.Format(time.RFC3339)
}
}
transitionList := make([]transition, 0, len(lifecycle.Transitions))
for _, trans := range lifecycle.Transitions {
t := transition{
Days: trans.Days,
StorageClass: trans.StorageClass,
}
if !trans.Date.IsZero() {
t.Date = trans.Date.Format(time.RFC3339)
}
transitionList = append(transitionList, t)
}
lifecycleInput = append(lifecycleInput, lifecycleRule{
ID: lifecycle.ID,
Prefix: lifecycle.Prefix,
Status: lifecycle.Status,
Transitions: transitionList,
Expiration: exp,
NonCurrentVersionTransition: lifecycle.NonCurrentVersionTransition,
NoCurrentVersionExpiration: lifecycle.NoCurrentVersionExpiration,
Tag: lifecycle.Tag,
AbortInCompleteMultipartUpload: lifecycle.AbortInCompleteMultipartUpload,
})
}
return putBucketLifecycleInput{Rules: lifecycleInput}
}
func (cli *ClientV2) PutBucketLifecycle(ctx context.Context, input *PutBucketLifecycleInput) (*PutLifecycleOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutBucketLifecycleInput", cli.parseLifecycleInput(input))
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("lifecycle", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutLifecycleOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketLifecycle(ctx context.Context, input *GetBucketLifecycleInput) (*GetBucketLifecycleOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("lifecycle", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketLifecycleOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) DeleteBucketLifecycle(ctx context.Context, input *DeleteBucketLifecycleInput) (*DeleteBucketLifecycleOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("lifecycle", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteBucketLifecycleOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

9
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/logger.go generated vendored Normal file
View File

@@ -0,0 +1,9 @@
package tos
type Logger interface {
Debug(args ...interface{})
Info(args ...interface{})
Warn(args ...interface{})
Error(args ...interface{})
Fatal(args ...interface{})
}

149
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/metadata.go generated vendored Normal file
View File

@@ -0,0 +1,149 @@
package tos
import (
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
)
// ObjectMeta object metadata
type ObjectMeta struct {
ContentLength int64 `json:"ContentLength,omitempty"`
ContentType string `json:"ContentType,omitempty"`
ContentMD5 string `json:"ContentMD5,omitempty"`
ContentLanguage string `json:"ContentLanguage,omitempty"`
ContentEncoding string `json:"ContentEncoding,omitempty"`
ContentDisposition string `json:"ContentDisposition,omitempty"`
LastModified string `json:"LastModified,omitempty"`
CacheControl string `json:"CacheControl,omitempty"`
Expires string `json:"Expires,omitempty"`
ETag string `json:"ETag,omitempty"`
VersionID string `json:"VersionId,omitempty"`
DeleteMarker bool `json:"DeleteMarker,omitempty"`
ObjectType string `json:"ObjectType,omitempty"` // "" or "Appendable"
StorageClass string `json:"StorageClass,omitempty"`
Restore string `json:"Restore,omitempty"`
Metadata map[string]string `json:"Metadata,omitempty"`
Tag string `json:"Tag,omitempty"`
SSECustomerAlgorithm string `json:"SSECustomerAlgorithm,omitempty"`
SSECustomerKeyMD5 string `json:"SSECustomerKeyMD5,omitempty"`
CSType string `json:"CSType,omitempty"`
}
type ObjectMetaV2 struct {
ETag string
LastModified time.Time
DeleteMarker bool
SSECAlgorithm string
SSECKeyMD5 string
VersionID string
WebsiteRedirectLocation string
ObjectType string
HashCrc64ecma uint64
StorageClass enum.StorageClassType
Meta Metadata
ContentLength int64
ContentType string
CacheControl string
ContentDisposition string
ContentEncoding string
ContentLanguage string
Expires time.Time
ServerSideEncryption string
ServerSideEncryptionKeyID string
}
func (om *ObjectMeta) fromResponse(res *Response) {
om.ETag = res.Header.Get(HeaderETag)
om.LastModified = res.Header.Get(HeaderLastModified)
om.DeleteMarker, _ = strconv.ParseBool(res.Header.Get(HeaderDeleteMarker))
om.SSECustomerAlgorithm = res.Header.Get(HeaderSSECustomerAlgorithm)
om.SSECustomerKeyMD5 = res.Header.Get(HeaderSSECustomerKeyMD5)
om.VersionID = res.Header.Get(HeaderVersionID)
om.ObjectType = res.Header.Get(HeaderObjectType)
om.StorageClass = res.Header.Get(HeaderStorageClass)
om.Metadata = userMetadata(res.Header)
om.ContentLength = res.ContentLength
om.ContentType = res.Header.Get(HeaderContentType)
om.CacheControl = res.Header.Get(HeaderCacheControl)
om.ContentDisposition = res.Header.Get(HeaderContentDisposition)
om.ContentEncoding = res.Header.Get(HeaderContentEncoding)
om.ContentLanguage = res.Header.Get(HeaderContentLanguage)
om.Expires = res.Header.Get(HeaderExpires)
om.ContentMD5 = res.Header.Get(HeaderContentMD5)
om.Restore = res.Header.Get(HeaderRestore)
om.Tag = res.Header.Get(HeaderTag)
om.CSType = res.Header.Get(HeaderCSType)
}
func (om *ObjectMetaV2) fromResponseV2(res *Response) {
lastModified, _ := time.ParseInLocation(http.TimeFormat, res.Header.Get(HeaderLastModified), time.UTC)
deleteMarker, _ := strconv.ParseBool(res.Header.Get(HeaderDeleteMarker))
// If s is empty or contains invalid digits, err.Err = ErrSyntax and the returned value is 0;
crc64, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64)
length, _ := strconv.ParseInt(res.Header.Get(HeaderContentLength), 10, 64)
expires, _ := time.ParseInLocation(http.TimeFormat, res.Header.Get(HeaderExpires), time.UTC)
om.ETag = res.Header.Get(HeaderETag)
om.LastModified = lastModified
om.DeleteMarker = deleteMarker
om.SSECAlgorithm = res.Header.Get(HeaderSSECustomerAlgorithm)
om.SSECKeyMD5 = res.Header.Get(HeaderContentMD5)
om.VersionID = res.Header.Get(HeaderVersionID)
om.WebsiteRedirectLocation = res.Header.Get(HeaderWebsiteRedirectLocation)
om.ObjectType = res.Header.Get(HeaderObjectType)
om.HashCrc64ecma = crc64
om.StorageClass = enum.StorageClassType(res.Header.Get(HeaderStorageClass))
om.Meta = &CustomMeta{m: userMetadata(res.Header)}
om.ContentLength = length
om.ContentType = res.Header.Get(HeaderContentType)
om.CacheControl = res.Header.Get(HeaderCacheControl)
om.ContentDisposition, _ = url.QueryUnescape(res.Header.Get(HeaderContentDisposition))
om.ContentEncoding = res.Header.Get(HeaderContentEncoding)
om.ContentLanguage = res.Header.Get(HeaderContentLanguage)
om.Expires = expires
om.ServerSideEncryption = res.Header.Get(HeaderServerSideEncryption)
om.ServerSideEncryptionKeyID = res.Header.Get(HeaderServerSideEncryptionKmsKeyID)
}
func userMetadata(header http.Header) map[string]string {
meta := make(map[string]string)
for key := range header {
if strings.HasPrefix(key, HeaderMetaPrefix) {
kk, err := url.QueryUnescape(key[len(HeaderMetaPrefix):])
if err != nil {
kk = key[len(HeaderMetaPrefix):]
}
meta[strings.ToLower(kk)], err = url.QueryUnescape(header.Get(key))
if err != nil {
meta[strings.ToLower(kk)] = header.Get(key)
}
}
}
return meta
}
func parseUserMetaData(userMeta []userMeta) Metadata {
if len(userMeta) == 0 {
return nil
}
metas := make(map[string]string, len(userMeta))
for _, meta := range userMeta {
kk, err := url.QueryUnescape(meta.Key)
if err != nil {
kk = meta.Key
}
metas[strings.ToLower(kk)], err = url.QueryUnescape(meta.Value)
if err != nil {
metas[strings.ToLower(kk)] = meta.Value
}
}
return &CustomMeta{metas}
}

572
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mime.go generated vendored Normal file
View File

@@ -0,0 +1,572 @@
package tos
import "path"
var mime = map[string]string{
"3gp": "video/3gpp",
"7z": "application/x-7z-compressed",
"abw": "application/x-abiword",
"ai": "application/postscript",
"aif": "audio/x-aiff",
"aifc": "audio/x-aiff",
"aiff": "audio/x-aiff",
"alc": "chemical/x-alchemy",
"amr": "audio/amr",
"anx": "application/annodex",
"apk": "application/vnd.android.package-archive",
"appcache": "text/cache-manifest",
"art": "image/x-jg",
"asc": "text/plain",
"asf": "video/x-ms-asf",
"aso": "chemical/x-ncbi-asn1-binary",
"asx": "video/x-ms-asf",
"atom": "application/atom+xml",
"atomcat": "application/atomcat+xml",
"atomsrv": "application/atomserv+xml",
"au": "audio/basic",
"avi": "video/x-msvideo",
"awb": "audio/amr-wb",
"axa": "audio/annodex",
"axv": "video/annodex",
"b": "chemical/x-molconn-Z",
"bak": "application/x-trash",
"bat": "application/x-msdos-program",
"bcpio": "application/x-bcpio",
"bib": "text/x-bibtex",
"bin": "application/octet-stream",
"bmp": "image/x-ms-bmp",
"boo": "text/x-boo",
"book": "application/x-maker",
"brf": "text/plain",
"bsd": "chemical/x-crossfire",
"c": "text/x-csrc",
"c++": "text/x-c++src",
"c3d": "chemical/x-chem3d",
"cab": "application/x-cab",
"cac": "chemical/x-cache",
"cache": "chemical/x-cache",
"cap": "application/vnd.tcpdump.pcap",
"cascii": "chemical/x-cactvs-binary",
"cat": "application/vnd.ms-pki.seccat",
"cbin": "chemical/x-cactvs-binary",
"cbr": "application/x-cbr",
"cbz": "application/x-cbz",
"cc": "text/x-c++src",
"cda": "application/x-cdf",
"cdf": "application/x-cdf",
"cdr": "image/x-coreldraw",
"cdt": "image/x-coreldrawtemplate",
"cdx": "chemical/x-cdx",
"cdy": "application/vnd.cinderella",
"cef": "chemical/x-cxf",
"cer": "chemical/x-cerius",
"chm": "chemical/x-chemdraw",
"chrt": "application/x-kchart",
"cif": "chemical/x-cif",
"class": "application/java-vm",
"cls": "text/x-tex",
"cmdf": "chemical/x-cmdf",
"cml": "chemical/x-cml",
"cod": "application/vnd.rim.cod",
"com": "application/x-msdos-program",
"cpa": "chemical/x-compass",
"cpio": "application/x-cpio",
"cpp": "text/x-c++src",
"cpt": "application/mac-compactpro",
"cr2": "image/x-canon-cr2",
"crl": "application/x-pkcs7-crl",
"crt": "application/x-x509-ca-cert",
"crw": "image/x-canon-crw",
"csd": "audio/csound",
"csf": "chemical/x-cache-csf",
"csh": "application/x-csh",
"csm": "chemical/x-csml",
"csml": "chemical/x-csml",
"css": "text/css",
"csv": "text/csv",
"ctab": "chemical/x-cactvs-binary",
"ctx": "chemical/x-ctx",
"cu": "application/cu-seeme",
"cub": "chemical/x-gaussian-cube",
"cxf": "chemical/x-cxf",
"cxx": "text/x-c++src",
"d": "text/x-dsrc",
"davmount": "application/davmount+xml",
"dcm": "application/dicom",
"dcr": "application/x-director",
"ddeb": "application/vnd.debian.binary-package",
"dif": "video/dv",
"diff": "text/x-diff",
"dir": "application/x-director",
"djv": "image/vnd.djvu",
"djvu": "image/vnd.djvu",
"dl": "video/dl",
"dll": "application/x-msdos-program",
"dmg": "application/x-apple-diskimage",
"dms": "application/x-dms",
"doc": "application/msword",
"docm": "application/vnd.ms-word.document.macroEnabled.12",
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"dot": "application/msword",
"dotm": "application/vnd.ms-word.template.macroEnabled.12",
"dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
"dv": "video/dv",
"dvi": "application/x-dvi",
"dx": "chemical/x-jcamp-dx",
"dxr": "application/x-director",
"emb": "chemical/x-embl-dl-nucleotide",
"embl": "chemical/x-embl-dl-nucleotide",
"eml": "message/rfc822",
"eot": "application/vnd.ms-fontobject",
"eps": "application/postscript",
"eps2": "application/postscript",
"eps3": "application/postscript",
"epsf": "application/postscript",
"epsi": "application/postscript",
"erf": "image/x-epson-erf",
"es": "application/ecmascript",
"etx": "text/x-setext",
"exe": "application/x-msdos-program",
"ez": "application/andrew-inset",
"fb": "application/x-maker",
"fbdoc": "application/x-maker",
"fch": "chemical/x-gaussian-checkpoint",
"fchk": "chemical/x-gaussian-checkpoint",
"fig": "application/x-xfig",
"flac": "audio/flac",
"fli": "video/fli",
"flv": "video/x-flv",
"fm": "application/x-maker",
"frame": "application/x-maker",
"frm": "application/x-maker",
"gal": "chemical/x-gaussian-log",
"gam": "chemical/x-gamess-input",
"gamin": "chemical/x-gamess-input",
"gan": "application/x-ganttproject",
"gau": "chemical/x-gaussian-input",
"gcd": "text/x-pcs-gcd",
"gcf": "application/x-graphing-calculator",
"gcg": "chemical/x-gcg8-sequence",
"gen": "chemical/x-genbank",
"gf": "application/x-tex-gf",
"gif": "image/gif",
"gjc": "chemical/x-gaussian-input",
"gjf": "chemical/x-gaussian-input",
"gl": "video/gl",
"gnumeric": "application/x-gnumeric",
"gpt": "chemical/x-mopac-graph",
"gsf": "application/x-font",
"gsm": "audio/x-gsm",
"gtar": "application/x-gtar",
"gz": "application/gzip",
"h": "text/x-chdr",
"h++": "text/x-c++hdr",
"hdf": "application/x-hdf",
"hh": "text/x-c++hdr",
"hin": "chemical/x-hin",
"hpp": "text/x-c++hdr",
"hqx": "application/mac-binhex40",
"hs": "text/x-haskell",
"hta": "application/hta",
"htc": "text/x-component",
"htm": "text/html",
"html": "text/html",
"hwp": "application/x-hwp",
"hxx": "text/x-c++hdr",
"ica": "application/x-ica",
"ice": "x-conference/x-cooltalk",
"ico": "image/vnd.microsoft.icon",
"ics": "text/calendar",
"icz": "text/calendar",
"ief": "image/ief",
"iges": "model/iges",
"igs": "model/iges",
"iii": "application/x-iphone",
"info": "application/x-info",
"inp": "chemical/x-gamess-input",
"ins": "application/x-internet-signup",
"iso": "application/x-iso9660-image",
"isp": "application/x-internet-signup",
"ist": "chemical/x-isostar",
"istr": "chemical/x-isostar",
"jad": "text/vnd.sun.j2me.app-descriptor",
"jam": "application/x-jam",
"jar": "application/java-archive",
"java": "text/x-java",
"jdx": "chemical/x-jcamp-dx",
"jmz": "application/x-jmol",
"jng": "image/x-jng",
"jnlp": "application/x-java-jnlp-file",
"jp2": "image/jp2",
"jpe": "image/jpeg",
"jpeg": "image/jpeg",
"jpf": "image/jpx",
"jpg": "image/jpeg",
"jpg2": "image/jp2",
"jpm": "image/jpm",
"jpx": "image/jpx",
"js": "application/javascript",
"json": "application/json",
"kar": "audio/midi",
"key": "application/pgp-keys",
"kil": "application/x-killustrator",
"kin": "chemical/x-kinemage",
"kml": "application/vnd.google-earth.kml+xml",
"kmz": "application/vnd.google-earth.kmz",
"kpr": "application/x-kpresenter",
"kpt": "application/x-kpresenter",
"ksp": "application/x-kspread",
"kwd": "application/x-kword",
"kwt": "application/x-kword",
"latex": "application/x-latex",
"lha": "application/x-lha",
"lhs": "text/x-literate-haskell",
"lin": "application/bbolin",
"lsf": "video/x-la-asf",
"lsx": "video/x-la-asf",
"ltx": "text/x-tex",
"ly": "text/x-lilypond",
"lyx": "application/x-lyx",
"lzh": "application/x-lzh",
"lzx": "application/x-lzx",
"m3g": "application/m3g",
"m3u": "audio/x-mpegurl",
"m3u8": "application/x-mpegURL",
"m4a": "audio/mpeg",
"maker": "application/x-maker",
"man": "application/x-troff-man",
"mbox": "application/mbox",
"mcif": "chemical/x-mmcif",
"mcm": "chemical/x-macmolecule",
"mdb": "application/msaccess",
"me": "application/x-troff-me",
"mesh": "model/mesh",
"mid": "audio/midi",
"midi": "audio/midi",
"mif": "application/x-mif",
"mkv": "video/x-matroska",
"mm": "application/x-freemind",
"mmd": "chemical/x-macromodel-input",
"mmf": "application/vnd.smaf",
"mml": "text/mathml",
"mmod": "chemical/x-macromodel-input",
"mng": "video/x-mng",
"moc": "text/x-moc",
"mol": "chemical/x-mdl-molfile",
"mol2": "chemical/x-mol2",
"moo": "chemical/x-mopac-out",
"mop": "chemical/x-mopac-input",
"mopcrt": "chemical/x-mopac-input",
"mov": "video/quicktime",
"movie": "video/x-sgi-movie",
"mp2": "audio/mpeg",
"mp3": "audio/mpeg",
"mp4": "video/mp4",
"mpc": "chemical/x-mopac-input",
"mpe": "video/mpeg",
"mpeg": "video/mpeg",
"mpega": "audio/mpeg",
"mpg": "video/mpeg",
"mpga": "audio/mpeg",
"mph": "application/x-comsol",
"mpv": "video/x-matroska",
"ms": "application/x-troff-ms",
"msh": "model/mesh",
"msi": "application/x-msi",
"mvb": "chemical/x-mopac-vib",
"mxf": "application/mxf",
"mxu": "video/vnd.mpegurl",
"nb": "application/mathematica",
"nbp": "application/mathematica",
"nc": "application/x-netcdf",
"nef": "image/x-nikon-nef",
"nwc": "application/x-nwc",
"o": "application/x-object",
"oda": "application/oda",
"odb": "application/vnd.oasis.opendocument.database",
"odc": "application/vnd.oasis.opendocument.chart",
"odf": "application/vnd.oasis.opendocument.formula",
"odg": "application/vnd.oasis.opendocument.graphics",
"odi": "application/vnd.oasis.opendocument.image",
"odm": "application/vnd.oasis.opendocument.text-master",
"odp": "application/vnd.oasis.opendocument.presentation",
"ods": "application/vnd.oasis.opendocument.spreadsheet",
"odt": "application/vnd.oasis.opendocument.text",
"oga": "audio/ogg",
"ogg": "audio/ogg",
"ogv": "video/ogg",
"ogx": "application/ogg",
"old": "application/x-trash",
"one": "application/onenote",
"onepkg": "application/onenote",
"onetmp": "application/onenote",
"onetoc2": "application/onenote",
"opf": "application/oebps-package+xml",
"opus": "audio/ogg",
"orc": "audio/csound",
"orf": "image/x-olympus-orf",
"otf": "application/font-sfnt",
"otg": "application/vnd.oasis.opendocument.graphics-template",
"oth": "application/vnd.oasis.opendocument.text-web",
"otp": "application/vnd.oasis.opendocument.presentation-template",
"ots": "application/vnd.oasis.opendocument.spreadsheet-template",
"ott": "application/vnd.oasis.opendocument.text-template",
"oza": "application/x-oz-application",
"p": "text/x-pascal",
"p7r": "application/x-pkcs7-certreqresp",
"pac": "application/x-ns-proxy-autoconfig",
"pas": "text/x-pascal",
"pat": "image/x-coreldrawpattern",
"patch": "text/x-diff",
"pbm": "image/x-portable-bitmap",
"pcap": "application/vnd.tcpdump.pcap",
"pcf": "application/x-font-pcf",
"pcf.Z": "application/x-font-pcf",
"pcx": "image/pcx",
"pdb": "chemical/x-pdb",
"pdf": "application/pdf",
"pfa": "application/x-font",
"pfb": "application/x-font",
"pfr": "application/font-tdpfr",
"pgm": "image/x-portable-graymap",
"pgn": "application/x-chess-pgn",
"pgp": "application/pgp-encrypted",
"php": "#application/x-httpd-php",
"php3": "#application/x-httpd-php3",
"php3p": "#application/x-httpd-php3-preprocessed",
"php4": "#application/x-httpd-php4",
"php5": "#application/x-httpd-php5",
"phps": "#application/x-httpd-php-source",
"pht": "#application/x-httpd-php",
"phtml": "#application/x-httpd-php",
"pk": "application/x-tex-pk",
"pl": "text/x-perl",
"pls": "audio/x-scpls",
"pm": "text/x-perl",
"png": "image/png",
"pnm": "image/x-portable-anymap",
"pot": "text/plain",
"potm": "application/vnd.ms-powerpoint.template.macroEnabled.12",
"potx": "application/vnd.openxmlformats-officedocument.presentationml.template",
"ppam": "application/vnd.ms-powerpoint.addin.macroEnabled.12",
"ppm": "image/x-portable-pixmap",
"pps": "application/vnd.ms-powerpoint",
"ppsm": "application/vnd.ms-powerpoint.slideshow.macroEnabled.12",
"ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
"ppt": "application/vnd.ms-powerpoint",
"pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
"prf": "application/pics-rules",
"prt": "chemical/x-ncbi-asn1-ascii",
"ps": "application/postscript",
"psd": "image/x-photoshop",
"py": "text/x-python",
"pyc": "application/x-python-code",
"pyo": "application/x-python-code",
"qgs": "application/x-qgis",
"qt": "video/quicktime",
"qtl": "application/x-quicktimeplayer",
"ra": "audio/x-pn-realaudio",
"ram": "audio/x-pn-realaudio",
"rar": "application/rar",
"ras": "image/x-cmu-raster",
"rb": "application/x-ruby",
"rd": "chemical/x-mdl-rdfile",
"rdf": "application/rdf+xml",
"rdp": "application/x-rdp",
"rgb": "image/x-rgb",
"rhtml": "#application/x-httpd-eruby",
"rm": "audio/x-pn-realaudio",
"roff": "application/x-troff",
"ros": "chemical/x-rosdal",
"rpm": "application/x-redhat-package-manager",
"rss": "application/x-rss+xml",
"rtf": "application/rtf",
"rtx": "text/richtext",
"rxn": "chemical/x-mdl-rxnfile",
"scala": "text/x-scala",
"sce": "application/x-scilab",
"sci": "application/x-scilab",
"sco": "audio/csound",
"scr": "application/x-silverlight",
"sct": "text/scriptlet",
"sd": "chemical/x-mdl-sdfile",
"sd2": "audio/x-sd2",
"sda": "application/vnd.stardivision.draw",
"sdc": "application/vnd.stardivision.calc",
"sdd": "application/vnd.stardivision.impress",
"sds": "application/vnd.stardivision.chart",
"sdw": "application/vnd.stardivision.writer",
"ser": "application/java-serialized-object",
"sfd": "application/vnd.font-fontforge-sfd",
"sfv": "text/x-sfv",
"sgf": "application/x-go-sgf",
"sgl": "application/vnd.stardivision.writer-global",
"sh": "application/x-sh",
"shar": "application/x-shar",
"shp": "application/x-qgis",
"shtml": "text/html",
"shx": "application/x-qgis",
"sid": "audio/prs.sid",
"sig": "application/pgp-signature",
"sik": "application/x-trash",
"silo": "model/mesh",
"sis": "application/vnd.symbian.install",
"sisx": "x-epoc/x-sisx-app",
"sit": "application/x-stuffit",
"sitx": "application/x-stuffit",
"skd": "application/x-koan",
"skm": "application/x-koan",
"skp": "application/x-koan",
"skt": "application/x-koan",
"sldm": "application/vnd.ms-powerpoint.slide.macroEnabled.12",
"sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide",
"smi": "application/smil+xml",
"smil": "application/smil+xml",
"snd": "audio/basic",
"spc": "chemical/x-galactic-spc",
"spl": "application/x-futuresplash",
"spx": "audio/ogg",
"sql": "application/x-sql",
"src": "application/x-wais-source",
"srt": "text/plain",
"stc": "application/vnd.sun.xml.calc.template",
"std": "application/vnd.sun.xml.draw.template",
"sti": "application/vnd.sun.xml.impress.template",
"stw": "application/vnd.sun.xml.writer.template",
"sty": "text/x-tex",
"sv4cpio": "application/x-sv4cpio",
"sv4crc": "application/x-sv4crc",
"svg": "image/svg+xml",
"svgz": "image/svg+xml",
"sw": "chemical/x-swissprot",
"swf": "application/x-shockwave-flash",
"swfl": "application/x-shockwave-flash",
"sxc": "application/vnd.sun.xml.calc",
"sxd": "application/vnd.sun.xml.draw",
"sxg": "application/vnd.sun.xml.writer.global",
"sxi": "application/vnd.sun.xml.impress",
"sxm": "application/vnd.sun.xml.math",
"sxw": "application/vnd.sun.xml.writer",
"t": "application/x-troff",
"tar": "application/x-tar",
"taz": "application/x-gtar-compressed",
"tcl": "application/x-tcl",
"tex": "text/x-tex",
"texi": "application/x-texinfo",
"texinfo": "application/x-texinfo",
"text": "text/plain",
"tgf": "chemical/x-mdl-tgf",
"tgz": "application/x-gtar-compressed",
"thmx": "application/vnd.ms-officetheme",
"tif": "image/tiff",
"tiff": "image/tiff",
"tk": "text/x-tcl",
"tm": "text/texmacs",
"torrent": "application/x-bittorrent",
"tr": "application/x-troff",
"ts": "video/MP2T",
"tsp": "application/dsptype",
"tsv": "text/tab-separated-values",
"ttf": "application/font-sfnt",
"ttl": "text/turtle",
"txt": "text/plain",
"uls": "text/iuls",
"ustar": "application/x-ustar",
"val": "chemical/x-ncbi-asn1-binary",
"vcard": "text/vcard",
"vcd": "application/x-cdlink",
"vcf": "text/vcard",
"vcs": "text/x-vcalendar",
"vmd": "chemical/x-vmd",
"vms": "chemical/x-vamas-iso14976",
"vrm": "x-world/x-vrml",
"vrml": "model/vrml",
"vsd": "application/vnd.visio",
"vss": "application/vnd.visio",
"vst": "application/vnd.visio",
"vsw": "application/vnd.visio",
"wad": "application/x-doom",
"wasm": "application/wasm",
"wav": "audio/x-wav",
"wax": "audio/x-ms-wax",
"wbmp": "image/vnd.wap.wbmp",
"wbxml": "application/vnd.wap.wbxml",
"webm": "video/webm",
"wk": "application/x-123",
"wm": "video/x-ms-wm",
"wma": "audio/x-ms-wma",
"wmd": "application/x-ms-wmd",
"wml": "text/vnd.wap.wml",
"wmlc": "application/vnd.wap.wmlc",
"wmls": "text/vnd.wap.wmlscript",
"wmlsc": "application/vnd.wap.wmlscriptc",
"wmv": "video/x-ms-wmv",
"wmx": "video/x-ms-wmx",
"wmz": "application/x-ms-wmz",
"woff": "application/font-woff",
"wp5": "application/vnd.wordperfect5.1",
"wpd": "application/vnd.wordperfect",
"wrl": "model/vrml",
"wsc": "text/scriptlet",
"wvx": "video/x-ms-wvx",
"wz": "application/x-wingz",
"x3d": "model/x3d+xml",
"x3db": "model/x3d+binary",
"x3dv": "model/x3d+vrml",
"xbm": "image/x-xbitmap",
"xcf": "application/x-xcf",
"xcos": "application/x-scilab-xcos",
"xht": "application/xhtml+xml",
"xhtml": "application/xhtml+xml",
"xlam": "application/vnd.ms-excel.addin.macroEnabled.12",
"xlb": "application/vnd.ms-excel",
"xls": "application/vnd.ms-excel",
"xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
"xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12",
"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"xlt": "application/vnd.ms-excel",
"xltm": "application/vnd.ms-excel.template.macroEnabled.12",
"xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
"xml": "application/xml",
"xpi": "application/x-xpinstall",
"xpm": "image/x-xpixmap",
"xsd": "application/xml",
"xsl": "application/xslt+xml",
"xslt": "application/xslt+xml",
"xspf": "application/xspf+xml",
"xtel": "chemical/x-xtel",
"xul": "application/vnd.mozilla.xul+xml",
"xwd": "image/x-xwindowdump",
"xyz": "chemical/x-xyz",
"xz": "application/x-xz",
"zip": "application/zip",
}
type ContentTypeRecognizer interface {
ContentType(objectKey string) string
}
type ExtensionBasedContentTypeRecognizer struct{}
func (er ExtensionBasedContentTypeRecognizer) ContentType(objectKey string) string {
if len(objectKey) == 0 {
return ""
}
extName := path.Ext(objectKey)
if len(extName) > 0 && extName[0] == '.' {
extName = extName[1:]
}
contentType, ok := mime[extName]
if ok {
return contentType
}
return "binary/octet-stream"
}
type EmptyContentTypeRecognizer struct{}
func (er EmptyContentTypeRecognizer) ContentType(objectKey string) string {
_ = objectKey
return ""
}

76
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/mirror_back.go generated vendored Normal file
View File

@@ -0,0 +1,76 @@
package tos
import (
"bytes"
"context"
"net/http"
)
func (cli *ClientV2) PutBucketMirrorBack(ctx context.Context, input *PutBucketMirrorBackInput) (*PutBucketMirrorBackOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutBucketMirrorBackInput", putBucketMirrorBackInput{
Rules: input.Rules,
})
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("mirror", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketMirrorBackOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketMirrorBack(ctx context.Context, input *GetBucketMirrorBackInput) (*GetBucketMirrorBackOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("mirror", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketMirrorBackOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) DeleteBucketMirrorBack(ctx context.Context, input *DeleteBucketMirrorBackInput) (*DeleteBucketMirrorBackOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("mirror", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteBucketMirrorBackOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

513
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/multipart.go generated vendored Normal file
View File

@@ -0,0 +1,513 @@
package tos
import (
"bytes"
"context"
"encoding/json"
"fmt"
"hash"
"io"
"io/ioutil"
"net/http"
"os"
"sort"
"strconv"
)
// CreateMultipartUpload create a multipart upload operation
// objectKey: the name of object
// options: WithContentType set Content-Type,
// WithContentDisposition set Content-Disposition,
// WithContentLanguage set Content-Language,
// WithContentEncoding set Content-Encoding,
// WithCacheControl set Cache-Control,
// WithExpires set Expires,
// WithMeta set meta header(s),
// WithContentSHA256 set Content-Sha256,
// WithContentMD5 set Content-MD5
// WithExpires set Expires,
// WithServerSideEncryptionCustomer set server side encryption options
// WithACL WithACLGrantFullControl WithACLGrantRead WithACLGrantReadAcp WithACLGrantWrite WithACLGrantWriteAcp set object acl
//
// Deprecated: use CreateMultipartUpload of ClientV2 instead
func (bkt *Bucket) CreateMultipartUpload(ctx context.Context, objectKey string, options ...Option) (*CreateMultipartUploadOutput, error) {
if err := isValidKey(objectKey); err != nil {
return nil, err
}
res, err := bkt.client.newBuilder(bkt.name, objectKey, options...).
WithQuery("uploads", "").
WithRetry(nil, ServerErrorClassifier{}).
Request(ctx, http.MethodPost, nil, bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
var upload multipartUpload
if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &upload); err != nil {
return nil, err
}
return &CreateMultipartUploadOutput{
RequestInfo: res.RequestInfo(),
Bucket: upload.Bucket,
Key: upload.Key,
UploadID: upload.UploadID,
SSECustomerAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm),
SSECustomerKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5),
}, nil
}
// CreateMultipartUploadV2 create a multipart upload operation
func (cli *ClientV2) CreateMultipartUploadV2(
ctx context.Context,
input *CreateMultipartUploadV2Input) (*CreateMultipartUploadV2Output, error) {
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
if err := isValidKey(input.Key); err != nil {
return nil, err
}
if err := isValidSSECAlgorithm(input.SSECAlgorithm); len(input.SSECAlgorithm) != 0 && err != nil {
return nil, err
}
if err := isValidACL(input.ACL); len(input.ACL) != 0 && err != nil {
return nil, err
}
if err := isValidStorageClass(input.StorageClass); len(input.StorageClass) != 0 && err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key).
WithQuery("uploads", "").
WithParams(*input).
WithRetry(nil, ServerErrorClassifier{}).
Request(ctx, http.MethodPost, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
var upload multipartUpload
if err = marshalOutput(res.RequestInfo().RequestID, res.Body, &upload); err != nil {
return nil, err
}
return &CreateMultipartUploadV2Output{
RequestInfo: res.RequestInfo(),
Bucket: upload.Bucket,
Key: upload.Key,
UploadID: upload.UploadID,
SSECAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm),
SSECKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5),
EncodingType: res.Header.Get(HeaderContentEncoding),
ServerSideEncryption: res.Header.Get(HeaderServerSideEncryption),
ServerSideEncryptionKeyID: res.Header.Get(HeaderServerSideEncryptionKmsKeyID),
}, nil
}
// UploadPart upload a part for a multipart upload operation
// input: the parameters, some fields is required, e.g. Key, UploadID, PartNumber and PartNumber
//
// If uploading 'Content' with known Content-Length, please add option tos.WithContentLength
//
// Deprecated: use UploadPart of ClientV2 instead
func (bkt *Bucket) UploadPart(ctx context.Context, input *UploadPartInput, options ...Option) (*UploadPartOutput, error) {
if err := isValidKey(input.Key); err != nil {
return nil, err
}
var (
onRetry func(req *Request) error = nil
cf classifier
content = input.Content
)
cf = NoRetryClassifier{}
if seeker, ok := content.(io.Seeker); ok {
start, err := seeker.Seek(0, io.SeekCurrent)
if err == nil {
onRetry = func(req *Request) error {
// PutObject/UploadPartV2 can be treated as an idempotent semantics if the request message body
// supports a reset operation. e.g. the request message body is a string,
// a local file handle, binary data in memory
if seeker, ok := req.Content.(io.Seeker); ok {
_, err := seeker.Seek(start, io.SeekStart)
if err != nil {
return err
}
} else {
return newTosClientError("Io Reader not support retry", nil)
}
return nil
}
cf = StatusCodeClassifier{}
}
}
res, err := bkt.client.newBuilder(bkt.name, input.Key, options...).
WithQuery("uploadId", input.UploadID).
WithQuery("partNumber", strconv.Itoa(input.PartNumber)).
WithRetry(onRetry, cf).
Request(ctx, http.MethodPut, input.Content, bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
return &UploadPartOutput{
RequestInfo: res.RequestInfo(),
PartNumber: input.PartNumber,
ETag: res.Header.Get(HeaderETag),
SSECustomerAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm),
SSECustomerKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5),
}, nil
}
// UploadPartV2 upload a part for a multipart upload operation
func (cli *ClientV2) UploadPartV2(ctx context.Context, input *UploadPartV2Input) (*UploadPartV2Output, error) {
if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
return nil, err
}
if err := isValidSSECAlgorithm(input.SSECAlgorithm); len(input.SSECAlgorithm) != 0 && err != nil {
return nil, err
}
var (
checker hash.Hash64
content = input.Content
contentLength = input.ContentLength
)
if input == nil {
return nil, InputInvalidClientError
}
if input.PartNumber == 0 {
return nil, InvalidPartNumber
}
if input.UploadID == "" {
return nil, InvalidUploadID
}
if contentLength == 0 {
contentLength = tryResolveLength(content)
}
if cli.enableCRC {
checker = NewCRC(DefaultCrcTable(), 0)
}
var (
onRetry func(req *Request) error = nil
cf classifier
)
if content != nil {
content = wrapReader(content, contentLength, input.DataTransferListener, input.RateLimiter, &crcChecker{checker: checker})
}
cf = NoRetryClassifier{}
if seeker, ok := content.(io.Seeker); ok {
start, err := seeker.Seek(0, io.SeekCurrent)
if err == nil {
onRetry = func(req *Request) error {
// PutObject/UploadPartV2 can be treated as an idempotent semantics if the request message body
// supports a reset operation. e.g. the request message body is a string,
// a local file handle, binary data in memory
if seeker, ok := req.Content.(io.Seeker); ok {
_, err := seeker.Seek(start, io.SeekStart)
if err != nil {
return err
}
} else {
return newTosClientError("Io Reader not support retry", nil)
}
return nil
}
cf = StatusCodeClassifier{}
}
}
res, err := cli.newBuilder(input.Bucket, input.Key).
WithParams(*input).
WithContentLength(input.ContentLength).
WithRetry(onRetry, cf).
Request(ctx, http.MethodPut, content, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
if err = checkCrc64(res, checker); err != nil {
return nil, err
}
checksum, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64)
return &UploadPartV2Output{
RequestInfo: res.RequestInfo(),
PartNumber: input.PartNumber,
ETag: res.Header.Get(HeaderETag),
SSECAlgorithm: res.Header.Get(HeaderSSECustomerAlgorithm),
SSECKeyMD5: res.Header.Get(HeaderSSECustomerKeyMD5),
HashCrc64ecma: checksum,
ServerSideEncryptionKeyID: res.Header.Get(HeaderServerSideEncryptionKmsKeyID),
ServerSideEncryption: res.Header.Get(HeaderServerSideEncryption),
}, nil
}
// UploadPartFromFile upload a part for a multipart upload operation from file
func (cli *ClientV2) UploadPartFromFile(ctx context.Context, input *UploadPartFromFileInput) (*UploadPartFromFileOutput, error) {
file, err := os.Open(input.FilePath)
if err != nil {
return nil, err
}
_, err = file.Seek(int64(input.Offset), io.SeekStart)
if err != nil {
return nil, err
}
output, err := cli.UploadPartV2(ctx, &UploadPartV2Input{
UploadPartBasicInput: input.UploadPartBasicInput,
Content: file,
ContentLength: input.PartSize,
})
if err != nil {
return nil, err
}
return &UploadPartFromFileOutput{*output}, nil
}
// CompleteMultipartUpload complete a multipart upload operation
// input: input.Key the object name,
// input.UploadID the uploadID got from CreateMultipartUpload
// input.UploadedParts upload part output got from UploadPart or UploadPartCopy
//
// Deprecated: use CompleteMultipartUpload of ClientV2 instead
func (bkt *Bucket) CompleteMultipartUpload(ctx context.Context, input *CompleteMultipartUploadInput, options ...Option) (*CompleteMultipartUploadOutput, error) {
if err := isValidKey(input.Key); err != nil {
return nil, err
}
multipart := partsToComplete{Parts: make(uploadedParts, 0, len(input.UploadedParts))}
for _, p := range input.UploadedParts {
multipart.Parts = append(multipart.Parts, p.uploadedPart())
}
sort.Sort(multipart.Parts)
data, err := json.Marshal(&multipart)
if err != nil {
return nil, InvalidMarshal
}
res, err := bkt.client.newBuilder(bkt.name, input.Key, options...).
WithQuery("uploadId", input.UploadID).
WithRetry(OnRetryFromStart, ServerErrorClassifier{}).
Request(ctx, http.MethodPost, bytes.NewReader(data), bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
return &CompleteMultipartUploadOutput{
RequestInfo: res.RequestInfo(),
VersionID: res.Header.Get(HeaderVersionID),
}, nil
}
// CompleteMultipartUploadV2 complete a multipart upload operation
func (cli *ClientV2) CompleteMultipartUploadV2(
ctx context.Context, input *CompleteMultipartUploadV2Input) (*CompleteMultipartUploadV2Output, error) {
if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
return nil, err
}
reqBuilder := cli.newBuilder(input.Bucket, input.Key).
WithParams(*input)
var err error
var res *Response
if input.CompleteAll {
if len(input.Parts) != 0 {
return nil, InvalidCompleteAllPartsLength
}
reqBuilder.WithHeader("x-tos-complete-all", "yes")
res, err = reqBuilder.WithRetry(nil, ServerErrorClassifier{}).Request(ctx, http.MethodPost, nil, cli.roundTripper(http.StatusOK))
} else {
if len(input.Parts) == 0 {
return nil, InvalidPartsLength
}
multipart := partsToComplete{Parts: make(uploadedParts, 0, len(input.Parts))}
for _, p := range input.Parts {
multipart.Parts = append(multipart.Parts, p.uploadedPart())
}
sort.Sort(multipart.Parts)
data, marshalErr := json.Marshal(&multipart)
if marshalErr != nil {
return nil, InvalidMarshal
}
res, err = reqBuilder.WithRetry(OnRetryFromStart, ServerErrorClassifier{}).Request(ctx, http.MethodPost, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
}
if err != nil {
return nil, err
}
defer res.Close()
crc64, _ := strconv.ParseUint(res.Header.Get(HeaderHashCrc64ecma), 10, 64)
output := &CompleteMultipartUploadV2Output{
RequestInfo: res.RequestInfo(),
VersionID: res.Header.Get(HeaderVersionID),
HashCrc64ecma: crc64,
}
var callbackResult string
if input.Callback == "" {
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
} else {
callbackRes, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, &TosServerError{
TosError: TosError{Message: fmt.Sprintf("tos: read callback result err:%s", err.Error())},
RequestInfo: res.RequestInfo(),
}
}
if len(callbackRes) > 0 {
callbackResult = string(callbackRes)
}
output.ETag = res.Header.Get(HeaderETag)
output.Location = res.Header.Get(HeaderLocation)
}
output.CallbackResult = callbackResult
output.ServerSideEncryption = res.Header.Get(HeaderServerSideEncryption)
output.ServerSideEncryptionKeyID = res.Header.Get(HeaderServerSideEncryptionKmsKeyID)
return output, nil
}
// AbortMultipartUpload abort a multipart upload operation
//
// Deprecated: use AbortMultipartUpload of ClientV2 instead
func (bkt *Bucket) AbortMultipartUpload(ctx context.Context, input *AbortMultipartUploadInput, options ...Option) (*AbortMultipartUploadOutput, error) {
if err := isValidKey(input.Key); err != nil {
return nil, err
}
res, err := bkt.client.newBuilder(bkt.name, input.Key, options...).
WithQuery("uploadId", input.UploadID).
WithRetry(nil, ServerErrorClassifier{}).
Request(ctx, http.MethodDelete, nil, bkt.client.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
return &AbortMultipartUploadOutput{RequestInfo: res.RequestInfo()}, nil
}
// AbortMultipartUpload abort a multipart upload operation
func (cli *ClientV2) AbortMultipartUpload(ctx context.Context, input *AbortMultipartUploadInput) (*AbortMultipartUploadOutput, error) {
if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key).
WithParams(*input).
WithRetry(nil, ServerErrorClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
return &AbortMultipartUploadOutput{RequestInfo: res.RequestInfo()}, nil
}
// ListUploadedParts List Uploaded Parts
// objectKey: the object name
// input: key, uploadID and other parameters
//
// Deprecated: use ListParts of ClientV2 instead
func (bkt *Bucket) ListUploadedParts(ctx context.Context, input *ListUploadedPartsInput, options ...Option) (*ListUploadedPartsOutput, error) {
if err := isValidKey(input.Key); err != nil {
return nil, err
}
res, err := bkt.client.newBuilder(bkt.name, input.Key, options...).
WithQuery("uploadId", input.UploadID).
WithQuery("max-parts", strconv.Itoa(input.MaxParts)).
WithQuery("part-number-marker", strconv.Itoa(input.PartNumberMarker)).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := ListUploadedPartsOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
// ListParts List Uploaded Parts
func (cli *ClientV2) ListParts(ctx context.Context, input *ListPartsInput) (*ListPartsOutput, error) {
if err := isValidNames(input.Bucket, input.Key, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key).
WithParams(*input).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := ListPartsOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
// ListMultipartUploads list multipart uploads
//
// Deprecated: use ListMultipartUploads of ClientV2 instead
func (bkt *Bucket) ListMultipartUploads(ctx context.Context, input *ListMultipartUploadsInput, options ...Option) (*ListMultipartUploadsOutput, error) {
res, err := bkt.client.newBuilder(bkt.name, "", options...).
WithQuery("uploads", "").
WithQuery("prefix", input.Prefix).
WithQuery("delimiter", input.Delimiter).
WithQuery("key-marker", input.KeyMarker).
WithQuery("upload-id-marker", input.UploadIDMarker).
WithQuery("max-uploads", strconv.Itoa(input.MaxUploads)).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, bkt.client.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := ListMultipartUploadsOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
// ListMultipartUploadsV2 list multipart uploads
func (cli *ClientV2) ListMultipartUploadsV2(
ctx context.Context,
input *ListMultipartUploadsV2Input) (*ListMultipartUploadsV2Output, error) {
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("uploads", "").
WithParams(*input).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := ListMultipartUploadsV2Output{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}

54
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/notification.go generated vendored Normal file
View File

@@ -0,0 +1,54 @@
package tos
import (
"bytes"
"context"
"net/http"
)
func (cli *ClientV2) PutBucketNotification(ctx context.Context, input *PutBucketNotificationInput) (*PutBucketNotificationOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutBucketNotification", putBucketNotificationInput{CloudFunctionConfigurations: input.CloudFunctionConfigurations, RocketMQConfigurations: input.RocketMQConfigurations})
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("notification", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketNotificationOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketNotification(ctx context.Context, input *GetBucketNotificationInput) (*GetBucketNotificationOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("notification", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketNotificationOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}

1121
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/object.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

274
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/options.go generated vendored Normal file
View File

@@ -0,0 +1,274 @@
package tos
import (
"net/http"
"time"
)
type Option func(*requestBuilder)
// WithContentType set Content-Type header
// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta
func WithContentType(contentType string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderContentType, contentType)
}
}
// WithContentLength set Content-Length header
// used in Bucket.PutObject Bucket.AppendObject Bucket.UploadPart
//
// If the length of the content is known, it is better to add this option when Put, Append or Upload.
func WithContentLength(length int64) Option {
return func(rb *requestBuilder) {
rb.WithContentLength(length)
}
}
// WithCacheControl set Cache-Control header
// used in Bucket.PutObject Bucket.AppendObject
// Bucket.CreateMultipartUpload Bucket.SetObjectMeta
func WithCacheControl(cacheControl string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderCacheControl, cacheControl)
}
}
// WithContentDisposition set Content-Disposition header
// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta
func WithContentDisposition(contentDisposition string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderContentDisposition, contentDisposition)
}
}
// WithContentEncoding set Content-Encoding header
// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta
func WithContentEncoding(contentEncoding string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderContentEncoding, contentEncoding)
}
}
// WithContentLanguage set Content-Language header
// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta
func WithContentLanguage(contentLanguage string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderContentLanguage, contentLanguage)
}
}
// WithContentMD5 set Content-MD5 header
func WithContentMD5(contentMD5 string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderContentMD5, contentMD5)
}
}
// WithContentSHA256 set X-Tos-Content-Sha256 header
func WithContentSHA256(contentSHA256 string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderContentSha256, contentSHA256)
}
}
// WithExpires set Expires header
// used in Bucket.PutObject Bucket.AppendObject Bucket.CreateMultipartUpload Bucket.SetObjectMeta
func WithExpires(expires time.Time) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderExpires, expires.Format(http.TimeFormat))
}
}
// WithServerSideEncryptionCustomer set server-side-encryption parameters
// used in Bucket.PutObject Bucket.CreateMultipartUpload
func WithServerSideEncryptionCustomer(ssecAlgorithm, ssecKey, ssecKeyMD5 string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderSSECustomerAlgorithm, ssecAlgorithm)
rb.Header.Set(HeaderSSECustomerKey, ssecKey)
rb.Header.Set(HeaderSSECustomerKeyMD5, ssecKeyMD5)
}
}
// WithIfModifiedSince set If-Modified-Since header
// used in Bucket.GetObject Bucket.HeadObject
func WithIfModifiedSince(since time.Time) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderIfModifiedSince, since.Format(http.TimeFormat))
}
}
// WithIfUnmodifiedSince set If-Unmodified-Since header
// used in Bucket.GetObject Bucket.HeadObject
func WithIfUnmodifiedSince(since time.Time) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderIfUnmodifiedSince, since.Format(http.TimeFormat))
}
}
// WithIfMatch set If-Match header
func WithIfMatch(ifMatch string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderIfMatch, ifMatch)
}
}
// WithIfNoneMatch set If-None-Match header
func WithIfNoneMatch(ifNoneMatch string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderIfNoneMatch, ifNoneMatch)
}
}
// WithCopySourceIfMatch set X-Tos-Copy-Source-If-Match header
// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom Bucket.UploadPartCopy
func WithCopySourceIfMatch(ifMatch string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderCopySourceIfMatch, ifMatch)
}
}
// WithCopySourceIfNoneMatch set X-Tos-Copy-Source-If-None-Match
// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom Bucket.UploadPartCopy
func WithCopySourceIfNoneMatch(ifNoneMatch string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderCopySourceIfNoneMatch, ifNoneMatch)
}
}
// WithCopySourceIfModifiedSince set X-Tos-Copy-Source-If-Modified-Since header
// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom Bucket.UploadPartCopy
func WithCopySourceIfModifiedSince(ifModifiedSince string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderCopySourceIfModifiedSince, ifModifiedSince)
}
}
// WithCopySourceIfUnmodifiedSince set X-Tos-Copy-Source-If-Unmodified-Since header
// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom Bucket.UploadPartCopy
func WithCopySourceIfUnmodifiedSince(ifUnmodifiedSince string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderCopySourceIfUnmodifiedSince, ifUnmodifiedSince)
}
}
// WithMeta set meta header
// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject Bucket.SetObjectMeta
func WithMeta(key, value string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderMetaPrefix+key, value)
}
}
// WithRange set Range header
// used in Bucket.GetObject Bucket.HeadObject
func WithRange(start, end int64) Option {
return func(rb *requestBuilder) {
rb.Range = &Range{Start: start, End: end}
rb.Header.Set(HeaderRange, rb.Range.String())
}
}
// WithVersionID set version parameter
// used in Bucket.GetObject Bucket.HeadObject Bucket.DeleteObject
// Bucket.GetObjectAcl Bucket.SetObjectMeta
// Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom
// Client.PreSignedURL
func WithVersionID(versionID string) Option {
return func(rb *requestBuilder) {
rb.Query.Add("versionId", versionID)
}
}
// WithMetadataDirective set X-Tos-Metadata-Directive header
// used in Bucket.CopyObject Bucket.CopyObjectTo Bucket.CopyObjectFrom
func WithMetadataDirective(directive string) Option {
return func(rb *requestBuilder) {
rb.Header.Add(HeaderMetadataDirective, directive)
}
}
// WithACL set X-Tos-Acl header
// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject
func WithACL(acl string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderACL, acl)
}
}
// WithACLGrantFullControl X-Tos-Grant-Full-Control header
// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject
func WithACLGrantFullControl(grantFullControl string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderGrantFullControl, grantFullControl)
}
}
// WithACLGrantRead set X-Tos-Grant-Read header
// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject
func WithACLGrantRead(grantRead string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderGrantRead, grantRead)
}
}
// WithACLGrantReadAcp set X-Tos-Grant-Read-Acp header
// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject
func WithACLGrantReadAcp(grantReadAcp string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderGrantReadAcp, grantReadAcp)
}
}
// WithACLGrantWrite set X-Tos-Grant-Write header
// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject
func WithACLGrantWrite(grantWrite string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderGrantWrite, grantWrite)
}
}
// WithACLGrantWriteAcp set X-Tos-Grant-Write-Acp header
// used in Bucket.PutObject Bucket.CreateMultipartUpload Bucket.AppendObject
func WithACLGrantWriteAcp(grantWriteAcp string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderGrantWriteAcp, grantWriteAcp)
}
}
// WithWebsiteRedirectLocation set X-Tos-Website-Redirect-Location header
func WithWebsiteRedirectLocation(redirectLocation string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(HeaderWebsiteRedirectLocation, redirectLocation)
}
}
// WithPerRequestSigner set Signer for a request
//
// use this option when you need set request-level signature parameter(s).
// for example, use different ak and sk for each request.
//
// if 'signer' set to nil, the request will not be signed.
func WithPerRequestSigner(signer Signer) Option {
return func(rb *requestBuilder) {
rb.Signer = signer
}
}
// WithHeader add request http header.
//
// NOTICE: use it carefully.
func WithHeader(key, value string) Option {
return func(rb *requestBuilder) {
rb.Header.Set(key, value)
}
}
// WithQuery add request query parameter
//
// NOTICE: use it carefully.
func WithQuery(key, value string) Option {
return func(rb *requestBuilder) {
rb.Query.Set(key, value)
}
}

249
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/parse_output.go generated vendored Normal file
View File

@@ -0,0 +1,249 @@
package tos
import (
"net/http"
"strconv"
)
// ParseListObjectsType2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *ListObjectsType2Output type.
func ParseListObjectsType2Output(httpRes *http.Response) (*ListObjectsType2Output, error) {
res := &Response{
StatusCode: httpRes.StatusCode,
ContentLength: httpRes.ContentLength,
Header: httpRes.Header,
Body: httpRes.Body,
}
err := checkError(res, true, 200)
if err != nil {
return nil, err
}
defer res.Close()
temp := listObjectsType2Output{
RequestInfo: res.RequestInfo(),
}
if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil {
return nil, err
}
contents := make([]ListedObjectV2, 0, len(temp.Contents))
for _, object := range temp.Contents {
var hashCrc uint64
if len(object.HashCrc64ecma) == 0 {
hashCrc = 0
} else {
hashCrc, err = strconv.ParseUint(object.HashCrc64ecma, 10, 64)
if err != nil {
return nil, &TosServerError{
TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"},
RequestInfo: RequestInfo{RequestID: temp.RequestID},
}
}
}
contents = append(contents, ListedObjectV2{
Key: object.Key,
LastModified: object.LastModified,
ETag: object.ETag,
Size: object.Size,
Owner: object.Owner,
StorageClass: object.StorageClass,
HashCrc64ecma: hashCrc,
})
}
output := ListObjectsType2Output{
RequestInfo: temp.RequestInfo,
Name: temp.Name,
ContinuationToken: temp.ContinuationToken,
Prefix: temp.Prefix,
MaxKeys: temp.MaxKeys,
KeyCount: temp.KeyCount,
Delimiter: temp.Delimiter,
IsTruncated: temp.IsTruncated,
EncodingType: temp.EncodingType,
CommonPrefixes: temp.CommonPrefixes,
NextContinuationToken: temp.NextContinuationToken,
Contents: contents,
}
return &output, nil
}
// ParseListObjectsV2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *ListObjectsV2Output type.
func ParseListObjectsV2Output(httpRes *http.Response) (*ListObjectsV2Output, error) {
res := &Response{
StatusCode: httpRes.StatusCode,
ContentLength: httpRes.ContentLength,
Header: httpRes.Header,
Body: httpRes.Body,
}
err := checkError(res, true, 200)
if err != nil {
return nil, err
}
defer res.Close()
temp := listObjectsV2Output{
RequestInfo: res.RequestInfo(),
}
if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil {
return nil, err
}
contents := make([]ListedObjectV2, 0, len(temp.Contents))
for _, object := range temp.Contents {
var hashCrc uint64
if len(object.HashCrc64ecma) == 0 {
hashCrc = 0
} else {
hashCrc, err = strconv.ParseUint(object.HashCrc64ecma, 10, 64)
if err != nil {
return nil, &TosServerError{
TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"},
RequestInfo: RequestInfo{RequestID: temp.RequestID},
}
}
}
contents = append(contents, ListedObjectV2{
Key: object.Key,
LastModified: object.LastModified,
ETag: object.ETag,
Size: object.Size,
Owner: object.Owner,
StorageClass: object.StorageClass,
HashCrc64ecma: uint64(hashCrc),
})
}
output := ListObjectsV2Output{
RequestInfo: temp.RequestInfo,
Name: temp.Name,
Prefix: temp.Prefix,
Marker: temp.Marker,
MaxKeys: temp.MaxKeys,
NextMarker: temp.NextMarker,
Delimiter: temp.Delimiter,
IsTruncated: temp.IsTruncated,
EncodingType: temp.EncodingType,
CommonPrefixes: temp.CommonPrefixes,
Contents: contents,
}
return &output, nil
}
// ParseListObjectVersionsV2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *ListObjectVersionsV2Output type.
func ParseListObjectVersionsV2Output(httpRes *http.Response) (*ListObjectVersionsV2Output, error) {
res := &Response{
StatusCode: httpRes.StatusCode,
ContentLength: httpRes.ContentLength,
Header: httpRes.Header,
Body: httpRes.Body,
}
err := checkError(res, true, 200)
if err != nil {
return nil, err
}
defer res.Close()
temp := listObjectVersionsV2Output{RequestInfo: res.RequestInfo()}
if err = marshalOutput(temp.RequestID, res.Body, &temp); err != nil {
return nil, err
}
versions := make([]ListedObjectVersionV2, 0, len(temp.Versions))
for _, version := range temp.Versions {
var hashCrc uint64
if len(version.HashCrc64ecma) == 0 {
hashCrc = 0
} else {
hashCrc, err = strconv.ParseUint(version.HashCrc64ecma, 10, 64)
if err != nil {
return nil, &TosServerError{
TosError: TosError{Message: "tos: server returned invalid HashCrc64Ecma"},
RequestInfo: RequestInfo{RequestID: temp.RequestID},
}
}
}
versions = append(versions, ListedObjectVersionV2{
Key: version.Key,
LastModified: version.LastModified,
ETag: version.ETag,
IsLatest: version.IsLatest,
Size: version.Size,
Owner: version.Owner,
StorageClass: version.StorageClass,
VersionID: version.VersionID,
HashCrc64ecma: hashCrc,
})
}
output := ListObjectVersionsV2Output{
RequestInfo: temp.RequestInfo,
Name: temp.Name,
Prefix: temp.Prefix,
KeyMarker: temp.KeyMarker,
VersionIDMarker: temp.VersionIDMarker,
Delimiter: temp.Delimiter,
EncodingType: temp.EncodingType,
MaxKeys: temp.MaxKeys,
NextKeyMarker: temp.NextKeyMarker,
NextVersionIDMarker: temp.NextVersionIDMarker,
IsTruncated: temp.IsTruncated,
CommonPrefixes: temp.CommonPrefixes,
DeleteMarkers: temp.DeleteMarkers,
Versions: versions,
}
return &output, nil
}
// ParseHeadObjectV2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *HeadObjectV2Output type.
func ParseHeadObjectV2Output(httpRes *http.Response) (*HeadObjectV2Output, error) {
res := &Response{
StatusCode: httpRes.StatusCode,
ContentLength: httpRes.ContentLength,
Header: httpRes.Header,
Body: httpRes.Body,
}
err := checkError(res, false, 200)
if err != nil {
return nil, err
}
defer res.Close()
output := HeadObjectV2Output{
RequestInfo: res.RequestInfo(),
}
output.ObjectMetaV2.fromResponseV2(res)
return &output, nil
}
// ParseGetObjectV2Output Parse the incoming parameters of *http.Response type, and respond to the return value of *GetObjectV2Output type.
func ParseGetObjectV2Output(httpRes *http.Response, expectedCode int) (*GetObjectV2Output, error) {
res := &Response{
StatusCode: httpRes.StatusCode,
ContentLength: httpRes.ContentLength,
Header: httpRes.Header,
Body: httpRes.Body,
}
err := checkError(res, true, expectedCode)
if err != nil {
return nil, err
}
basic := GetObjectBasicOutput{
RequestInfo: res.RequestInfo(),
ContentRange: res.Header.Get(HeaderContentRange),
}
basic.ObjectMetaV2.fromResponseV2(res)
output := GetObjectV2Output{
GetObjectBasicOutput: basic,
Content: wrapReader(res.Body, res.ContentLength, nil, nil, nil),
}
return &output, nil
}

180
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/policy.go generated vendored Normal file
View File

@@ -0,0 +1,180 @@
package tos
import (
"bytes"
"context"
"io/ioutil"
"net/http"
"strings"
)
type BucketPolicy struct {
Policy string `json:"Policy,omitempty"`
}
type GetBucketPolicyOutput struct {
RequestInfo `json:"-"`
Policy string `json:"Policy,omitempty"`
}
type PutBucketPolicyOutput struct {
RequestInfo `json:"-"`
}
type DeleteBucketPolicyOutput struct {
RequestInfo `json:"-"`
}
type GetBucketPolicyV2Input struct {
Bucket string `json:"-"`
}
type GetBucketPolicyV2Output struct {
RequestInfo `json:"-"`
Policy string `json:"Policy,omitempty"`
}
type putBucketPolicyV2Input struct {
Policy string `json:"Policy,omitempty"`
}
type PutBucketPolicyV2Input struct {
Bucket string `json:"-"`
Policy string `json:"Policy,omitempty"`
}
type PutBucketPolicyV2Output struct {
RequestInfo `json:"-"`
}
type DeleteBucketPolicyV2Input struct {
Bucket string `json:"-"`
}
type DeleteBucketPolicyV2Output struct {
RequestInfo
}
// GetBucketPolicy get bucket access policy
func (cli *Client) GetBucketPolicy(ctx context.Context, bucket string) (*GetBucketPolicyOutput, error) {
if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(bucket, "").
WithQuery("policy", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
return &GetBucketPolicyOutput{
RequestInfo: res.RequestInfo(),
Policy: string(data),
}, nil
}
// PutBucketPolicy set bucket access policy
func (cli *Client) PutBucketPolicy(ctx context.Context, bucket string, policy *BucketPolicy) (*PutBucketPolicyOutput, error) {
if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(bucket, "").
WithQuery("policy", "").
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, strings.NewReader(policy.Policy), cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
return &PutBucketPolicyOutput{RequestInfo: res.RequestInfo()}, nil
}
// DeleteBucketPolicy delete bucket access policy
func (cli *Client) DeleteBucketPolicy(ctx context.Context, bucket string) (*DeleteBucketPolicyOutput, error) {
if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(bucket, "").
WithQuery("policy", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
return &DeleteBucketPolicyOutput{RequestInfo: res.RequestInfo()}, nil
}
func (cli *ClientV2) PutBucketPolicyV2(ctx context.Context, input *PutBucketPolicyV2Input) (*PutBucketPolicyV2Output, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("policy", "").
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader([]byte(input.Policy)), cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketPolicyV2Output{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketPolicyV2(ctx context.Context, input *GetBucketPolicyV2Input) (*GetBucketPolicyV2Output, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("policy", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketPolicyV2Output{RequestInfo: res.RequestInfo()}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}
output.Policy = string(data)
return &output, nil
}
func (cli *ClientV2) DeleteBucketPolicyV2(ctx context.Context, input *DeleteBucketPolicyV2Input) (*DeleteBucketPolicyV2Output, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("policy", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteBucketPolicyV2Output{RequestInfo: res.RequestInfo()}
return &output, nil
}

55
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/proxy.go generated vendored Normal file
View File

@@ -0,0 +1,55 @@
package tos
import (
"net/url"
"strconv"
)
type Proxy struct {
proxyHost string
proxyUserName string
proxyPassword string
proxyPort int
}
func (p *Proxy) getRawUrl() string {
return p.proxyHost + ":" + strconv.Itoa(p.proxyPort)
}
func (p *Proxy) Url() *url.URL {
proxyURL, _ := url.Parse(p.getRawUrl())
if p.proxyUserName != "" && p.proxyPassword != "" {
proxyURL.User = url.UserPassword(p.proxyUserName, p.proxyPassword)
} else if p.proxyUserName != "" {
proxyURL.User = url.User(p.proxyUserName)
}
return proxyURL
}
func NewProxy(proxyHost string, proxyPort int) (*Proxy, error) {
proxyUrl, err := url.Parse(proxyHost)
if err != nil {
return nil, ProxyUrlInvalid.withCause(err)
}
if proxyUrl.Scheme == "" {
proxyHost = "http://" + proxyHost
}
if proxyUrl.Scheme == "https" {
return nil, ProxyNotSupportHttps
}
if _, err := url.Parse(proxyHost + ":" + strconv.Itoa(proxyPort)); err != nil {
if err != nil {
return nil, err
}
}
return &Proxy{
proxyHost: proxyHost,
proxyPort: proxyPort,
}, nil
}
func (p *Proxy) WithAuth(username string, password string) {
p.proxyUserName = username
p.proxyPassword = password
}

58
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rate_limiter.go generated vendored Normal file
View File

@@ -0,0 +1,58 @@
package tos
import (
"sync"
"time"
)
const (
minRate = 1024
minCapacity = 10 * 1024
)
type defaultRateLimit struct {
rate int64
capacity int64
currentAmount int64
sync.Mutex
lastConsumeTime time.Time
}
func NewDefaultRateLimit(rate int64, capacity int64) RateLimiter {
if rate < minRate {
rate = minRate
}
if capacity < minCapacity {
capacity = minCapacity
}
return &defaultRateLimit{
rate: rate,
capacity: capacity,
lastConsumeTime: time.Now(),
currentAmount: capacity,
Mutex: sync.Mutex{},
}
}
func (d *defaultRateLimit) Acquire(want int64) (ok bool, timeToWait time.Duration) {
d.Lock()
defer d.Unlock()
if want > d.capacity {
want = d.capacity
}
increment := int64(time.Since(d.lastConsumeTime).Seconds() * float64(d.rate))
if increment+d.currentAmount > d.capacity {
d.currentAmount = d.capacity
} else {
d.currentAmount += increment
}
if want > d.currentAmount {
timeToWaitSec := float64(want-d.currentAmount) / float64(d.rate)
return false, time.Duration(timeToWaitSec * float64(time.Second))
}
d.lastConsumeTime = time.Now()
d.currentAmount -= want
return true, 0
}

76
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/realtime_log.go generated vendored Normal file
View File

@@ -0,0 +1,76 @@
package tos
import (
"bytes"
"context"
"net/http"
)
func (cli *ClientV2) PutBucketRealTimeLog(ctx context.Context, input *PutBucketRealTimeLogInput) (*PutBucketRealTimeLogOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
body := putBucketRealTimeLogInput{
Configuration: input.Configuration,
}
data, contentMD5, err := marshalInput("PutBucketRealTimeLogInput", body)
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("realtimeLog", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketRealTimeLogOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketRealTimeLog(ctx context.Context, input *GetBucketRealTimeLogInput) (*GetBucketRealTimeLogOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("realtimeLog", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketRealTimeLogOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) DeleteBucketRealTimeLog(ctx context.Context, input *DeleteBucketRealTimeLogInput) (*DeleteBucketRealTimeLogOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("realtimeLog", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteBucketRealTimeLogOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

129
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/rename.go generated vendored Normal file
View File

@@ -0,0 +1,129 @@
package tos
import (
"bytes"
"context"
"net/http"
)
type PutBucketRenameInput struct {
Bucket string `json:"-"`
RenameEnable bool `json:"RenameEnable"`
}
type PutBucketRenameOutput struct {
RequestInfo
}
type GetBucketRenameInput struct {
Bucket string
}
type GetBucketRenameOutput struct {
RequestInfo
RenameEnable bool
}
type DeleteBucketRenameInput struct {
Bucket string
}
type DeleteBucketRenameOutput struct {
RequestInfo
}
func (cli *ClientV2) PutBucketRename(ctx context.Context, input *PutBucketRenameInput) (*PutBucketRenameOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutBucketRename", input)
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("rename", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketRenameOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketRename(ctx context.Context, input *GetBucketRenameInput) (*GetBucketRenameOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
req := cli.newBuilder(input.Bucket, "").
WithQuery("rename", "").
WithRetry(nil, StatusCodeClassifier{})
res, err := req.Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketRenameOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) DeleteBucketRename(ctx context.Context, input *DeleteBucketRenameInput) (*DeleteBucketRenameOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("rename", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteBucketRenameOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) RenameObject(ctx context.Context, input *RenameObjectInput) (*RenameObjectOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, input.Key).
WithQuery("rename", "").
WithParams(*input).
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := RenameObjectOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

79
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/replication.go generated vendored Normal file
View File

@@ -0,0 +1,79 @@
package tos
import (
"bytes"
"context"
"net/http"
)
func (cli *ClientV2) PutBucketReplication(ctx context.Context, input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
data, contentMD5, err := marshalInput("PutBucketReplication", putBucketReplicationInput{Role: input.Role, Rules: input.Rules})
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("replication", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketReplicationOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketReplication(ctx context.Context, input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
req := cli.newBuilder(input.Bucket, "").
WithQuery("replication", "").
WithQuery("progress", "").
WithRetry(nil, StatusCodeClassifier{})
if input.RuleID != "" {
req.WithQuery("rule-id", input.RuleID)
}
res, err := req.Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketReplicationOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) DeleteBucketReplication(ctx context.Context, input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("replication", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteBucketReplicationOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

425
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/request.go generated vendored Normal file
View File

@@ -0,0 +1,425 @@
package tos
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"reflect"
"strconv"
"strings"
"time"
)
type urlMode int
const (
// urlModePath url pattern is http(s)://{bucket}.domain/{object}
urlModeDefault = 0
// urlModePath url pattern is http(s)://domain/{bucket}/{object}
urlModePath = 1
)
type Request struct {
Scheme string
Method string
Host string
Path string
ContentLength *int64
Content io.Reader
Query url.Values
Header http.Header
}
func (req *Request) URL() string {
u := url.URL{
Scheme: req.Scheme,
Host: req.Host,
Path: req.Path,
RawQuery: req.Query.Encode(),
}
return u.String()
}
func OnRetryFromStart(req *Request) error {
if seek, ok := req.Content.(io.Seeker); ok {
_, err := seek.Seek(0, io.SeekStart)
return err
}
return nil
}
// Range represents a range of an object
type Range struct {
Start int64
End int64
}
// HTTP Range header
func (hr *Range) String() string {
return fmt.Sprintf("bytes=%d-%d", hr.Start, hr.End)
}
type CopySource struct {
srcBucket string
srcObjectKey string
}
type requestBuilder struct {
Signer Signer
Scheme string
Host string
Bucket string
Object string
URLMode urlMode
ContentLength *int64
Range *Range
Query url.Values
Header http.Header
Retry *retryer
OnRetry func(req *Request) error
Classifier classifier
CopySource *CopySource
IsCustomDomain bool
// CheckETag bool
// CheckCRC32 bool
}
func (rb *requestBuilder) WithRetry(onRetry func(req *Request) error, classifier classifier) *requestBuilder {
if onRetry == nil {
rb.OnRetry = func(req *Request) error { return nil }
} else {
rb.OnRetry = onRetry
}
if classifier == nil {
rb.Classifier = NoRetryClassifier{}
} else {
rb.Classifier = classifier
}
return rb
}
func (rb *requestBuilder) WithCopySource(srcBucket, srcObjectKey string) *requestBuilder {
rb.CopySource = &CopySource{
srcBucket: srcBucket,
srcObjectKey: srcObjectKey,
}
return rb
}
func (rb *requestBuilder) WithQuery(key, value string) *requestBuilder {
rb.Query.Add(key, value)
return rb
}
func (rb *requestBuilder) WithHeader(key, value string) *requestBuilder {
if len(value) > 0 {
rb.Header.Set(key, value)
}
return rb
}
func convertToString(iface interface{}, tag *reflect.StructTag) string {
// return empty string if value is zero except filed with "default" tag
var result string
switch v := iface.(type) {
case string:
result = v
case int:
if v != 0 {
result = strconv.Itoa(v)
} else {
result = tag.Get("default")
}
case int64:
if v != 0 {
result = strconv.Itoa(int(v))
} else {
result = tag.Get("default")
}
case time.Time:
if !v.IsZero() {
result = v.Format(http.TimeFormat)
}
case bool:
result = strconv.FormatBool(v)
default:
if reflect.TypeOf(iface).Kind() == reflect.String {
result = reflect.ValueOf(iface).String()
}
}
return result
}
// WithParams will set filed with tag "header" in input to rb.Header.
func (rb *requestBuilder) WithParams(input interface{}) *requestBuilder {
t := reflect.TypeOf(input)
v := reflect.ValueOf(input)
for i := 0; i < v.NumField(); i++ {
filed := t.Field(i)
if filed.Type.Kind() == reflect.Struct {
rb.WithParams(v.Field(i).Interface())
}
location := filed.Tag.Get("location")
switch location {
case "header":
value := convertToString(v.Field(i).Interface(), &filed.Tag)
if filed.Tag.Get("encodeChinese") == "true" {
value = headerEncode(value)
}
rb.WithHeader(filed.Tag.Get("locationName"), value)
case "headers":
if headers, ok := v.Field(i).Interface().(map[string]string); ok {
for k, v := range headers {
rb.Header.Set(HeaderMetaPrefix+headerEncode(k), headerEncode(v))
}
return rb
}
case "query":
v := convertToString(v.Field(i).Interface(), &filed.Tag)
if len(v) > 0 {
rb.WithQuery(filed.Tag.Get("locationName"), v)
}
}
}
return rb
}
func (rb *requestBuilder) WithContentLength(length int64) *requestBuilder {
rb.ContentLength = &length
return rb
}
func (rb *requestBuilder) hostPath() (string, string) {
if rb.IsCustomDomain {
if len(rb.Object) > 0 {
return rb.Host, "/" + rb.Object
}
return rb.Host, "/"
}
if rb.URLMode == urlModePath {
if len(rb.Object) > 0 {
return rb.Host, "/" + rb.Bucket + "/" + rb.Object
}
return rb.Host, "/" + rb.Bucket // rb.Bucket may be empty ""
}
// URLModeDefault
if len(rb.Bucket) == 0 {
return rb.Host, "/"
}
return rb.Bucket + "." + rb.Host, "/" + rb.Object
}
func (rb *requestBuilder) build(method string, content io.Reader) *Request {
host, path := rb.hostPath()
req := &Request{
Scheme: rb.Scheme,
Method: method,
Host: host,
Path: path,
Content: content,
Query: rb.Query,
Header: rb.Header,
}
if content != nil {
if rb.ContentLength != nil {
req.ContentLength = rb.ContentLength
} else if length := tryResolveLength(content); length >= 0 {
req.ContentLength = &length
}
}
return req
}
func (rb *requestBuilder) Build(method string, content io.Reader) *Request {
req := rb.build(method, content)
if rb.CopySource != nil {
versionID := req.Query.Get("versionId")
req.Query.Del("versionId")
req.Header.Add(HeaderCopySource, copySource(rb.CopySource.srcBucket, rb.CopySource.srcObjectKey, versionID))
}
if rb.Signer != nil {
signed := rb.Signer.SignHeader(req)
for key, values := range signed {
req.Header[key] = values
}
}
return req
}
type roundTripper func(ctx context.Context, req *Request) (*Response, error)
func (rb *requestBuilder) Request(ctx context.Context, method string,
content io.Reader, roundTripper roundTripper) (*Response, error) {
var (
req *Request
res *Response
err error
)
req = rb.Build(method, content)
if rb.Retry != nil {
work := func() (err error) {
err = rb.OnRetry(req)
if err != nil {
return err
}
res, err = roundTripper(ctx, req)
return err
}
err = rb.Retry.Run(ctx, work, rb.Classifier)
if err != nil {
return nil, err
}
return res, err
}
res, err = roundTripper(ctx, req)
return res, err
}
func (rb *requestBuilder) PreSignedURL(method string, ttl time.Duration) (string, error) {
req := rb.build(method, nil)
if rb.Signer == nil {
return "", errors.New("tos: credentials is not set when the tos.Client was created")
}
query := rb.Signer.SignQuery(req, ttl)
for k, v := range query {
req.Query[k] = v
}
return req.URL(), nil
}
type RequestInfo struct {
RequestID string
ID2 string
StatusCode int
Header http.Header
}
type Response struct {
StatusCode int
ContentLength int64
Header http.Header
Body io.ReadCloser
}
func (r *Response) RequestInfo() RequestInfo {
return RequestInfo{
RequestID: r.Header.Get(HeaderRequestID),
ID2: r.Header.Get(HeaderID2),
StatusCode: r.StatusCode,
Header: r.Header,
}
}
func (r *Response) Close() error {
if r.Body != nil {
return r.Body.Close()
}
return nil
}
func marshalOutput(requestID string, reader io.Reader, output interface{}) error {
// Although status code is ok, we need to check if response body is valid.
// If response body is invalid, TosServerError should be raised. But we can't
// unmarshal error from response body now.
data, err := ioutil.ReadAll(reader)
if err != nil {
return &TosServerError{
TosError: TosError{Message: "tos: unmarshal response body failed."},
RequestInfo: RequestInfo{RequestID: requestID},
}
}
data = bytes.TrimSpace(data)
if len(data) == 0 {
return &TosServerError{
TosError: TosError{Message: "server returns empty result"},
RequestInfo: RequestInfo{RequestID: requestID},
}
}
if err = json.Unmarshal(data, output); err != nil {
return &TosServerError{
TosError: TosError{Message: err.Error()},
RequestInfo: RequestInfo{RequestID: requestID},
}
}
return nil
}
func marshalInput(name string, input interface{}) ([]byte, string, error) {
data, err := json.Marshal(input)
if err != nil {
return nil, "", InvalidMarshal
}
sum := md5.Sum(data)
return data, base64.StdEncoding.EncodeToString(sum[:]), nil
}
func fileUnreadLength(file *os.File) (int64, error) {
offset, err := file.Seek(0, io.SeekCurrent)
if err != nil {
return 0, err
}
stat, err := file.Stat()
if err != nil {
return 0, err
}
size := stat.Size()
if offset > size || offset < 0 {
return 0, newTosClientError("tos: unexpected file size and(or) offset", nil)
}
return size - offset, nil
}
func tryResolveLength(reader io.Reader) int64 {
switch v := reader.(type) {
case *bytes.Buffer:
return int64(v.Len())
case *bytes.Reader:
return int64(v.Len())
case *strings.Reader:
return int64(v.Len())
case *os.File:
length, err := fileUnreadLength(v)
if err != nil {
return -1
}
return length
case *io.LimitedReader:
return v.N
case *net.Buffers:
if v != nil {
length := int64(0)
for _, p := range *v {
length += int64(len(p))
}
return length
}
return 0
default:
return -1
}
}
func Int64(value int64) *int64 { return &value }

289
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/resumable_copy.go generated vendored Normal file
View File

@@ -0,0 +1,289 @@
package tos
import (
"context"
"fmt"
"os"
"path/filepath"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
)
func parseResumableCopyObjectPath(input *ResumableCopyObjectInput) {
isDirRes := isDir(input.CheckpointFile)
if isDirRes || input.CheckpointFile == "" {
input.CheckpointFile = filepath.Clean(filepath.Join(input.CheckpointFile, fmt.Sprintf("%s.%s.%s.%s.%s", input.SrcBucket, input.SrcKey, input.SrcVersionID, input.Bucket, input.Key)))
}
}
func loadExistCopyCheckPoint(ctx context.Context, cli *ClientV2, input *ResumableCopyObjectInput, headOutput *HeadObjectV2Output) (*copyObjectCheckpoint, bool) {
checkpoint := &copyObjectCheckpoint{}
var err error
loadCheckPoint(input.CheckpointFile, checkpoint)
if checkpoint.Valid(input, headOutput) {
return checkpoint, true
} else if checkpoint.Bucket != "" && checkpoint.Key != "" && checkpoint.UploadID != "" {
// 尝试去 abort
_, err = cli.AbortMultipartUpload(ctx,
&AbortMultipartUploadInput{
Bucket: checkpoint.Bucket,
Key: checkpoint.Key,
UploadID: checkpoint.UploadID})
if err != nil && cli.logger != nil {
cli.logger.Debug("fail to abort upload task: %s, err:%s", checkpoint.UploadID, err.Error())
}
}
return nil, false
}
func getResumableCopyObjectCheckpoint(ctx context.Context, cli *ClientV2, input *ResumableCopyObjectInput, headOutput *HeadObjectV2Output, init func() (*copyObjectCheckpoint, error)) (checkpoint *copyObjectCheckpoint, err error) {
if !input.EnableCheckpoint {
return init()
}
parseResumableCopyObjectPath(input)
checkpoint, exist := loadExistCopyCheckPoint(ctx, cli, input, headOutput)
if exist {
return checkpoint, nil
}
err = checkAndCreateDir(input.CheckpointFile)
if err != nil {
return nil, InvalidCheckpointFilePath.withCause(err)
}
file, err := os.Create(input.CheckpointFile)
if err != nil {
return nil, newTosClientError("tos: create checkpoint file failed", err)
}
_ = file.Close()
checkpoint, err = init()
if err != nil {
return nil, err
}
err = checkpoint.WriteToFile()
if err != nil {
return nil, err
}
return
}
func initCopyPartsInfo(headOutput *HeadObjectV2Output, partSize int64) ([]copyPartInfo, error) {
if headOutput.ContentLength == 0 {
return []copyPartInfo{{
PartNumber: 1,
IsZeroSize: true,
}}, nil
}
partCount := headOutput.ContentLength / partSize
remainder := headOutput.ContentLength % partSize
if remainder != 0 {
partCount++
}
if partCount > 10000 {
return nil, InvalidFilePartNum
}
parts := make([]copyPartInfo, 0, partCount)
for i := int64(0); i < partCount; i++ {
part := copyPartInfo{
PartNumber: i + 1,
CopySourceRangeStart: i * partSize,
CopySourceRangeEnd: (i+1)*partSize - 1,
CopySourceRange: fmt.Sprintf("bytes=%d-%d", i*partSize, (i+1)*partSize-1),
}
parts = append(parts, part)
}
if remainder != 0 {
parts[partCount-1].CopySourceRangeEnd = (partCount-1)*partSize + remainder - 1
parts[partCount-1].CopySourceRange = fmt.Sprintf("bytes=%d-%d", (partCount-1)*partSize, (partCount-1)*partSize+remainder-1)
}
return parts, nil
}
func initCopyCheckpoint(input *ResumableCopyObjectInput, headOutput *HeadObjectV2Output) (*copyObjectCheckpoint, error) {
parts, err := initCopyPartsInfo(headOutput, input.PartSize)
if err != nil {
return nil, err
}
cp := &copyObjectCheckpoint{
Bucket: input.Bucket,
Key: input.Key,
SrcBucket: input.SrcBucket,
SrcVersionID: input.SrcVersionID,
PartSize: input.PartSize,
UploadID: "",
CopySourceIfMatch: input.CopySourceIfMatch,
CopySourceIfModifiedSince: input.CopySourceIfModifiedSince,
CopySourceIfNoneMatch: input.CopySourceIfNoneMatch,
CopySourceIfUnmodifiedSince: input.CopySourceIfUnmodifiedSince,
CopySourceSSECAlgorithm: input.CopySourceSSECAlgorithm,
CopySourceSSECKeyMD5: input.CopySourceSSECKeyMD5,
SSECAlgorithm: input.SSECAlgorithm,
SSECKeyMD5: input.SSECKeyMD5,
EncodingType: input.EncodingType,
CopySourceObjectInfo: objectInfo{
Etag: headOutput.ETag,
HashCrc64ecma: headOutput.HashCrc64ecma,
LastModified: headOutput.LastModified,
ObjectSize: headOutput.ContentLength,
},
PartsInfo: parts,
CheckpointPath: input.CheckpointFile,
}
return cp, nil
}
func prepareCopyTasks(cli *ClientV2, ctx context.Context, checkpoint *copyObjectCheckpoint, input *ResumableCopyObjectInput) []task {
tasks := make([]task, 0)
for _, part := range checkpoint.PartsInfo {
if !part.IsCompleted {
tasks = append(tasks, &copyTask{
cli: cli,
ctx: ctx,
input: input,
UploadID: checkpoint.UploadID,
PartNumber: part.PartNumber,
PartInfo: part,
})
}
}
return tasks
}
func (cli *ClientV2) copyPart(ctx context.Context, cp *copyObjectCheckpoint, input *ResumableCopyObjectInput, event *copyEvent) (*ResumableCopyObjectOutput, error) {
tasks := prepareCopyTasks(cli, ctx, cp, input)
routinesNum := min(input.TaskNum, len(tasks))
cancelHandle := getCancelHandle(input.CancelHook)
tg := newTaskGroup(cancelHandle, routinesNum, cp, event, input.EnableCheckpoint, tasks)
abort := func() error {
_, err := cli.AbortMultipartUpload(ctx,
&AbortMultipartUploadInput{
Bucket: input.Bucket,
Key: input.Key,
UploadID: cp.UploadID})
return err
}
bindCancelHookWithAborter(input.CancelHook, abort)
tg.RunWorker()
tg.Scheduler()
success, taskErr := tg.Wait()
if taskErr != nil {
if err := abort(); err != nil {
return nil, err
}
return nil, taskErr
}
// handle results
if success < len(tasks) {
return nil, newTosClientError("tos: some tasks copy failed.", nil)
}
complete, err := cli.CompleteMultipartUploadV2(ctx, &CompleteMultipartUploadV2Input{
Bucket: input.Bucket,
Key: input.Key,
UploadID: cp.UploadID,
Parts: cp.GetParts(),
})
if err != nil {
event.postCopyEvent(&CopyEvent{
Type: enum.CopyEventCompleteMultipartUploadFailed,
Err: err,
Bucket: input.Bucket,
Key: input.Key,
UploadID: &cp.UploadID,
SrcBucket: input.SrcBucket,
SrcKey: input.SrcKey,
SrcVersionID: input.SrcVersionID,
CheckpointFile: &input.CheckpointFile,
})
return nil, err
}
event.postCopyEvent(&CopyEvent{
Type: enum.CopyEventCompleteMultipartUploadSucceed,
Err: err,
Bucket: input.Bucket,
Key: input.Key,
UploadID: &cp.UploadID,
SrcBucket: input.SrcBucket,
SrcKey: input.SrcKey,
SrcVersionID: input.SrcVersionID,
CheckpointFile: &input.CheckpointFile,
})
_ = os.Remove(input.CheckpointFile)
return &ResumableCopyObjectOutput{
RequestInfo: complete.RequestInfo,
Bucket: complete.Bucket,
Key: complete.Key,
UploadID: cp.UploadID,
Etag: complete.ETag,
Location: complete.Location,
VersionID: complete.VersionID,
HashCrc64ecma: complete.HashCrc64ecma,
SSECAlgorithm: cp.SSECAlgorithm,
SSECKeyMD5: cp.SSECKeyMD5,
EncodingType: cp.EncodingType,
}, nil
}
func (cli *ClientV2) ResumableCopyObject(ctx context.Context, input *ResumableCopyObjectInput) (*ResumableCopyObjectOutput, error) {
rawInput := *input
copyInput := &rawInput
headOutput, err := cli.HeadObjectV2(ctx, &HeadObjectV2Input{
Bucket: copyInput.SrcBucket,
Key: copyInput.SrcKey,
VersionID: copyInput.SrcVersionID,
SSECAlgorithm: copyInput.CopySourceSSECAlgorithm,
SSECKey: copyInput.CopySourceSSECKey,
SSECKeyMD5: copyInput.CopySourceSSECKeyMD5,
IfModifiedSince: copyInput.CopySourceIfModifiedSince,
IfNoneMatch: copyInput.CopySourceIfNoneMatch,
IfUnmodifiedSince: copyInput.CopySourceIfUnmodifiedSince,
IfMatch: copyInput.CopySourceIfMatch,
})
if err != nil {
return nil, err
}
event := &copyEvent{input: copyInput}
init := func() (*copyObjectCheckpoint, error) {
return initCopyCheckpoint(copyInput, headOutput)
}
cp, err := getResumableCopyObjectCheckpoint(ctx, cli, copyInput, headOutput, init)
if err != nil {
return nil, err
}
if cp.UploadID == "" {
created, err := cli.CreateMultipartUploadV2(ctx, &copyInput.CreateMultipartUploadV2Input)
if err != nil {
event.postCopyEvent(&CopyEvent{
Type: enum.CopyEventCreateMultipartUploadFailed,
Err: err,
Bucket: copyInput.Bucket,
Key: copyInput.Key,
SrcBucket: copyInput.SrcBucket,
SrcKey: copyInput.SrcKey,
SrcVersionID: copyInput.SrcVersionID,
CheckpointFile: &copyInput.CheckpointFile,
})
return nil, err
}
event.uploadID = created.UploadID
event.postCopyEvent(&CopyEvent{
Type: enum.CopyEventCreateMultipartUploadSucceed,
Bucket: copyInput.Bucket,
Key: copyInput.Key,
SrcBucket: copyInput.SrcBucket,
SrcKey: copyInput.SrcKey,
SrcVersionID: copyInput.SrcVersionID,
CheckpointFile: &copyInput.CheckpointFile,
})
cp.UploadID = created.UploadID
}
cleaner := func() {
_ = os.Remove(copyInput.CheckpointFile)
}
bindCancelHookWithCleaner(copyInput.CancelHook, cleaner)
return cli.copyPart(ctx, cp, copyInput, event)
}

414
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/sign_v4.go generated vendored Normal file
View File

@@ -0,0 +1,414 @@
package tos
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"time"
)
const (
emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
unsignedPayload = "UNSIGNED-PAYLOAD"
signPrefix = "TOS4-HMAC-SHA256"
iso8601Layout = "20060102T150405Z"
yyMMdd = "20060102"
serverTimeFormat = "2006-01-02T15:04:05Z"
authorization = "Authorization"
v4Algorithm = "X-Tos-Algorithm"
v4Credential = "X-Tos-Credential"
v4Date = "X-Tos-Date"
v4Expires = "X-Tos-Expires"
v4SignedHeaders = "X-Tos-SignedHeaders"
v4Signature = "X-Tos-Signature"
v4SignatureLower = "x-tos-signature"
v4ContentSHA256 = "X-Tos-Content-Sha256"
v4SecurityToken = "X-Tos-Security-Token"
v4Prefix = "x-tos"
)
func defaultSigningQueryV4(key string) bool {
return key != v4SignatureLower
}
func defaultSigningHeaderV4(key string, isSigningQuery bool) bool {
return (key == "content-type" && !isSigningQuery) || strings.HasPrefix(key, v4Prefix)
}
func UTCNow() time.Time {
return time.Now().UTC()
}
type Signer interface {
SignHeader(req *Request) http.Header
SignQuery(req *Request, ttl time.Duration) url.Values
}
type SigningKeyInfo struct {
Date string
Region string
Credential *Credential
}
type SignV4 struct {
credentials Credentials
region string
signingHeader func(key string, isSigningQuery bool) bool
signingQuery func(key string) bool
now func() time.Time
signingKey func(*SigningKeyInfo) []byte
logger Logger
}
type signedRes struct {
CanonicalString string
StringToSign string
Sign string
}
type signedHeader struct {
CanonicalString string
StringToSign string
Header http.Header
}
type signedQuery struct {
CanonicalString string
StringToSign string
Query url.Values
}
type SignV4Option func(*SignV4)
func (sv *SignV4) WithSignLogger(logger Logger) {
sv.logger = logger
}
// NewSignV4 create SignV4
// use WithSignKey to set self-defined sign-key generator
// use WithSignLogger to set logger
func NewSignV4(credentials Credentials, region string) *SignV4 {
signV4 := &SignV4{
credentials: credentials,
region: region,
signingHeader: defaultSigningHeaderV4,
signingQuery: defaultSigningQueryV4,
now: UTCNow,
signingKey: SigningKey,
}
return signV4
}
// WithSigningKey for self-defined sign-key generator
func (sv *SignV4) WithSigningKey(signingKey func(*SigningKeyInfo) []byte) {
sv.signingKey = signingKey
}
func (sv *SignV4) signedHeader(header http.Header, isSignedQuery bool) KVs {
var signed = make(KVs, 0, 10)
for key, values := range header {
kk := strings.ToLower(key)
if sv.signingHeader(kk, isSignedQuery) {
vv := make([]string, 0, len(values))
for _, value := range values {
vv = append(vv, strings.Join(strings.Fields(value), " "))
}
signed = append(signed, KV{Key: kk, Values: vv})
}
}
return signed
}
func (sv *SignV4) signedQuery(query url.Values, extra url.Values) KVs {
var signed = make(KVs, 0, len(query)+len(extra))
for key, values := range query {
if sv.signingQuery(strings.ToLower(key)) {
signed = append(signed, KV{Key: key, Values: values})
}
}
for key, values := range extra {
if sv.signingQuery(strings.ToLower(key)) {
signed = append(signed, KV{Key: key, Values: values})
}
}
return signed
}
func (sv *SignV4) canonicalRequest(method, path, contentSha256 string, header, query KVs) string {
const split = byte('\n')
var buf bytes.Buffer
buf.Grow(512)
// Method
buf.WriteString(method)
buf.WriteByte(split)
// URI
buf.Write(encodePath(path))
buf.WriteByte(split)
// query
buf.Write(encodeQuery(query))
buf.WriteByte(split)
// canonical headers
keys := make([]string, 0, len(header))
for _, kv := range header {
keys = append(keys, kv.Key)
buf.WriteString(kv.Key)
buf.WriteByte(':')
buf.WriteString(strings.Join(kv.Values, ","))
buf.WriteByte('\n')
}
buf.WriteByte(split)
// signed headers
buf.WriteString(strings.Join(keys, ";"))
buf.WriteByte(split)
if len(contentSha256) > 0 {
buf.WriteString(contentSha256)
} else {
buf.WriteString(emptySHA256)
}
return buf.String()
}
func SigningKey(info *SigningKeyInfo) []byte {
date := hmacSHA256([]byte(info.Credential.AccessKeySecret), []byte(info.Date))
region := hmacSHA256(date, []byte(info.Region))
service := hmacSHA256(region, []byte("tos"))
return hmacSHA256(service, []byte("request"))
}
func (sv *SignV4) doSign(method, path, contentSha256 string, header, query KVs, now time.Time, cred *Credential) signedRes {
const split = byte('\n')
canonicalStr := sv.canonicalRequest(method, path, contentSha256, header, query)
var buf bytes.Buffer
buf.Grow(len(signPrefix) + 128)
buf.WriteString(signPrefix)
buf.WriteByte(split)
buf.WriteString(now.Format(iso8601Layout))
buf.WriteByte(split)
date := now.Format(yyMMdd)
buf.WriteString(date) // yyMMdd + '/' + region + '/' + service + '/' + request
buf.WriteByte('/')
buf.WriteString(sv.region)
buf.WriteString("/tos/request")
buf.WriteByte(split)
sum := sha256.Sum256([]byte(canonicalStr))
buf.WriteString(hex.EncodeToString(sum[:]))
signK := sv.signingKey(&SigningKeyInfo{Date: date, Region: sv.region, Credential: cred})
sign := hmacSHA256(signK, buf.Bytes())
return signedRes{
CanonicalString: canonicalStr,
StringToSign: buf.String(),
Sign: hex.EncodeToString(sign),
}
}
func (sv *SignV4) SignHeader(req *Request) http.Header {
signed := make(http.Header, 4)
now := sv.now()
date := now.Format(iso8601Layout)
contentSha256 := req.Header.Get(v4ContentSHA256)
signedHeader := sv.signedHeader(req.Header, false)
signedHeader = append(signedHeader, KV{Key: strings.ToLower(v4Date), Values: []string{date}})
signedHeader = append(signedHeader, KV{Key: "date", Values: []string{date}})
signedHeader = append(signedHeader, KV{Key: "host", Values: []string{req.Host}})
// if len(contentSha256) == 0 {
// signedHeader = append(signedHeader, KV{Key: strings.ToLower(v4ContentSHA256), Values: []string{unsignedPayload}})
// signed.Set(v4ContentSHA256, unsignedPayload)
// }
cred := sv.credentials.Credential()
if sts := cred.SecurityToken; len(sts) > 0 {
signedHeader = append(signedHeader, KV{Key: strings.ToLower(v4SecurityToken), Values: []string{sts}})
signed.Set(v4SecurityToken, sts)
}
sort.Sort(signedHeader)
signedQuery := sv.signedQuery(req.Query, nil)
signRes := sv.doSign(req.Method, req.Path, contentSha256, signedHeader, signedQuery, now, &cred)
credential := fmt.Sprintf("%s/%s/%s/tos/request", cred.AccessKeyID, now.Format(yyMMdd), sv.region)
auth := fmt.Sprintf("TOS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", credential, joinKeys(signedHeader), signRes.Sign)
signed.Set(authorization, auth)
signed.Set(v4Date, date)
signed.Set("Date", date)
if sv.logger != nil {
sv.logger.Debug("[tos] CanonicalString:" + "\n" + signRes.CanonicalString + "\n")
sv.logger.Debug("[tos] StringToSign:" + "\n" + signRes.StringToSign + "\n")
}
return signed
}
func (sv *SignV4) SignQuery(req *Request, ttl time.Duration) url.Values {
now := sv.now()
date := now.Format(iso8601Layout)
query := req.Query
extra := make(url.Values)
cred := sv.credentials.Credential()
credential := fmt.Sprintf("%s/%s/%s/tos/request", cred.AccessKeyID, now.Format(yyMMdd), sv.region)
extra.Add(v4Algorithm, signPrefix)
extra.Add(v4Credential, credential)
extra.Add(v4Date, date)
extra.Add(v4Expires, strconv.FormatInt(ttl.Milliseconds()/1000, 10))
if sts := cred.SecurityToken; len(sts) > 0 {
extra.Add(v4SecurityToken, sts)
}
signedHeader := sv.signedHeader(req.Header, true)
signedHeader = append(signedHeader, KV{Key: "host", Values: []string{req.Host}})
sort.Sort(signedHeader)
extra.Add(v4SignedHeaders, joinKeys(signedHeader))
signedQuery := sv.signedQuery(query, extra)
signRes := sv.doSign(req.Method, req.Path, unsignedPayload, signedHeader, signedQuery, now, &cred)
extra.Add(v4Signature, signRes.Sign)
if sv.logger != nil {
sv.logger.Debug("[tos] CanonicalString:" + "\n" + signRes.CanonicalString + "\n")
sv.logger.Debug("[tos] StringToSign:" + "\n" + signRes.StringToSign + "\n")
}
return extra
}
type KV struct {
Key string
Values []string
}
type KVs []KV
func (kvs KVs) Len() int { return len(kvs) }
func (kvs KVs) Swap(i, j int) { kvs[i], kvs[j] = kvs[j], kvs[i] }
func (kvs KVs) Less(i, j int) bool { return kvs[i].Key < kvs[j].Key }
func joinKeys(kvs KVs) string {
keys := make([]string, 0, len(kvs))
for i := range kvs {
keys = append(keys, kvs[i].Key)
}
sort.Strings(keys)
return strings.Join(keys, ";")
}
func hmacSHA256(key []byte, value []byte) []byte {
h := hmac.New(sha256.New, key)
h.Write(value)
return h.Sum(nil)
}
var (
nonEscape [256]bool
)
// ((ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' || ch == '~' || ch == '.')
func init() {
for i := 'a'; i <= 'z'; i++ {
nonEscape[i] = true
}
for i := 'A'; i <= 'Z'; i++ {
nonEscape[i] = true
}
for i := '0'; i <= '9'; i++ {
nonEscape[i] = true
}
nonEscape['-'] = true
nonEscape['_'] = true
nonEscape['.'] = true
nonEscape['~'] = true
}
func encodePath(path string) []byte {
if len(path) == 0 {
return []byte{'/'}
}
return URIEncode(path, false)
}
func encodeQuery(query KVs) []byte {
if len(query) == 0 {
return make([]byte, 0)
}
var buf bytes.Buffer
buf.Grow(512)
sort.Sort(query)
for _, kv := range query {
keyEscaped := URIEncode(kv.Key, true)
for _, v := range kv.Values {
if buf.Len() > 0 {
buf.WriteByte('&')
}
buf.Write(keyEscaped)
buf.WriteByte('=')
buf.Write(URIEncode(v, true))
}
}
return buf.Bytes()
}
func URIEncode(in string, encodeSlash bool) []byte {
hexCount := 0
for i := 0; i < len(in); i++ {
c := uint8(in[i])
if c == '/' {
if encodeSlash {
hexCount++
}
} else if !nonEscape[c] {
hexCount++
}
}
encoded := make([]byte, len(in)+2*hexCount)
for i, j := 0, 0; i < len(in); i++ {
c := uint8(in[i])
if c == '/' {
if encodeSlash {
encoded[j] = '%'
encoded[j+1] = '2'
encoded[j+2] = 'F'
j += 3
} else {
encoded[j] = c
j++
}
} else if !nonEscape[c] {
encoded[j] = '%'
encoded[j+1] = "0123456789ABCDEF"[c>>4]
encoded[j+2] = "0123456789ABCDEF"[c&15]
j += 3
} else {
encoded[j] = c
j++
}
}
return encoded
}

17
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/tagging.go generated vendored Normal file
View File

@@ -0,0 +1,17 @@
package tos
import (
"context"
)
func (cli *ClientV2) PutObjectTagging(ctx context.Context, input *PutObjectTaggingInput) (*PutObjectTaggingOutput, error) {
return cli.baseClient.PutObjectTagging(ctx, input)
}
func (cli *ClientV2) GetObjectTagging(ctx context.Context, input *GetObjectTaggingInput) (*GetObjectTaggingOutput, error) {
return cli.baseClient.GetObjectTagging(ctx, input)
}
func (cli *ClientV2) DeleteObjectTagging(ctx context.Context, input *DeleteObjectTaggingInput) (*DeleteObjectTaggingOutput, error) {
return cli.baseClient.DeleteObjectTagging(ctx, input)
}

254
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/transport.go generated vendored Normal file
View File

@@ -0,0 +1,254 @@
package tos
import (
"context"
"crypto/tls"
"math/rand"
"net"
"net/http"
"net/http/httptrace"
"time"
)
type TransportConfig struct {
// MaxIdleConns same as http.Transport MaxIdleConns. Default is 1024.
MaxIdleConns int
// MaxIdleConnsPerHost same as http.Transport MaxIdleConnsPerHost. Default is 1024.
MaxIdleConnsPerHost int
// MaxConnsPerHost same as http.Transport MaxConnsPerHost. Default is no limit.
MaxConnsPerHost int
// RequestTimeout same as http.Client Timeout
// Deprecated: use ReadTimeout or WriteTimeout instead
RequestTimeout time.Duration
// DialTimeout same as net.Dialer Timeout
DialTimeout time.Duration
// KeepAlive same as net.Dialer KeepAlive
KeepAlive time.Duration
// IdleConnTimeout same as http.Transport IdleConnTimeout
IdleConnTimeout time.Duration
// TLSHandshakeTimeout same as http.Transport TLSHandshakeTimeout
TLSHandshakeTimeout time.Duration
// ResponseHeaderTimeout same as http.Transport ResponseHeaderTimeout
ResponseHeaderTimeout time.Duration
// ExpectContinueTimeout same as http.Transport ExpectContinueTimeout
ExpectContinueTimeout time.Duration
// ReadTimeout see net.Conn SetReadDeadline
ReadTimeout time.Duration
// WriteTimeout set net.Conn SetWriteDeadline
WriteTimeout time.Duration
// InsecureSkipVerify set tls.Config InsecureSkipVerify
InsecureSkipVerify bool
// DNSCacheTime Set Dns Cache Time.
DNSCacheTime time.Duration
// Proxy Set http proxy for http client.
Proxy *Proxy
}
type Transport interface {
RoundTrip(context.Context, *Request) (*Response, error)
}
type DefaultTransport struct {
client http.Client
logger Logger
}
func (d *DefaultTransport) WithDefaultTransportLogger(logger Logger) {
d.logger = logger
}
// NewDefaultTransport create a DefaultTransport with config
func NewDefaultTransport(config *TransportConfig) *DefaultTransport {
var r *resolver
if config.DNSCacheTime >= time.Minute {
r = newResolver(config.DNSCacheTime)
}
transport := &http.Transport{
DialContext: (&TimeoutDialer{
Dialer: net.Dialer{
Timeout: config.DialTimeout,
KeepAlive: config.KeepAlive,
},
resolver: r,
ReadTimeout: config.ReadTimeout,
WriteTimeout: config.WriteTimeout,
}).DialContext,
MaxIdleConns: config.MaxIdleConns,
MaxIdleConnsPerHost: config.MaxIdleConnsPerHost,
MaxConnsPerHost: config.MaxConnsPerHost,
IdleConnTimeout: config.IdleConnTimeout,
TLSHandshakeTimeout: config.TLSHandshakeTimeout,
ResponseHeaderTimeout: config.ResponseHeaderTimeout,
ExpectContinueTimeout: config.ExpectContinueTimeout,
DisableCompression: true,
// #nosec G402
TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify},
}
if config.Proxy != nil && config.Proxy.proxyHost != "" {
transport.Proxy = http.ProxyURL(config.Proxy.Url())
}
return &DefaultTransport{
client: http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
Transport: transport,
},
}
}
// newDefaultTranposrtWithHTTPTransport
func newDefaultTranposrtWithHTTPTransport(transport http.RoundTripper) *DefaultTransport {
return &DefaultTransport{
client: http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
Transport: transport,
},
}
}
// NewDefaultTransportWithClient crate a DefaultTransport with a http.Client
func NewDefaultTransportWithClient(client http.Client) *DefaultTransport {
return &DefaultTransport{client: client}
}
func (dt *DefaultTransport) RoundTrip(ctx context.Context, req *Request) (*Response, error) {
hr, err := http.NewRequestWithContext(ctx, req.Method, req.URL(), req.Content)
if err != nil {
return nil, newTosClientError(err.Error(), err)
}
if req.ContentLength != nil {
hr.ContentLength = *req.ContentLength
}
for key, values := range req.Header {
hr.Header[key] = values
}
var accessLog *accessLogRequest
if dt.logger != nil {
var trace *httptrace.ClientTrace
trace, accessLog = getClientTrace(GetUnixTimeMs())
ctx = httptrace.WithClientTrace(ctx, trace)
hr = hr.WithContext(ctx)
}
res, err := dt.client.Do(hr)
if accessLog != nil {
accessLog.PrintAccessLog(dt.logger, hr, res)
}
if err != nil {
return nil, newTosClientError(err.Error(), err)
}
return &Response{
StatusCode: res.StatusCode,
ContentLength: res.ContentLength,
Header: res.Header,
Body: res.Body,
}, nil
}
type TimeoutConn struct {
net.Conn
readTimeout time.Duration
writeTimeout time.Duration
zero time.Time
}
func NewTimeoutConn(conn net.Conn, readTimeout, writeTimeout time.Duration) *TimeoutConn {
return &TimeoutConn{
Conn: conn,
readTimeout: readTimeout,
writeTimeout: writeTimeout,
}
}
func (tc *TimeoutConn) Read(b []byte) (n int, err error) {
timeout := tc.readTimeout > 0
if timeout {
_ = tc.SetReadDeadline(time.Now().Add(tc.readTimeout))
}
n, err = tc.Conn.Read(b)
if timeout {
_ = tc.SetReadDeadline(time.Now().Add(tc.readTimeout * 5))
}
return n, err
}
func (tc *TimeoutConn) Write(b []byte) (n int, err error) {
timeout := tc.writeTimeout > 0
if timeout {
_ = tc.SetWriteDeadline(time.Now().Add(tc.writeTimeout))
}
n, err = tc.Conn.Write(b)
if tc.readTimeout > 0 {
_ = tc.SetReadDeadline(time.Now().Add(tc.readTimeout * 5))
}
return n, err
}
type TimeoutDialer struct {
net.Dialer
resolver *resolver
ReadTimeout time.Duration
WriteTimeout time.Duration
}
func (d *TimeoutDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
if d.resolver != nil {
host, port, err := net.SplitHostPort(address)
if err != nil {
return nil, err
}
ipList, err := d.resolver.GetIpList(ctx, host)
if err != nil {
return nil, err
}
// 随机打乱 IP List
rand.Shuffle(len(ipList), func(i, j int) {
ipList[i], ipList[j] = ipList[j], ipList[i]
})
for _, ip := range ipList {
conn, err := d.Dialer.DialContext(ctx, network, ip+":"+port)
if err == nil {
return NewTimeoutConn(conn, d.ReadTimeout, d.WriteTimeout), nil
} else {
d.resolver.Remove(address, ip)
}
}
}
conn, err := d.Dialer.DialContext(ctx, network, address)
if err != nil {
return nil, err
}
return NewTimeoutConn(conn, d.ReadTimeout, d.WriteTimeout), nil
}

2221
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

1009
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/type_internal.go generated vendored Normal file
View File

File diff suppressed because it is too large Load Diff

428
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/upload_file.go generated vendored Normal file
View File

@@ -0,0 +1,428 @@
package tos
import (
"context"
"os"
"path/filepath"
"strings"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
)
// initUploadPartsInfo initialize parts info from file stat,return TosClientError if failed
func initUploadPartsInfo(uploadFileStat os.FileInfo, partSize int64) ([]uploadPartInfo, error) {
partCount := uploadFileStat.Size() / partSize
lastPartSize := uploadFileStat.Size() % partSize
if lastPartSize != 0 {
partCount++
}
if partCount > 10000 {
return nil, InvalidFilePartNum
}
parts := make([]uploadPartInfo, 0, partCount)
for i := int64(0); i < partCount; i++ {
part := uploadPartInfo{
PartNumber: int(i + 1),
PartSize: partSize,
Offset: uint64(i * partSize),
}
parts = append(parts, part)
}
if lastPartSize != 0 {
parts[partCount-1].PartSize = lastPartSize
}
if uploadFileStat.Size() == 0 {
parts = append(parts, uploadPartInfo{PartNumber: 1, PartSize: 0, Offset: 0})
}
return parts, nil
}
// initUploadCheckpoint initialize checkpoint file, return TosClientError if failed
func initUploadCheckpoint(input *UploadFileInput, stat os.FileInfo) (*uploadCheckpoint, error) {
parts, err := initUploadPartsInfo(stat, input.PartSize)
if err != nil {
return nil, err
}
checkPoint := &uploadCheckpoint{
checkpointPath: input.CheckpointFile,
PartsInfo: parts,
Bucket: input.Bucket,
Key: input.Key,
PartSize: input.PartSize,
SSECAlgorithm: input.SSECAlgorithm,
SSECKeyMD5: input.SSECKeyMD5,
EncodingType: input.ContentEncoding,
FilePath: input.FilePath,
FileInfo: fileInfo{
Size: stat.Size(),
LastModified: stat.ModTime().Unix(),
},
}
return checkPoint, nil
}
func getUploadCheckpointFilePath(checkpointPath, filePath string, bucket, key string) string {
fileName := strings.Join([]string{filepath.Base(filePath), checkpointPathMd5(bucket, key, ""), "upload"}, ".")
if len(checkpointPath) == 0 {
dirName := filepath.Dir(filePath)
return filepath.Join(dirName, fileName)
}
return withSuffixIfDir(checkpointPath, fileName)
}
// validateUploadInput validate upload input, return TosClientError failed
func validateUploadInput(input *UploadFileInput, stat os.FileInfo, isCustomDomain bool) error {
if err := isValidNames(input.Bucket, input.Key, isCustomDomain); err != nil {
return err
}
if input.PartSize == 0 {
input.PartSize = DefaultPartSize
}
if input.PartSize < MinPartSize || input.PartSize > MaxPartSize {
return InvalidPartSize
}
if stat.IsDir() {
return newTosClientError("tos: does not support directory, please specific your file path.", nil)
}
if input.EnableCheckpoint {
// get correct checkpoint path
input.CheckpointFile = getUploadCheckpointFilePath(input.CheckpointFile, input.FilePath, input.Bucket, input.Key)
}
if input.TaskNum < 1 {
input.TaskNum = 1
}
if input.TaskNum > 1000 {
input.TaskNum = 1000
}
return nil
}
func (u *uploadPostEvent) postUploadEvent(event *UploadEvent) {
if u.input.UploadEventListener != nil {
u.input.UploadEventListener.EventChange(event)
}
}
func loadExistUploadCheckPoint(ctx context.Context, cli *ClientV2, input *UploadFileInput, srcFile os.FileInfo) (*uploadCheckpoint, bool) {
checkpoint := &uploadCheckpoint{}
var err error
loadCheckPoint(input.CheckpointFile, checkpoint)
if checkpoint.Valid(srcFile, input.Bucket, input.Key, input.FilePath) {
return checkpoint, true
} else if checkpoint.Bucket != "" && checkpoint.Key != "" && checkpoint.UploadID != "" {
// 尝试去 abort
_, err = cli.AbortMultipartUpload(ctx,
&AbortMultipartUploadInput{
Bucket: checkpoint.Bucket,
Key: checkpoint.Key,
UploadID: checkpoint.UploadID})
if err != nil && cli.logger != nil {
cli.logger.Debug("fail to abort upload task: %s, err:%s", checkpoint.UploadID, err.Error())
}
}
return nil, false
}
// getUploadCheckpoint get struct checkpoint from checkpoint file if checkpointPath is valid,
// or initialize from scratch with function init
func getUploadCheckpoint(ctx context.Context, cli *ClientV2, input *UploadFileInput, srcFile os.FileInfo, init func() (*uploadCheckpoint, error)) (checkpoint *uploadCheckpoint, err error) {
if !input.EnableCheckpoint {
return init()
}
checkpoint, exist := loadExistUploadCheckPoint(ctx, cli, input, srcFile)
if exist {
return checkpoint, nil
}
err = checkAndCreateDir(input.CheckpointFile)
if err != nil {
return nil, InvalidCheckpointFilePath.withCause(err)
}
file, err := os.Create(input.CheckpointFile)
if err != nil {
return nil, newTosClientError("tos: create checkpoint file failed", err)
}
_ = file.Close()
checkpoint, err = init()
if err != nil {
return nil, err
}
err = checkpoint.WriteToFile()
if err != nil {
return nil, err
}
return
}
func bindCancelHookWithAborter(hook CancelHook, aborter func() error) {
if hook == nil {
return
}
cancel := hook.(*canceler)
cancel.aborter = aborter
}
func bindCancelHookWithCleaner(hook CancelHook, cleaner func()) {
if hook == nil {
return
}
cancel := hook.(*canceler)
cancel.cleaner = cleaner
}
func (cli *ClientV2) UploadFile(ctx context.Context, input *UploadFileInput) (output *UploadFileOutput, err error) {
// avoid modifying on origin pointer
input = &(*input)
stat, err := os.Stat(input.FilePath)
if err != nil {
return nil, InvalidSrcFilePath
}
if err = validateUploadInput(input, stat, cli.isCustomDomain); err != nil {
return nil, err
}
init := func() (*uploadCheckpoint, error) {
return initUploadCheckpoint(input, stat)
}
// if the checkpoint file not exist, here we will create it
checkpoint, err := getUploadCheckpoint(ctx, cli, input, stat, init)
if err != nil {
return nil, err
}
event := &uploadPostEvent{
input: input,
checkPoint: checkpoint,
}
if checkpoint.UploadID == "" {
// create multipart upload task
created, err := cli.CreateMultipartUploadV2(ctx, &input.CreateMultipartUploadV2Input)
if err != nil {
event.postUploadEvent(&UploadEvent{
Type: enum.UploadEventCreateMultipartUploadFailed,
Err: err,
Bucket: input.Bucket,
Key: input.Key,
CheckpointFile: &input.CheckpointFile,
})
return nil, err
}
event.postUploadEvent(&UploadEvent{
Type: enum.UploadEventCreateMultipartUploadSucceed,
Bucket: input.Bucket,
Key: input.Key,
UploadID: &created.UploadID,
CheckpointFile: &input.CheckpointFile,
})
checkpoint.UploadID = created.UploadID
}
cleaner := func() {
_ = os.Remove(input.CheckpointFile)
}
event.checkPoint = checkpoint
bindCancelHookWithCleaner(input.CancelHook, cleaner)
return cli.uploadPart(ctx, checkpoint, input, event)
}
func prepareUploadTasks(cli *ClientV2, ctx context.Context, checkpoint *uploadCheckpoint, input *UploadFileInput) []task {
tasks := make([]task, 0)
consumed := int64(0)
subtotal := int64(0)
for _, part := range checkpoint.PartsInfo {
if !part.IsCompleted {
tasks = append(tasks, &uploadTask{
cli: cli,
ctx: ctx,
input: input,
total: checkpoint.FileInfo.Size,
UploadID: checkpoint.UploadID,
PartNumber: part.PartNumber,
subtotal: &subtotal,
consumed: &consumed,
Offset: part.Offset,
PartSize: part.PartSize,
})
} else {
consumed += part.PartSize
}
}
return tasks
}
func (u *uploadPostEvent) newUploadPartSucceedEvent(input *UploadFileInput, part uploadPartInfo) *UploadEvent {
return &UploadEvent{
Type: enum.UploadEventUploadPartSucceed,
Bucket: input.Bucket,
Key: input.Key,
UploadID: part.uploadID,
CheckpointFile: &input.CheckpointFile,
UploadPartInfo: &UploadPartInfo{
PartNumber: part.PartNumber,
PartSize: part.PartSize,
Offset: int64(part.Offset),
ETag: &part.ETag,
HashCrc64ecma: &part.HashCrc64ecma,
},
}
}
func (u *uploadPostEvent) newUploadPartAbortedEvent(input *UploadFileInput, uploadID string, err error) *UploadEvent {
return &UploadEvent{
Type: enum.UploadEventUploadPartAborted,
Err: err,
Bucket: input.Bucket,
Key: input.Key,
UploadID: &uploadID,
CheckpointFile: &input.CheckpointFile,
}
}
func (u *uploadPostEvent) newUploadPartFailedEvent(input *UploadFileInput, uploadID string, err error) *UploadEvent {
return &UploadEvent{
Type: enum.UploadEventUploadPartFailed,
Err: err,
Bucket: input.Bucket,
Key: input.Key,
UploadID: &uploadID,
CheckpointFile: &input.CheckpointFile,
}
}
func (u *uploadPostEvent) newCompleteMultipartUploadFailedEvent(input *UploadFileInput, uploadID string, err error) *UploadEvent {
return &UploadEvent{
Type: enum.UploadEventCompleteMultipartUploadFailed,
Err: err,
Bucket: input.Bucket,
Key: input.Key,
UploadID: &uploadID,
CheckpointFile: &input.CheckpointFile,
}
}
func newCompleteMultipartUploadSucceedEvent(input *UploadFileInput, uploadID string) *UploadEvent {
return &UploadEvent{
Type: enum.UploadEventCompleteMultipartUploadSucceed,
Bucket: input.Bucket,
Key: input.Key,
UploadID: &uploadID,
CheckpointFile: &input.CheckpointFile,
}
}
func postDataTransferStatus(listener DataTransferListener, status *DataTransferStatus) {
if listener != nil {
listener.DataTransferStatusChange(status)
}
}
func getCancelHandle(hook CancelHook) chan struct{} {
if c, ok := hook.(*canceler); ok {
return c.cancelHandle
}
return make(chan struct{})
}
func combineCRCInDownload(parts []downloadPartInfo) uint64 {
if len(parts) == 0 {
return 0
}
crc := parts[0].HashCrc64ecma
for i := 1; i < len(parts); i++ {
crc = CRC64Combine(crc, parts[i].HashCrc64ecma, uint64(parts[i].RangeEnd-parts[i].RangeStart+1))
}
return crc
}
// combineCRCInParts calculates the total CRC of continuous parts
func combineCRCInParts(parts []uploadPartInfo) uint64 {
if parts == nil || len(parts) == 0 {
return 0
}
crc := parts[0].HashCrc64ecma
for i := 1; i < len(parts); i++ {
crc = CRC64Combine(crc, parts[i].HashCrc64ecma, uint64(parts[i].PartSize))
}
return crc
}
func (cli *ClientV2) uploadPart(ctx context.Context, checkpoint *uploadCheckpoint, input *UploadFileInput, event *uploadPostEvent) (*UploadFileOutput, error) {
// prepare tasks
// if amount of tasks >= 10000, err "tos: part count too many" will be raised.
tasks := prepareUploadTasks(cli, ctx, checkpoint, input)
routinesNum := min(input.TaskNum, len(tasks))
cancelHandle := getCancelHandle(input.CancelHook)
tg := newTaskGroup(cancelHandle, routinesNum, checkpoint, event, input.EnableCheckpoint, tasks)
abort := func() error {
_, err := cli.AbortMultipartUpload(ctx,
&AbortMultipartUploadInput{
Bucket: input.Bucket,
Key: input.Key,
UploadID: checkpoint.UploadID})
_ = os.Remove(input.CheckpointFile)
return err
}
bindCancelHookWithAborter(input.CancelHook, abort)
tg.RunWorker()
// start adding tasks
postDataTransferStatus(input.DataTransferListener, &DataTransferStatus{
TotalBytes: checkpoint.FileInfo.Size,
Type: enum.DataTransferStarted,
})
tg.Scheduler()
success, taskErr := tg.Wait()
if taskErr != nil {
if err := abort(); err != nil {
return nil, err
}
return nil, taskErr
}
// handle results
if success < len(tasks) {
return nil, newTosClientError("tos: some upload tasks failed.", nil)
}
complete, err := cli.CompleteMultipartUploadV2(ctx, &CompleteMultipartUploadV2Input{
Bucket: input.Bucket,
Key: input.Key,
UploadID: checkpoint.UploadID,
Parts: checkpoint.GetParts(),
})
if err != nil {
event.postUploadEvent(event.newCompleteMultipartUploadFailedEvent(input, checkpoint.UploadID, err))
return nil, err
}
event.postUploadEvent(newCompleteMultipartUploadSucceedEvent(input, checkpoint.UploadID))
if cli.enableCRC && complete.HashCrc64ecma != 0 && combineCRCInParts(checkpoint.PartsInfo) != complete.HashCrc64ecma {
return nil, newTosClientError("tos: crc of entire file mismatch.", nil)
}
_ = os.Remove(input.CheckpointFile)
return &UploadFileOutput{
RequestInfo: complete.RequestInfo,
Bucket: complete.Bucket,
Key: complete.Key,
UploadID: checkpoint.UploadID,
ETag: complete.ETag,
Location: complete.Location,
VersionID: complete.VersionID,
HashCrc64ecma: complete.HashCrc64ecma,
SSECAlgorithm: checkpoint.SSECAlgorithm,
SSECKeyMD5: checkpoint.SSECKeyMD5,
EncodingType: checkpoint.EncodingType,
}, nil
}

165
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/util.go generated vendored Normal file
View File

@@ -0,0 +1,165 @@
package tos
import (
"fmt"
"os"
"time"
)
func min(a int, b int) int {
if a < b {
return a
}
return b
}
const (
EventPartSucceed = 3
EventPartFailed = 4
EventPartAborted = 5 // The task needs to be interrupted in case of 403, 404, 405 errors
)
type task interface {
do() (interface{}, error)
getBaseInput() interface{}
}
type checkPoint interface {
WriteToFile() error
UpdatePartsInfo(result interface{})
GetCheckPointFilePath() string
}
type taskGroup interface {
// Wait 等待执行结果, success 是此次成功的 task 数量
Wait() (success int, err error)
// RunWorker 启动worker
RunWorker()
// Scheduler 分发任务
Scheduler()
}
type postEvent interface {
PostEvent(eventType int, result interface{}, taskErr error)
}
type taskGroupImpl struct {
cancelHandle chan struct{}
abortHandle chan struct{}
errCh chan error
resultsCh chan interface{}
tasksCh chan task
routinesNum int
tasks []task
checkPoint checkPoint
enableCheckPoint bool
postEvent postEvent
}
func (t *taskGroupImpl) Wait() (int, error) {
successNum := 0
failNum := 0
Loop:
for successNum+failNum < len(t.tasks) {
select {
case <-t.abortHandle:
break Loop
case <-t.cancelHandle:
break Loop
case part := <-t.resultsCh:
successNum++
t.checkPoint.UpdatePartsInfo(part)
if t.enableCheckPoint {
t.checkPoint.WriteToFile()
}
t.postEvent.PostEvent(EventPartSucceed, part, nil)
case taskErr := <-t.errCh:
if StatusCode(taskErr) == 403 || StatusCode(taskErr) == 404 || StatusCode(taskErr) == 405 {
close(t.abortHandle)
_ = os.Remove(t.checkPoint.GetCheckPointFilePath())
t.postEvent.PostEvent(EventPartAborted, nil, taskErr)
return successNum, fmt.Errorf("status code not service error, err:%s. ", taskErr.Error())
}
t.postEvent.PostEvent(EventPartFailed, nil, taskErr)
failNum++
}
}
return successNum, nil
}
func newTaskGroup(cancelHandle chan struct{}, routinesNum int, checkPoint checkPoint, postEvent postEvent, enableCheckPoint bool, tasks []task) taskGroup {
taskBufferSize := min(routinesNum, DefaultTaskBufferSize)
tasksCh := make(chan task, taskBufferSize)
return &taskGroupImpl{
cancelHandle: cancelHandle,
abortHandle: make(chan struct{}),
errCh: make(chan error),
resultsCh: make(chan interface{}),
tasksCh: tasksCh,
routinesNum: routinesNum,
tasks: tasks,
checkPoint: checkPoint,
enableCheckPoint: enableCheckPoint,
postEvent: postEvent,
}
}
func (t *taskGroupImpl) RunWorker() {
for i := 0; i < t.routinesNum; i++ {
go t.worker()
}
}
func (t *taskGroupImpl) Scheduler() {
go func() {
for _, task := range t.tasks {
select {
case <-t.cancelHandle:
return
case <-t.abortHandle:
return
default:
t.tasksCh <- task
}
}
close(t.tasksCh)
}()
}
func (t *taskGroupImpl) worker() {
for {
select {
case <-t.cancelHandle:
return
case <-t.abortHandle:
return
case task, ok := <-t.tasksCh:
if !ok {
return
}
result, err := task.do()
if err != nil {
t.errCh <- err
}
if result != nil {
t.resultsCh <- result
}
}
}
}
func GetUnixTimeMs() int64 {
return ToMillis(time.Now())
}
func ToMillis(t time.Time) int64 {
return t.UnixNano() / int64(time.Millisecond)
}
func StringPtr(input string) *string {
return &input
}

38
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/versioning.go generated vendored Normal file
View File

@@ -0,0 +1,38 @@
package tos
import (
"context"
"net/http"
)
const (
BucketVersioningEnable = "Enabled"
BucketVersioningSuspended = "Suspended"
)
type GetBucketVersioningOutput struct {
RequestInfo `json:"-"`
Status string `json:"Status"`
}
// GetBucketVersioning get the multi-version status of a bucket
func (cli *Client) GetBucketVersioning(ctx context.Context, bucket string) (*GetBucketVersioningOutput, error) {
if err := isValidBucketName(bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(bucket, "").
WithQuery("versioning", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketVersioningOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}

82
vendor/github.com/volcengine/ve-tos-golang-sdk/v2/tos/website.go generated vendored Normal file
View File

@@ -0,0 +1,82 @@
package tos
import (
"bytes"
"context"
"net/http"
)
func (cli *ClientV2) PutBucketWebsite(ctx context.Context, input *PutBucketWebsiteInput) (*PutBucketWebsiteOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
body := putBucketWebsiteInput{
RedirectAllRequestsTo: input.RedirectAllRequestsTo,
IndexDocument: input.IndexDocument,
ErrorDocument: input.ErrorDocument,
}
if input.RoutingRules != nil {
body.RoutingRules = input.RoutingRules.Rules
}
data, contentMD5, err := marshalInput("PutBucketWebsiteInput", body)
if err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("website", "").
WithHeader(HeaderContentMD5, contentMD5).
WithRetry(OnRetryFromStart, StatusCodeClassifier{}).
Request(ctx, http.MethodPut, bytes.NewReader(data), cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := PutBucketWebsiteOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}
func (cli *ClientV2) GetBucketWebsite(ctx context.Context, input *GetBucketWebsiteInput) (*GetBucketWebsiteOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("website", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodGet, nil, cli.roundTripper(http.StatusOK))
if err != nil {
return nil, err
}
defer res.Close()
output := GetBucketWebsiteOutput{RequestInfo: res.RequestInfo()}
if err = marshalOutput(output.RequestID, res.Body, &output); err != nil {
return nil, err
}
return &output, nil
}
func (cli *ClientV2) DeleteBucketWebsite(ctx context.Context, input *DeleteBucketWebsiteInput) (*DeleteBucketWebsiteOutput, error) {
if input == nil {
return nil, InputIsNilClientError
}
if err := isValidBucketName(input.Bucket, cli.isCustomDomain); err != nil {
return nil, err
}
res, err := cli.newBuilder(input.Bucket, "").
WithQuery("website", "").
WithRetry(nil, StatusCodeClassifier{}).
Request(ctx, http.MethodDelete, nil, cli.roundTripper(http.StatusNoContent))
if err != nil {
return nil, err
}
defer res.Close()
output := DeleteBucketWebsiteOutput{RequestInfo: res.RequestInfo()}
return &output, nil
}

51
vendor/github.com/volcengine/volc-sdk-golang/base/aes.go generated vendored Normal file
View File

@@ -0,0 +1,51 @@
package base
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"encoding/base64"
"errors"
"fmt"
)
// AES CBC 加密
func aesEncryptCBC(origData, key []byte) (crypted []byte, err error) {
defer func() {
if r := recover(); r != nil {
crypted = nil
err = errors.New(fmt.Sprintf("%v", r))
}
}()
block, err := aes.NewCipher(key)
if err != nil {
return
}
blockSize := block.BlockSize()
origData = zeroPadding(origData, blockSize)
blockMode := cipher.NewCBCEncrypter(block, key[:blockSize])
crypted = make([]byte, len(origData))
blockMode.CryptBlocks(crypted, origData)
return
}
// AES CBC 加密后做一次Base64加密
func aesEncryptCBCWithBase64(origData, key []byte) (string, error) {
cbc, err := aesEncryptCBC(origData, key)
if err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(cbc), nil
}
func zeroPadding(ciphertext []byte, blockSize int) []byte {
padding := blockSize - len(ciphertext)%blockSize
if padding == 0 {
return ciphertext
}
padtext := bytes.Repeat([]byte{byte(0)}, padding)
return append(ciphertext, padtext...)
}

290
vendor/github.com/volcengine/volc-sdk-golang/base/client.go generated vendored Normal file
View File

@@ -0,0 +1,290 @@
package base
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
"strings"
"time"
)
const (
accessKey = "VOLC_ACCESSKEY"
secretKey = "VOLC_SECRETKEY"
defaultScheme = "http"
)
var _GlobalClient *http.Client
func init() {
_GlobalClient = &http.Client{
Transport: &http.Transport{
MaxIdleConns: 1000,
MaxIdleConnsPerHost: 100,
IdleConnTimeout: 10 * time.Second,
},
}
}
// Client 基础客户端
type Client struct {
Client *http.Client
SdkVersion string
ServiceInfo *ServiceInfo
ApiInfoList map[string]*ApiInfo
}
// NewClient 生成一个客户端
func NewClient(info *ServiceInfo, apiInfoList map[string]*ApiInfo) *Client {
client := &Client{Client: _GlobalClient, ServiceInfo: info.Clone(), ApiInfoList: apiInfoList}
if client.ServiceInfo.Scheme == "" {
client.ServiceInfo.Scheme = defaultScheme
}
if os.Getenv(accessKey) != "" && os.Getenv(secretKey) != "" {
client.ServiceInfo.Credentials.AccessKeyID = os.Getenv(accessKey)
client.ServiceInfo.Credentials.SecretAccessKey = os.Getenv(secretKey)
} else if _, err := os.Stat(os.Getenv("HOME") + "/.volc/config"); err == nil {
if content, err := ioutil.ReadFile(os.Getenv("HOME") + "/.volc/config"); err == nil {
m := make(map[string]string)
json.Unmarshal(content, &m)
if accessKey, ok := m["ak"]; ok {
client.ServiceInfo.Credentials.AccessKeyID = accessKey
}
if secretKey, ok := m["sk"]; ok {
client.ServiceInfo.Credentials.SecretAccessKey = secretKey
}
}
}
content, err := ioutil.ReadFile("VERSION")
if err == nil {
client.SdkVersion = strings.TrimSpace(string(content))
client.ServiceInfo.Header.Set("User-Agent", strings.Join([]string{"volc-sdk-golang", client.SdkVersion}, "/"))
}
return client
}
func (serviceInfo *ServiceInfo) Clone() *ServiceInfo {
ret := new(ServiceInfo)
//base info
ret.Timeout = serviceInfo.Timeout
ret.Host = serviceInfo.Host
ret.Scheme = serviceInfo.Scheme
//credential
ret.Credentials = serviceInfo.Credentials.Clone()
// header
ret.Header = serviceInfo.Header.Clone()
return ret
}
func (cred Credentials) Clone() Credentials {
return Credentials{
Service: cred.Service,
Region: cred.Region,
SecretAccessKey: cred.SecretAccessKey,
AccessKeyID: cred.AccessKeyID,
SessionToken: cred.SessionToken,
}
}
// SetAccessKey 设置AK
func (client *Client) SetAccessKey(ak string) {
if ak != "" {
client.ServiceInfo.Credentials.AccessKeyID = ak
}
}
// SetSecretKey 设置SK
func (client *Client) SetSecretKey(sk string) {
if sk != "" {
client.ServiceInfo.Credentials.SecretAccessKey = sk
}
}
// SetSessionToken
func (client *Client) SetSessionToken(token string) {
if token != "" {
client.ServiceInfo.Credentials.SessionToken = token
}
}
// SetHost 设置Host
func (client *Client) SetHost(host string) {
if host != "" {
client.ServiceInfo.Host = host
}
}
func (client *Client) SetScheme(scheme string) {
if scheme != "" {
client.ServiceInfo.Scheme = scheme
}
}
// SetCredential 设置Credentials
func (client *Client) SetCredential(c Credentials) {
if c.AccessKeyID != "" {
client.ServiceInfo.Credentials.AccessKeyID = c.AccessKeyID
}
if c.SecretAccessKey != "" {
client.ServiceInfo.Credentials.SecretAccessKey = c.SecretAccessKey
}
if c.Region != "" {
client.ServiceInfo.Credentials.Region = c.Region
}
if c.SessionToken != "" {
client.ServiceInfo.Credentials.SessionToken = c.SessionToken
}
}
func (client *Client) SetTimeout(timeout time.Duration) {
if timeout > 0 {
client.ServiceInfo.Timeout = timeout
}
}
// GetSignUrl 获取签名字符串
func (client *Client) GetSignUrl(api string, query url.Values) (string, error) {
apiInfo := client.ApiInfoList[api]
if apiInfo == nil {
return "", errors.New("相关api不存在")
}
query = mergeQuery(query, apiInfo.Query)
u := url.URL{
Scheme: client.ServiceInfo.Scheme,
Host: client.ServiceInfo.Host,
Path: apiInfo.Path,
RawQuery: query.Encode(),
}
req, err := http.NewRequest(strings.ToUpper(apiInfo.Method), u.String(), nil)
if err != nil {
return "", errors.New("构建request失败")
}
return client.ServiceInfo.Credentials.SignUrl(req), nil
}
// SignSts2 生成sts信息
func (client *Client) SignSts2(inlinePolicy *Policy, expire time.Duration) (*SecurityToken2, error) {
var err error
sts := new(SecurityToken2)
if sts.AccessKeyID, sts.SecretAccessKey, err = createTempAKSK(); err != nil {
return nil, err
}
if expire < time.Minute {
expire = time.Minute
}
now := time.Now()
expireTime := now.Add(expire)
sts.CurrentTime = now.Format(time.RFC3339)
sts.ExpiredTime = expireTime.Format(time.RFC3339)
innerToken, err := createInnerToken(client.ServiceInfo.Credentials, sts, inlinePolicy, expireTime.Unix())
if err != nil {
return nil, err
}
b, _ := json.Marshal(innerToken)
sts.SessionToken = "STS2" + base64.StdEncoding.EncodeToString(b)
return sts, nil
}
// Query 发起Get的query请求
func (client *Client) Query(api string, query url.Values) ([]byte, int, error) {
return client.requestWithContentType(api, query, "", "")
}
// Json 发起Json的post请求
func (client *Client) Json(api string, query url.Values, body string) ([]byte, int, error) {
return client.requestWithContentType(api, query, body, "application/json")
}
// PostWithContentType 发起自定义 Content-Type 的 post 请求，Content-Type 不可以为空
func (client *Client) PostWithContentType(api string, query url.Values, body string, ct string) ([]byte, int, error) {
return client.requestWithContentType(api, query, body, ct)
}
func (client *Client) requestWithContentType(api string, query url.Values, body string, ct string) ([]byte, int, error) {
apiInfo := client.ApiInfoList[api]
if apiInfo == nil {
return []byte(""), 500, errors.New("相关api不存在")
}
timeout := getTimeout(client.ServiceInfo.Timeout, apiInfo.Timeout)
header := mergeHeader(client.ServiceInfo.Header, apiInfo.Header)
query = mergeQuery(query, apiInfo.Query)
u := url.URL{
Scheme: client.ServiceInfo.Scheme,
Host: client.ServiceInfo.Host,
Path: apiInfo.Path,
RawQuery: query.Encode(),
}
var requestBody io.Reader
if body != "" {
requestBody = strings.NewReader(body)
}
req, err := http.NewRequest(strings.ToUpper(apiInfo.Method), u.String(), requestBody)
if err != nil {
return []byte(""), 500, errors.New("构建request失败")
}
req.Header = header
if ct != "" {
req.Header.Set("Content-Type", ct)
}
return client.makeRequest(api, req, timeout)
}
// Post 发起Post请求
func (client *Client) Post(api string, query url.Values, form url.Values) ([]byte, int, error) {
apiInfo := client.ApiInfoList[api]
form = mergeQuery(form, apiInfo.Form)
return client.requestWithContentType(api, query, form.Encode(), "application/x-www-form-urlencoded")
}
func (client *Client) makeRequest(api string, req *http.Request, timeout time.Duration) ([]byte, int, error) {
req = client.ServiceInfo.Credentials.Sign(req)
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
req = req.WithContext(ctx)
resp, err := client.Client.Do(req)
if err != nil {
return []byte(""), 500, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return []byte(""), resp.StatusCode, err
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return body, resp.StatusCode, fmt.Errorf("api %s http code %d body %s", api, resp.StatusCode, string(body))
}
return body, resp.StatusCode, nil
}

109
vendor/github.com/volcengine/volc-sdk-golang/base/model.go generated vendored Normal file
View File

@@ -0,0 +1,109 @@
package base
import (
"net/http"
"net/url"
"time"
)
const (
RegionCnNorth1 = "cn-north-1"
RegionUsEast1 = "us-east-1"
RegionApSingapore = "ap-singapore-1"
timeFormatV4 = "20060102T150405Z"
)
type ServiceInfo struct {
Timeout time.Duration
Scheme string
Host string
Header http.Header
Credentials Credentials
}
type ApiInfo struct {
Method string
Path string
Query url.Values
Form url.Values
Timeout time.Duration
Header http.Header
}
type Credentials struct {
AccessKeyID string
SecretAccessKey string
Service string
Region string
SessionToken string
}
type metadata struct {
algorithm string
credentialScope string
signedHeaders string
date string
region string
service string
}
// 统一的JSON返回结果
type CommonResponse struct {
ResponseMetadata ResponseMetadata
Result interface{} `json:"Result,omitempty"`
}
type BaseResp struct {
Status string
CreatedTime int64
UpdatedTime int64
}
type ErrorObj struct {
CodeN int
Code string
Message string
}
type ResponseMetadata struct {
RequestId string
Service string `json:",omitempty"`
Region string `json:",omitempty"`
Action string `json:",omitempty"`
Version string `json:",omitempty"`
Error *ErrorObj `json:",omitempty"`
}
type Policy struct {
Statement []*Statement
}
const (
StatementEffectAllow = "Allow"
StatementEffectDeny = "Deny"
)
type Statement struct {
Effect string
Action []string
Resource []string
Condition string `json:",omitempty"`
}
type SecurityToken2 struct {
AccessKeyID string
SecretAccessKey string
SessionToken string
ExpiredTime string
CurrentTime string
}
type InnerToken struct {
LTAccessKeyId string
AccessKeyId string
SignedSecretAccessKey string
ExpiredTime int64
PolicyString string
Signature string
}

288
vendor/github.com/volcengine/volc-sdk-golang/base/sign.go generated vendored Normal file
View File

@@ -0,0 +1,288 @@
package base
import (
"bytes"
"crypto/hmac"
"crypto/md5"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"sort"
"strings"
"time"
)
func (c Credentials) Sign(request *http.Request) *http.Request {
query := request.URL.Query()
request.URL.RawQuery = query.Encode()
return Sign4(request, c)
}
func (c Credentials) SignUrl(request *http.Request) string {
query := request.URL.Query()
ldt := timestampV4()
sdt := ldt[:8]
meta := new(metadata)
meta.date, meta.service, meta.region, meta.signedHeaders, meta.algorithm = sdt, c.Service, c.Region, "", "HMAC-SHA256"
meta.credentialScope = concat("/", meta.date, meta.region, meta.service, "request")
query.Set("X-Date", ldt)
query.Set("X-NotSignBody", "")
query.Set("X-Credential", c.AccessKeyID+"/"+meta.credentialScope)
query.Set("X-Algorithm", meta.algorithm)
query.Set("X-SignedHeaders", meta.signedHeaders)
query.Set("X-SignedQueries", "")
keys := make([]string, 0, len(query))
for k := range query {
keys = append(keys, k)
}
sort.Strings(keys)
query.Set("X-SignedQueries", strings.Join(keys, ";"))
if c.SessionToken != "" {
query.Set("X-Security-Token", c.SessionToken)
}
// Task 1
hashedCanonReq := hashedSimpleCanonicalRequestV4(request, query, meta)
// Task 2
stringToSign := concat("\n", meta.algorithm, ldt, meta.credentialScope, hashedCanonReq)
// Task 3
signingKey := signingKeyV4(c.SecretAccessKey, meta.date, meta.region, meta.service)
signature := signatureV4(signingKey, stringToSign)
query.Set("X-Signature", signature)
return query.Encode()
}
// Sign4 signs a request with Signed Signature Version 4.
func Sign4(request *http.Request, credential Credentials) *http.Request {
keys := credential
prepareRequestV4(request)
meta := new(metadata)
meta.service, meta.region = keys.Service, keys.Region
// Task 0 设置SessionToken的header
if credential.SessionToken != "" {
request.Header.Set("X-Security-Token", credential.SessionToken)
}
// Task 1
hashedCanonReq := hashedCanonicalRequestV4(request, meta)
// Task 2
stringToSign := stringToSignV4(request, hashedCanonReq, meta)
// Task 3
signingKey := signingKeyV4(keys.SecretAccessKey, meta.date, meta.region, meta.service)
signature := signatureV4(signingKey, stringToSign)
request.Header.Set("Authorization", buildAuthHeaderV4(signature, meta, keys))
return request
}
func hashedSimpleCanonicalRequestV4(request *http.Request, query url.Values, meta *metadata) string {
payloadHash := hashSHA256([]byte(""))
if request.URL.Path == "" {
request.URL.Path = "/"
}
canonicalRequest := concat("\n", request.Method, normuri(request.URL.Path), normquery(query), "\n", meta.signedHeaders, payloadHash)
return hashSHA256([]byte(canonicalRequest))
}
func hashedCanonicalRequestV4(request *http.Request, meta *metadata) string {
payload := readAndReplaceBody(request)
payloadHash := hashSHA256(payload)
request.Header.Set("X-Content-Sha256", payloadHash)
request.Header.Set("Host", request.Host)
var sortedHeaderKeys []string
for key := range request.Header {
switch key {
case "Content-Type", "Content-Md5", "Host", "X-Security-Token":
default:
if !strings.HasPrefix(key, "X-") {
continue
}
}
sortedHeaderKeys = append(sortedHeaderKeys, strings.ToLower(key))
}
sort.Strings(sortedHeaderKeys)
var headersToSign string
for _, key := range sortedHeaderKeys {
value := strings.TrimSpace(request.Header.Get(key))
if key == "host" {
if strings.Contains(value, ":") {
split := strings.Split(value, ":")
port := split[1]
if port == "80" || port == "443" {
value = split[0]
}
}
}
headersToSign += key + ":" + value + "\n"
}
meta.signedHeaders = concat(";", sortedHeaderKeys...)
canonicalRequest := concat("\n", request.Method, normuri(request.URL.Path), normquery(request.URL.Query()), headersToSign, meta.signedHeaders, payloadHash)
return hashSHA256([]byte(canonicalRequest))
}
func stringToSignV4(request *http.Request, hashedCanonReq string, meta *metadata) string {
requestTs := request.Header.Get("X-Date")
meta.algorithm = "HMAC-SHA256"
meta.date = tsDateV4(requestTs)
meta.credentialScope = concat("/", meta.date, meta.region, meta.service, "request")
return concat("\n", meta.algorithm, requestTs, meta.credentialScope, hashedCanonReq)
}
func signatureV4(signingKey []byte, stringToSign string) string {
return hex.EncodeToString(hmacSHA256(signingKey, stringToSign))
}
func prepareRequestV4(request *http.Request) *http.Request {
necessaryDefaults := map[string]string{
"Content-Type": "application/x-www-form-urlencoded; charset=utf-8",
"X-Date": timestampV4(),
}
for header, value := range necessaryDefaults {
if request.Header.Get(header) == "" {
request.Header.Set(header, value)
}
}
if request.URL.Path == "" {
request.URL.Path += "/"
}
return request
}
func signingKeyV4(secretKey, date, region, service string) []byte {
kDate := hmacSHA256([]byte(secretKey), date)
kRegion := hmacSHA256(kDate, region)
kService := hmacSHA256(kRegion, service)
kSigning := hmacSHA256(kService, "request")
return kSigning
}
func buildAuthHeaderV4(signature string, meta *metadata, keys Credentials) string {
credential := keys.AccessKeyID + "/" + meta.credentialScope
return meta.algorithm +
" Credential=" + credential +
", SignedHeaders=" + meta.signedHeaders +
", Signature=" + signature
}
func timestampV4() string {
return now().Format(timeFormatV4)
}
func tsDateV4(timestamp string) string {
return timestamp[:8]
}
func hmacSHA256(key []byte, content string) []byte {
mac := hmac.New(sha256.New, key)
mac.Write([]byte(content))
return mac.Sum(nil)
}
func hashSHA256(content []byte) string {
h := sha256.New()
h.Write(content)
return fmt.Sprintf("%x", h.Sum(nil))
}
func hashMD5(content []byte) string {
h := md5.New()
h.Write(content)
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
func readAndReplaceBody(request *http.Request) []byte {
if request.Body == nil {
return []byte{}
}
payload, _ := ioutil.ReadAll(request.Body)
request.Body = ioutil.NopCloser(bytes.NewReader(payload))
return payload
}
func concat(delim string, str ...string) string {
return strings.Join(str, delim)
}
var now = func() time.Time {
return time.Now().UTC()
}
func normuri(uri string) string {
parts := strings.Split(uri, "/")
for i := range parts {
parts[i] = encodePathFrag(parts[i])
}
return strings.Join(parts, "/")
}
func encodePathFrag(s string) string {
hexCount := 0
for i := 0; i < len(s); i++ {
c := s[i]
if shouldEscape(c) {
hexCount++
}
}
t := make([]byte, len(s)+2*hexCount)
j := 0
for i := 0; i < len(s); i++ {
c := s[i]
if shouldEscape(c) {
t[j] = '%'
t[j+1] = "0123456789ABCDEF"[c>>4]
t[j+2] = "0123456789ABCDEF"[c&15]
j += 3
} else {
t[j] = c
j++
}
}
return string(t)
}
func shouldEscape(c byte) bool {
if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' {
return false
}
if '0' <= c && c <= '9' {
return false
}
if c == '-' || c == '_' || c == '.' || c == '~' {
return false
}
return true
}
func normquery(v url.Values) string {
queryString := v.Encode()
return strings.Replace(queryString, "+", "%20", -1)
}

180
vendor/github.com/volcengine/volc-sdk-golang/base/utils.go generated vendored Normal file
View File

@@ -0,0 +1,180 @@
package base
import (
"crypto/md5"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"math/rand"
"net/http"
"net/url"
"reflect"
"strconv"
"strings"
"time"
"github.com/google/uuid"
)
var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
func init() {
rand.Seed(time.Now().Unix())
}
func createTempAKSK() (accessKeyId string, plainSk string, err error) {
if accessKeyId, err = generateAccessKeyId("AKTP"); err != nil {
return
}
plainSk, err = generateSecretKey()
if err != nil {
return
}
return
}
func generateAccessKeyId(prefix string) (string, error) {
uuid := uuid.New()
uidBase64 := base64.StdEncoding.EncodeToString([]byte(strings.Replace(uuid.String(), "-", "", -1)))
s := strings.Replace(uidBase64, "=", "", -1)
s = strings.Replace(s, "/", "", -1)
s = strings.Replace(s, "+", "", -1)
s = strings.Replace(s, "-", "", -1)
return prefix + s, nil
}
func randStringRunes(n int) string {
b := make([]rune, n)
for i := range b {
b[i] = letterRunes[rand.Intn(len(letterRunes))]
}
return string(b)
}
func generateSecretKey() (string, error) {
randString32 := randStringRunes(32)
return aesEncryptCBCWithBase64([]byte(randString32), []byte("bytedance-isgood"))
}
func createInnerToken(credentials Credentials, sts *SecurityToken2, inlinePolicy *Policy, t int64) (*InnerToken, error) {
var err error
innerToken := new(InnerToken)
innerToken.LTAccessKeyId = credentials.AccessKeyID
innerToken.AccessKeyId = sts.AccessKeyID
innerToken.ExpiredTime = t
key := md5.Sum([]byte(credentials.SecretAccessKey))
innerToken.SignedSecretAccessKey, err = aesEncryptCBCWithBase64([]byte(sts.SecretAccessKey), key[:])
if err != nil {
return nil, err
}
if inlinePolicy != nil {
b, _ := json.Marshal(inlinePolicy)
innerToken.PolicyString = string(b)
}
signStr := fmt.Sprintf("%s|%s|%d|%s|%s", innerToken.LTAccessKeyId, innerToken.AccessKeyId, innerToken.ExpiredTime, innerToken.SignedSecretAccessKey, innerToken.PolicyString)
innerToken.Signature = hex.EncodeToString(hmacSHA256(key[:], signStr))
return innerToken, nil
}
func getTimeout(serviceTimeout, apiTimeout time.Duration) time.Duration {
timeout := time.Second
if serviceTimeout != time.Duration(0) {
timeout = serviceTimeout
}
if apiTimeout != time.Duration(0) {
timeout = apiTimeout
}
return timeout
}
func mergeQuery(query1, query2 url.Values) (query url.Values) {
query = url.Values{}
if query1 != nil {
for k, vv := range query1 {
for _, v := range vv {
query.Add(k, v)
}
}
}
if query2 != nil {
for k, vv := range query2 {
for _, v := range vv {
query.Add(k, v)
}
}
}
return
}
func mergeHeader(header1, header2 http.Header) (header http.Header) {
header = http.Header{}
if header1 != nil {
for k, v := range header1 {
header.Set(k, strings.Join(v, ";"))
}
}
if header2 != nil {
for k, v := range header2 {
header.Set(k, strings.Join(v, ";"))
}
}
return
}
func NewAllowStatement(actions, resources []string) *Statement {
sts := new(Statement)
sts.Effect = "Allow"
sts.Action = actions
sts.Resource = resources
return sts
}
func NewDenyStatement(actions, resources []string) *Statement {
sts := new(Statement)
sts.Effect = "Deny"
sts.Action = actions
sts.Resource = resources
return sts
}
func ToUrlValues(i interface{}) (values url.Values) {
values = url.Values{}
iVal := reflect.ValueOf(i).Elem()
typ := iVal.Type()
for i := 0; i < iVal.NumField(); i++ {
f := iVal.Field(i)
// You ca use tags here...
// tag := typ.Field(i).Tag.Get("tagname")
// Convert each type into a string for the url.Values string map
var v string
switch f.Interface().(type) {
case int, int8, int16, int32, int64:
v = strconv.FormatInt(f.Int(), 10)
case uint, uint8, uint16, uint32, uint64:
v = strconv.FormatUint(f.Uint(), 10)
case float32:
v = strconv.FormatFloat(f.Float(), 'f', 4, 32)
case float64:
v = strconv.FormatFloat(f.Float(), 'f', 4, 64)
case []byte:
v = string(f.Bytes())
case string:
v = f.String()
}
values.Set(typ.Field(i).Name, v)
}
return
}

205
vendor/golang.org/x/sync/singleflight/singleflight.go generated vendored Normal file
View File

@@ -0,0 +1,205 @@
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package singleflight provides a duplicate function call suppression
// mechanism.
package singleflight // import "golang.org/x/sync/singleflight"
import (
"bytes"
"errors"
"fmt"
"runtime"
"runtime/debug"
"sync"
)
// errGoexit indicates the runtime.Goexit was called in
// the user given function.
var errGoexit = errors.New("runtime.Goexit was called")
// A panicError is an arbitrary value recovered from a panic
// with the stack trace during the execution of given function.
type panicError struct {
value interface{}
stack []byte
}
// Error implements error interface.
func (p *panicError) Error() string {
return fmt.Sprintf("%v\n\n%s", p.value, p.stack)
}
func newPanicError(v interface{}) error {
stack := debug.Stack()
// The first line of the stack trace is of the form "goroutine N [status]:"
// but by the time the panic reaches Do the goroutine may no longer exist
// and its status will have changed. Trim out the misleading line.
if line := bytes.IndexByte(stack[:], '\n'); line >= 0 {
stack = stack[line+1:]
}
return &panicError{value: v, stack: stack}
}
// call is an in-flight or completed singleflight.Do call
type call struct {
wg sync.WaitGroup
// These fields are written once before the WaitGroup is done
// and are only read after the WaitGroup is done.
val interface{}
err error
// These fields are read and written with the singleflight
// mutex held before the WaitGroup is done, and are read but
// not written after the WaitGroup is done.
dups int
chans []chan<- Result
}
// Group represents a class of work and forms a namespace in
// which units of work can be executed with duplicate suppression.
type Group struct {
mu sync.Mutex // protects m
m map[string]*call // lazily initialized
}
// Result holds the results of Do, so they can be passed
// on a channel.
type Result struct {
Val interface{}
Err error
Shared bool
}
// Do executes and returns the results of the given function, making
// sure that only one execution is in-flight for a given key at a
// time. If a duplicate comes in, the duplicate caller waits for the
// original to complete and receives the same results.
// The return value shared indicates whether v was given to multiple callers.
func (g *Group) Do(key string, fn func() (interface{}, error)) (v interface{}, err error, shared bool) {
g.mu.Lock()
if g.m == nil {
g.m = make(map[string]*call)
}
if c, ok := g.m[key]; ok {
c.dups++
g.mu.Unlock()
c.wg.Wait()
if e, ok := c.err.(*panicError); ok {
panic(e)
} else if c.err == errGoexit {
runtime.Goexit()
}
return c.val, c.err, true
}
c := new(call)
c.wg.Add(1)
g.m[key] = c
g.mu.Unlock()
g.doCall(c, key, fn)
return c.val, c.err, c.dups > 0
}
// DoChan is like Do but returns a channel that will receive the
// results when they are ready.
//
// The returned channel will not be closed.
func (g *Group) DoChan(key string, fn func() (interface{}, error)) <-chan Result {
ch := make(chan Result, 1)
g.mu.Lock()
if g.m == nil {
g.m = make(map[string]*call)
}
if c, ok := g.m[key]; ok {
c.dups++
c.chans = append(c.chans, ch)
g.mu.Unlock()
return ch
}
c := &call{chans: []chan<- Result{ch}}
c.wg.Add(1)
g.m[key] = c
g.mu.Unlock()
go g.doCall(c, key, fn)
return ch
}
// doCall handles the single call for a key.
func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) {
normalReturn := false
recovered := false
// use double-defer to distinguish panic from runtime.Goexit,
// more details see https://golang.org/cl/134395
defer func() {
// the given function invoked runtime.Goexit
if !normalReturn && !recovered {
c.err = errGoexit
}
g.mu.Lock()
defer g.mu.Unlock()
c.wg.Done()
if g.m[key] == c {
delete(g.m, key)
}
if e, ok := c.err.(*panicError); ok {
// In order to prevent the waiting channels from being blocked forever,
// needs to ensure that this panic cannot be recovered.
if len(c.chans) > 0 {
go panic(e)
select {} // Keep this goroutine around so that it will appear in the crash dump.
} else {
panic(e)
}
} else if c.err == errGoexit {
// Already in the process of goexit, no need to call again
} else {
// Normal return
for _, ch := range c.chans {
ch <- Result{c.val, c.err, c.dups > 0}
}
}
}()
func() {
defer func() {
if !normalReturn {
// Ideally, we would wait to take a stack trace until we've determined
// whether this is a panic or a runtime.Goexit.
//
// Unfortunately, the only way we can distinguish the two is to see
// whether the recover stopped the goroutine from terminating, and by
// the time we know that, the part of the stack trace relevant to the
// panic has been discarded.
if r := recover(); r != nil {
c.err = newPanicError(r)
}
}
}()
c.val, c.err = fn()
normalReturn = true
}()
if !normalReturn {
recovered = true
}
}
// Forget tells the singleflight to forget about a key. Future calls
// to Do for this key will call the function rather than waiting for
// an earlier call to complete.
func (g *Group) Forget(key string) {
g.mu.Lock()
delete(g.m, key)
g.mu.Unlock()
}

15
vendor/modules.txt vendored
View File

@@ -915,6 +915,13 @@ github.com/vmware/govmomi/vim25/progress
github.com/vmware/govmomi/vim25/soap
github.com/vmware/govmomi/vim25/types
github.com/vmware/govmomi/vim25/xml
# github.com/volcengine/ve-tos-golang-sdk/v2 v2.6.2
## explicit; go 1.13
github.com/volcengine/ve-tos-golang-sdk/v2/tos
github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum
# github.com/volcengine/volc-sdk-golang v1.0.23
## explicit; go 1.14
github.com/volcengine/volc-sdk-golang/base
# github.com/willf/bitset v1.1.9
## explicit
github.com/willf/bitset
@@ -1051,6 +1058,7 @@ golang.org/x/oauth2/jwt
# golang.org/x/sync v0.1.0
## explicit
golang.org/x/sync/errgroup
golang.org/x/sync/singleflight
# golang.org/x/sys v0.9.0
## explicit; go 1.17
golang.org/x/sys/cpu
@@ -1243,8 +1251,6 @@ gopkg.in/alexcesaro/quotedprintable.v3
# gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d
## explicit
gopkg.in/asn1-ber.v1
# gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
## explicit; go 1.11
# gopkg.in/fatih/set.v0 v0.2.1
## explicit
gopkg.in/fatih/set.v0
@@ -1432,7 +1438,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
# sigs.k8s.io/yaml v1.2.0
## explicit; go 1.12
sigs.k8s.io/yaml
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231012115531-f16047235f44
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231012115531-f16047235f44 => /root/projects/cloudmux
## explicit; go 1.18
yunion.io/x/cloudmux/pkg/apis
yunion.io/x/cloudmux/pkg/apis/billing
@@ -1512,6 +1518,8 @@ yunion.io/x/cloudmux/pkg/multicloud/remotefile
yunion.io/x/cloudmux/pkg/multicloud/remotefile/provider
yunion.io/x/cloudmux/pkg/multicloud/ucloud
yunion.io/x/cloudmux/pkg/multicloud/ucloud/provider
yunion.io/x/cloudmux/pkg/multicloud/volcengine
yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider
yunion.io/x/cloudmux/pkg/multicloud/zstack
yunion.io/x/cloudmux/pkg/multicloud/zstack/provider
# yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32
@@ -1596,3 +1604,4 @@ yunion.io/x/sqlchemy/backends/sqlite
# yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
## explicit; go 1.12
yunion.io/x/structarg
# yunion.io/x/cloudmux => /root/projects/cloudmux

2
vendor/yunion.io/x/cloudmux/pkg/apis/compute/cloudaccount_const.go generated vendored
View File

@@ -44,6 +44,7 @@ const (
CLOUD_PROVIDER_BAIDU = "Baidu"
CLOUD_PROVIDER_CUCLOUD = "ChinaUnion"
CLOUD_PROVIDER_QINGCLOUD = "QingCloud"
CLOUD_PROVIDER_VOLCENGINE = "VolcEngine"
CLOUD_PROVIDER_GENERICS3 = "S3"
CLOUD_PROVIDER_CEPH = "Ceph"
@@ -73,4 +74,5 @@ const (
CLOUD_ACCESS_ENV_CTYUN_CHINA = CLOUD_PROVIDER_CTYUN
CLOUD_ACCESS_ENV_ECLOUD_CHINA = CLOUD_PROVIDER_ECLOUD
CLOUD_ACCESS_ENV_JDCLOUD_CHINA = CLOUD_PROVIDER_JDCLOUD
CLOUD_ACCESS_ENV_VOLCENGINE_CHINA = CLOUD_PROVIDER_VOLCENGINE
)

1
vendor/yunion.io/x/cloudmux/pkg/apis/compute/guest_const.go generated vendored
View File

@@ -77,6 +77,7 @@ const (
HYPERVISOR_BAIDU = "baidu"
HYPERVISOR_CUCLOUD = "cucloud"
HYPERVISOR_QINGCLOUD = "qingcloud"
HYPERVISOR_VOLCENGINE = "volcengine"
)
const (

1
vendor/yunion.io/x/cloudmux/pkg/apis/compute/host_const.go generated vendored
View File

@@ -46,6 +46,7 @@ const (
HOST_TYPE_BAIDU = "baidu"
HOST_TYPE_CUCLOUD = "cucloud"
HOST_TYPE_QINGCLOUD = "qingcloud"
HOST_TYPE_VOLCENGINE = "volcengine"
// # possible status
HOST_ONLINE = "online"

5
vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go generated vendored
View File

@@ -105,6 +105,11 @@ const (
STORAGE_ECLOUD_SSD = "ssd" // 高性能盘
STORAGE_ECLOUD_SSDEBS = "ssdebs" // 性能优化盘
STORAGE_ECLOUD_SYSTEM = "system" // 系统盘
// volcengine storage type
STORAGE_VOLCENGINE_FlexPL = "ESSD_FlexPL" // 极速型SSD(单盘最大IOPS 5万)
STORAGE_VOLCENGINE_PL0 = "ESSD_PL0" // 极速型SSD(单盘最大IOPS 1万)
STORAGE_VOLCENGINE_PTSSD = "PTSSD" // 性能型SSD(上一代产品)
)
const (

1
vendor/yunion.io/x/cloudmux/pkg/multicloud/loader/loader.go generated vendored
View File

@@ -43,6 +43,7 @@ import (
_ "yunion.io/x/cloudmux/pkg/multicloud/remotefile/provider" // private clouds
_ "yunion.io/x/cloudmux/pkg/multicloud/ucloud/provider" // object storages
_ "yunion.io/x/cloudmux/pkg/multicloud/zstack/provider" // private clouds
_ "yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider"
)
func init() {

416
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/bucket.go generated vendored Normal file
View File

@@ -0,0 +1,416 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"time"
tos "github.com/volcengine/ve-tos-golang-sdk/v2/tos"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/fileutils"
)
type SBucket struct {
multicloud.SBaseBucket
VolcEngineTags
region *SRegion
Name string
Location string
CreationDate time.Time
StorageClass string
}
func (b *SBucket) GetProjectId() string {
return ""
}
func (b *SBucket) GetGlobalId() string {
return b.Name
}
func (b *SBucket) GetName() string {
return b.Name
}
func (b *SBucket) GetLocation() string {
return b.Location
}
func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion {
return b.region
}
func (b *SBucket) GetCreatedAt() time.Time {
return b.CreationDate
}
func (b *SBucket) GetStorageClass() string {
return b.StorageClass
}
func (b *SBucket) GetStats() cloudprovider.SBucketStats {
stats, _ := cloudprovider.GetIBucketStats(b)
return stats
}
func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
ret := []cloudprovider.SBucketAccessUrl{
{
Url: fmt.Sprintf("%s.%s", b.Name, b.region.getTOSExternalDomain()),
Description: "ExtranetEndpoint",
Primary: true,
},
{
Url: fmt.Sprintf("%s.%s", b.Name, b.region.getTOSInternalDomain()),
Description: "IntranetEndpoint",
},
}
return ret
}
func grantToCannedAcl(acls []tos.GrantV2) cloudprovider.TBucketACLType {
switch {
case len(acls) == 1:
if acls[0].Permission == enum.PermissionFullControl {
return cloudprovider.ACLPrivate
}
case len(acls) == 2:
for _, g := range acls {
if g.GranteeV2.Type == enum.GranteeGroup && g.GranteeV2.Canned == enum.CannedAuthenticatedUsers && g.Permission == enum.PermissionRead {
return cloudprovider.ACLAuthRead
}
if g.GranteeV2.Type == enum.GranteeGroup && g.GranteeV2.Canned == enum.CannedAllUsers && g.Permission == enum.PermissionRead {
return cloudprovider.ACLPublicRead
}
}
case len(acls) == 3:
for _, g := range acls {
if g.GranteeV2.Type == enum.GranteeGroup && g.GranteeV2.Canned == enum.CannedAllUsers && g.Permission == enum.PermissionRead {
return cloudprovider.ACLPublicReadWrite
}
}
}
return cloudprovider.ACLUnknown
}
func (b *SBucket) GetAcl() cloudprovider.TBucketACLType {
acl := cloudprovider.ACLPrivate
toscli, err := b.region.GetTosClient()
if err != nil {
log.Errorf("GetTosClient fail %s", err)
return acl
}
input := tos.GetBucketACLInput{}
input.Bucket = b.Name
output, err := toscli.GetBucketACL(context.Background(), &input)
if err != nil {
log.Errorf("toscli.GetBucketAcl fail %s", err)
return acl
}
return grantToCannedAcl(output.Grants)
}
func (b *SBucket) SetAcl(aclStr cloudprovider.TBucketACLType) error {
toscli, err := b.region.GetTosClient()
if err != nil {
return errors.Wrapf(err, "Get TosClient")
}
input := tos.PutBucketACLInput{}
input.Bucket = b.Name
input.ACLType = enum.ACLType(aclStr)
_, err = toscli.PutBucketACL(context.Background(), &input)
if err != nil {
return errors.Wrapf(err, "PutBucketAcl")
}
return nil
}
func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) (string, error) {
return "", errors.ErrNotImplemented
}
func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error {
toscli, err := b.region.GetTosClient()
if err != nil {
return errors.Wrapf(err, "GetTosClient")
}
_, err = toscli.AbortMultipartUpload(ctx, &tos.AbortMultipartUploadInput{Bucket: b.Name, Key: key, UploadID: uploadId})
if err != nil {
return errors.Wrapf(err, "AbortMultipartUploadWithContext")
}
return nil
}
func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error {
toscli, err := b.region.GetTosClient()
if err != nil {
return errors.Wrap(err, "GetTosClient")
}
parts := make([]tos.UploadedPartV2, len(partEtags))
for i := range partEtags {
parts[i].PartNumber = int(i + 1)
parts[i].ETag = partEtags[i]
}
_, err = toscli.CompleteMultipartUploadV2(ctx, &tos.CompleteMultipartUploadV2Input{Bucket: b.Name, Key: key, UploadID: uploadId, Parts: parts})
if err != nil {
return errors.Wrapf(err, "CompleteMultipartUploadV2")
}
return nil
}
func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error {
toscli, err := b.region.GetTosClient()
if err != nil {
return errors.Wrap(err, "GetTosClient")
}
if len(cannedAcl) == 0 {
cannedAcl = b.GetAcl()
}
var metaDir string
metaHdr := make(map[string]string)
cacheControl := ""
if meta != nil {
for k, v := range meta {
if len(v) == 0 || len(v[0]) == 0 {
continue
}
switch http.CanonicalHeaderKey(k) {
case cloudprovider.META_HEADER_CACHE_CONTROL:
cacheControl = v[0]
case cloudprovider.META_HEADER_CONTENT_TYPE:
cacheControl = v[0]
case cloudprovider.META_HEADER_CONTENT_LANGUAGE:
cacheControl = v[0]
case cloudprovider.META_HEADER_CONTENT_ENCODING:
cacheControl = v[0]
case cloudprovider.META_HEADER_CONTENT_DISPOSITION:
cacheControl = v[0]
default:
metaHdr[k] = v[0]
}
}
metaDir = "REPLACE"
} else {
metaDir = "COPY"
}
input := tos.CopyObjectInput{Bucket: b.Name, Key: destKey, SrcKey: fmt.Sprintf("%s/%s", srcBucket, url.PathEscape(srcKey)), StorageClass: enum.StorageClassType(storageClassStr), ACL: enum.ACLType(cannedAcl), MetadataDirective: enum.MetadataDirectiveType(metaDir)}
if len(cacheControl) > 0 {
input.CacheControl = cacheControl
}
if len(metaHdr) > 0 {
input.Meta = metaHdr
}
_, err = toscli.CopyObject(ctx, &input)
if err != nil {
return errors.Wrapf(err, "CopyObject")
}
return nil
}
func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) {
toscli, err := b.region.GetTosClient()
if err != nil {
return "", errors.Wrap(err, "GetTosClient")
}
input := tos.UploadPartCopyV2Input{}
input.Bucket = b.Name
input.Key = key
input.UploadID = uploadId
input.PartNumber = partNumber
input.SrcBucket = srcBucket
input.SrcKey = srcKey
if srcLength > 0 {
input.CopySourceRange = fmt.Sprintf("bytes=%d-%d", srcOffset, srcOffset+srcLength-1)
}
output, err := toscli.UploadPartCopyV2(ctx, &input)
if err != nil {
return "", errors.Wrapf(err, "CopyPart")
}
return output.ETag, nil
}
func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, part io.Reader, partSize int64, offset, totalSize int64) (string, error) {
toscli, err := b.region.GetTosClient()
if err != nil {
return "", errors.Wrap(err, "GetTosClient")
}
input := tos.UploadPartV2Input{}
input.Bucket = b.Name
input.Key = key
input.UploadID = uploadId
input.PartNumber = int(partIndex)
seeker, err := fileutils.NewReadSeeker(part, partSize)
if err != nil {
return "", errors.Wrap(err, "newFakeSeeker")
}
defer seeker.Close()
input.Content = seeker
input.ContentLength = partSize
output, err := toscli.UploadPartV2(ctx, &input)
if err != nil {
return "", errors.Wrapf(err, "UploadPart")
}
return output.ETag, nil
}
func (b *SBucket) DeleteObject(ctx context.Context, key string) error {
toscli, err := b.region.GetTosClient()
if err != nil {
return errors.Wrap(err, "GetTosClient")
}
input := tos.DeleteObjectV2Input{}
_, err = toscli.DeleteObjectV2(ctx, &input)
if err != nil {
return errors.Wrap(err, "DeleteObject")
}
return nil
}
func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) {
toscli, err := b.region.GetTosClient()
if err != nil {
return nil, errors.Wrap(err, "GetTosClient")
}
input := tos.GetObjectV2Input{}
output, err := toscli.GetObjectV2(ctx, &input)
if err != nil {
return nil, errors.Wrap(err, "DeleteObject")
}
return output.Content, nil
}
func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}
toscli, err := b.region.GetTosClient()
if err != nil {
return result, errors.Wrap(err, "GetTosClient")
}
input := tos.ListObjectsV2Input{}
input.Bucket = b.Name
if len(prefix) > 0 {
input.Prefix = prefix
}
if len(marker) > 0 {
input.Marker = marker
}
if len(delimiter) > 0 {
input.Delimiter = delimiter
}
if maxCount > 0 {
input.MaxKeys = maxCount
}
output, err := toscli.ListObjectsV2(context.Background(), &input)
if err != nil {
return result, errors.Wrap(err, "ListObjects")
}
for _, object := range output.Contents {
obj := &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{
StorageClass: string(object.StorageClass),
Key: object.Key,
SizeBytes: object.Size,
ETag: object.ETag,
LastModified: object.LastModified,
},
}
result.Objects = append(result.Objects, obj)
}
if output.CommonPrefixes != nil {
result.CommonPrefixes = make([]cloudprovider.ICloudObject, len(output.CommonPrefixes))
for i, commonPrefix := range output.CommonPrefixes {
result.CommonPrefixes[i] = &SObject{
bucket: b,
SBaseCloudObject: cloudprovider.SBaseCloudObject{Key: commonPrefix.Prefix},
}
}
}
result.IsTruncated = output.IsTruncated
result.NextMarker = output.NextMarker
return result, nil
}
func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
return "", errors.ErrNotImplemented
}
func (b *SBucket) PutObject(ctx context.Context, key string, body io.Reader, sizeBytes int64, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error {
if sizeBytes < 0 {
return errors.Error("context length expected")
}
toscli, err := b.region.GetTosClient()
if err != nil {
return errors.Wrapf(err, "GetTosClient")
}
input := tos.PutObjectV2Input{}
input.Bucket = b.Name
input.Key = key
seeker, err := fileutils.NewReadSeeker(body, sizeBytes)
if err != nil {
return errors.Wrap(err, "newFakeSeeker")
}
defer seeker.Close()
input.Content = body
input.ContentLength = sizeBytes
if meta != nil {
metaHdr := make(map[string]string)
for k, v := range meta {
if len(v) == 0 || len(v[0]) == 0 {
continue
}
switch http.CanonicalHeaderKey(k) {
case cloudprovider.META_HEADER_CACHE_CONTROL:
input.CacheControl = v[0]
case cloudprovider.META_HEADER_CONTENT_TYPE:
input.ContentType = v[0]
case cloudprovider.META_HEADER_CONTENT_MD5:
input.ContentMD5 = v[0]
case cloudprovider.META_HEADER_CONTENT_LANGUAGE:
input.ContentEncoding = v[0]
case cloudprovider.META_HEADER_CONTENT_ENCODING:
input.ContentDisposition = v[0]
default:
metaHdr[k] = v[0]
}
}
if len(metaHdr) > 0 {
input.Meta = metaHdr
}
}
if len(cannedAcl) > 0 {
cannedAcl = b.GetAcl()
}
input.ACL = enum.ACLType(cannedAcl)
if len(storageClassStr) > 0 {
input.StorageClass = enum.StorageClassType(storageClassStr)
}
_, err = toscli.PutObjectV2(ctx, &input)
if err != nil {
return errors.Wrapf(err, "PutObject")
}
return nil
}

48
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/charge.go generated vendored Normal file
View File

@@ -0,0 +1,48 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"time"
api "yunion.io/x/cloudmux/pkg/apis/billing"
)
const (
PrePaidInstanceChargeType TChargeType = "PrePaid"
PostPaidInstanceChargeType TChargeType = "PostPaid"
DefaultInstanceChargeType = PostPaidInstanceChargeType
)
func convertChargeType(ct TChargeType) string {
switch ct {
case PrePaidInstanceChargeType:
return api.BILLING_TYPE_PREPAID
case PostPaidInstanceChargeType:
return api.BILLING_TYPE_POSTPAID
default:
return ""
}
}
func convertExpiredAt(expired time.Time) time.Time {
if !expired.IsZero() {
now := time.Now()
if expired.Sub(now) < time.Hour*24*365*6 {
return expired
}
}
return time.Time{}
}

334
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/disk.go generated vendored Normal file
View File

@@ -0,0 +1,334 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"context"
"fmt"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
)
type SDisk struct {
storage *SStorage
multicloud.SDisk
VolcEngineTags
ZoneId string
VolumeId string
VolumeName string
VolumeType string
Description string
InstanceId string
ImageId string
Size int
Status string
Kind string
CreatedAt time.Time
UpdatedAt time.Time
BillingType TChargeType
PayType string
TradeStatus int
ExpiredTime time.Time
DeleteWithInstance bool
}
func (disk *SDisk) GetId() string {
return disk.VolumeId
}
func (disk *SDisk) Delete(ctx context.Context) error {
_, err := disk.storage.zone.region.getDisk(disk.VolumeId)
if err != nil {
if errors.Cause(err) == cloudprovider.ErrNotFound {
return nil
}
return errors.Wrapf(err, "Failed to find disk %s when delete", disk.VolumeId)
}
for {
err := disk.storage.zone.region.DeleteDisk(disk.VolumeId)
if err != nil {
if isError(err, "IncorrectDiskStatus") {
log.Infof("The disk is initializing, try later ...")
time.Sleep(10 * time.Second)
} else {
return errors.Wrapf(err, "DeleteDisk fail")
}
} else {
break
}
}
return cloudprovider.WaitDeleted(disk, 10*time.Second, 300*time.Second) // 5minutes
}
func (disk *SDisk) Resize(ctx context.Context, sizeMb int64) error {
return disk.storage.zone.region.ResizeDisk(disk.VolumeId, sizeMb/1024)
}
func (disk *SDisk) GetName() string {
if len(disk.VolumeName) > 0 {
return disk.VolumeName
}
return disk.VolumeId
}
func (disk *SDisk) GetGlobalId() string {
return disk.VolumeId
}
func (disk *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) {
return disk.storage, nil
}
func (disk *SDisk) GetStatus() string {
switch disk.Status {
case "creating":
return api.DISK_ALLOCATING
default:
return api.DISK_READY
}
}
func (disk *SDisk) Refresh() error {
new, err := disk.storage.zone.region.getDisk(disk.VolumeId)
if err != nil {
return err
}
return jsonutils.Update(disk, new)
}
func (disk *SDisk) ResizeDisk(newSize int64) error {
// newSize 单位为 GB. 只能扩容，不能缩减。范围参考下面链接。
// https://www.volcengine.com/docs/6396/76561
return disk.storage.zone.region.ResizeDisk(disk.VolumeId, newSize)
}
func (disk *SDisk) GetDiskFormat() string {
return "vhd"
}
func (disk *SDisk) GetDiskSizeMB() int {
return disk.Size * 1024
}
func (disk *SDisk) GetIsAutoDelete() bool {
return disk.DeleteWithInstance
}
func (disk *SDisk) GetTemplateId() string {
return disk.ImageId
}
func (disk *SDisk) GetDiskType() string {
switch disk.Kind {
case "system":
return api.DISK_TYPE_SYS
case "data":
return api.DISK_TYPE_DATA
default:
return api.DISK_TYPE_DATA
}
}
func (disk *SDisk) GetFsFormat() string {
return ""
}
func (disk *SDisk) GetIsNonPersistent() bool {
return false
}
func (disk *SDisk) GetDriver() string {
return "scsi"
}
func (disk *SDisk) GetCacheMode() string {
return "none"
}
func (disk *SDisk) GetMountpoint() string {
return ""
}
func (disk *SDisk) GetISnapshot(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetISnapshot")
}
func (disk *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetISnapshots")
}
func (disk *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) {
return "", disk.storage.zone.region.resetDisk(disk.VolumeId, snapshotId)
}
func (disk *SDisk) GetBillingType() string {
return convertChargeType(disk.BillingType)
}
func (disk *SDisk) GetCreatedAt() time.Time {
return disk.CreatedAt
}
func (disk *SDisk) GetExtSnapshotPolicyIds() ([]string, error) {
return nil, errors.ErrNotImplemented
}
func (disk *SDisk) GetExpiredAt() time.Time {
return convertExpiredAt(disk.ExpiredTime)
}
func (disk *SDisk) GetAccessPath() string {
return ""
}
func (disk *SDisk) Rebuild(ctx context.Context) error {
return errors.Wrapf(cloudprovider.ErrNotImplemented, "Rebuild")
}
func (disk *SDisk) GetProjectId() string {
return ""
}
// Snapshot API is not supported, refer to
// https://www.volcengine.com/docs/6460/195549
func (disk *SDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) {
snapshotId, err := disk.storage.zone.region.CreateSnapshot(disk.VolumeId, name, desc)
if err != nil {
return nil, errors.Wrapf(err, "CreateSnapshot")
}
snapshot, err := disk.storage.zone.region.GetISnapshotById(snapshotId)
if err != nil {
return nil, errors.Wrapf(err, "getSnapshot(%s)", snapshotId)
}
err = cloudprovider.WaitStatus(snapshot, api.SNAPSHOT_READY, 15*time.Second, 3600*time.Second)
if err != nil {
return nil, errors.Wrapf(err, "cloudprovider.WaitStatus")
}
return snapshot, nil
}
// region
func (region *SRegion) GetDisks(instanceId string, zoneId string, category string, diskIds []string, pageNumber int, pageSize int) ([]SDisk, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
if len(instanceId) > 0 {
params["InstanceId"] = instanceId
}
if len(zoneId) > 0 {
params["ZoneId"] = zoneId
}
if len(category) > 0 {
params["VolumeType"] = category
}
if len(diskIds) > 0 {
for index, id := range diskIds {
key := fmt.Sprintf("VolumeIds.%d", index+1)
params[key] = id
}
}
body, err := region.storageRequest("DescribeVolumes", params)
if err != nil {
return nil, 0, errors.Wrap(err, "GetDisks fail")
}
disks := make([]SDisk, 0)
err = body.Unmarshal(&disks, "Result", "Volumes")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal disk details fail")
}
total, _ := body.Int("Result", "TotalCount")
return disks, int(total), nil
}
func (region *SRegion) CreateDisk(zoneId string, category string, name string, sizeGb int, desc string, projectId string) (string, error) {
params := make(map[string]string)
params["ZoneId"] = zoneId
params["VolumeName"] = name
if len(desc) > 0 {
params["Description"] = desc
}
params["VolumeType"] = category
// only data disk is supported
params["Kind"] = "data"
params["Size"] = fmt.Sprintf("%d", sizeGb)
params["ClientToken"] = utils.GenRequestId(20)
body, err := region.storageRequest("CreateVolume", params)
if err != nil {
return "", err
}
return body.GetString("Result", "VolumeId")
}
func (region *SRegion) getDisk(diskId string) (*SDisk, error) {
disks, _, err := region.GetDisks("", "", "", []string{diskId}, 1, 50)
if err != nil {
return nil, errors.Wrapf(err, fmt.Sprint("%s not found", diskId))
}
for _, disk := range disks {
if disk.VolumeId == diskId {
return &disk, nil
}
}
return nil, errors.Wrapf(cloudprovider.ErrNotFound, fmt.Sprint("%s not found", diskId))
}
func (region *SRegion) DeleteDisk(diskId string) error {
params := make(map[string]string)
params["VolumeId"] = diskId
_, err := region.storageRequest("DeleteVolume", params)
return err
}
func (region *SRegion) ResizeDisk(diskId string, sizeGb int64) error {
params := make(map[string]string)
params["VolumeId"] = diskId
params["NewSize"] = fmt.Sprintf("%d", sizeGb)
_, err := region.storageRequest("ExtendVolume", params)
if err != nil {
return errors.Wrapf(err, "Resizing disk (%s) to %d GiB failed", diskId, sizeGb)
}
return nil
}
func (region *SRegion) resetDisk(diskId, snapshotId string) error {
// not supported API
return errors.Wrapf(cloudprovider.ErrNotImplemented, "resetDisk")
}
func (region *SRegion) CreateSnapshot(diskId, name, desc string) (string, error) {
return "", errors.Wrapf(cloudprovider.ErrNotImplemented, "CreateSnapshot")
}

388
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/eip.go generated vendored Normal file
View File

@@ -0,0 +1,388 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"strings"
"time"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
)
type TBillingType int
const (
BillingByPrePaid TBillingType = iota + 1
BillingByBandwidth
BillingByTraffic
)
const (
EIP_STATUS_ATTACHING = "Attaching"
EIP_STATUS_DETACHING = "Detaching"
EIP_STATUS_ATTACHED = "Attached"
EIP_STATUS_AVAILABLE = "Available"
EIP_STATUS_DELETING = "Deleting"
EIP_INSTANCE_TYPE_NAT = "Nat" // NAT网关
EIP_INSTANCE_TYPE_ENI = "NetworkInterface" // 辅助网卡
EIP_INSTANCE_TYPE_CLB = "ClbInstance" // 负载均衡
EIP_INSTANCE_TYPE_ALB = "Albinstance" // 应用型负载均衡
EIP_INSTANCE_TYPE_ECS = "EcsInstance" // 云服务器
EIP_INSTANCE_TYPE_HAVIP = "HaVip" // 高可用虚拟IP
)
type SEipAddress struct {
region *SRegion
multicloud.SEipBase
VolcEngineTags
Name string
AllocationId string
BillingType TBillingType
EipAddress string
Status string
InstanceType string
InstanceId string
Bandwidth int /* Mbps */
BusinessStatus string
AllocationTime time.Time
Description string
ISP string
LockReason string
ExpiredTime time.Time
ProjectName string
}
func (eipaddr *SEipAddress) GetId() string {
return eipaddr.AllocationId
}
func (eipaddr *SEipAddress) GetName() string {
if len(eipaddr.Name) > 0 {
return eipaddr.Name
}
return eipaddr.EipAddress
}
func (eipaddr *SEipAddress) GetGlobalId() string {
return eipaddr.AllocationId
}
func (eipaddr *SEipAddress) GetStatus() string {
switch eipaddr.Status {
case EIP_STATUS_ATTACHED, EIP_STATUS_AVAILABLE:
return api.EIP_STATUS_READY
case EIP_STATUS_ATTACHING:
return api.EIP_STATUS_ASSOCIATE
case EIP_STATUS_DETACHING:
return api.EIP_STATUS_DISSOCIATE
case EIP_STATUS_DELETING:
return api.EIP_STATUS_DEALLOCATE
default:
return api.EIP_STATUS_UNKNOWN
}
}
func (eipaddr *SEipAddress) Refresh() error {
if eipaddr.IsEmulated() {
return nil
}
new, err := eipaddr.region.GetEip(eipaddr.AllocationId)
if err != nil {
return err
}
return jsonutils.Update(eipaddr, new)
}
func (eipaddr *SEipAddress) GetProjectId() string {
return eipaddr.ProjectName
}
func (eipaddr *SEipAddress) GetIpAddr() string {
return eipaddr.EipAddress
}
func (eipaddr *SEipAddress) GetMode() string {
if eipaddr.InstanceId == eipaddr.AllocationId {
return api.EIP_MODE_INSTANCE_PUBLICIP
} else {
return api.EIP_MODE_STANDALONE_EIP
}
}
func (eipaddr *SEipAddress) GetINetworkId() string {
return ""
}
func (eipaddr *SEipAddress) GetAssociationType() string {
switch eipaddr.InstanceType {
case EIP_INSTANCE_TYPE_ECS, EIP_INSTANCE_TYPE_ENI:
return api.EIP_ASSOCIATE_TYPE_SERVER
case EIP_INSTANCE_TYPE_NAT:
return api.EIP_ASSOCIATE_TYPE_NAT_GATEWAY
case EIP_INSTANCE_TYPE_ALB, EIP_INSTANCE_TYPE_CLB:
return api.EIP_ASSOCIATE_TYPE_LOADBALANCER
default:
return eipaddr.InstanceType
}
}
func (eipaddr *SEipAddress) GetAssociationExternalId() string {
return eipaddr.InstanceId
}
func (eipaddr *SEipAddress) GetBandwidth() int {
return eipaddr.Bandwidth
}
func (eipaddr *SEipAddress) GetInternetChargeType() string {
switch eipaddr.BillingType {
case BillingByPrePaid, BillingByBandwidth:
return api.EIP_CHARGE_TYPE_BY_BANDWIDTH
case BillingByTraffic:
return api.EIP_CHARGE_TYPE_BY_TRAFFIC
default:
return api.EIP_CHARGE_TYPE_BY_BANDWIDTH
}
}
func (eipaddr *SEipAddress) Delete() error {
return eipaddr.region.DeallocateEIP(eipaddr.AllocationId)
}
func (eipaddr *SEipAddress) Associate(conf *cloudprovider.AssociateConfig) error {
_ = cloudprovider.Wait(20*time.Second, time.Minute, func() (bool, error) {
err := eipaddr.region.AssociateEip(eipaddr.AllocationId, conf.InstanceId)
if err != nil {
if isError(err, "IncorrectInstanceStatus") {
return false, nil
}
return false, errors.Wrapf(err, "region.AssociateEip")
}
return true, nil
})
err := cloudprovider.WaitStatus(eipaddr, api.EIP_STATUS_READY, 10*time.Second, 180*time.Second)
return err
}
func (eipaddr *SEipAddress) Dissociate() error {
err := eipaddr.region.DissociateEip(eipaddr.AllocationId, eipaddr.InstanceId)
if err != nil {
return err
}
err = cloudprovider.WaitStatus(eipaddr, api.EIP_STATUS_READY, 10*time.Second, 180*time.Second)
return err
}
func (eipaddr *SEipAddress) ChangeBandwidth(bw int) error {
return eipaddr.region.UpdateEipBandwidth(eipaddr.AllocationId, bw)
}
func getInstanceType(instanceId string) (string, error) {
prefixMap := map[string]string{
"i-": EIP_INSTANCE_TYPE_ECS,
"clb-": EIP_INSTANCE_TYPE_CLB,
"alb-": EIP_INSTANCE_TYPE_ALB,
"ngw-": EIP_INSTANCE_TYPE_NAT,
"eni-": EIP_INSTANCE_TYPE_ENI,
"havip-": EIP_INSTANCE_TYPE_HAVIP,
}
for prefix, instanceType := range prefixMap {
if strings.HasPrefix(instanceId, prefix) {
return instanceType, nil
}
}
return "", errors.Errorf("Unknown instance type for %s", instanceId)
}
func (region *SRegion) GetEips(eipIds []string, associatedId string, addresses []string, pageNumber int, pageSize int) ([]SEipAddress, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
for index, addr := range addresses {
params[fmt.Sprintf("EipAddresses.%d", index+1)] = addr
}
for index, eipId := range eipIds {
params[fmt.Sprintf("AllocationIds.%d", index+1)] = eipId
}
if len(associatedId) > 0 {
params["AssociatedInstanceId"] = associatedId
associatedType, err := getInstanceType(associatedId)
if err != nil {
return nil, 0, errors.Wrapf(err, "Unknown associated type")
}
params["AssociatedInstanceType"] = associatedType
}
body, err := region.vpcRequest("DescribeEipAddresses", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "DescribeEipAddresses fail")
}
eips := make([]SEipAddress, 0)
err = body.Unmarshal(&eips, "Result", "EipAddresses")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal EipAddress details fail")
}
total, _ := body.Int("TotalCount")
for i := 0; i < len(eips); i += 1 {
eips[i].region = region
}
return eips, int(total), nil
}
func (region *SRegion) GetEip(eipId string) (*SEipAddress, error) {
eips, _, err := region.GetEips([]string{eipId}, "", make([]string, 0), 1, 1)
if err != nil {
return nil, err
}
for i := range eips {
if eips[i].AllocationId == eipId {
eips[i].region = region
return &eips[i], nil
}
}
return nil, errors.Wrapf(cloudprovider.ErrNotFound, eipId)
}
func (region *SRegion) AllocateEIP(opts *cloudprovider.SEip) (*SEipAddress, error) {
params := make(map[string]string)
if len(opts.Name) > 0 {
params["Name"] = opts.Name
}
params["Bandwidth"] = fmt.Sprintf("%d", opts.BandwidthMbps)
switch opts.ChargeType {
case api.EIP_CHARGE_TYPE_BY_TRAFFIC:
params["BillingType"] = fmt.Sprintf("%d", BillingByTraffic)
case api.EIP_CHARGE_TYPE_BY_BANDWIDTH:
params["BillingType"] = fmt.Sprintf("%d", BillingByBandwidth)
}
params["ClientToken"] = utils.GenRequestId(20)
if len(opts.ProjectId) > 0 {
params["ProjectName"] = opts.ProjectId
}
params["ISP"] = "BGP"
index := 1
for key, value := range opts.Tags {
params[fmt.Sprintf("Tags.%d.Key", index)] = key
params[fmt.Sprintf("Tags.%d.Value", index)] = value
index++
}
body, err := region.vpcRequest("AllocateEipAddress", params)
if err != nil {
return nil, errors.Wrapf(err, "AllocateEipAddress fail")
}
eipId, err := body.GetString("Result", "AllocationId")
if err != nil {
return nil, errors.Wrapf(err, "get AllocationId after created fail")
}
err = cloudprovider.Wait(5*time.Second, time.Minute, func() (bool, error) {
_, err := region.GetEip(eipId)
if errors.Cause(err) == cloudprovider.ErrNotFound {
return false, nil
} else {
return true, err
}
})
if err != nil {
return nil, errors.Wrapf(err, "cannot find eip after create")
}
return region.GetEip(eipId)
}
func (region *SRegion) CreateEIP(opts *cloudprovider.SEip) (cloudprovider.ICloudEIP, error) {
return region.AllocateEIP(opts)
}
func (region *SRegion) DeallocateEIP(eipId string) error {
params := make(map[string]string)
params["AllocationId"] = eipId
_, err := region.vpcRequest("ReleaseEipAddress", params)
if err != nil {
err = errors.Wrapf(err, "ReleaseEipAddress fail")
}
return err
}
func (region *SRegion) AssociateEip(eipId string, instanceId string) error {
params := make(map[string]string)
params["AllocationId"] = eipId
params["InstanceId"] = instanceId
instanceType, err := getInstanceType(instanceId)
if err != nil {
return errors.Wrapf(err, "Unknown instance type")
}
params["InstanceType"] = instanceType
_, err = region.vpcRequest("AssociateEipAddress", params)
return errors.Wrapf(err, "AssociateEipAddress fail")
}
func (region *SRegion) DissociateEip(eipId string, instanceId string) error {
params := make(map[string]string)
params["AllocationId"] = eipId
params["InstanceId"] = instanceId
instanceType, err := getInstanceType(instanceId)
if err != nil {
return errors.Wrapf(err, "Unknown instance type")
}
params["InstanceType"] = instanceType
_, err = region.vpcRequest("DisassociateEipAddress", params)
if err != nil {
err = errors.Wrapf(err, "DisassociateEipAddress fail")
}
return err
}
func (region *SRegion) UpdateEipBandwidth(eipId string, bw int) error {
params := make(map[string]string)
params["AllocationId"] = eipId
params["Bandwidth"] = fmt.Sprintf("%d", bw)
_, err := region.vpcRequest("ModifyEipAddressAttributes", params)
if err != nil {
err = errors.Wrapf(err, "ModifyEipAddressAttributes fail")
}
return err
}

29
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/errors.go generated vendored Normal file
View File

@@ -0,0 +1,29 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"strings"
)
func isError(err error, code string) bool {
errStr := fmt.Sprintf("%s", err)
if strings.Index(errStr, code) > 0 {
return true
} else {
return false
}
}

250
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/host.go generated vendored Normal file
View File

@@ -0,0 +1,250 @@
package volcengine
import (
"fmt"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/billing"
)
type SHost struct {
multicloud.SHostBase
zone *SZone
}
func (host *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) {
return host.zone.GetIWires()
}
func (host *SHost) GetIStorages() ([]cloudprovider.ICloudStorage, error) {
return host.zone.GetIStorages()
}
func (host *SHost) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) {
return host.zone.GetIStorageById(id)
}
func (host *SHost) GetId() string {
return fmt.Sprintf("%s-%s", host.zone.region.client.cpcfg.Id, host.zone.GetId())
}
func (host *SHost) GetName() string {
return fmt.Sprintf("%s-%s", host.zone.region.client.cpcfg.Name, host.zone.GetId())
}
func (host *SHost) GetGlobalId() string {
return fmt.Sprintf("%s-%s", host.zone.region.client.cpcfg.Id, host.zone.GetId())
}
func (host *SHost) IsEmulated() bool {
return true
}
func (host *SHost) GetStatus() string {
return api.HOST_STATUS_RUNNING
}
func (host *SHost) Refresh() error {
return nil
}
func (host *SHost) GetHostStatus() string {
return api.HOST_ONLINE
}
func (host *SHost) GetEnabled() bool {
return true
}
func (host *SHost) GetAccessIp() string {
return ""
}
func (host *SHost) GetAccessMac() string {
return ""
}
func (host *SHost) GetSysInfo() jsonutils.JSONObject {
info := jsonutils.NewDict()
info.Add(jsonutils.NewString(CLOUD_PROVIDER_VOLCENGINE), "manufacture")
return info
}
func (host *SHost) GetSN() string {
return ""
}
func (host *SHost) GetCpuCount() int {
return 0
}
func (host *SHost) GetNodeCount() int8 {
return 0
}
func (host *SHost) GetCpuDesc() string {
return ""
}
func (host *SHost) GetCpuMhz() int {
return 0
}
func (host *SHost) GetMemSizeMB() int {
return 0
}
func (host *SHost) GetStorageSizeMB() int {
return 0
}
func (host *SHost) GetStorageType() string {
return api.DISK_TYPE_HYBRID
}
func (host *SHost) GetHostType() string {
return api.HOST_TYPE_VOLCENGINE
}
func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) {
wires, err := host.zone.GetIWires()
if err != nil {
return nil, errors.Wrap(err, "GetIWires")
}
return cloudprovider.GetHostNetifs(host, wires), nil
}
func (host *SHost) GetIsMaintenance() bool {
return false
}
func (host *SHost) GetVersion() string {
return VOLCENGINE_API_VERSION
}
func (host *SHost) GetIVMs() ([]cloudprovider.ICloudVM, error) {
vms := make([]SInstance, 0)
token := ""
for {
parts, nextToken, err := host.zone.region.GetInstances(host.zone.ZoneId, nil, 10, token)
if err != nil {
return nil, err
}
vms = append(vms, parts...)
if len(nextToken) == 0 {
break
}
token = nextToken
}
ivms := make([]cloudprovider.ICloudVM, len(vms))
for i := 0; i < len(vms); i += 1 {
vms[i].host = host
ivms[i] = &vms[i]
}
return ivms, nil
}
func (host *SHost) GetIVMById(gid string) (cloudprovider.ICloudVM, error) {
id := gid
parts, _, err := host.zone.region.GetInstances(host.zone.ZoneId, []string{id}, 1, "")
if err != nil {
return nil, err
}
if len(parts) == 0 {
return nil, cloudprovider.ErrNotFound
}
if len(parts) > 1 {
return nil, cloudprovider.ErrDuplicateId
}
parts[0].host = host
return &parts[0], nil
}
func (host *SHost) GetInstanceById(instanceId string) (*SInstance, error) {
inst, err := host.zone.region.GetInstance(instanceId)
if err != nil {
return nil, err
}
inst.host = host
return inst, nil
}
func (host *SHost) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (cloudprovider.ICloudVM, error) {
vmId, err := host._createVM(desc.Name, desc.Hostname, desc.ExternalImageId, desc.SysDisk, desc.Cpu, desc.MemoryMB,
desc.InstanceType, desc.ExternalNetworkId, desc.IpAddr, desc.Description, desc.Password,
desc.DataDisks, desc.PublicKey, desc.ExternalSecgroupId, desc.UserData, desc.BillingCycle,
desc.ProjectId, desc.Tags, desc.SPublicIpInfo)
if err != nil {
return nil, err
}
vm, err := host.GetInstanceById(vmId)
if err != nil {
return nil, errors.Wrapf(err, "GetInstanceById")
}
return vm, nil
}
func (host *SHost) _createVM(name string, hostname string, imgId string,
sysDisk cloudprovider.SDiskInfo, cpu int, memMB int, instanceType string,
networkID string, ipAddr string, desc string, passwd string,
dataDisks []cloudprovider.SDiskInfo, publicKey string, secgroupId string,
userData string, bc *billing.SBillingCycle, projectId string,
tags map[string]string, publicIp cloudprovider.SPublicIpInfo,
) (string, error) {
var err error
keypair := ""
if len(publicKey) > 0 {
keypair, err = host.zone.region.syncKeypair(publicKey)
if err != nil {
return "", err
}
}
img, err := host.zone.region.GetImage(imgId)
if err != nil {
return "", errors.Wrapf(err, "GetImage fail")
}
if img.Status != ImageStatusAvailable {
log.Errorf("image %s status %s", imgId, img.Status)
return "", fmt.Errorf("image not ready")
}
disks := make([]SDisk, len(dataDisks)+1)
disks[0].Size = img.Size
if sysDisk.SizeGB > 0 && sysDisk.SizeGB > img.Size {
disks[0].Size = sysDisk.SizeGB
}
storage, err := host.zone.getStorageByCategory(sysDisk.StorageType)
if err != nil {
return "", fmt.Errorf("storage %s not avaiable: %s", sysDisk.StorageType, err)
}
disks[0].VolumeType = storage.storageType
for i, dataDisk := range dataDisks {
disks[i+1].Size = dataDisk.SizeGB
storage, err := host.zone.getStorageByCategory(dataDisk.StorageType)
if err != nil {
return "", fmt.Errorf("storage %s not avaiable: %s", dataDisk.StorageType, err)
}
disks[i+1].VolumeType = storage.storageType
}
_, err = host.zone.region.GetSecurityGroupDetails(secgroupId)
if err != nil {
return "", errors.Wrapf(err, "GetSecurityGroup fail")
}
if len(instanceType) == 0 {
return "", fmt.Errorf("instance type must be specified")
}
vmId, err := host.zone.region.CreateInstance(name, hostname, imgId, instanceType, secgroupId, host.zone.ZoneId, desc, passwd, disks, networkID, ipAddr, keypair, userData, bc, projectId, tags)
if err != nil {
return "", errors.Wrapf(err, "Failed to create %s", instanceType)
}
return vmId, nil
}

320
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/image.go generated vendored Normal file
View File

@@ -0,0 +1,320 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"context"
"fmt"
"strings"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/imagetools"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
)
type ImageStatusType string
const (
ImageStatusCreating ImageStatusType = "creating"
ImageStatusAvailable ImageStatusType = "available"
ImageStatusError ImageStatusType = "error"
)
type ImageOwnerType string
const (
ImageOwnerPrivate ImageOwnerType = "private"
ImageOwnerShared ImageOwnerType = "shared"
ImageOwnerPublic ImageOwnerType = "public"
)
type SImage struct {
multicloud.SImageBase
VolcEngineTags
storageCache *SStoragecache
// normalized image info
imgInfo *imagetools.ImageInfo
Architecture string
CreationTime time.Time
Description string
ImageId string
ImageName string
OSName string
OSType string
Visibility string
IsSupportCloudinit bool
IsSupportIoOptimized bool
Platform string
Size int
Status ImageStatusType
Usage string
}
func (img *SImage) GetMinRamSizeMb() int {
return 0
}
func (img *SImage) GetId() string {
return img.ImageId
}
func (img *SImage) GetName() string {
return img.ImageName
}
func (img *SImage) Delete(ctx context.Context) error {
return img.storageCache.region.DeleteImage(img.ImageId)
}
func (img *SImage) GetGlobalId() string {
return img.ImageId
}
func (img *SImage) GetIStoragecache() cloudprovider.ICloudStoragecache {
return img.storageCache
}
func (img *SImage) GetStatus() string {
switch img.Status {
case ImageStatusCreating:
return api.CACHED_IMAGE_STATUS_SAVING
case ImageStatusAvailable:
return api.CACHED_IMAGE_STATUS_ACTIVE
case ImageStatusError:
return api.CACHED_IMAGE_STATUS_CACHE_FAILED
default:
return api.CACHED_IMAGE_STATUS_CACHE_FAILED
}
}
func (region *SRegion) ImportImage(name string, osArch string, osType string, platform, platformVersion string, bucket string, key string) (string, error) {
params := map[string]string{
"Architecture": osArch,
"OsType": osType,
"Platform": platform,
"PlatformVersion": platformVersion,
"Tags.1.Key": "Name",
"Tags.2.Value": name,
"Url": fmt.Sprintf("https://%s.%s/%s", bucket, region.getS3Endpoint(), key),
}
body, err := region.ecsRequest("ImportImage", params)
if err != nil {
return "", errors.Wrapf(err, "ImportImage")
}
imageId, err := body.GetString("Result", "ImageId")
if err != nil {
return "", errors.Wrap(err, "Unmarsh imageId failed")
}
return imageId, nil
}
func (region *SRegion) ExportImage(imageId, bucketName string) (string, error) {
params := make(map[string]string)
params["RegionId"] = region.RegionId
params["ImageId"] = imageId
params["OssBucket"] = bucketName
params["OssPrefix"] = fmt.Sprintf("%sexport", strings.Replace(imageId, "-", "", -1))
body, err := region.ecsRequest("ExportImage", params)
if err != nil {
return "", errors.Wrapf(err, "ExportImage")
}
taskId, err := body.GetString("Result", "TaskId")
if err != nil {
return "", errors.Wrapf(err, "Unmarshal")
}
return taskId, nil
}
func (img *SImage) GetImageStatus() string {
switch img.Status {
case ImageStatusCreating:
return cloudprovider.IMAGE_STATUS_QUEUED
case ImageStatusAvailable:
return cloudprovider.IMAGE_STATUS_ACTIVE
case ImageStatusError:
return cloudprovider.IMAGE_STATUS_DELETED
default:
return cloudprovider.IMAGE_STATUS_KILLED
}
}
func (img *SImage) Refresh() error {
new, err := img.storageCache.region.GetImage(img.ImageId)
if err != nil {
return err
}
return jsonutils.Update(img, new)
}
func (img *SImage) GetImageType() cloudprovider.TImageType {
switch img.Visibility {
case string(ImageOwnerPublic):
return cloudprovider.ImageTypeSystem
case string(ImageOwnerPrivate), string(ImageOwnerShared):
return cloudprovider.ImageTypeCustomized
default:
return cloudprovider.ImageTypeCustomized
}
}
func (img *SImage) GetSizeByte() int64 {
return int64(img.Size) * 1024 * 1024 * 1024
}
func (img *SImage) GetOsType() cloudprovider.TOsType {
return cloudprovider.TOsType(img.getNormalizedImageInfo().OsType)
}
func (img *SImage) GetOsDist() string {
return img.getNormalizedImageInfo().OsDistro
}
func (img *SImage) getNormalizedImageInfo() *imagetools.ImageInfo {
if img.imgInfo == nil {
imgInfo := imagetools.NormalizeImageInfo(img.OSName, img.Architecture, img.OSType, img.Platform, "")
img.imgInfo = &imgInfo
}
return img.imgInfo
}
func (img *SImage) GetFullOsName() string {
return img.OSName
}
func (img *SImage) GetOsVersion() string {
return img.getNormalizedImageInfo().OsVersion
}
func (img *SImage) GetOsLang() string {
return img.getNormalizedImageInfo().OsLang
}
func (img *SImage) GetOsArch() string {
return img.getNormalizedImageInfo().OsArch
}
func (img *SImage) GetBios() cloudprovider.TBiosType {
return cloudprovider.ToBiosType(img.getNormalizedImageInfo().OsBios)
}
func (img *SImage) GetMinOsDiskSizeGb() int {
return 40
}
func (img *SImage) GetImageFormat() string {
return "vhd"
}
func (img *SImage) GetCreatedAt() time.Time {
return img.CreationTime
}
func (region *SRegion) GetImage(imageId string) (*SImage, error) {
images, _, err := region.GetImages("", "", []string{imageId}, "", 1, "")
if err != nil {
return nil, err
}
if len(images) == 0 {
return nil, cloudprovider.ErrNotFound
}
return &images[0], nil
}
func (region *SRegion) GetImageByName(name string) (*SImage, error) {
images, _, err := region.GetImages("", "", nil, name, 1, "")
if err != nil {
return nil, err
}
if len(images) == 0 {
return nil, cloudprovider.ErrNotFound
}
return &images[0], nil
}
func (region *SRegion) GetImageStatus(imageId string) (ImageStatusType, error) {
image, err := region.GetImage(imageId)
if err != nil {
return "", err
}
return image.Status, nil
}
func (region *SRegion) GetImages(status ImageStatusType, owner ImageOwnerType, imageId []string, name string, limit int, token string) ([]SImage, string, error) {
if limit > 100 || limit <= 0 {
limit = 100
}
params := make(map[string]string)
params["MaxResults"] = fmt.Sprintf("%d", limit)
if len(token) > 0 {
params["NextToken"] = token
}
if len(status) > 0 {
params["Status"] = string(status)
} else {
allStatus := []string{"available", "creating", "error"}
for idx, status := range allStatus {
params[fmt.Sprintf("Status.%d", idx+1)] = status
}
}
if len(imageId) > 0 {
params["ImageId"] = strings.Join(imageId, ",")
}
if len(owner) > 0 {
params["ImageOwnerAlias"] = string(owner)
}
if len(name) > 0 {
params["ImageName"] = name
}
return region.getImages(params)
}
func (region *SRegion) getImages(params map[string]string) ([]SImage, string, error) {
body, err := region.ecsRequest("DescribeImages", params)
if err != nil {
return nil, "", errors.Wrapf(err, "DescribeImages fail")
}
images := make([]SImage, 0)
err = body.Unmarshal(&images, "Result", "Images")
if err != nil {
return nil, "", errors.Wrapf(err, "Unmarshal images fail")
}
nextToken, _ := body.GetString("Result", "NextToken")
return images, nextToken, nil
}
func (region *SRegion) DeleteImage(imageId string) error {
params := make(map[string]string)
params["RegionId"] = region.RegionId
params["ImageId"] = imageId
params["Force"] = "true"
_, err := region.ecsRequest("DeleteImage", params)
if err != nil {
return errors.Wrapf(err, "DeleteImage fail")
}
return nil
}

832
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instance.go generated vendored Normal file
View File

@@ -0,0 +1,832 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"context"
"encoding/base64"
"fmt"
"strings"
"time"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/billing"
"yunion.io/x/pkg/util/cloudinit"
"yunion.io/x/pkg/util/imagetools"
"yunion.io/x/pkg/util/osprofile"
"yunion.io/x/pkg/utils"
)
const (
InstanceStatusCreating = "CREATING"
InstanceStatusRunning = "RUNNING"
InstanceStatusStopping = "STOPPING"
InstanceStatusStopped = "STOPPED"
InstanceStatusRebooting = "REBOOTING"
InstanceStatusStarting = "STARTING"
InstanceStatusRebuilding = "REBUILDING"
InstanceStatusResizing = "RESIZING"
InstanceStatusError = "ERROR"
InstanceStatusDeleting = "DELETING"
)
type TChargeType string
type SSecurityGroupIds []string
type SRdmaIPAddress []string
type SInstance struct {
multicloud.SInstanceBase
VolcEngineTags
host *SHost
osInfo *imagetools.ImageInfo
CreatedAt time.Time
UpdatedAt time.Time
InstanceId string
ZoneId string
ImageId string
Status string
InstanceName string
Description string
Hostname string
VpcId string
InstanceTypeId string
Cpus int
MemorySize int
OsName string
OsType string
NetworkInterfaces []SNetworkInterface
RdmaIpAddress SRdmaIPAddress
KeyPairName string
KeyPairId string
InstanceChargeType TChargeType
StoppedMode string
SpotStrategy string
DeploymentSetId string
EipAddress SEipAddress
ExpiredAt time.Time
Uuid string
ProjectName string
}
func billingCycle2Params(bc *billing.SBillingCycle, params map[string]string) error {
if bc.GetMonths() > 0 {
params["PeriodUnit"] = "Month"
params["Period"] = fmt.Sprintf("%d", bc.GetMonths())
} else if bc.GetWeeks() > 0 {
params["PeriodUnit"] = "Week"
params["Period"] = fmt.Sprintf("%d", bc.GetWeeks())
// renew by week is not currently supported
return fmt.Errorf("invalid renew time period %s", bc.String())
} else {
return fmt.Errorf("invalid renew time period %s", bc.String())
}
return nil
}
func (instance *SInstance) UpdatePassword(passwd string) error {
params := make(map[string]string)
params["Password"] = passwd
return instance.host.zone.region.modifyInstanceAttribute(instance.InstanceId, params)
}
func (instance *SInstance) UpdateUserData(userData string) error {
params := make(map[string]string)
params["UserData"] = userData
return instance.host.zone.region.modifyInstanceAttribute(instance.InstanceId, params)
}
func (instance *SInstance) GetUserData() (string, error) {
params := make(map[string]string)
params["InstanceId"] = instance.InstanceId
body, err := instance.host.zone.region.ecsRequest("DescribeUserData", params)
if err != nil {
return "", errors.Wrapf(err, "GetUserData")
}
userData, err := body.GetString("Result", "UserData")
if err != nil {
return "", errors.Wrapf(err, "GetUserData")
}
return userData, nil
}
func (region *SRegion) GetInstance(instanceId string) (*SInstance, error) {
instances, _, err := region.GetInstances("", []string{instanceId}, 1, "")
if err != nil {
return nil, err
}
if len(instances) == 0 {
return nil, cloudprovider.ErrNotFound
}
return &instances[0], nil
}
func (region *SRegion) GetInstances(zoneId string, ids []string, limit int, token string) ([]SInstance, string, error) {
if limit > 10 || limit <= 0 {
limit = 10
}
params := make(map[string]string)
params["MaxResults"] = fmt.Sprintf("%d", limit)
if len(token) > 0 {
params["NextToken"] = token
}
if len(zoneId) > 0 {
params["ZoneId"] = zoneId
}
if len(ids) > 0 {
for index, id := range ids {
key := fmt.Sprintf("InstanceIds.%d", index+1)
params[key] = id
}
}
body, err := region.ecsRequest("DescribeInstances", params)
if err != nil {
return nil, "", errors.Wrapf(err, "GetInstances fail")
}
instances := make([]SInstance, 0)
err = body.Unmarshal(&instances, "Result", "Instances")
if err != nil {
return nil, "", errors.Wrapf(err, "Unmarshal details fail")
}
nextToken, _ := body.GetString("Result", "NextToken")
return instances, nextToken, nil
}
func (instance *SInstance) GetIHost() cloudprovider.ICloudHost {
return instance.host
}
func (instance *SInstance) GetIHostId() string {
return instance.host.GetGlobalId()
}
func (instance *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) {
pageNumber := 1
disks := make([]SDisk, 0)
for {
parts, total, err := instance.host.zone.region.GetDisks(instance.InstanceId, "", "", nil, pageNumber, 50)
if err != nil {
return nil, err
}
disks = append(disks, parts...)
if len(disks) >= total {
break
}
pageNumber += 1
}
idisks := make([]cloudprovider.ICloudDisk, len(disks))
for i := 0; i < len(disks); i += 1 {
store, err := instance.host.zone.getStorageByCategory(disks[i].VolumeType)
if err != nil {
return nil, errors.Wrap(err, "getStorageByCategory")
}
disks[i].storage = store
idisks[i] = &disks[i]
}
return idisks, nil
}
func (instance *SInstance) GetIEIP() (cloudprovider.ICloudEIP, error) {
if len(instance.EipAddress.AllocationId) > 0 {
return instance.host.zone.region.GetEip(instance.EipAddress.AllocationId)
}
for _, nic := range instance.NetworkInterfaces {
if len(nic.AssociatedElasticIp.EipAddress) > 0 {
eip := SEipAddress{region: instance.host.zone.region}
eip.region = instance.host.zone.region
eip.EipAddress = nic.AssociatedElasticIp.EipAddress
eip.InstanceId = instance.InstanceId
eip.AllocationId = instance.InstanceId
return &eip, nil
}
}
return nil, cloudprovider.ErrNotFound
}
func (instance *SInstance) GetINics() ([]cloudprovider.ICloudNic, error) {
networkInterfaces := instance.NetworkInterfaces
nics := make([]cloudprovider.ICloudNic, 0)
for _, ni := range networkInterfaces {
nic := SInstanceNic{
instance: instance,
id: ni.NetworkInterfaceId,
ipAddr: ni.PrimaryIpAddress,
macAddr: ni.MacAddress,
}
nics = append(nics, &nic)
}
return nics, nil
}
func (instance *SInstance) GetId() string {
return instance.InstanceId
}
func (instance *SInstance) GetName() string {
if len(instance.InstanceName) > 0 {
return instance.InstanceName
}
return instance.InstanceId
}
func (instance *SInstance) GetHostname() string {
return instance.Hostname
}
func (instance *SInstance) GetGlobalId() string {
return instance.InstanceId
}
func (instance *SInstance) GetInstanceType() string {
return instance.InstanceTypeId
}
func (instance *SInstance) GetSecurityGroupIds() ([]string, error) {
ret := []string{}
for _, net := range instance.NetworkInterfaces {
ret = append(ret, net.SecurityGroupIds...)
}
return ret, nil
}
func (instance *SInstance) GetVcpuCount() int {
return instance.Cpus
}
func (instance *SInstance) GetVmemSizeMB() int {
return instance.MemorySize
}
func (instance *SInstance) GetBootOrder() string {
return "dcn"
}
func (instance *SInstance) GetVga() string {
return "std"
}
func (instance *SInstance) GetVdi() string {
return "vnc"
}
func (ins *SInstance) getNormalizedOsInfo() *imagetools.ImageInfo {
if ins.osInfo == nil {
osInfo := imagetools.NormalizeImageInfo(ins.OsName, "", ins.OsType, "", "")
ins.osInfo = &osInfo
}
return ins.osInfo
}
func (instance *SInstance) GetOsType() cloudprovider.TOsType {
return cloudprovider.TOsType(osprofile.NormalizeOSType(instance.OsType))
}
func (instance *SInstance) GetFullOsName() string {
return instance.OsName
}
func (ins *SInstance) GetBios() cloudprovider.TBiosType {
return cloudprovider.ToBiosType(ins.getNormalizedOsInfo().OsBios)
}
func (ins *SInstance) GetOsArch() string {
return ins.getNormalizedOsInfo().OsArch
}
func (ins *SInstance) GetOsDist() string {
return ins.getNormalizedOsInfo().OsDistro
}
func (ins *SInstance) GetOsVersion() string {
return ins.getNormalizedOsInfo().OsVersion
}
func (ins *SInstance) GetOsLang() string {
return ins.getNormalizedOsInfo().OsLang
}
func (instance *SInstance) GetMachine() string {
return "pc"
}
func (instance *SInstance) GetStatus() string {
switch instance.Status {
case InstanceStatusRunning:
return api.VM_RUNNING
case InstanceStatusStarting:
return api.VM_STARTING
case InstanceStatusStopping:
return api.VM_STOPPING
case InstanceStatusStopped:
return api.VM_READY
case InstanceStatusDeleting:
return api.VM_DELETING
default:
return api.VM_UNKNOWN
}
}
func (instance *SInstance) Refresh() error {
ins, err := instance.host.zone.region.GetInstance(instance.InstanceId)
if err != nil {
return err
}
return jsonutils.Update(instance, ins)
}
func (instance *SInstance) GetHypervisor() string {
return api.HYPERVISOR_VOLCENGINE
}
func (instance *SInstance) GetCreatedAt() time.Time {
return instance.CreatedAt
}
func (instance *SInstance) GetExpiredAt() time.Time {
// return instance.ExpiredAt
return time.Time{}
}
func (instance *SInstance) AssignSecurityGroup(secgroupId string) error {
return errors.Wrapf(cloudprovider.ErrNotImplemented, "AssignSecurityGroup")
}
func (instance *SInstance) SetSecurityGroups(secgroupIds []string) error {
return errors.Wrapf(cloudprovider.ErrNotImplemented, "SetSecurityGroups")
}
func (instance *SInstance) GetError() error {
return nil
}
func (instance *SInstance) ChangeConfig(ctx context.Context, config *cloudprovider.SManagedVMChangeConfig) error {
if config.InstanceType == "nil" {
return errors.Wrapf(cloudprovider.ErrInputParameter, "InstanceType")
}
return instance.host.zone.region.ChangeConfig(instance.InstanceId, config.InstanceType)
}
func (instance *SInstance) GetVNCInfo(input *cloudprovider.ServerVncInput) (*cloudprovider.ServerVncOutput, error) {
return nil, cloudprovider.ErrNotSupported
}
func (instance *SInstance) StartVM(ctx context.Context) error {
err := instance.host.zone.region.StartVM(instance.InstanceId)
return err
}
func (instance *SInstance) StopVM(ctx context.Context, opts *cloudprovider.ServerStopOptions) error {
err := instance.host.zone.region.StopVM(instance.InstanceId, opts.IsForce, opts.StopCharging)
if err != nil {
return err
}
return cloudprovider.WaitStatus(instance, api.VM_READY, 10*time.Second, 300*time.Second)
}
func (instance *SInstance) DeleteVM(ctx context.Context) error {
for {
err := instance.host.zone.region.DeleteVM(instance.InstanceId)
if err != nil {
if isError(err, "IncorrectInstanceStatus.Initializing") {
log.Infof("The instance is initializing, try later ...")
time.Sleep(10 * time.Second)
} else {
return errors.Wrapf(err, "DeleteVM fail")
}
} else {
break
}
}
return cloudprovider.WaitDeleted(instance, 10*time.Second, 300*time.Second)
}
func (instance *SInstance) UpdateVM(ctx context.Context, input cloudprovider.SInstanceUpdateOptions) error {
return instance.host.zone.region.UpdateVM(instance.InstanceId, input.NAME, input.Description)
}
func (instance *SInstance) DeployVM(ctx context.Context, name string, username string, password string, publicKey string, deleteKeypair bool, description string) error {
var keypairName string
if len(publicKey) > 0 {
var err error
keypairName, err = instance.host.zone.region.syncKeypair(publicKey)
if err != nil {
return err
}
}
return instance.host.zone.region.DeployVM(instance.InstanceId, name, password, keypairName, deleteKeypair, description)
}
func (instance *SInstance) AttachDisk(ctx context.Context, diskId string) error {
return instance.host.zone.region.AttachDisk(instance.InstanceId, diskId)
}
func (instance *SInstance) DetachDisk(ctx context.Context, diskId string) error {
return cloudprovider.RetryOnError(
func() error {
return instance.host.zone.region.DetachDisk(instance.InstanceId, diskId)
},
[]string{
`"Code":"InvalidOperation.Conflict"`,
},
4)
}
func (instance *SInstance) GetProjectId() string {
return instance.ProjectName
}
func (instance *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) {
udata, err := instance.GetUserData()
if err != nil {
return "", err
}
image, err := instance.host.zone.region.GetImage(desc.ImageId)
if err != nil {
return "", errors.Wrapf(err, "GetImage fail")
}
keypairName := instance.KeyPairName
if len(desc.PublicKey) > 0 {
keypairName, err = instance.host.zone.region.syncKeypair(desc.PublicKey)
if err != nil {
return "", fmt.Errorf("RebuildRoot.syncKeypair %s", err)
}
}
userdata := ""
srcOsType := strings.ToLower(string(instance.GetOsType()))
destOsType := strings.ToLower(string(image.GetOsType()))
winOS := strings.ToLower(osprofile.OS_TYPE_WINDOWS)
cloudconfig := &cloudinit.SCloudConfig{}
if srcOsType != winOS && len(udata) > 0 {
_cloudconfig, err := cloudinit.ParseUserDataBase64(udata)
if err != nil {
log.Debugf("RebuildRoot invalid instance user data %s", udata)
} else {
cloudconfig = _cloudconfig
}
}
if (srcOsType != winOS && destOsType != winOS) || (srcOsType == winOS && destOsType != winOS) {
// linux/windows to linux
loginUser := cloudinit.NewUser(api.VM_AWS_DEFAULT_LOGIN_USER)
loginUser.SudoPolicy(cloudinit.USER_SUDO_NOPASSWD)
if len(desc.PublicKey) > 0 {
loginUser.SshKey(desc.PublicKey)
cloudconfig.MergeUser(loginUser)
} else if len(desc.Password) > 0 {
cloudconfig.SshPwauth = cloudinit.SSH_PASSWORD_AUTH_ON
loginUser.Password(desc.Password)
cloudconfig.MergeUser(loginUser)
}
userdata = cloudconfig.UserDataBase64()
} else {
// linux/windows to windows
data := ""
if len(desc.Password) > 0 {
cloudconfig.SshPwauth = cloudinit.SSH_PASSWORD_AUTH_ON
loginUser := cloudinit.NewUser(api.VM_AWS_DEFAULT_WINDOWS_LOGIN_USER)
loginUser.SudoPolicy(cloudinit.USER_SUDO_NOPASSWD)
loginUser.Password(desc.Password)
cloudconfig.MergeUser(loginUser)
data = fmt.Sprintf("<powershell>%s</powershell>", cloudconfig.UserDataPowerShell())
} else {
if len(udata) > 0 {
data = fmt.Sprintf("<powershell>%s</powershell>", udata)
}
}
userdata = base64.StdEncoding.EncodeToString([]byte(data))
}
diskId, err := instance.host.zone.region.ReplaceSystemDisk(ctx, instance.InstanceId, desc.ImageId, desc.Password, keypairName, userdata)
if err != nil {
return "", err
}
return diskId, nil
}
func (instance *SInstance) SaveImage(opts *cloudprovider.SaveImageOptions) (cloudprovider.ICloudImage, error) {
image, err := instance.host.zone.region.SaveImage(instance.InstanceId, opts)
if err != nil {
return nil, errors.Wrapf(err, "SaveImage %s", opts.Name)
}
return image, nil
}
// region
func (region *SRegion) CreateInstance(
name string,
hostname string,
imageId string,
instanceType string,
securityGroupId string,
zoneId string,
desc string,
passwd string,
disks []SDisk,
networkID string,
ipAddr string,
keypair string,
userData string,
bc *billing.SBillingCycle,
projectId string,
tags map[string]string,
) (string, error) {
params := make(map[string]string)
params["RegionId"] = region.RegionId
params["ImageId"] = imageId
params["InstanceType"] = instanceType
params["ZoneId"] = zoneId
params["InstanceName"] = name
params["ProjectName"] = projectId
if len(hostname) > 0 {
params["HostName"] = hostname
}
params["Description"] = desc
if len(passwd) > 0 {
params["Password"] = passwd
} else {
params["KeepImageCredential"] = "True"
}
if len(keypair) > 0 {
params["KeyPairName"] = keypair
}
if len(userData) > 0 {
params["UserData"] = userData
}
if len(tags) > 0 {
tagIdx := 1
for k, v := range tags {
params[fmt.Sprintf("Tag.%d.Key", tagIdx)] = k
params[fmt.Sprintf("Tag.%d.Value", tagIdx)] = v
tagIdx += 1
}
}
if len(disks) > 0 {
for idx, disk := range disks {
diskIdx := idx + 1
params[fmt.Sprintf("Volumes.%d.Size", diskIdx)] = fmt.Sprintf("%d", disk.Size)
params[fmt.Sprintf("Volumes.%d.VolumeType", diskIdx)] = disk.VolumeType
}
}
params["NetworkInterfaces.1.SubnetId"] = ipAddr
// currently only support binding the first NetworkInterface securitygroup
params["NetworkInterfaces.1.SecurityGroupIds.1"] = securityGroupId
if bc != nil {
params["InstanceChargeType"] = "PrePaid"
err := billingCycle2Params(bc, params)
if err != nil {
return "", err
}
if bc.AutoRenew {
params["AutoRenew"] = "true"
params["AutoRenewPeriod"] = "1"
} else {
params["AutoRenew"] = "False"
}
} else {
params["InstanceChargeType"] = "PostPaid"
params["SpotStrategy"] = "NoSpot"
}
params["ClientToken"] = utils.GenRequestId(20)
body, err := region.ecsRequest("CreateInstance", params)
if err != nil {
return "", errors.Wrapf(err, "CreateInstance fail")
}
instanceId, _ := body.GetString("InstanceId")
return instanceId, nil
}
func (region *SRegion) RenewInstance(instanceId string, bc billing.SBillingCycle) error {
params := make(map[string]string)
params["InstanceId"] = instanceId
err := billingCycle2Params(&bc, params)
if err != nil {
return err
}
params["ClientToken"] = utils.GenRequestId(20)
_, err = region.ecsRequest("RenewInstance", params)
if err != nil {
return errors.Wrapf(err, "RenewInstance fail")
}
return nil
}
func (region *SRegion) ChangeConfig(instanceId string, instanceTypeId string) error {
params := make(map[string]string)
params["InstanceTypeId"] = instanceTypeId
return region.instanceOperation(instanceId, "ModifyInstanceSpec", params)
}
func (region *SRegion) StartVM(instanceId string) error {
status, err := region.GetInstanceStatus(instanceId)
if err != nil {
return errors.Wrapf(err, "Fail to get instance status on StartVM")
}
if status != InstanceStatusStopped {
return errors.Wrapf(cloudprovider.ErrInvalidStatus, "StartVM: vm status is %s expect %s", status, InstanceStatusStopped)
}
return region.doStartVM(instanceId)
}
func (region *SRegion) StopVM(instanceId string, isForce, stopCharging bool) error {
status, err := region.GetInstanceStatus(instanceId)
if err != nil {
return errors.Wrapf(err, "Fail to get instance status on StopVM")
}
if status == InstanceStatusStopped {
return nil
}
if status != InstanceStatusRunning {
return errors.Wrapf(cloudprovider.ErrInvalidStatus, "StartVM: vm status is %s expect %s", status, InstanceStatusRunning)
}
return region.doStopVM(instanceId, isForce, stopCharging)
}
func (region *SRegion) DeleteVM(instanceId string) error {
status, err := region.GetInstanceStatus(instanceId)
if err != nil {
return errors.Wrapf(err, "Fail to get instance status on DeleteVM")
}
log.Debugf("Instance status on delete is %s", status)
if status != InstanceStatusStopped {
log.Warningf("DeleteVM: vm status is %s expect %s", status, InstanceStatusStopped)
}
return region.doDeleteVM(instanceId)
}
func (region *SRegion) doStartVM(instanceId string) error {
return region.instanceOperation(instanceId, "StartInstance", nil)
}
func (region *SRegion) doStopVM(instanceId string, isForce, stopCharging bool) error {
params := make(map[string]string)
if isForce {
params["ForceStop"] = "true"
} else {
params["ForceStop"] = "false"
}
params["StoppedMode"] = "KeepCharging"
if stopCharging {
params["StoppedMode"] = "StopCharging"
}
return region.instanceOperation(instanceId, "StopInstance", params)
}
func (region *SRegion) doDeleteVM(instanceId string) error {
return region.instanceOperation(instanceId, "DeleteInstance", nil)
}
func (region *SRegion) modifyInstanceAttribute(instanceId string, params map[string]string) error {
return region.instanceOperation(instanceId, "ModifyInstanceAttribute", params)
}
func (region *SRegion) UpdateVM(instanceId string, name, description string) error {
params := make(map[string]string)
params["InstanceName"] = name
params["Description"] = description
return region.modifyInstanceAttribute(instanceId, params)
}
func (region *SRegion) DeployVM(instanceId string, name string, password string, keypairName string, deleteKeypair bool, description string) error {
instance, err := region.GetInstance(instanceId)
if err != nil {
return err
}
if deleteKeypair {
err = region.DetachKeyPair(instanceId, instance.KeyPairName)
if err != nil {
return err
}
}
if len(keypairName) > 0 {
err = region.AttachKeypair(instanceId, keypairName)
if err != nil {
return err
}
}
params := make(map[string]string)
if len(password) > 0 {
params["Password"] = password
}
if len(name) > 0 && instance.InstanceName != name {
params["InstanceName"] = name
}
if len(description) > 0 && instance.Description != description {
params["Description"] = description
}
if len(params) > 0 {
return region.modifyInstanceAttribute(instanceId, params)
} else {
return nil
}
}
func (region *SRegion) DetachDisk(instanceId string, diskId string) error {
params := make(map[string]string)
params["InstanceId"] = instanceId
params["VolumeId"] = diskId
log.Infof("Detach instance %s disk %s", instanceId, diskId)
_, err := region.storageRequest("DetachVolume", params)
if err != nil {
return errors.Wrap(err, "DetachDisk")
}
return nil
}
func (region *SRegion) AttachDisk(instanceId string, diskId string) error {
params := make(map[string]string)
params["InstanceId"] = instanceId
params["VolumeId"] = diskId
_, err := region.storageRequest("AttachVolume", params)
if err != nil {
return errors.Wrapf(err, "AttachDisk %s to %s fail", diskId, instanceId)
}
return nil
}
func (region *SRegion) ReplaceSystemDisk(ctx context.Context, instanceId string, imageId string, passwd string, keypairName string, userdata string) (string, error) {
params := make(map[string]string)
params["InstanceId"] = instanceId
params["ImageId"] = imageId
if len(passwd) > 0 {
params["Password"] = passwd
} else {
params["KeepImageCredential"] = "True"
}
if len(keypairName) > 0 {
params["KeyPairName"] = keypairName
}
_, err := region.ecsRequest("ReplaceSystemVolume", params)
if err != nil {
return "", err
}
// volcengine does not return volumeId
return "", nil
}
func (region *SRegion) SaveImage(instanceId string, opts *cloudprovider.SaveImageOptions) (*SImage, error) {
params := map[string]string{
"InstanceId": instanceId,
"ImageName": opts.Name,
"Description": opts.Notes,
"ClientToken": utils.GenRequestId(20),
}
body, err := region.ecsRequest("CreateImage", params)
if err != nil {
return nil, errors.Wrapf(err, "CreateImage")
}
imageId, err := body.GetString("Result", "IamgeId")
if err != nil {
return nil, errors.Wrapf(err, "Unmarshal")
}
image, err := region.GetImage(imageId)
if err != nil {
return nil, errors.Wrapf(err, "GetImage %s", imageId)
}
return image, nil
}

122
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/instancenic.go generated vendored Normal file
View File

@@ -0,0 +1,122 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"github.com/golang-plus/errors"
"yunion.io/x/cloudmux/pkg/cloudprovider"
)
type SInstanceNic struct {
cloudprovider.DummyICloudNic
instance *SInstance
id string
ipAddr string
macAddr string
}
func (nic *SInstanceNic) GetId() string {
return nic.id
}
func (nic *SInstanceNic) GetIP() string {
return nic.ipAddr
}
func (nic *SInstanceNic) GetMAC() string {
return nic.macAddr
}
func (nic *SInstanceNic) InClassicNetwork() bool {
return false
}
func (nic *SInstanceNic) GetDriver() string {
return "virtio"
}
func (nic *SInstanceNic) GetINetworkId() string {
return nic.instance.NetworkInterfaces[0].SubnetId
}
func (nic *SInstanceNic) GetSubAddress() ([]string, error) {
return nic.instance.host.zone.region.GetSubAddress(nic.id)
}
func (nic *SInstanceNic) AssignAddress(ipAddrs []string) error {
return nic.instance.host.zone.region.AssignAddres(nic.id, ipAddrs)
}
func (nic *SInstanceNic) UnassignAddress(ipAddrs []string) error {
return nic.instance.host.zone.region.UnassignAddress(nic.id, ipAddrs)
}
func (region *SRegion) GetSubAddress(nicId string) ([]string, error) {
params := map[string]string{
"NetworkInterfaceId.1": nicId,
}
body, err := region.vpcRequest("DescribeNetworkInterfaces", params)
if err != nil {
return nil, errors.Wrapf(err, "DescribeNetworkInterfaces")
}
interfaces := []SNetworkInterface{}
err = body.Unmarshal(&interfaces, "Result", "NetworkInterfaceSets")
if err != nil {
return nil, errors.Wrapf(err, "Unmarshal")
}
ipAddrs := []string{}
for _, net := range interfaces {
if net.NetworkInterfaceId != nicId {
continue
}
for _, addr := range net.PrivateIpSets.PrivateIpSet {
if !addr.Primary {
ipAddrs = append(ipAddrs, addr.PrivateIpAddress)
}
}
}
return ipAddrs, nil
}
func (region *SRegion) AssignAddres(nicId string, ipAddrs []string) error {
params := make(map[string]string)
params["NetworkInterfaceId"] = nicId
for idx, addr := range ipAddrs {
params[fmt.Sprintf("PrivateIpAddress.%d", idx+1)] = addr
}
_, err := region.vpcRequest("AssignPrivateIpAddresses", params)
if err != nil {
return errors.Wrapf(err, "AssignPrivateIpAddresses")
}
return nil
}
func (region *SRegion) UnassignAddress(nicId string, ipAddrs []string) error {
params := make(map[string]string)
params["NetworkInterfaceId"] = nicId
for idx, addr := range ipAddrs {
params[fmt.Sprintf("PrivateIpAddress.%d", idx+1)] = addr
}
_, err := region.vpcRequest("UnassignPrivateAddress", params)
if err != nil {
return errors.Wrapf(err, "UnassignPrivateAdress")
}
return nil
}

143
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/keypairs.go generated vendored Normal file
View File

@@ -0,0 +1,143 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/aokoli/goutils"
"golang.org/x/crypto/ssh"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
)
type SKeypair struct {
KeyPairFingerPrint string
KeyPairName string
}
func (region *SRegion) GetKeypairs(finger string, name string, limit int, token string) ([]SKeypair, string, error) {
if limit > 500 || limit <= 0 {
limit = 500
}
params := make(map[string]string)
params["MaxResults"] = fmt.Sprintf("%d", limit)
if len(token) > 0 {
params["NextToken"] = token
}
if len(finger) > 0 {
params["FingerPrint"] = finger
}
if len(name) > 0 {
params["KeyPairName"] = name
}
body, err := region.ecsRequest("DescribeKeyPairs", params)
if err != nil {
return nil, "", errors.Wrapf(err, "GetKeypairs fail")
}
keypairs := make([]SKeypair, 0)
err = body.Unmarshal(&keypairs, "Result", "KeyPairs")
if err != nil {
return nil, "", errors.Wrapf(err, "Unmarshal keypair fail")
}
nextToken, _ := body.GetString("Result", "NextToken")
return keypairs, nextToken, nil
}
func (region *SRegion) ImportKeypair(name string, pubKey string) (*SKeypair, error) {
params := make(map[string]string)
params["PublicKey"] = pubKey
params["KeyPairName"] = name
body, err := region.ecsRequest("ImportKeyPair", params)
if err != nil {
return nil, errors.Wrapf(err, "ImportKeypair fail")
}
log.Debugf("%s", body)
keypair := SKeypair{}
err = body.Unmarshal(&keypair, "Result")
if err != nil {
return nil, errors.Wrapf(err, "Unmarshal keypair fail")
}
return &keypair, nil
}
func (region *SRegion) AttachKeypair(instanceId string, name string) error {
params := make(map[string]string)
params["KeyPairName"] = name
params["InstanceIds.1"] = instanceId
_, err := region.ecsRequest("AttachKeyPair", params)
if err != nil {
return errors.Wrapf(err, "AttachKeyPair fail")
}
return nil
}
func (region *SRegion) DetachKeyPair(instanceId string, name string) error {
params := make(map[string]string)
params["KeyPairName"] = name
params["InstanceIds.1"] = instanceId
_, err := region.ecsRequest("DetachKeyPair", params)
if err != nil {
return errors.Wrapf(err, "DetachKeyPair fail")
}
return nil
}
func (region *SRegion) lookUpVolcEngineKeypair(publicKey string) (string, error) {
pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(publicKey))
if err != nil {
return "", fmt.Errorf("publicKey error %s", err)
}
fingerprint := strings.Replace(ssh.FingerprintLegacyMD5(pk), ":", "", -1)
ks, _, err := region.GetKeypairs(fingerprint, "*", 0, "")
if len(ks) < 1 {
return "", fmt.Errorf("keypair not found %s", err)
} else {
return ks[0].KeyPairName, nil
}
}
func (region *SRegion) importVolcEngineKeypair(publicKey string) (string, error) {
prefix, e := goutils.RandomAlphabetic(6)
if e != nil {
return "", fmt.Errorf("publicKey error %s", e)
}
name := prefix + strconv.FormatInt(time.Now().Unix(), 10)
if k, e := region.ImportKeypair(name, publicKey); e != nil {
return "", fmt.Errorf("keypair import error %s", e)
} else {
return k.KeyPairName, nil
}
}
func (region *SRegion) syncKeypair(publicKey string) (string, error) {
name, e := region.lookUpVolcEngineKeypair(publicKey)
if e == nil {
return name, nil
}
return region.importVolcEngineKeypair(publicKey)
}

26
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/latitude_and_longitude.go generated vendored Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
)
var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{
"cn-guangzhou": api.RegionGuangzhou,
"cn-beijing": api.RegionBeijing,
"cn-shanghai": api.RegionShanghai,
}

176
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natdtable.go generated vendored Normal file
View File

@@ -0,0 +1,176 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"strconv"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/pkg/errors"
)
type SDNATEntry struct {
multicloud.SResourceBase
VolcEngineTags
nat *SNatGateway
NatGatewayId string
DnatEntryId string
DnatEntryName string
Protocol string
InternalIp string
InternalPort string
ExternalIp string
ExternalPort string
Status string
}
func (dentry *SDNATEntry) GetName() string {
if len(dentry.DnatEntryName) > 0 {
return dentry.DnatEntryName
}
return dentry.DnatEntryId
}
func (dentry *SDNATEntry) GetId() string {
return dentry.DnatEntryId
}
func (dentry *SDNATEntry) GetGlobalId() string {
return dentry.DnatEntryId
}
func (dentry *SDNATEntry) GetStatus() string {
switch dentry.Status {
case "Creating":
return api.NAT_STATUS_ALLOCATE
case "Available":
return api.NAT_STAUTS_AVAILABLE
case "Deleteting":
return api.NAT_STATUS_DELETING
default:
return api.NAT_STATUS_UNKNOWN
}
}
func (dentry *SDNATEntry) GetIpProtocol() string {
return dentry.Protocol
}
func (dentry *SDNATEntry) GetExternalIp() string {
return dentry.ExternalIp
}
func (dentry *SDNATEntry) GetExternalPort() int {
port, _ := strconv.Atoi(dentry.ExternalPort)
return port
}
func (dentry *SDNATEntry) GetInternalIp() string {
return dentry.InternalIp
}
func (dentry *SDNATEntry) GetInternalPort() int {
port, _ := strconv.Atoi(dentry.InternalPort)
return port
}
func (dentry *SDNATEntry) Delete() error {
return dentry.nat.vpc.region.DeleteDnatEntry(dentry.DnatEntryId)
}
func (nat *SNatGateway) getDnatEntries() ([]SDNATEntry, error) {
entries := make([]SDNATEntry, 0)
pageNumber := 1
for {
parts, total, err := nat.vpc.region.GetDnatEntries(nat.NatGatewayId, pageNumber, 50)
if err != nil {
return nil, err
}
entries = append(entries, parts...)
if len(entries) >= total {
break
}
pageNumber += 1
}
return entries, nil
}
func (region *SRegion) GetDnatEntries(natGatewayId string, pageNumber int, pageSize int) ([]SDNATEntry, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["NatGatewayId"] = natGatewayId
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
body, err := region.natRequest("DescribeDnatEntries", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "DescribeDnatEntries fail")
}
entries := make([]SDNATEntry, 0)
err = body.Unmarshal(&entries, "Result", "DnatEntries")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal entries fail")
}
total, _ := body.Int("Result", "TotalCount")
return entries, int(total), nil
}
func (region *SRegion) GetDnatEntry(natGatewayId string, dnatEntryID string) (SDNATEntry, error) {
params := make(map[string]string)
params["NatGatewayId"] = natGatewayId
params["DnatEntryIds.1"] = dnatEntryID
body, err := region.natRequest("DescribeDnatEntries", params)
if err != nil {
return SDNATEntry{}, errors.Wrapf(err, "DescribeDnatEntries fail")
}
entries := make([]SDNATEntry, 0)
err = body.Unmarshal(&entries, "Result", "DnatEntries")
if err != nil {
return SDNATEntry{}, errors.Wrapf(err, "Unmarshal entries fail")
}
if len(entries) == 0 {
return SDNATEntry{}, cloudprovider.ErrNotFound
}
return entries[0], nil
}
func (region *SRegion) CreateDnatEntry(rule cloudprovider.SNatDRule, natGatewayId string) (string, error) {
params := make(map[string]string)
params["NatGatewayId"] = natGatewayId
params["ExternalIp"] = rule.ExternalIP
params["ExternalPort"] = strconv.Itoa(rule.ExternalPort)
params["InternalIp"] = rule.InternalIP
params["InternalPort"] = strconv.Itoa(rule.InternalPort)
params["Protocol"] = rule.Protocol
body, err := region.natRequest("CreateDnatEntry", params)
if err != nil {
return "", err
}
entryID, _ := body.GetString("Result", "DnatEntryId")
return entryID, nil
}
func (region *SRegion) DeleteDnatEntry(dnatEntryId string) error {
params := make(map[string]string)
params["DnatEntryId"] = dnatEntryId
_, err := region.natRequest("DeleteDnatEntry", params)
return err
}

293
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natgateway.go generated vendored Normal file
View File

@@ -0,0 +1,293 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"time"
billing "yunion.io/x/cloudmux/pkg/apis/billing"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
)
type SNatGateway struct {
multicloud.SNatGatewayBase
VolcEngineTags
vpc *SVpc
NatGatewayId string
NatGatewayName string
Description string
Spec string
BillingType int
VpcId string
SubnetId string
ZoneId string
NetworkInterfaceId string
ProjectName string
Status string
BusinessStatus string
LockReason string
CreationTime time.Time
UpdatedAt time.Time
ExpiredTime time.Time
OverdueTime time.Time
DeletedTime time.Time
PrivateIP string
}
func (nat *SNatGateway) GetId() string {
return nat.NatGatewayId
}
func (nat *SNatGateway) GetGlobalId() string {
return nat.NatGatewayId
}
func (nat *SNatGateway) GetName() string {
if len(nat.NatGatewayName) > 0 {
return nat.NatGatewayName
}
return nat.NatGatewayId
}
func (nat *SNatGateway) GetStatus() string {
switch nat.Status {
case "Creating":
return api.NAT_STATUS_ALLOCATE
case "Available":
return api.NAT_STAUTS_AVAILABLE
case "Pending":
return api.NAT_STATUS_DEPLOYING
case "Deleteting":
return api.NAT_STATUS_DELETING
default:
return api.NAT_STATUS_UNKNOWN
}
}
func (nat *SNatGateway) GetINetworkId() string {
return nat.SubnetId
}
func (nat *SNatGateway) GetIpAddr() string {
if len(nat.PrivateIP) > 0 {
return nat.PrivateIP
}
return ""
}
func (nat *SNatGateway) GetBandwidthMb() int {
return 0
}
func (nat *SNatGateway) Delete() error {
return nat.vpc.region.DeleteNatGateway(nat.NatGatewayId, false)
}
func (nat *SNatGateway) GetBillingType() string {
switch nat.BillingType {
case 1:
return billing.BILLING_TYPE_PREPAID
case 2:
return billing.BILLING_TYPE_POSTPAID
default:
return ""
}
}
func (nat *SNatGateway) GetNatSpec() string {
if len(nat.Spec) == 0 {
return "Small"
}
return nat.Spec
}
func (nat *SNatGateway) Refresh() error {
newNat, _, err := nat.vpc.region.GetNatGateways("", nat.NatGatewayId, 1, 1)
if err != nil {
return errors.Wrapf(err, "GetNatGateways")
}
for _, nt := range newNat {
if nt.NatGatewayId == nat.NatGatewayId {
return jsonutils.Update(nat, nt)
}
}
return errors.Wrapf(cloudprovider.ErrNotFound, "%s not found", nat.NatGatewayId)
}
func (nat *SNatGateway) GetCreatedAt() time.Time {
return nat.CreationTime
}
func (nat *SNatGateway) GetExpiredAt() time.Time {
return nat.ExpiredTime
}
func (nat *SNatGateway) GetINatDTable() ([]cloudprovider.ICloudNatDEntry, error) {
stables, err := nat.getDnatEntries()
if err != nil {
return nil, err
}
itables := []cloudprovider.ICloudNatDEntry{}
for i := 0; i < len(stables); i++ {
stables[i].nat = nat
itables = append(itables, &stables[i])
}
return itables, nil
}
func (nat *SNatGateway) GetINatSTable() ([]cloudprovider.ICloudNatSEntry, error) {
stables, err := nat.getSnatEntries()
if err != nil {
return nil, err
}
itables := []cloudprovider.ICloudNatSEntry{}
for i := 0; i < len(stables); i++ {
stables[i].nat = nat
itables = append(itables, &stables[i])
}
return itables, nil
}
func (nat *SNatGateway) GetINatDEntryByID(id string) (cloudprovider.ICloudNatDEntry, error) {
dNATEntry, err := nat.vpc.region.GetDnatEntry(nat.NatGatewayId, id)
if err != nil {
return nil, cloudprovider.ErrNotFound
}
dNATEntry.nat = nat
return &dNATEntry, nil
}
func (nat *SNatGateway) GetINatSEntryByID(id string) (cloudprovider.ICloudNatSEntry, error) {
sNATEntry, err := nat.vpc.region.GetSnatEntry(nat.NatGatewayId, id)
if err != nil {
return nil, cloudprovider.ErrNotFound
}
sNATEntry.nat = nat
return &sNATEntry, nil
}
func (nat *SNatGateway) CreateINatDEntry(rule cloudprovider.SNatDRule) (cloudprovider.ICloudNatDEntry, error) {
entryID, err := nat.vpc.region.CreateDnatEntry(rule, nat.NatGatewayId)
if err != nil {
return nil, errors.Wrapf(err, `create dnat rule for nat gateway %q`, nat.GetId())
}
return nat.GetINatDEntryByID(entryID)
}
func (nat *SNatGateway) CreateINatSEntry(rule cloudprovider.SNatSRule) (cloudprovider.ICloudNatSEntry, error) {
entryID, err := nat.vpc.region.CreateSnatEntry(rule, nat.NatGatewayId)
if err != nil {
return nil, errors.Wrapf(err, `create snat rule for nat gateway %q`, nat.GetId())
}
return nat.GetINatSEntryByID(entryID)
}
func (region *SRegion) GetNatGateways(vpcId string, natGatewayId string, pageNumber int, pageSize int) ([]SNatGateway, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
if len(vpcId) > 0 {
params["VpcId"] = vpcId
}
if len(natGatewayId) > 0 {
params["NatGatewayId.1"] = natGatewayId
}
body, err := region.natRequest("DescribeNatGateways", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "DescribeNatGateways")
}
gateways := make([]SNatGateway, 0)
err = body.Unmarshal(&gateways, "Result", "NatGateways")
if err != nil {
return nil, 0, errors.Wrapf(err, "body.Unmarshal")
}
total, _ := body.Int("Result", "TotalCount")
return gateways, int(total), nil
}
func (region *SRegion) CreateNatGateway(opts *cloudprovider.NatGatewayCreateOptions) (*SNatGateway, error) {
params := map[string]string{
"VpcId": opts.VpcId,
"SubnetId": opts.NetworkId,
"NatGatewayName": opts.Name,
"Description": opts.Desc,
"ClientToken": utils.GenRequestId(20),
"BillingType": fmt.Sprintf("%d", 2),
}
if len(opts.NatSpec) != 0 {
params["Spec"] = opts.NatSpec
}
if opts.BillingCycle != nil {
params["BillingType"] = fmt.Sprintf("%d", 1)
params["Period"] = fmt.Sprintf("%d", 1)
params["PeriodUnit"] = "Month"
if opts.BillingCycle.GetYears() > 0 {
params["PeriodUnit"] = "Year"
params["Period"] = fmt.Sprintf("%d", opts.BillingCycle.GetYears())
} else if opts.BillingCycle.GetMonths() > 0 {
params["PeriodUnit"] = "Month"
params["Period"] = fmt.Sprintf("%d", opts.BillingCycle.GetMonths())
}
}
resp, err := region.natRequest("CreateNatGateway", params)
if err != nil {
return nil, errors.Wrapf(err, "CreateNatGateway")
}
natId, err := resp.GetString("Result", "NatGatewayId")
if err != nil {
return nil, errors.Wrapf(err, "resp.Get(NatGatewayId)")
}
if len(natId) == 0 {
return nil, errors.Errorf("empty NatGatewayId after created")
}
err = cloudprovider.Wait(time.Second*5, time.Minute*15, func() (bool, error) {
_, _, err := region.GetNatGateways("", natId, 1, 1)
if errors.Cause(err) == cloudprovider.ErrNotFound {
return false, nil
} else {
return true, err
}
})
if err != nil {
return nil, errors.Wrapf(err, "cannot find nat gateway after create")
}
nats, _, err := region.GetNatGateways("", natId, 1, 1)
for _, nat := range nats {
if nat.NatGatewayId == natId {
return &nat, nil
}
}
return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s not found", natId)
}
func (region *SRegion) DeleteNatGateway(natId string, isForce bool) error {
params := make(map[string]string)
params["NatGatewayId"] = natId
_, err := region.natRequest("DeleteNatGateway", params)
return errors.Wrapf(err, "DeleteNatGateway")
}

194
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/natstable.go generated vendored Normal file
View File

@@ -0,0 +1,194 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
)
type SSNATEntry struct {
multicloud.SResourceBase
VolcEngineTags
nat *SNatGateway
NatGatewayId string
SnatEntryId string
SnatEntryName string
SubnetId string
SourceCidr string
EipId string
EipAddress string
Status string
}
func (sentry *SSNATEntry) GetName() string {
if len(sentry.SnatEntryName) > 0 {
return sentry.SnatEntryName
}
return sentry.SnatEntryId
}
func (sentry *SSNATEntry) GetId() string {
return sentry.SnatEntryId
}
func (sentry *SSNATEntry) GetGlobalId() string {
return sentry.SnatEntryId
}
func (sentry *SSNATEntry) GetStatus() string {
switch sentry.Status {
case "Creating":
return api.NAT_STATUS_ALLOCATE
case "Available":
return api.NAT_STAUTS_AVAILABLE
case "Deleteting":
return api.NAT_STATUS_DELETING
default:
return api.NAT_STATUS_UNKNOWN
}
}
func (sentry *SSNATEntry) GetIP() string {
return sentry.EipAddress
}
func (sentry *SSNATEntry) GetSourceCIDR() string {
return sentry.SourceCidr
}
func (sentry *SSNATEntry) GetNetworkId() string {
return sentry.SubnetId
}
func (sentry *SSNATEntry) Delete() error {
return sentry.nat.vpc.region.DeleteSnatEntry(sentry.SnatEntryId)
}
func (sentry *SSNATEntry) Refresh() error {
new, err := sentry.nat.vpc.region.GetSnatEntry(sentry.NatGatewayId, sentry.SnatEntryId)
if err != nil {
return err
}
return jsonutils.Update(sentry, new)
}
func (nat *SNatGateway) getSnatEntries() ([]SSNATEntry, error) {
entries := make([]SSNATEntry, 0)
pageNumber := 1
for {
parts, total, err := nat.vpc.region.GetSnatEntries(nat.NatGatewayId, pageNumber, 50)
if err != nil {
return nil, err
}
entries = append(entries, parts...)
if len(entries) >= total {
break
}
pageNumber += 1
}
return entries, nil
}
func (nat *SNatGateway) dissociateWithSubnet(subnetId string) error {
entries, err := nat.getSnatEntries()
if err != nil {
return err
}
for i := range entries {
if entries[i].SubnetId == subnetId {
err := nat.vpc.region.DeleteSnatEntry(entries[i].SnatEntryId)
if err != nil {
return nil
}
}
}
return nil
}
func (region *SRegion) GetSnatEntries(natGatewayId string, pageNumber int, pageSize int) ([]SSNATEntry, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["NatGatewayId"] = natGatewayId
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
body, err := region.natRequest("DescribeSnatEntries", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "DescribeSNATEntries fail")
}
entries := make([]SSNATEntry, 0)
err = body.Unmarshal(&entries, "Result", "SnatEntries")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal entries fail")
}
total, _ := body.Int("Result", "TotalCount")
return entries, int(total), nil
}
func (region *SRegion) GetSnatEntry(natGatewayId string, snatEntryID string) (SSNATEntry, error) {
params := make(map[string]string)
params["NatGatewayId"] = natGatewayId
params["SnatEntryIds.1"] = snatEntryID
body, err := region.natRequest("DescribeSnatEntries", params)
if err != nil {
return SSNATEntry{}, errors.Wrapf(err, "DescribeSnatEntries fail")
}
entries := make([]SSNATEntry, 0)
err = body.Unmarshal(&entries, "Result", "SnatEntries")
if err != nil {
return SSNATEntry{}, errors.Wrapf(err, "Unmarshal entries fail")
}
if len(entries) == 0 {
return SSNATEntry{}, cloudprovider.ErrNotFound
}
return entries[0], nil
}
func (region *SRegion) CreateSnatEntry(rule cloudprovider.SNatSRule, natGatewayId string) (string, error) {
params := make(map[string]string)
params["NatGatewayId"] = natGatewayId
params["SubnetId"] = rule.NetworkID
eips, _, err := region.GetEips(nil, rule.ExternalIP, nil, 1, 1)
if err != nil {
return "", err
}
params["EipId"] = eips[0].AllocationId
if len(rule.SourceCIDR) != 0 {
params["SourceCidr"] = rule.SourceCIDR
}
body, err := region.natRequest("CreateSnatEntry", params)
if err != nil {
return "", err
}
entryID, _ := body.GetString("Result", "SnatEntryId")
return entryID, nil
}
func (region *SRegion) DeleteSnatEntry(snatEntryId string) error {
params := make(map[string]string)
params["SnatEntryId"] = snatEntryId
_, err := region.natRequest("DeleteSnatEntry", params)
return errors.Wrapf(err, "DeleteSnatEntry")
}

255
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/network.go generated vendored Normal file
View File

@@ -0,0 +1,255 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"strings"
"time"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/netutils"
"yunion.io/x/pkg/util/rbacscope"
"yunion.io/x/pkg/utils"
)
type SNetwork struct {
multicloud.SResourceBase
VolcEngineTags
wire *SWire
AccountId string
SubnetId string
VpcId string
Status string
CidrBlock string
Ipv6CidrBlock string
ZoneId string
AvailableIpAddressCount int
Description string
SubnetName string
CreationTime time.Time
UpdateTime time.Time
TotalIpv4Count int
NetworkAclId string
IsDefault bool
RouteTable SRouteTable
ProjectName string
}
func (subnet *SNetwork) GetId() string {
return subnet.SubnetId
}
func (subnet *SNetwork) GetName() string {
if len(subnet.SubnetName) > 0 {
return subnet.SubnetName
}
return subnet.SubnetId
}
func (subnet *SNetwork) GetGlobalId() string {
return subnet.SubnetId
}
func (subnet *SNetwork) GetStatus() string {
return strings.ToLower(subnet.Status)
}
func (subnet *SNetwork) Refresh() error {
log.Debugf("Subnet refresh %s", subnet.SubnetId)
new, err := subnet.wire.zone.region.GetSubnetAttributes(subnet.SubnetId)
if err != nil {
return err
}
return jsonutils.Update(subnet, new)
}
func (subnet *SNetwork) GetIWire() cloudprovider.ICloudWire {
return subnet.wire
}
func (subnet *SNetwork) GetProjectId() string {
return subnet.ProjectName
}
func (subnet *SNetwork) GetIpStart() string {
pref, _ := netutils.NewIPV4Prefix(subnet.CidrBlock)
startIp := pref.Address.NetAddr(pref.MaskLen)
startIp = startIp.StepUp()
return startIp.String()
}
func (subnet *SNetwork) GetIpEnd() string {
pref, _ := netutils.NewIPV4Prefix(subnet.CidrBlock)
endIp := pref.Address.BroadcastAddr(pref.MaskLen)
endIp = endIp.StepDown()
endIp = endIp.StepDown()
endIp = endIp.StepDown()
return endIp.String()
}
func (subnet *SNetwork) GetIpMask() int8 {
pref, _ := netutils.NewIPV4Prefix(subnet.CidrBlock)
return pref.MaskLen
}
func (subnet *SNetwork) GetGateway() string {
pref, _ := netutils.NewIPV4Prefix(subnet.CidrBlock)
endIp := pref.Address.BroadcastAddr(pref.MaskLen)
endIp = endIp.StepDown()
return endIp.String()
}
func (subnet *SNetwork) GetServerType() string {
return api.NETWORK_TYPE_GUEST
}
func (subnet *SNetwork) GetIsPublic() bool {
return subnet.IsDefault
}
func (subnet *SNetwork) GetPublicScope() rbacscope.TRbacScope {
return rbacscope.ScopeDomain
}
func (region *SRegion) CreateSubnet(zoneId string, vpcId string, name string, cidr string, desc string) (string, error) {
params := make(map[string]string)
params["ZoneId"] = zoneId
params["VpcId"] = vpcId
params["CidrBlock"] = cidr
params["SubnetName"] = name
if len(desc) > 0 {
params["Description"] = desc
}
params["ClientToken"] = utils.GenRequestId(20)
body, err := region.vpcRequest("CreateSubnet", params)
if err != nil {
return "", err
}
return body.GetString("Result", "SubnetId")
}
func (region *SRegion) DeleteSubnet(SubnetId string) error {
params := make(map[string]string)
params["SubnetId"] = SubnetId
_, err := region.vpcRequest("DeleteSubnet", params)
return err
}
func (subnet *SNetwork) dissociateWithSNAT() error {
natgatways, err := subnet.wire.vpc.getNatGateways()
if err != nil {
return err
}
for i := range natgatways {
err = natgatways[i].dissociateWithSubnet(subnet.SubnetId)
if err != nil {
return err
}
}
return nil
}
func (subnet *SNetwork) Delete() error {
err := subnet.Refresh()
if err != nil {
log.Errorf("refresh Subnet fail %s", err)
return err
}
if len(subnet.RouteTable.RouteTableId) > 0 && !subnet.RouteTable.IsSystem() {
err = subnet.wire.zone.region.UnassociateRouteTable(subnet.RouteTable.RouteTableId, subnet.SubnetId)
if err != nil {
log.Errorf("unassociate routetable fail %s", err)
return err
}
}
err = subnet.dissociateWithSNAT()
if err != nil {
log.Errorf("fail to dissociateWithSNAT")
return err
}
err = cloudprovider.Wait(10*time.Second, time.Minute, func() (bool, error) {
err := subnet.wire.zone.region.DeleteSubnet(subnet.SubnetId)
if err != nil {
if isError(err, "DependencyViolation") {
return false, nil
}
return false, err
} else {
return true, nil
}
})
return err
}
func (subnet *SNetwork) GetAllocTimeoutSeconds() int {
return 120
}
func (region *SRegion) GetSubnets(ids []string, zoneId string, vpcId string, pageNumber int, pageSize int) ([]SNetwork, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
for idx, id := range ids {
params[fmt.Sprintf("SubnetIds.%d", idx)] = id
}
if len(zoneId) > 0 {
params["ZoneId"] = zoneId
}
if len(vpcId) > 0 {
params["VpcId"] = vpcId
}
body, err := region.vpcRequest("DescribeSubnets", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "GetSubnets fail")
}
subnets := make([]SNetwork, 0)
err = body.Unmarshal(&subnets, "Result", "Subnets")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal subnets fail")
}
total, _ := body.Int("Result", "TotalCount")
return subnets, int(total), nil
}
func (region *SRegion) GetSubnetAttributes(SubnetId string) (*SNetwork, error) {
params := make(map[string]string)
params["SubnetId"] = SubnetId
body, err := region.vpcRequest("DescribeSubnetAttributes", params)
if err != nil {
return nil, errors.Wrapf(err, "DescribeSubnetAttributes fail")
}
if region.client.debug {
log.Debugf("%s", body.PrettyString())
}
subnet := SNetwork{}
err = body.Unmarshal(&subnet, "Result")
if err != nil {
return nil, errors.Wrapf(err, "Unmarshal subnet fail")
}
return &subnet, nil
}

174
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/networkinterfaces.go generated vendored Normal file
View File

@@ -0,0 +1,174 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"time"
"yunion.io/x/pkg/errors"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
)
type SPrivateIp struct {
nic *SNetworkInterface
Primary bool
PrivateIpAddress string
}
func (ip *SPrivateIp) GetGlobalId() string {
return ip.PrivateIpAddress
}
func (ip *SPrivateIp) GetINetworkId() string {
return ip.nic.SubnetId
}
func (ip *SPrivateIp) GetIP() string {
return ip.PrivateIpAddress
}
func (ip *SPrivateIp) IsPrimary() bool {
return ip.Primary
}
type SAssociatedElasticIp struct {
AllocationId string
EipAddress string
}
type SPrivateIpSets struct {
PrivateIpSet []SPrivateIp
}
type SNetworkInterface struct {
multicloud.SNetworkInterfaceBase
VolcEngineTags
region *SRegion
InstanceId string
NetworkInterfaceId string
VpcId string
SubnetId string
PrimaryIpAddress string
Type string
MacAddress string
CreationTime time.Time
NetworkInterfaceName string
PrivateIpSets SPrivateIpSets
ResourceGroupId string
SecurityGroupIds SSecurityGroupIds
Status string
ZoneId string
PrivateIpAddresses []string
AssociatedElasticIp SAssociatedElasticIp
}
func (nic *SNetworkInterface) GetName() string {
return nic.NetworkInterfaceName
}
func (nic *SNetworkInterface) GetId() string {
return nic.NetworkInterfaceId
}
func (nic *SNetworkInterface) GetGlobalId() string {
return nic.NetworkInterfaceId
}
func (nic *SNetworkInterface) GetAssociateId() string {
return nic.InstanceId
}
func (nic *SNetworkInterface) GetAssociateType() string {
return api.NETWORK_INTERFACE_ASSOCIATE_TYPE_SERVER
}
func (nic *SNetworkInterface) GetMacAddress() string {
return nic.MacAddress
}
func (nic *SNetworkInterface) GetStatus() string {
switch nic.Status {
case "Available":
return api.NETWORK_INTERFACE_STATUS_AVAILABLE
}
return nic.Status
}
func (region *SRegion) GetINetworkInterfaces() ([]cloudprovider.ICloudNetworkInterface, error) {
interfaces := []SNetworkInterface{}
pageNumber := 1
for {
parts, total, err := region.GetNetworkInterfaces("", pageNumber, 50)
if err != nil {
return nil, err
}
interfaces = append(interfaces, parts...)
if len(interfaces) >= total {
break
}
pageNumber += 1
}
ret := []cloudprovider.ICloudNetworkInterface{}
for i := 0; i < len(interfaces); i++ {
if len(interfaces[i].InstanceId) == 0 {
interfaces[i].region = region
ret = append(ret, &interfaces[i])
}
}
return ret, nil
}
func (nic *SNetworkInterface) GetICloudInterfaceAddresses() ([]cloudprovider.ICloudInterfaceAddress, error) {
address := []cloudprovider.ICloudInterfaceAddress{}
for i := 0; i < len(nic.PrivateIpSets.PrivateIpSet); i++ {
nic.PrivateIpSets.PrivateIpSet[i].nic = nic
address = append(address, &nic.PrivateIpSets.PrivateIpSet[i])
}
return address, nil
}
func (region *SRegion) GetNetworkInterfaces(instanceId string, pageNumber int, pageSize int) ([]SNetworkInterface, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := map[string]string{
"RegionId": region.RegionId,
"PageSize": fmt.Sprintf("%d", pageSize),
"PageNumber": fmt.Sprintf("%d", pageNumber),
}
if len(instanceId) > 0 {
params["InstanceId"] = instanceId
}
body, err := region.vpcRequest("DescribeNetworkInterfaces", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "DescribeNetworkInterfaces")
}
interfaces := []SNetworkInterface{}
err = body.Unmarshal(&interfaces, "Result", "NetworkInterfaceSets")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal")
}
total, _ := body.Int("Result", "TotalCount")
return interfaces, int(total), nil
}

102
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/objects.go generated vendored Normal file
View File

@@ -0,0 +1,102 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"context"
"net/http"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
)
const (
OSS_META_HEADER = "x-oss-meta-"
)
type SObject struct {
bucket *SBucket
cloudprovider.SBaseCloudObject
}
func (obj *SObject) GetIBucket() cloudprovider.ICloudBucket {
return obj.bucket
}
func (obj *SObject) GetAcl() cloudprovider.TBucketACLType {
acl := cloudprovider.ACLPrivate
toscli, err := obj.bucket.region.GetTosClient()
if err != nil {
log.Errorf("Get Client %s", err)
return acl
}
result, err := toscli.GetObjectACL(context.Background(), &tos.GetObjectACLInput{Bucket: obj.bucket.Name, Key: obj.Key})
if err != nil {
log.Errorf("GetObjectAcl %s", err)
}
grants := result.Grants
return grantToCannedAcl(grants)
}
func (obj *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error {
toscli, err := obj.bucket.region.GetTosClient()
if err != nil {
return errors.Wrap(err, "GetTosClient")
}
_, err = toscli.PutObjectACL(context.Background(), &tos.PutObjectACLInput{Key: obj.Key, ACL: enum.ACLType(aclStr)})
if err != nil {
return errors.Wrapf(err, "PutObjectACL")
}
return nil
}
func (obj *SObject) GetMeta() http.Header {
if obj.Meta != nil {
return obj.Meta
}
toscli, err := obj.bucket.region.GetTosClient()
if err != nil {
log.Errorf("Get Client %s", err)
return nil
}
result, err := toscli.GetObjectV2(context.Background(), &tos.GetObjectV2Input{Bucket: obj.bucket.Name, Key: obj.Key})
if err != nil {
log.Errorf("Get Object error %s", err)
return nil
}
newHeader := http.Header{}
meta := result.GetObjectBasicOutput.ObjectMetaV2.Meta
for _, key := range meta.AllKeys() {
value, exist := meta.Get(key)
if !exist {
log.Errorf("Key missing in meta data %s", key)
} else {
newHeader.Add(key, value)
}
}
obj.Meta = cloudprovider.FetchMetaFromHttpHeader(
OSS_META_HEADER,
newHeader,
)
return obj.Meta
}
func (obj *SObject) SetMeta(ctx context.Context, meta http.Header) error {
return cloudprovider.ObjectSetMeta(ctx, obj.bucket, obj, meta)
}

140
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/project.go generated vendored Normal file
View File

@@ -0,0 +1,140 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"time"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
)
type SProject struct {
multicloud.SProjectBase
VolcEngineTags
client *SVolcEngineClient
AccountId int
ProjectName string
ParentProjectName string
Path string
DisplayName string
Description string
CreateDate time.Time
UpdateDate time.Time
Status string
}
func (project *SProject) GetGlobalId() string {
return project.ProjectName
}
func (project *SProject) GetId() string {
return project.ProjectName
}
func (project *SProject) GetName() string {
if len(project.DisplayName) > 0 {
return project.DisplayName
}
return project.ProjectName
}
func (project *SProject) Refresh() error {
group, err := project.client.GetProject(project.ProjectName)
if err != nil {
return errors.Wrap(err, "GetProject")
}
return jsonutils.Update(project, group)
}
func (project *SProject) GetStatus() string {
switch project.Status {
case "active":
return api.EXTERNAL_PROJECT_STATUS_AVAILABLE
default:
return api.EXTERNAL_PROJECT_STATUS_UNKNOWN
}
}
func (client *SVolcEngineClient) GetProject(name string) (*SProject, error) {
params := map[string]string{
"ProjectName": name,
}
body, err := client.iamRequest("", "GetProject", params)
if err != nil {
return nil, err
}
project := &SProject{client: client}
err = body.Unmarshal(project, "Result")
if err != nil {
return nil, errors.Wrap(err, "resp.Unmarshal")
}
return project, nil
}
func (client *SVolcEngineClient) ListProjects(limit int, offset int) ([]SProject, int, error) {
if limit > 50 || limit <= 0 {
limit = 50
}
params := map[string]string{
"Limit": fmt.Sprintf("%d", limit),
"Offset": fmt.Sprintf("%d", offset),
}
resp, err := client.iamRequest("", "ListProjects", params)
if err != nil {
return nil, 0, errors.Wrap(err, "iamRequest.ListProjects")
}
projects := []SProject{}
err = resp.Unmarshal(&projects, "Result", "Projects")
if err != nil {
return nil, 0, errors.Wrap(err, "resp.Unmarshal")
}
total, _ := resp.Int("Result", "Total")
return projects, int(total), nil
}
func (client *SVolcEngineClient) CreateIProject(name string) (cloudprovider.ICloudProject, error) {
group, err := client.CreateProject(name)
if err != nil {
return nil, errors.Wrap(err, "CreateProject")
}
return group, nil
}
func (client *SVolcEngineClient) CreateProject(name string) (*SProject, error) {
params := map[string]string{
"DisplayName": name,
"ProjectName": name,
}
resp, err := client.iamRequest("", "CreateProject", params)
if err != nil {
return nil, errors.Wrap(err, "CreateProject")
}
group := &SProject{client: client}
err = resp.Unmarshal(group, "Project")
if err != nil {
return nil, errors.Wrap(err, "resp.Unmarshal")
}
err = cloudprovider.WaitStatus(group, api.EXTERNAL_PROJECT_STATUS_AVAILABLE, time.Second*5, time.Minute*3)
if err != nil {
return nil, errors.Wrap(err, "WaitStatus")
}
return group, nil
}

4
pkg/keystone/cache/doc.go → vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/doc.go generated vendored
View File

@@ -1,4 +1,4 @@
// Copyright 2019 Yunion
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -12,4 +12,4 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package cache // import "yunion.io/x/onecloud/pkg/keystone/cache"
package volcengine

189
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/provider.go generated vendored Normal file
View File

@@ -0,0 +1,189 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"context"
"strings"
"github.com/pkg/errors"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud/volcengine"
"yunion.io/x/jsonutils"
)
type SVolcEngineProviderFactory struct {
cloudprovider.SPublicCloudBaseProviderFactory
}
func (self *SVolcEngineProviderFactory) GetId() string {
return volcengine.CLOUD_PROVIDER_VOLCENGINE
}
func (self *SVolcEngineProviderFactory) GetName() string {
return volcengine.CLOUD_PROVIDER_VOLCENGINE_CN
}
func (self *SVolcEngineProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
output := cloudprovider.SCloudaccount{}
if len(input.AccessKeyId) == 0 {
return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_id")
}
if len(input.AccessKeySecret) == 0 {
return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_secret")
}
output.Account = input.AccessKeyId
output.Secret = input.AccessKeySecret
return output, nil
}
func (f *SVolcEngineProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
output := cloudprovider.SCloudaccount{}
if len(input.AccessKeyId) == 0 {
return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_id")
}
if len(input.AccessKeySecret) == 0 {
return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_secret")
}
output = cloudprovider.SCloudaccount{
Account: input.AccessKeyId,
Secret: input.AccessKeySecret,
}
return output, nil
}
func validateClientCloudenv(client *volcengine.SVolcEngineClient) error {
regions := client.GetIRegions()
if len(regions) == 0 {
return nil
}
return nil
}
func parseAccount(account string) (accessKey string, projectId string) {
segs := strings.Split(account, "::")
if len(segs) == 2 {
accessKey = segs[0]
projectId = segs[1]
} else {
accessKey = account
projectId = ""
}
return
}
func (self *SVolcEngineProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) {
accessKey, accountId := parseAccount(cfg.Account)
client, err := volcengine.NewVolcEngineClient(
volcengine.NewVolcEngineClientConfig(
accessKey,
cfg.Secret,
).CloudproviderConfig(cfg).AccountId(accountId),
)
if err != nil {
return nil, err
}
err = validateClientCloudenv(client)
if err != nil {
return nil, errors.Wrap(err, "validateClientCloudenv")
}
return &SVolcEngineProvider{
SBaseProvider: cloudprovider.NewBaseProvider(self),
client: client,
}, nil
}
func (self *SVolcEngineProviderFactory) GetClientRC(info cloudprovider.SProviderInfo) (map[string]string, error) {
accessKey, accountId := parseAccount(info.Account)
return map[string]string{
"VOLCENGINE_ACCESS_KEY": accessKey,
"VOLCENGINE_SECRET_KEY": info.Secret,
"VOLCENGINE_REGION": volcengine.VOLCENGINE_DEFAULT_REGION,
"VOLCENGINE_ACCOUNT_ID": accountId,
}, nil
}
func init() {
factory := SVolcEngineProviderFactory{}
cloudprovider.RegisterFactory(&factory)
}
type SVolcEngineProvider struct {
cloudprovider.SBaseProvider
client *volcengine.SVolcEngineClient
}
func (self *SVolcEngineProvider) GetAccountId() string {
return self.client.GetAccountId()
}
func (self *SVolcEngineProvider) GetSysInfo() (jsonutils.JSONObject, error) {
regions := self.client.GetIRegions()
info := jsonutils.NewDict()
info.Add(jsonutils.NewInt(int64(len(regions))), "region_count")
info.Add(jsonutils.NewString(volcengine.VOLCENGINE_API_VERSION), "api_version")
return info, nil
}
func (self *SVolcEngineProvider) GetBalance() (*cloudprovider.SBalanceInfo, error) {
// GetBalance is not currently open
return &cloudprovider.SBalanceInfo{
Amount: 0.0,
Currency: "CNY",
Status: api.CLOUD_PROVIDER_HEALTH_NORMAL,
}, nil
}
func (self *SVolcEngineProvider) GetBucketCannedAcls(regionId string) []string {
return nil
}
func (self *SVolcEngineProvider) GetCapabilities() []string {
return self.client.GetCapabilities()
}
func (self *SVolcEngineProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) {
return self.client.GetIProjects()
}
func (self *SVolcEngineProvider) GetIRegionById(extId string) (cloudprovider.ICloudRegion, error) {
return self.client.GetIRegionById(extId)
}
func (self *SVolcEngineProvider) GetIRegions() []cloudprovider.ICloudRegion {
return self.client.GetIRegions()
}
func (self *SVolcEngineProvider) GetObjectCannedAcls(regionId string) []string {
return nil
}
func (self *SVolcEngineProvider) GetStorageClasses(regionId string) []string {
return nil
}
func (self *SVolcEngineProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
return self.client.GetSubAccounts()
}
func (self *SVolcEngineProvider) GetVersion() string {
return volcengine.VOLCENGINE_API_VERSION
}

680
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/region.go generated vendored Normal file
View File

@@ -0,0 +1,680 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"context"
"fmt"
"strings"
"time"
tos "github.com/volcengine/ve-tos-golang-sdk/v2/tos"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
sdk "github.com/volcengine/volc-sdk-golang/base"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
)
var RegionLocations = map[string]string{
"cn-beijing": "中国(北京)",
"cn-shanghai": "中国(上海)",
"cn-guangzhou": "中国(广州)",
}
var RegionLocationsEN = map[string]string{
"cn-beijing": "China (Beijing)",
"cn-shanghai": "China (Shanghai)",
"cn-guangzhou": "China (Guangzhou)",
}
var RegionEndpoint = map[string]string{
"cn-beijing": "cn-beijing.volces.com",
"cn-shanghai": "cn-shanghai.volces.com",
"cn-guangzhou": "cn-beijing.volces.com",
}
type SRegion struct {
multicloud.SRegion
multicloud.SNoLbRegion
client *SVolcEngineClient
tosClient *tos.ClientV2
RegionId string
ivpcs []cloudprovider.ICloudVpc
storageCache *SStoragecache
}
func (region *SRegion) GetClient() *SVolcEngineClient {
return region.client
}
func (region *SRegion) Refresh() error {
return nil
}
func (region *SRegion) GetProvider() string {
return CLOUD_PROVIDER_VOLCENGINE
}
func (region *SRegion) GetCloudEnv() string {
return region.client.cloudEnv
}
func (region *SRegion) GetId() string {
return region.RegionId
}
func (region *SRegion) GetName() string {
if localName, ok := RegionLocations[region.RegionId]; ok {
return fmt.Sprintf("%s %s", CLOUD_PROVIDER_VOLCENGINE_CN, localName)
}
return fmt.Sprintf("%s %s", CLOUD_PROVIDER_VOLCENGINE_CN, region.RegionId)
}
func (region *SRegion) GetGlobalId() string {
return fmt.Sprintf("%s/%s", region.client.GetAccessEnv(), region.RegionId)
}
func (region *SRegion) GetI18n() cloudprovider.SModelI18nTable {
var en string
if localName, ok := RegionLocationsEN[region.RegionId]; ok {
en = fmt.Sprintf("%s %s", CLOUD_PROVIDER_VOLCENGINE_EN, localName)
} else {
en = fmt.Sprintf("%s %s", CLOUD_PROVIDER_VOLCENGINE_EN, region.RegionId)
}
table := cloudprovider.SModelI18nTable{}
table["name"] = cloudprovider.NewSModelI18nEntry(region.GetName()).CN(region.GetName()).EN(en)
return table
}
func (region *SRegion) GetStatus() string {
return api.CLOUD_REGION_STATUS_INSERVER
}
func (region *SRegion) GetGeographicInfo() cloudprovider.SGeographicInfo {
if info, ok := LatitudeAndLongitude[region.RegionId]; ok {
return info
}
return cloudprovider.SGeographicInfo{}
}
func (region *SRegion) getStoragecache() *SStoragecache {
if region.storageCache == nil {
region.storageCache = &SStoragecache{region: region}
}
return region.storageCache
}
func (region *SRegion) GetZones(id string) ([]SZone, error) {
params := map[string]string{}
params["DestinationResource"] = "InstanceType"
// DedicatedHost is not supported
if len(id) > 0 {
params["ZoneId"] = id
}
body, err := region.ecsRequest("DescribeAvailableResource", params)
if err != nil {
return nil, err
}
ret := []SZone{}
err = body.Unmarshal(&ret, "Result", "AvailableZones")
if err != nil {
return nil, err
}
return ret, nil
}
func (region *SRegion) GetIZones() ([]cloudprovider.ICloudZone, error) {
zones, err := region.GetZones("")
if err != nil {
return nil, errors.Wrapf(err, "GetZones")
}
ret := []cloudprovider.ICloudZone{}
for i := range zones {
zones[i].region = region
ret = append(ret, &zones[i])
}
return ret, nil
}
func (region *SRegion) GetIZoneById(id string) (cloudprovider.ICloudZone, error) {
zones, err := region.GetZones("")
if err != nil {
return nil, errors.Wrap(err, "GetZones")
}
for i := range zones {
zones[i].region = region
if zones[i].GetId() == id || zones[i].GetGlobalId() == id {
return &zones[i], nil
}
}
return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s", id)
}
// vpc
func (region *SRegion) CreateIVpc(opts *cloudprovider.VpcCreateOptions) (cloudprovider.ICloudVpc, error) {
vpc, err := region.CreateVpc(opts)
if err != nil {
return nil, err
}
return vpc, nil
}
func (region *SRegion) CreateVpc(opts *cloudprovider.VpcCreateOptions) (*SVpc, error) {
params := make(map[string]string)
if len(opts.CIDR) > 0 {
params["CidrBlock"] = opts.CIDR
}
if len(opts.NAME) > 0 {
params["VpcName"] = opts.NAME
}
if len(opts.Desc) > 0 {
params["Description"] = opts.Desc
}
params["ClientToken"] = utils.GenRequestId(20)
body, err := region.vpcRequest("CreateVpc", params)
if err != nil {
return nil, err
}
vpcId, err := body.GetString("Result", "VpcId")
if err != nil {
return nil, err
}
err = cloudprovider.Wait(5*time.Second, time.Minute, func() (bool, error) {
_, err = region.getVpc(vpcId)
if errors.Cause(err) == cloudprovider.ErrNotFound {
return false, nil
} else {
return true, err
}
})
if err != nil {
return nil, errors.Wrapf(err, "cannot find networks after create")
}
return region.getVpc(vpcId)
}
func (region *SRegion) DeleteVpc(vpcId string) error {
params := make(map[string]string)
params["VpcId"] = vpcId
_, err := region.vpcRequest("DeleteVpc", params)
return err
}
func (region *SRegion) getVpc(vpcId string) (*SVpc, error) {
vpcs, _, err := region.GetVpcs([]string{vpcId}, 1, 50)
if err != nil {
return nil, err
}
for _, vpc := range vpcs {
if vpc.VpcId == vpcId {
vpc.region = region
return &vpc, nil
}
}
return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s not found", vpcId)
}
func (region *SRegion) GetVpcs(vpcIds []string, pageNumber int, pageSize int) ([]SVpc, int, error) {
params := make(map[string]string)
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
if len(vpcIds) > 0 {
for index, id := range vpcIds {
key := fmt.Sprintf("VpcIds.%d", index+1)
params[key] = id
}
}
body, err := region.vpcRequest("DescribeVpcs", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "GetVpcs fail")
}
vpcs := make([]SVpc, 0)
err = body.Unmarshal(&vpcs, "Result", "Vpcs")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal vpcs fail")
}
total, _ := body.Int("Result", "TotalCount")
return vpcs, int(total), nil
}
func (region *SRegion) GetIVpcs() ([]cloudprovider.ICloudVpc, error) {
if region.ivpcs == nil {
vpcs, err := region.GetAllVpcs()
if err != nil {
return nil, err
}
region.ivpcs = make([]cloudprovider.ICloudVpc, len(vpcs))
for i := 0; i < len(vpcs); i += 1 {
vpcs[i].region = region
region.ivpcs[i] = &vpcs[i]
}
}
return region.ivpcs, nil
}
func (region *SRegion) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) {
ivpcs, err := region.GetIVpcs()
if err != nil {
return nil, err
}
for i := 0; i < len(ivpcs); i += 1 {
if ivpcs[i].GetGlobalId() == id {
return ivpcs[i], nil
}
}
return nil, cloudprovider.ErrNotFound
}
func (region *SRegion) GetAllVpcs() ([]SVpc, error) {
vpcs := make([]SVpc, 0)
pageNumber := 1
for {
part, total, err := region.GetVpcs(nil, pageNumber, 50)
if err != nil {
return nil, err
}
vpcs = append(vpcs, part...)
if len(vpcs) >= total {
break
}
pageNumber += 1
}
return vpcs, nil
}
// EIP
func (region *SRegion) GetIEipById(eipId string) (cloudprovider.ICloudEIP, error) {
eip, err := region.GetEip(eipId)
if err != nil {
return nil, err
}
return eip, nil
}
func (region *SRegion) GetIEips() ([]cloudprovider.ICloudEIP, error) {
pageNumber := 1
eips, total, err := region.GetEips(make([]string, 0), "", make([]string, 0), pageNumber, 100)
if err != nil {
return nil, err
}
for len(eips) < total {
var parts []SEipAddress
pageNumber++
parts, total, err = region.GetEips(make([]string, 0), "", make([]string, 0), pageNumber, 100)
if err != nil {
return nil, err
}
eips = append(eips, parts...)
}
ret := make([]cloudprovider.ICloudEIP, len(eips))
for i := 0; i < len(eips); i += 1 {
ret[i] = &eips[i]
}
return ret, nil
}
func (region *SRegion) FetchSubnets(ids []string, zoneId string, vpcId string) ([]SNetwork, error) {
pageNumber := 1
nets := make([]SNetwork, 0)
for {
parts, total, err := region.GetSubnets(ids, zoneId, vpcId, pageNumber, 50)
if err != nil {
return nil, err
}
nets = append(nets, parts...)
if len(nets) >= total {
break
}
pageNumber += 1
}
return nets, nil
}
// IBucket
func (region *SRegion) IBucketExist(name string) (bool, error) {
toscli, err := region.GetTosClient()
if err != nil {
return false, errors.Wrap(err, "region.GetTosClient")
}
_, err = toscli.HeadBucket(context.Background(), &tos.HeadBucketInput{Bucket: name})
if err != nil || tos.StatusCode(err) != 404 {
return false, errors.Wrap(err, "IsBucketExist")
}
return true, nil
}
func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
toscli, err := region.GetTosClient()
if err != nil {
return errors.Wrap(err, "region.GetTosClient")
}
_, err = toscli.CreateBucketV2(context.Background(), &tos.CreateBucketV2Input{Bucket: name, ACL: enum.ACLType(aclStr), StorageClass: enum.StorageClassType(storageClassStr)})
if err != nil {
return errors.Wrap(err, "tos.CreateBucketV2")
}
region.client.invalidateIBuckets()
return nil
}
func (region *SRegion) DeleteIBucket(name string) error {
toscli, err := region.GetTosClient()
if err != nil {
return errors.Wrapf(err, "region.GetOssClient")
}
_, err = toscli.DeleteBucket(context.Background(), &tos.DeleteBucketInput{Bucket: name})
if err != nil {
if tos.StatusCode(err) == 404 {
return nil
}
return errors.Wrap(err, "DeleteBucket")
}
region.client.invalidateIBuckets()
return nil
}
func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
toscli, err := region.GetTosClient()
if err != nil {
return nil, errors.Wrapf(err, "region.GetOssClient")
}
out, err := toscli.ListBuckets(context.Background(), &tos.ListBucketsInput{})
if err != nil {
return nil, errors.Wrap(err, "ListBucket")
}
for _, bucket := range out.Buckets {
if bucket.Name == name {
t, err := time.Parse(time.RFC3339, bucket.CreationDate)
if err != nil {
return nil, errors.Wrapf(err, "Prase CreationDate error")
}
b := SBucket{
region: region,
Name: name,
Location: bucket.Location,
CreationDate: t,
}
return &b, nil
}
}
return nil, errors.Wrapf(cloudprovider.ErrNotFound, "Bucket Not Found")
}
func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
return region.GetIBucketById(name)
}
func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
iBuckets, err := region.client.getIBuckets()
if err != nil {
return nil, errors.Wrap(err, "getIBuckets")
}
ret := make([]cloudprovider.ICloudBucket, 0)
for i := range iBuckets {
if iBuckets[i].GetIRegion().GetId() != region.GetId() {
continue
}
ret = append(ret, iBuckets[i])
}
return ret, nil
}
func (region *SRegion) GetCapabilities() []string {
return region.client.GetCapabilities()
}
// Security Group
func (region *SRegion) CreateISecurityGroup(conf *cloudprovider.SecurityGroupCreateInput) (cloudprovider.ICloudSecurityGroup, error) {
externalId, err := region.CreateSecurityGroup(conf.VpcId, conf.Name, conf.Desc, conf.ProjectId)
if err != nil {
return nil, err
}
if conf.OnCreated != nil {
conf.OnCreated(externalId)
}
outRules := conf.OutRules
if len(outRules) > 0 && outRules[0].String() == "out:allow any" {
outRules = outRules[1:]
}
rules := append(conf.InRules, outRules...)
for _, rule := range rules {
rule.Priority = 101 - rule.Priority
err = region.addSecurityGroupRule(externalId, rule)
if err != nil {
return nil, err
}
}
err = cloudprovider.Wait(5*time.Second, time.Minute, func() (bool, error) {
_, err := region.GetISecurityGroupById(externalId)
if errors.Cause(err) == cloudprovider.ErrNotFound {
return false, nil
} else {
return true, err
}
})
if err != nil {
return nil, errors.Wrapf(err, "cannot find security group after create")
}
return region.GetISecurityGroupById(externalId)
}
func (region *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.ICloudSecurityGroup, error) {
return region.GetSecurityGroupDetails(secgroupId)
}
func (region *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) {
secgroups, _, err := region.GetSecurityGroups(opts.VpcId, opts.Name, nil, 1, 100)
if err != nil {
return nil, err
}
for _, secgroup := range secgroups {
if secgroup.SecurityGroupName == opts.Name {
secgroup.region = region
return &secgroup, nil
}
}
return nil, errors.Wrapf(cloudprovider.ErrNotFound, "%s not found", opts.Name)
}
func (region *SRegion) DeleteISecurityGroupById(secgroupId string) error {
return region.DeleteSecurityGroupById(secgroupId)
}
func (region *SRegion) getSdkCredential(service string, token string) sdk.Credentials {
return region.client.getSdkCredential(region.RegionId, service, token)
}
func (region *SRegion) ecsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) {
cred := region.getSdkCredential(VOLCENGINE_SERVICE_ECS, "")
return region.client.jsonRequest(cred, VOLCENGINE_API, VOLCENGINE_API_VERSION, apiName, params)
}
func (region *SRegion) vpcRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) {
cred := region.getSdkCredential(VOLCENGINE_SERVICE_VPC, "")
return region.client.jsonRequest(cred, VOLCENGINE_API, VOLCENGINE_API_VERSION, apiName, params)
}
func (region *SRegion) natRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) {
cred := region.getSdkCredential(VOLCENGINE_SERVICE_NAT, "")
return region.client.jsonRequest(cred, VOLCENGINE_API, VOLCENGINE_API_VERSION, apiName, params)
}
func (region *SRegion) storageRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) {
cred := region.getSdkCredential(VOLCENGINE_SERVICE_STORAGE, "")
return region.client.jsonRequest(cred, VOLCENGINE_API, VOLCENGINE_API_VERSION, apiName, params)
}
func (region *SRegion) GetTosClient() (*tos.ClientV2, error) {
if region.tosClient == nil {
cli, err := region.client.getTosClient(region.RegionId)
if err != nil {
return nil, errors.Wrap(err, "region.client.getOssClient")
}
region.tosClient = cli
}
return region.tosClient, nil
}
func (region *SRegion) UpdateInstancePassword(instanceId string, passwd string) error {
params := make(map[string]string)
params["Password"] = passwd
return region.modifyInstanceAttribute(instanceId, params)
}
func (region *SRegion) GetInstanceStatus(instanceId string) (string, error) {
instance, err := region.GetInstance(instanceId)
if err != nil {
return "", err
}
return instance.Status, nil
}
func (region *SRegion) instanceOperation(instanceId string, apiName string, extra map[string]string) error {
params := make(map[string]string)
params["RegionId"] = region.RegionId
params["InstanceId"] = instanceId
if len(extra) > 0 {
for k, v := range extra {
params[k] = v
}
}
_, err := region.ecsRequest(apiName, params)
return err
}
func (region *SRegion) getBaseEndpoint() string {
return RegionEndpoint[region.RegionId]
}
func (region *SRegion) getS3Endpoint() string {
base := region.getBaseEndpoint()
if len(base) > 0 {
return "tos-s3-" + base
}
return ""
}
func (region *SRegion) getTOSExternalDomain() string {
return getTOSExternalDomain(region.RegionId)
}
func (region *SRegion) getTOSInternalDomain() string {
return getTOSInternalDomain(region.RegionId)
}
func (region *SRegion) GetRouteTables(ids []string, pageNumber int, pageSize int) ([]SRouteTable, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
if len(ids) > 0 {
params["RouteTableId"] = strings.Join(ids, ",")
}
body, err := region.vpcRequest("DescribeRouteTableList", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "GetRoutseTables fail")
}
routetables := make([]SRouteTable, 0)
err = body.Unmarshal(&routetables, "RouteTables", "RouteTable")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal routetables fail")
}
total, _ := body.Int("Result", "TotalCount")
return routetables, int(total), nil
}
func (region *SRegion) GetIHostById(id string) (cloudprovider.ICloudHost, error) {
izones, err := region.GetIZones()
if err != nil {
return nil, err
}
for i := 0; i < len(izones); i += 1 {
ihost, err := izones[i].GetIHostById(id)
if err == nil {
return ihost, nil
} else if errors.Cause(err) != cloudprovider.ErrNotFound {
return nil, err
}
}
return nil, cloudprovider.ErrNotFound
}
func (regioin *SRegion) GetIHosts() ([]cloudprovider.ICloudHost, error) {
iHosts := make([]cloudprovider.ICloudHost, 0)
izones, err := regioin.GetIZones()
if err != nil {
return nil, err
}
for i := 0; i < len(izones); i += 1 {
iZoneHost, err := izones[i].GetIHosts()
if err != nil {
return nil, err
}
iHosts = append(iHosts, iZoneHost...)
}
return iHosts, nil
}
func (region *SRegion) GetIDiskById(id string) (cloudprovider.ICloudDisk, error) {
return region.getDisk(id)
}
func (region *SRegion) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) {
izones, err := region.GetIZones()
if err != nil {
return nil, err
}
for i := 0; i < len(izones); i += 1 {
istore, err := izones[i].GetIStorageById(id)
if err == nil {
return istore, nil
} else if errors.Cause(err) != cloudprovider.ErrNotFound {
return nil, err
}
}
return nil, cloudprovider.ErrNotFound
}
func (region *SRegion) GetIStorages() ([]cloudprovider.ICloudStorage, error) {
iStores := make([]cloudprovider.ICloudStorage, 0)
izones, err := region.GetIZones()
if err != nil {
return nil, err
}
for i := 0; i < len(izones); i += 1 {
iZoneStores, err := izones[i].GetIStorages()
if err != nil {
return nil, err
}
iStores = append(iStores, iZoneStores...)
}
return iStores, nil
}
func (region *SRegion) GetIVMById(id string) (cloudprovider.ICloudVM, error) {
return region.GetInstance(id)
}

290
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/routetable.go generated vendored Normal file
View File

@@ -0,0 +1,290 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"strings"
"time"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/pkg/errors"
)
type SRouteEntry struct {
multicloud.SResourceBase
VolcEngineTags
routeTable *SRouteTable
Description string
DestinationCidrBlock string
RouteEntryId string
RouteEntryName string
RouteTableId string
Status string
Type string
VpcId string
NextHopId string
NextHopName string
NextHopType string
}
type SRouteEntrys []*SRouteEntry
type SubnetIds []string
type SRouteTable struct {
multicloud.SResourceBase
VolcEngineTags
region *SRegion
vpc *SVpc
routes []cloudprovider.ICloudRoute
Description string
RouteTableId string
RouteTableName string
RouteTableType string
VpcId string
VpcName string
CreationTime time.Time
UpdateTime time.Time
AccountId string
ProjectName string
SubnetIds SubnetIds
RouteEntrys SRouteEntrys
}
func (route *SRouteEntry) GetId() string {
return fmt.Sprintf("%s-%s-%s", route.RouteTableId, route.DestinationCidrBlock, route.NextHopType)
}
func (route *SRouteEntry) GetName() string {
return route.RouteEntryName
}
func (route *SRouteEntry) GetGlobalId() string {
return route.GetId()
}
func (route *SRouteEntry) GetStatus() string {
return api.ROUTE_ENTRY_STATUS_AVAILIABLE
}
func (route *SRouteEntry) Refresh() error {
return nil
}
func (route *SRouteEntry) GetType() string {
return route.Type
}
func (route *SRouteEntry) GetCidr() string {
return route.DestinationCidrBlock
}
func (route *SRouteEntry) GetNextHopType() string {
switch route.NextHopType {
case "Instance":
return api.NEXT_HOP_TYPE_INSTANCE
case "HaVip":
return api.NEXT_HOP_TYPE_HAVIP
case "VpnGW":
return api.NEXT_HOP_TYPE_VPN
case "NatGW":
return api.NEXT_HOP_TYPE_NAT
case "NetworkInterface":
return api.NEXT_HOP_TYPE_NETWORK
case "IPv6GW":
return api.NEXT_HOP_TYPE_IPV6
case "TransitRouter":
return api.NEXT_HOP_TYPE_ROUTER
default:
return ""
}
}
func (route *SRouteEntry) GetNextHop() string {
return route.NextHopId
}
func (table *SRouteTable) GetDescription() string {
return table.Description
}
func (table *SRouteTable) GetId() string {
return table.RouteTableId
}
func (table *SRouteTable) GetGlobalId() string {
return table.RouteTableId
}
func (table *SRouteTable) GetName() string {
return table.RouteTableName
}
func (table *SRouteTable) GetRegionId() string {
return table.region.RegionId
}
func (table *SRouteTable) GetType() cloudprovider.RouteTableType {
switch table.RouteTableType {
case "System":
return cloudprovider.RouteTableTypeSystem
case "Custom":
return cloudprovider.RouteTableTypeCustom
default:
return cloudprovider.RouteTableTypeSystem
}
}
func (table *SRouteTable) GetVpcId() string {
return table.VpcId
}
func (table *SRouteTable) GetStatus() string {
return api.ROUTE_TABLE_AVAILABLE
}
func (table *SRouteTable) Refresh() error {
return nil
}
func (routeTable *SRouteTable) IsSystem() bool {
return strings.ToLower(routeTable.RouteTableType) == "system"
}
func (table *SRouteTable) RemoteGetRoutes(pageNumber int, pageSize int) ([]*SRouteEntry, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["RouteTableId"] = table.RouteTableId
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
body, err := table.region.vpcRequest("DescribeRouteEntryList", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "RemoteGetRoutes fail")
}
entries := SRouteEntrys{}
err = body.Unmarshal(&entries, "Result", "RouteEntries")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal routeEntrys fail")
}
total, _ := body.Int("Result", "TotalCount")
return entries, int(total), nil
}
func (table *SRouteTable) fetchRoutes() error {
routes := []*SRouteEntry{}
pageNumber := 1
for {
parts, total, err := table.RemoteGetRoutes(pageNumber, 50)
if err != nil {
return err
}
routes = append(routes, parts...)
if len(routes) >= total {
break
}
pageNumber += 1
}
table.routes = make([]cloudprovider.ICloudRoute, len(routes))
for i := 0; i < len(routes); i++ {
routes[i].routeTable = table
table.routes[i] = routes[i]
}
return nil
}
func (table *SRouteTable) GetIRoutes() ([]cloudprovider.ICloudRoute, error) {
if table.routes == nil {
err := table.fetchRoutes()
if err != nil {
return nil, err
}
}
return table.routes, nil
}
func (table *SRouteTable) GetAssociations() []cloudprovider.RouteTableAssociation {
result := []cloudprovider.RouteTableAssociation{}
for i := range table.SubnetIds {
association := cloudprovider.RouteTableAssociation{
AssociationId: table.RouteTableId + ":" + table.SubnetIds[i],
AssociationType: cloudprovider.RouteTableAssociaToSubnet,
AssociatedResourceId: table.SubnetIds[i],
}
result = append(result, association)
}
return result
}
func (table *SRouteTable) CreateRoute(route cloudprovider.RouteSet) error {
return cloudprovider.ErrNotSupported
}
func (table *SRouteTable) UpdateRoute(route cloudprovider.RouteSet) error {
return cloudprovider.ErrNotSupported
}
func (table *SRouteTable) RemoveRoute(route cloudprovider.RouteSet) error {
return cloudprovider.ErrNotSupported
}
func (vpc *SVpc) RemoteGetRouteTableList(pageNumber int, pageSize int) ([]*SRouteTable, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["VpcId"] = vpc.VpcId
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
body, err := vpc.region.vpcRequest("DescribeRouteTableList", params)
if err != nil {
return nil, 0, errors.Wrapf(err, "RemoteGetRouteTableList fail")
}
routeTables := make([]*SRouteTable, 0)
err = body.Unmarshal(&routeTables, "Result", "RouterTableList")
if err != nil {
return nil, 0, errors.Wrapf(err, "Unmarshal routeTables fail")
}
for _, routeTable := range routeTables {
routeTable.region = vpc.region
}
total, _ := body.Int("Result", "TotalCount")
return routeTables, int(total), nil
}
func (region *SRegion) AssociateRouteTable(rtableId string, SubnetId string) error {
params := make(map[string]string)
params["RouteTableId"] = rtableId
params["SubnetId"] = SubnetId
_, err := region.vpcRequest("AssociateRouteTable", params)
return err
}
func (region *SRegion) UnassociateRouteTable(rtableId string, SubnetId string) error {
params := make(map[string]string)
params["RouteTableId"] = rtableId
params["SubnetId"] = SubnetId
_, err := region.vpcRequest("UnassociateRouteTable", params)
return err
}

327
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/securitygroup.go generated vendored Normal file
View File

@@ -0,0 +1,327 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"strings"
"time"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/secrules"
"yunion.io/x/pkg/utils"
)
type SCidrList []string
type SSecurityGroupRule struct {
CreationTime time.Time
UpdateTime time.Time
Description string
Direction string
Protocol string
Policy string
PortStart int
PortEnd int
CidrIp string
PrefixListId string
PrefixListCidrs SCidrList
Priority int
SourceGroupId string
}
type SSecurityGroup struct {
multicloud.SSecurityGroup
VolcEngineTags
region *SRegion
Description string
Permissions []SSecurityGroupRule
SecurityGroupId string
SecurityGroupName string
VpcId string
CreationTime time.Time
UpdateTime time.Time
Type string
ProjectName string
ServiceManaged bool
Status string
}
func (region *SRegion) CreateSecurityGroup(vpcId string, name string, desc, projectName string) (string, error) {
params := make(map[string]string)
if len(vpcId) > 0 {
params["VpcId"] = vpcId
}
if len(projectName) > 0 {
params["ProjectName"] = projectName
}
if len(name) > 0 {
params["SecurityGroupName"] = name
}
if len(desc) > 0 {
params["Description"] = desc
}
params["ClientToken"] = utils.GenRequestId(20)
body, err := region.vpcRequest("CreateSecurityGroup", params)
if err != nil {
return "", errors.Wrap(err, "CreateSecurityGroup")
}
return body.GetString("Result", "SecurityGroupId")
}
func (region *SRegion) GetSecurityGroupDetails(secGroupId string) (*SSecurityGroup, error) {
params := make(map[string]string)
params["SecurityGroupId"] = secGroupId
body, err := region.vpcRequest("DescribeSecurityGroupAttributes", params)
if err != nil {
return nil, err
}
securitygroup := SSecurityGroup{}
err = body.Unmarshal(&securitygroup, "Result")
if err != nil {
return nil, errors.Wrapf(err, "Unmarshal security group details fail")
}
securitygroup.region = region
return &securitygroup, err
}
func (region *SRegion) DeleteSecurityGroupById(secGroupId string) error {
params := make(map[string]string)
params["SecurityGroupId"] = secGroupId
_, err := region.vpcRequest("DeleteSecurityGroupId", params)
return err
}
func (region *SRegion) AddSecurityGroupRules(secGrpId string, rule cloudprovider.SecurityRule) error {
if len(rule.Ports) != 0 {
for _, port := range rule.Ports {
rule.PortStart, rule.PortEnd = port, port
err := region.addSecurityGroupRule(secGrpId, rule)
if err != nil {
return errors.Wrapf(err, "addSecurityGroupRule %s", rule.String())
}
}
return nil
}
return region.addSecurityGroupRule(secGrpId, rule)
}
func (region *SRegion) addSecurityGroupRule(secGrpId string, rule cloudprovider.SecurityRule) error {
params := make(map[string]string)
params["RegionId"] = region.RegionId
params["SecurityGroupId"] = secGrpId
params["Description"] = rule.Description
params["PortStart"] = fmt.Sprintf("%d", rule.PortStart)
params["PortEnd"] = fmt.Sprintf("%d", rule.PortEnd)
protocol := rule.Protocol
if len(rule.Protocol) == 0 || rule.Protocol == secrules.PROTO_ANY {
protocol = "all"
}
params["Protocol"] = protocol
if rule.Action == secrules.SecurityRuleAllow {
params["Policy"] = "accept"
} else {
params["Policy"] = "drop"
}
params["Priority"] = fmt.Sprintf("%d", rule.Priority)
if rule.Direction == secrules.SecurityRuleIngress {
if rule.IPNet != nil {
params["CidrIp"] = rule.IPNet.String()
} else {
params["CidrIp"] = "0.0.0.0/0"
}
_, err := region.vpcRequest("AuthorizeSecurityGroupIngress", params)
return err
} else {
if rule.IPNet != nil {
params["CidrIp"] = rule.IPNet.String()
} else {
params["CidrIp"] = "0.0.0.0/0"
}
_, err := region.vpcRequest("AuthorizeSecurityGroupEgress", params)
return err
}
}
func (region *SRegion) GetSecurityGroups(vpcId, name string, securityGroupIds []string, pageSize int, pageNumber int) ([]SSecurityGroup, int, error) {
if pageSize > 100 || pageSize <= 0 {
pageSize = 100
}
params := make(map[string]string)
params["PageSize"] = fmt.Sprintf("%d", pageSize)
params["PageNumber"] = fmt.Sprintf("%d", pageNumber)
if len(vpcId) > 0 {
params["VpcId"] = vpcId
}
if len(name) > 0 {
params["SecurityGroupName"] = name
}
if len(securityGroupIds) > 0 {
params["SecurityGroupIds"] = jsonutils.Marshal(securityGroupIds).String()
}
body, err := region.vpcRequest("DescribeSecurityGroups", params)
if err != nil {
log.Errorf("GetSecurityGroups fail %s", err)
return nil, 0, err
}
secgrps := make([]SSecurityGroup, 0)
err = body.Unmarshal(&secgrps, "Result", "SecurityGroups")
if err != nil {
log.Errorf("Unmarshal security groups fail %s", err)
return nil, 0, err
}
total, _ := body.Int("Result", "TotalCount")
return secgrps, int(total), nil
}
func (rule *SSecurityGroupRule) toUniformRule() (cloudprovider.SecurityRule, error) {
uniformRule := cloudprovider.SecurityRule{
SecurityRule: secrules.SecurityRule{
Action: secrules.SecurityRuleDeny,
Direction: secrules.DIR_IN,
Priority: 101 - rule.Priority,
Description: rule.Description,
PortStart: -1,
PortEnd: -1,
},
}
if strings.ToLower(rule.Policy) == "accept" {
uniformRule.Action = secrules.SecurityRuleAllow
}
cidr := rule.CidrIp
if rule.Direction == "egress" {
uniformRule.Direction = secrules.DIR_OUT
}
uniformRule.ParseCIDR(cidr)
switch strings.ToLower(rule.Protocol) {
case "tcp", "udp", "icmp":
uniformRule.Protocol = strings.ToLower(rule.Protocol)
case "all":
uniformRule.Protocol = secrules.PROTO_ANY
default:
return uniformRule, fmt.Errorf("unsupported protocal %s", rule.Protocol)
}
port := ""
if rule.PortStart == rule.PortEnd {
if rule.PortStart != -1 {
port = fmt.Sprintf("%d", rule.PortStart)
}
} else if rule.PortStart != -1 && rule.PortEnd != 65535 {
port = fmt.Sprintf("%d-%d", rule.PortStart, rule.PortEnd)
}
err := uniformRule.ParsePorts(port)
if err != nil {
return uniformRule, errors.Wrapf(err, "ParsePorts(%s)", port)
}
return uniformRule, nil
}
func (secgroup *SSecurityGroup) GetId() string {
return secgroup.SecurityGroupId
}
func (secgroup *SSecurityGroup) GetName() string {
return secgroup.SecurityGroupName
}
func (secgroup *SSecurityGroup) GetGlobalId() string {
return secgroup.GetId()
}
func (secgroup *SSecurityGroup) GetCreatedAt() time.Time {
return secgroup.CreationTime
}
func (secgroup *SSecurityGroup) GetDescription() string {
return secgroup.Description
}
func (secgroup *SSecurityGroup) GetStatus() string {
return secgroup.Status
}
func (secgroup *SSecurityGroup) Refresh() error {
if body, err := secgroup.region.GetSecurityGroupDetails(secgroup.GetId()); err != nil {
return err
} else {
return jsonutils.Update(secgroup, body)
}
}
func (secgroup *SSecurityGroup) GetProjectId() string {
return secgroup.ProjectName
}
func (secgroup *SSecurityGroup) GetRules() ([]cloudprovider.SecurityRule, error) {
rules := make([]cloudprovider.SecurityRule, 0)
updatedSecgroup, err := secgroup.region.GetSecurityGroupDetails(secgroup.SecurityGroupId)
if err != nil {
return nil, err
}
outAllow := secrules.MustParseSecurityRule("out:allow any")
rules = append(rules, cloudprovider.SecurityRule{SecurityRule: *outAllow})
for _, permission := range updatedSecgroup.Permissions {
if len(permission.SourceGroupId) > 0 {
continue
}
if !utils.IsInStringArray(strings.ToLower(permission.Protocol), []string{"tcp", "udp", "icmp", "all"}) {
continue
}
rule, err := permission.toUniformRule()
if err != nil {
log.Errorf("convert rule %s for group %s(%s) error: %v", permission.Description, secgroup.SecurityGroupName, secgroup.SecurityGroupId, err)
continue
}
rules = append(rules, rule)
}
return rules, nil
}
func (secgroup *SSecurityGroup) GetVpcId() string {
return secgroup.VpcId
}
func (secgroup *SSecurityGroup) GetReferences() ([]cloudprovider.SecurityGroupReference, error) {
ret := []cloudprovider.SecurityGroupReference{}
return ret, errors.Wrapf(errors.ErrNotImplemented, "GetReferences not supported")
}
func (region *SRegion) DeleteSecurityGroup(secGrpId string) error {
params := make(map[string]string)
params["SecurityGroupId"] = secGrpId
_, err := region.vpcRequest("DeleteSecurityGroup", params)
if err != nil {
return errors.Wrapf(err, "Delete security group fail")
}
return nil
}
func (secgroup *SSecurityGroup) Delete() error {
return secgroup.region.DeleteSecurityGroupById(secgroup.SecurityGroupId)
}

155
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storage.go generated vendored Normal file
View File

@@ -0,0 +1,155 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"fmt"
"time"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
)
type SStorage struct {
multicloud.SStorageBase
VolcEngineTags
zone *SZone
storageType string
}
func (storage *SStorage) GetId() string {
return fmt.Sprintf("%s-%s-%s", storage.zone.region.client.cpcfg.Id, storage.zone.GetId(), storage.storageType)
}
func (storage *SStorage) GetName() string {
return fmt.Sprintf("%s-%s-%s", storage.zone.region.client.cpcfg.Name, storage.zone.GetId(), storage.storageType)
}
func (storage *SStorage) GetGlobalId() string {
return fmt.Sprintf("%s-%s-%s", storage.zone.region.client.cpcfg.Id, storage.zone.GetGlobalId(), storage.storageType)
}
func (storage *SStorage) IsEmulated() bool {
return true
}
func (storage *SStorage) GetIZone() cloudprovider.ICloudZone {
return storage.zone
}
func (storage *SStorage) GetIDisks() ([]cloudprovider.ICloudDisk, error) {
disks := make([]SDisk, 0)
pageNumber := 1
storageType := storage.storageType
for {
parts, total, err := storage.zone.region.GetDisks("", storage.zone.GetId(), storageType, nil, pageNumber, 50)
if err != nil {
return nil, errors.Wrapf(err, "GetDisks")
}
disks = append(disks, parts...)
if len(parts) >= total {
break
}
pageNumber += 1
}
idisks := make([]cloudprovider.ICloudDisk, len(disks))
for i := 0; i < len(disks); i += 1 {
disks[i].storage = storage
idisks[i] = &disks[i]
}
return idisks, nil
}
func (storage *SStorage) GetStorageType() string {
return storage.storageType
}
func (storage *SStorage) GetCapacityMB() int64 {
return 0
}
func (storage *SStorage) GetCapacityUsedMB() int64 {
return 0
}
func (storage *SStorage) GetMediumType() string {
return api.DISK_TYPE_SSD
}
func (storage *SStorage) GetStorageConf() jsonutils.JSONObject {
conf := jsonutils.NewDict()
return conf
}
func (storage *SStorage) GetStatus() string {
return api.STORAGE_ONLINE
}
func (storage *SStorage) Refresh() error {
return nil
}
func (storage *SStorage) GetEnabled() bool {
return true
}
func (storage *SStorage) GetIStoragecache() cloudprovider.ICloudStoragecache {
return storage.zone.region.getStoragecache()
}
func (storage *SStorage) CreateIDisk(conf *cloudprovider.DiskCreateConfig) (cloudprovider.ICloudDisk, error) {
diskId, err := storage.zone.region.CreateDisk(storage.zone.ZoneId, storage.storageType, conf.Name, conf.SizeGb, conf.Desc, conf.ProjectId)
if err != nil {
log.Errorf("createDisk fail %s", err)
return nil, err
}
err = cloudprovider.Wait(5*time.Second, time.Minute, func() (bool, error) {
_, err := storage.zone.region.getDisk(diskId)
if errors.Cause(err) == cloudprovider.ErrNotFound {
return false, nil
}
return true, err
})
if err != nil {
return nil, errors.Wrapf(err, "cannot find disk after create")
}
disk, err := storage.zone.region.getDisk(diskId)
if err != nil {
return nil, err
}
disk.storage = storage
return disk, nil
}
func (storage *SStorage) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) {
disk, err := storage.zone.region.getDisk(idStr)
if err != nil {
return nil, err
}
disk.storage = storage
return disk, nil
}
func (storage *SStorage) GetMountPoint() string {
return ""
}
func (storage *SStorage) IsSysDiskStore() bool {
return true
}

174
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storagecache.go generated vendored Normal file
View File

@@ -0,0 +1,174 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"context"
"fmt"
"strings"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/qemuimgfmt"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
)
type SStoragecache struct {
multicloud.SResourceBase
VolcEngineTags
region *SRegion
}
func GetBucketName(regionId string, imageId string) string {
return fmt.Sprintf("imgcache-%s-%s", strings.ToLower(regionId), imageId)
}
func (scache *SStoragecache) GetId() string {
return fmt.Sprintf("%s-%s", scache.region.client.cpcfg.Id, scache.region.GetId())
}
func (scache *SStoragecache) GetName() string {
return fmt.Sprintf("%s-%s", scache.region.client.cpcfg.Name, scache.region.GetId())
}
func (scache *SStoragecache) GetStatus() string {
return "available"
}
func (scache *SStoragecache) Refresh() error {
return nil
}
func (scache *SStoragecache) GetGlobalId() string {
return fmt.Sprintf("%s-%s", scache.region.client.cpcfg.Id, scache.region.GetGlobalId())
}
func (scache *SStoragecache) GetICloudImages() ([]cloudprovider.ICloudImage, error) {
return nil, cloudprovider.ErrNotImplemented
}
func (scache *SStoragecache) GetICustomizedCloudImages() ([]cloudprovider.ICloudImage, error) {
images := make([]SImage, 0)
token := ""
for {
parts, nextToken, err := scache.region.GetImages(ImageStatusType(""), ImageOwnerPrivate, nil, "", 50, token)
if err != nil {
return nil, errors.Wrapf(err, "GetImages")
}
images = append(images, parts...)
if len(nextToken) == 0 {
break
}
token = nextToken
}
ret := []cloudprovider.ICloudImage{}
for i := range images {
images[i].storageCache = scache
ret = append(ret, &images[i])
}
return ret, nil
}
func (scache *SStoragecache) GetIImageById(extId string) (cloudprovider.ICloudImage, error) {
img, err := scache.region.GetImage(extId)
if err != nil {
return nil, err
}
img.storageCache = scache
return img, nil
}
func (scache *SStoragecache) GetPath() string {
return ""
}
func (scache *SStoragecache) UploadImage(ctx context.Context, image *cloudprovider.SImageCreateOption, callback func(progress float32)) (string, error) {
return scache.uploadImage(ctx, image, callback)
}
func (scache *SStoragecache) uploadImage(ctx context.Context, image *cloudprovider.SImageCreateOption, callback func(progress float32)) (string, error) {
bucketName := GetBucketName(scache.region.GetId(), image.ImageId)
exist, err := scache.region.IBucketExist(bucketName)
if err != nil {
return "", errors.Wrapf(err, "IBucketExist")
}
if !exist {
err = scache.region.CreateIBucket(bucketName, "", "")
if err != nil {
return "", errors.Wrapf(err, "CreateIBucket")
}
}
defer scache.region.DeleteIBucket(bucketName)
reader, sizeBytes, err := image.GetReader(image.ImageId, string(qemuimgfmt.VMDK))
if err != nil {
return "", errors.Wrapf(err, "GetReader")
}
bucket, err := scache.region.GetIBucketByName(bucketName)
if err != nil {
return "", errors.Wrap(err, "GetIBucketByName")
}
body := multicloud.NewProgress(sizeBytes, 80, reader, callback)
err = cloudprovider.UploadObject(ctx, bucket, image.ImageId, 0, body, sizeBytes, "", "", nil, false)
if err != nil {
return "", errors.Wrap(err, "cloudprovider.UploadObject")
}
defer bucket.DeleteObject(ctx, image.ImageId)
imageBaseName := image.ImageId
if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' {
imageBaseName = fmt.Sprintf("img%s", image.ImageId)
}
imageName := imageBaseName
nameIdx := 1
for {
_, err = scache.region.GetImageByName(imageName)
if err != nil {
if errors.Cause(err) == cloudprovider.ErrNotFound {
break
} else {
return "", err
}
}
imageName = fmt.Sprintf("%s-%d", imageBaseName, nameIdx)
nameIdx += 1
log.Debugf("uploadImage Match remote name %s", imageName)
}
log.Debugf("Import image %s", imageName)
imageId, err := scache.region.ImportImage(imageName, image.OsArch, image.OsType, image.OsDistribution, image.OsVersion, bucketName, image.ImageId)
if err != nil {
return "", errors.Wrapf(err, "ImportImage %s %s", image.ImageId, bucketName)
}
return imageId, nil
}
func (region *SRegion) GetIStoragecaches() ([]cloudprovider.ICloudStoragecache, error) {
storageCache := region.getStoragecache()
return []cloudprovider.ICloudStoragecache{storageCache}, nil
}
func (region *SRegion) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) {
storageCache := region.getStoragecache()
if id == storageCache.GetGlobalId() {
return storageCache, nil
}
return nil, cloudprovider.ErrNotFound
}

62
vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/tag_base.go generated vendored Normal file
View File

@@ -0,0 +1,62 @@
// Copyright 2023 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package volcengine
import (
"strings"
"yunion.io/x/pkg/errors"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/cloudmux/pkg/multicloud"
)
type VolcEngineTags struct {
Tags []multicloud.STag
}
func (itag *VolcEngineTags) GetTags() (map[string]string, error) {
ret := map[string]string{}
for _, tag := range itag.Tags {
if len(tag.TagKey) > 0 {
ret[tag.TagKey] = tag.TagValue
} else if len(tag.Key) > 0 {
ret[tag.Key] = tag.Value
}
}
return ret, nil
}
func (itag *VolcEngineTags) GetSysTags() map[string]string {
ret := map[string]string{}
prefix := "volc:"
for _, tag := range itag.Tags {
if len(tag.TagKey) > 0 {
if strings.HasPrefix(tag.TagKey, prefix) {
ret[tag.TagKey] = tag.TagValue
}
}
if len(tag.Key) > 0 {
if strings.HasPrefix(tag.Key, prefix) {
ret[tag.Key] = tag.Value
}
}
}
return ret
}
func (itag *VolcEngineTags) SetTags(tags map[string]string, replace bool) error {
return errors.Wrap(cloudprovider.ErrNotImplemented, "SetTags")
}

Some files were not shown because too many files have changed in this diff Show More