mirror of
https://hubproxy.babadafafafafa.cn/https://github.com/yunionio/cloudpods.git
synced 2026-09-20 08:03:53 +08:00
fix(mcp-server): read the field tags through the reflectutils accessor (#25691)
Co-authored-by: Qiu Jian <qiujian@yunionyun.com>
This commit is contained in:
10
go.mod
10
go.mod
@@ -117,15 +117,15 @@ require (
|
||||
k8s.io/cri-api v0.28.15
|
||||
k8s.io/klog/v2 v2.90.1
|
||||
moul.io/http2curl/v2 v2.3.0
|
||||
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120446-39657d0a072d
|
||||
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101522-bf8649bcb146
|
||||
yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005
|
||||
yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c
|
||||
yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea
|
||||
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
|
||||
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
|
||||
yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f
|
||||
yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758
|
||||
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
|
||||
yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc
|
||||
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
|
||||
yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12
|
||||
yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac
|
||||
)
|
||||
|
||||
require (
|
||||
|
||||
26
go.sum
26
go.sum
@@ -1682,6 +1682,7 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -1999,28 +2000,29 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
|
||||
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
|
||||
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
|
||||
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
|
||||
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120446-39657d0a072d h1:z7q3rvsp8iTju0FuIKPBSrEU4peDulMNR1d7DwxEy1g=
|
||||
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120446-39657d0a072d/go.mod h1:FLNUcF+R7ep5AmH7cVHRw1xFfxEKahDzyDRWcV3naOc=
|
||||
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101522-bf8649bcb146 h1:DIGcOQoFp0McIOysABzOjcLJBFsyMTH/8qwBUECiMPE=
|
||||
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101522-bf8649bcb146/go.mod h1:FLNUcF+R7ep5AmH7cVHRw1xFfxEKahDzyDRWcV3naOc=
|
||||
yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005 h1:3sWwcjGXGjG9mLBWa7AyLq+QSi0udTAx21pfVQRFMBE=
|
||||
yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
|
||||
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
|
||||
yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c h1:FthZM1upg4/yF4vmYheo/J7a738sGPjWPKk5kY88sH0=
|
||||
yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
|
||||
yunion.io/x/jsonutils v0.0.0-20220106020632-953b71a4c3a8/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
|
||||
yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea h1:lB7Kwanf7REFaA19YmDc7p0HVkssVRoNQejMlqtpecw=
|
||||
yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea/go.mod h1:8ctyIiHT+Af1Ldr0xhvZZvuf+kyghlBUuvTy6pOKwbo=
|
||||
yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
|
||||
yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
|
||||
yunion.io/x/log v1.0.0/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
|
||||
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 h1:inY5o3LDa/zgsIZuPN0HmpzKIsu/lLgsBmMttuDPGj4=
|
||||
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
|
||||
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 h1:Hu/4ERvoWaN6aiFs4h4/yvVBMeVzb3eDaYTxkVx9Kdo=
|
||||
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
|
||||
yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
|
||||
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
|
||||
yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f h1:eRWHOwgnEK4dMoq7TQ61qZxly3LF0s7tH6nlcahvZiA=
|
||||
yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
|
||||
yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
|
||||
yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758 h1:3p/GCOPvDfvlSfovOh9GKxsSa7PPqp//DUjGRX8Wyus=
|
||||
yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758/go.mod h1:Crp6aUYB+Yq16DFjZPHhHq7ha5HL2DfyPvzCo2LJ+LU=
|
||||
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4=
|
||||
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
|
||||
yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc h1:di0CJ1Fu/aeY0Lfvr52sQx8VTOm0JyZXJz0aC+7p6No=
|
||||
yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc/go.mod h1:vCIZpqhZ5Jzaq3tFyrti/vv8BijQKtkzSgNT/uH4H5A=
|
||||
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ=
|
||||
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
|
||||
yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12 h1:xU8Z/Zg+zc199Y58eB3fA8mvOHsA28UCDfd7OuK3QKA=
|
||||
yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12/go.mod h1:RNiIM7ZFF7VFrk18DbwAa8PYR00YP7Y23XjqKIlfr3I=
|
||||
yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac h1:hJAthswj6YfWd3GCOBl9eM7mv0+4EUmjnD0b9jdBYi0=
|
||||
yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac/go.mod h1:DyMCdRyfV5PdEe/BDz2vBbemnmALUF3Rj1JmUUlu3KY=
|
||||
zombiezen.com/go/sqlite v0.13.1 h1:qDzxyWWmMtSSEH5qxamqBFmqA2BLSSbtODi3ojaE02o=
|
||||
zombiezen.com/go/sqlite v0.13.1/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4=
|
||||
|
||||
@@ -87,17 +87,16 @@ func collectMcpFields(optionsProto interface{}) map[string]mcpFieldMeta {
|
||||
if info == nil {
|
||||
continue
|
||||
}
|
||||
tagMap := info.Tags
|
||||
mcpVal, ok := tagMap[TagMCP]
|
||||
mcpVal, ok := info.Tag(TagMCP)
|
||||
if !ok || mcpVal == "" || mcpVal == "false" {
|
||||
continue
|
||||
}
|
||||
// 即便 json:"-"(Ignore)也保留:climc 仍可能用字段名作为 CLI token(如 MemSpec -> mem-spec)
|
||||
token, tokOK := tagMap["token"]
|
||||
token, tokOK := info.Tag("token")
|
||||
if !tokOK {
|
||||
if jsonName := tagMap["json"]; jsonName != "" && jsonName != "-" {
|
||||
if jsonName, _ := info.Tag("json"); jsonName != "" && jsonName != "-" {
|
||||
token = info.MarshalName()
|
||||
} else if alias := tagMap["alias"]; alias != "" {
|
||||
} else if alias, _ := info.Tag("alias"); alias != "" {
|
||||
token = alias
|
||||
} else {
|
||||
// Ignore 字段的 info.Name 为空,必须用 FieldName
|
||||
@@ -115,7 +114,7 @@ func collectMcpFields(optionsProto interface{}) map[string]mcpFieldMeta {
|
||||
out[fieldTok] = meta
|
||||
}
|
||||
// json 名与字段名不一致时(如 Region / prefer_region)两边都登记
|
||||
if jsonName := tagMap["json"]; jsonName != "" && jsonName != "-" {
|
||||
if jsonName, _ := info.Tag("json"); jsonName != "" && jsonName != "-" {
|
||||
if jsonTok := utils.CamelSplit(info.MarshalName(), "-"); jsonTok != "" && jsonTok != cliToken {
|
||||
out[jsonTok] = meta
|
||||
}
|
||||
|
||||
10
vendor/modules.txt
vendored
10
vendor/modules.txt
vendored
@@ -2709,7 +2709,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
|
||||
# sigs.k8s.io/yaml v1.3.0
|
||||
## explicit; go 1.12
|
||||
sigs.k8s.io/yaml
|
||||
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120446-39657d0a072d
|
||||
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101522-bf8649bcb146
|
||||
## explicit; go 1.24
|
||||
yunion.io/x/cloudmux/pkg/apis
|
||||
yunion.io/x/cloudmux/pkg/apis/billing
|
||||
@@ -2809,7 +2809,7 @@ yunion.io/x/cloudmux/pkg/multicloud/zstack/provider
|
||||
yunion.io/x/executor/apis
|
||||
yunion.io/x/executor/client
|
||||
yunion.io/x/executor/server
|
||||
# yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c
|
||||
# yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea
|
||||
## explicit; go 1.18
|
||||
yunion.io/x/jsonutils
|
||||
# yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
|
||||
@@ -2821,7 +2821,7 @@ yunion.io/x/log/hooks
|
||||
yunion.io/x/ovsdb/cli_util
|
||||
yunion.io/x/ovsdb/schema/ovn_nb
|
||||
yunion.io/x/ovsdb/types
|
||||
# yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f
|
||||
# yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758
|
||||
## explicit; go 1.18
|
||||
yunion.io/x/pkg/appctx
|
||||
yunion.io/x/pkg/errors
|
||||
@@ -2875,7 +2875,7 @@ yunion.io/x/pkg/utils
|
||||
# yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
|
||||
## explicit; go 1.12
|
||||
yunion.io/x/s3cli
|
||||
# yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc
|
||||
# yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12
|
||||
## explicit; go 1.17
|
||||
yunion.io/x/sqlchemy
|
||||
yunion.io/x/sqlchemy/backends
|
||||
@@ -2883,7 +2883,7 @@ yunion.io/x/sqlchemy/backends/clickhouse
|
||||
yunion.io/x/sqlchemy/backends/dameng
|
||||
yunion.io/x/sqlchemy/backends/mysql
|
||||
yunion.io/x/sqlchemy/backends/sqlite
|
||||
# yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
|
||||
# yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac
|
||||
## explicit; go 1.12
|
||||
yunion.io/x/structarg
|
||||
# zombiezen.com/go/sqlite v0.13.1
|
||||
|
||||
1
vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go
generated
vendored
1
vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go
generated
vendored
@@ -33,6 +33,7 @@ type SDiskConfig struct {
|
||||
UnitNumber int32
|
||||
Key int32
|
||||
ImagePath string
|
||||
DestPath string
|
||||
IsRoot bool
|
||||
Datastore *SDatastore
|
||||
Preallocation string
|
||||
|
||||
134
vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go
generated
vendored
134
vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go
generated
vendored
@@ -285,17 +285,105 @@ func (svm *SVirtualMachine) DoRebuildRoot(ctx context.Context, imagePath string,
|
||||
return svm.rebuildDisk(ctx, &svm.vdisks[0], imagePath, uefi)
|
||||
}
|
||||
|
||||
func rebuildRootDiskBackupPath(filename string) string {
|
||||
const suffix = ".vmdk"
|
||||
if strings.HasSuffix(strings.ToLower(filename), suffix) {
|
||||
return filename[:len(filename)-len(suffix)] + ".rebuild-bak.vmdk"
|
||||
}
|
||||
return filename + ".rebuild-bak.vmdk"
|
||||
}
|
||||
|
||||
func (svm *SVirtualMachine) moveVirtualDiskFile(ctx context.Context, ds *SDatastore, src, dst string) error {
|
||||
obj, err := ds.getDatastoreObj(ctx)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "getDatastoreObj")
|
||||
}
|
||||
fm := obj.NewFileManager(ds.datacenter.getObjectDatacenter(), true)
|
||||
err = fm.Move(ctx, src, dst)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "move %s -> %s", src, dst)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svm *SVirtualMachine) reattachNonRootDisks(ctx context.Context) error {
|
||||
for i := 1; i < len(svm.vdisks); i++ {
|
||||
err := svm.doDetachDisk(ctx, &svm.vdisks[i], false)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "doDetachDisk %d", i)
|
||||
}
|
||||
err = svm.doAttachDisk(ctx, &svm.vdisks[i])
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "doAttachDisk: %d", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svm *SVirtualMachine) attachDisksInOriginalOrder(ctx context.Context, root *SVirtualDisk) error {
|
||||
for i := 1; i < len(svm.vdisks); i++ {
|
||||
err := svm.doDetachDisk(ctx, &svm.vdisks[i], false)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "doDetachDisk %d", i)
|
||||
}
|
||||
}
|
||||
err := svm.doAttachDisk(ctx, root)
|
||||
if err != nil {
|
||||
for i := 1; i < len(svm.vdisks); i++ {
|
||||
if attachErr := svm.doAttachDisk(ctx, &svm.vdisks[i]); attachErr != nil {
|
||||
log.Errorf("reattach disk %d after root attach failed: %s", i, attachErr)
|
||||
}
|
||||
}
|
||||
return errors.Wrap(err, "doAttachDisk root")
|
||||
}
|
||||
for i := 1; i < len(svm.vdisks); i++ {
|
||||
err = svm.doAttachDisk(ctx, &svm.vdisks[i])
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "doAttachDisk: %d", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (svm *SVirtualMachine) restoreRootDiskAfterRebuild(ctx context.Context, disk *SVirtualDisk, ds *SDatastore, origPath, backupPath string) error {
|
||||
if err := ds.Delete2(ctx, origPath, false, true); err != nil {
|
||||
log.Errorf("delete rebuilt root disk %s before restore: %s", origPath, err)
|
||||
}
|
||||
if err := svm.moveVirtualDiskFile(ctx, ds, backupPath, origPath); err != nil {
|
||||
return errors.Wrap(err, "rename backup root disk")
|
||||
}
|
||||
return svm.attachDisksInOriginalOrder(ctx, disk)
|
||||
}
|
||||
|
||||
func (svm *SVirtualMachine) rebuildDisk(ctx context.Context, disk *SVirtualDisk, imagePath string, uefi bool) error {
|
||||
uuid := disk.GetId()
|
||||
sizeMb := disk.GetDiskSizeMB()
|
||||
diskKey := disk.getKey()
|
||||
ctlKey := disk.getControllerKey()
|
||||
unitNumber := *disk.dev.GetVirtualDevice().UnitNumber
|
||||
origPath := disk.GetFilename()
|
||||
backupPath := rebuildRootDiskBackupPath(origPath)
|
||||
|
||||
err := svm.doDetachAndDeleteDisk(ctx, disk)
|
||||
istorage, err := disk.GetIStorage()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "GetIStorage")
|
||||
}
|
||||
ds := istorage.(*SDatastore)
|
||||
|
||||
err = svm.doDetachDisk(ctx, disk, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = svm.moveVirtualDiskFile(ctx, ds, origPath, backupPath)
|
||||
if err != nil {
|
||||
if attachErr := svm.attachDisksInOriginalOrder(ctx, disk); attachErr != nil {
|
||||
log.Errorf("reattach disks in original order after backup failed: %s", attachErr)
|
||||
}
|
||||
return errors.Wrapf(err, "backup root disk %s", origPath)
|
||||
}
|
||||
log.Infof("backup root disk %s -> %s", origPath, backupPath)
|
||||
|
||||
err = svm.createDiskInternal(ctx, SDiskConfig{
|
||||
Uefi: uefi,
|
||||
SizeMb: int64(sizeMb),
|
||||
@@ -304,20 +392,21 @@ func (svm *SVirtualMachine) rebuildDisk(ctx context.Context, disk *SVirtualDisk,
|
||||
UnitNumber: unitNumber,
|
||||
Key: diskKey,
|
||||
ImagePath: imagePath,
|
||||
DestPath: origPath,
|
||||
IsRoot: len(imagePath) > 0,
|
||||
Datastore: ds,
|
||||
}, false)
|
||||
if err != nil {
|
||||
if restoreErr := svm.restoreRootDiskAfterRebuild(ctx, disk, ds, origPath, backupPath); restoreErr != nil {
|
||||
log.Errorf("restore root disk %s from %s failed: %s", origPath, backupPath, restoreErr)
|
||||
}
|
||||
return errors.Wrapf(err, "createDiskInternal")
|
||||
}
|
||||
for i := 1; i < len(svm.vdisks); i++ {
|
||||
err = svm.doDetachDisk(ctx, &svm.vdisks[i], false)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "doDetachDisk %d", i)
|
||||
}
|
||||
err = svm.doAttachDisk(ctx, &svm.vdisks[i])
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "doAttachDisk: %d", i)
|
||||
}
|
||||
if err = svm.reattachNonRootDisks(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = ds.Delete2(ctx, backupPath, false, true); err != nil {
|
||||
log.Errorf("delete backup root disk %s: %s", backupPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1256,10 +1345,24 @@ func (svm *SVirtualMachine) GetRootImagePath() (string, error) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (svm *SVirtualMachine) CopyRootDisk(ctx context.Context, imagePath string) (string, error) {
|
||||
newImagePath, datastore, err := svm.getDatastoreAndRootImagePath(false)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "GetRootImagePath")
|
||||
func (svm *SVirtualMachine) CopyRootDisk(ctx context.Context, imagePath, destPath string, datastore *SDatastore) (string, error) {
|
||||
var (
|
||||
newImagePath string
|
||||
err error
|
||||
)
|
||||
if len(destPath) > 0 {
|
||||
newImagePath = destPath
|
||||
if datastore == nil {
|
||||
_, datastore, err = svm.getDatastoreAndRootImagePath(false)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "GetRootImagePath")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
newImagePath, datastore, err = svm.getDatastoreAndRootImagePath(false)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "GetRootImagePath")
|
||||
}
|
||||
}
|
||||
ds, err := datastore.getDatastoreObj(ctx)
|
||||
if err != nil {
|
||||
@@ -1278,7 +1381,7 @@ func (svm *SVirtualMachine) createDiskWithDeviceChange(ctx context.Context, devi
|
||||
// copy disk
|
||||
if len(config.ImagePath) > 0 {
|
||||
config.IsRoot = true
|
||||
config.ImagePath, err = svm.CopyRootDisk(ctx, config.ImagePath)
|
||||
config.ImagePath, err = svm.CopyRootDisk(ctx, config.ImagePath, config.DestPath, config.Datastore)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to copyRootDisk")
|
||||
}
|
||||
@@ -1327,7 +1430,6 @@ func (svm *SVirtualMachine) createDiskWithDeviceChange(ctx context.Context, devi
|
||||
}
|
||||
|
||||
func (svm *SVirtualMachine) createDiskInternal(ctx context.Context, config SDiskConfig, check bool) error {
|
||||
|
||||
return svm.createDiskWithDeviceChange(ctx, nil, config, check)
|
||||
}
|
||||
|
||||
|
||||
125
vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go
generated
vendored
125
vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go
generated
vendored
@@ -220,6 +220,130 @@ func (disk *SDisk) GetProjectId() string {
|
||||
return disk.storage.zone.region.GetProjectId()
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func diskTypeName(storageType string) string {
|
||||
if idx := strings.LastIndex(storageType, "/"); idx >= 0 {
|
||||
return storageType[idx+1:]
|
||||
}
|
||||
return storageType
|
||||
}
|
||||
|
||||
func inRange(val, minVal, maxVal int) bool {
|
||||
return val >= minVal && val <= maxVal
|
||||
}
|
||||
|
||||
// hyperdiskBalancedIopsRange 计算 balanced / balanced-ha 的 IOPS 区间
|
||||
func hyperdiskBalancedIopsRange(sizeGb, maxIopsCap int) (minIops, maxIops int, ok bool) {
|
||||
switch {
|
||||
case sizeGb == 4:
|
||||
return 2000, 2000, true
|
||||
case sizeGb == 5:
|
||||
return 2500, 2500, true
|
||||
case sizeGb >= 6:
|
||||
return 3000, minInt(500*sizeGb, maxIopsCap), true
|
||||
default:
|
||||
return 0, 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func hyperdiskBalancedDefaultIops(sizeGb, maxIopsCap int) int {
|
||||
if sizeGb <= 6 {
|
||||
return 500 * sizeGb
|
||||
}
|
||||
// 6 GiB ~ 26.666667 TiB ≈ 27307 GiB
|
||||
if sizeGb <= 27307 {
|
||||
return minInt(6*sizeGb+3000, maxIopsCap)
|
||||
}
|
||||
return maxIopsCap
|
||||
}
|
||||
|
||||
func hyperdiskBalancedThroughputRange(iops int) (minTp, maxTp int) {
|
||||
return maxInt(140, iops/256), minInt(2400, iops/4)
|
||||
}
|
||||
|
||||
func setHyperdiskBalancedPerformance(params map[string]interface{}, sizeGb, iops, throughput, maxIopsCap int) {
|
||||
minIops, maxIops, ok := hyperdiskBalancedIopsRange(sizeGb, maxIopsCap)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
effectiveIops := iops
|
||||
if iops > 0 && inRange(iops, minIops, maxIops) {
|
||||
params["provisionedIops"] = iops
|
||||
} else if throughput > 0 {
|
||||
effectiveIops = hyperdiskBalancedDefaultIops(sizeGb, maxIopsCap)
|
||||
} else {
|
||||
return
|
||||
}
|
||||
if throughput > 0 {
|
||||
minTp, maxTp := hyperdiskBalancedThroughputRange(effectiveIops)
|
||||
if inRange(throughput, minTp, maxTp) {
|
||||
params["provisionedThroughput"] = throughput
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setDiskProvisionedPerformance 按 GCP 盘型区间+容量公式写入 IOPS/吞吐,超范围或不支持则丢弃该参数
|
||||
func setDiskProvisionedPerformance(params map[string]interface{}, storageType string, sizeGb, iops, throughput int) {
|
||||
if iops <= 0 && throughput <= 0 {
|
||||
return
|
||||
}
|
||||
diskType := diskTypeName(storageType)
|
||||
switch diskType {
|
||||
case api.STORAGE_GOOGLE_PD_EXTREME:
|
||||
if iops > 0 && inRange(iops, 2500, 120000) {
|
||||
params["provisionedIops"] = iops
|
||||
}
|
||||
case api.STORAGE_GOOGLE_HYPERDISK_EXTREME:
|
||||
if sizeGb < 64 || iops <= 0 {
|
||||
return
|
||||
}
|
||||
minIops := 2 * sizeGb
|
||||
maxIops := 350000
|
||||
if sizeGb <= 291 {
|
||||
maxIops = 1200 * sizeGb
|
||||
}
|
||||
if inRange(iops, minIops, maxIops) {
|
||||
params["provisionedIops"] = iops
|
||||
}
|
||||
case api.STORAGE_GOOGLE_HYPERDISK_BALANCED:
|
||||
setHyperdiskBalancedPerformance(params, sizeGb, iops, throughput, 160000)
|
||||
case "hyperdisk-balanced-high-availability":
|
||||
setHyperdiskBalancedPerformance(params, sizeGb, iops, throughput, 100000)
|
||||
case api.STORAGE_GOOGLE_HYPERDISK_THROUGHPUT:
|
||||
if throughput <= 0 || sizeGb < 2048 || sizeGb > 32768 {
|
||||
return
|
||||
}
|
||||
sizeTiB := float64(sizeGb) / 1024.0
|
||||
minTp := maxInt(20, int(5*sizeTiB))
|
||||
maxTp := minInt(int(90*sizeTiB), 2400)
|
||||
if inRange(throughput, minTp, maxTp) {
|
||||
params["provisionedThroughput"] = throughput
|
||||
}
|
||||
case api.STORAGE_GOOGLE_HYPERDISK_ML:
|
||||
if throughput <= 0 {
|
||||
return
|
||||
}
|
||||
minTp := maxInt(400, int(0.12*float64(sizeGb)))
|
||||
maxTp := minInt(2097152, 1600*sizeGb)
|
||||
if inRange(throughput, minTp, maxTp) {
|
||||
params["provisionedThroughput"] = throughput
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (region *SRegion) CreateDisk(zone string, storageType string, opts *cloudprovider.DiskCreateConfig) (*SDisk, error) {
|
||||
if !strings.HasPrefix(storageType, GOOGLE_COMPUTE_DOMAIN) {
|
||||
storageType = fmt.Sprintf("projects/%s/zones/%s/diskTypes/%s", region.GetProjectId(), zone, storageType)
|
||||
@@ -240,6 +364,7 @@ func (region *SRegion) CreateDisk(zone string, storageType string, opts *cloudpr
|
||||
if len(opts.ImageId) > 0 {
|
||||
body["sourceImage"] = opts.ImageId
|
||||
}
|
||||
setDiskProvisionedPerformance(body, storageType, opts.SizeGb, opts.Iops, opts.Throughput)
|
||||
disk := &SDisk{}
|
||||
resource := fmt.Sprintf("zones/%s/disks", zone)
|
||||
err := region.Insert(resource, jsonutils.Marshal(body), disk)
|
||||
|
||||
45
vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go
generated
vendored
45
vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go
generated
vendored
@@ -581,30 +581,35 @@ func (region *SRegion) _createVM(zone string, desc *cloudprovider.SManagedVMCrea
|
||||
labels[encode.EncodeGoogleLabel(k)] = encode.EncodeGoogleLabel(v)
|
||||
}
|
||||
|
||||
sysDiskParams := map[string]interface{}{
|
||||
"diskName": normalizeString(desc.SysDisk.Name),
|
||||
"sourceImage": desc.ExternalImageId,
|
||||
"diskSizeGb": desc.SysDisk.SizeGB,
|
||||
"diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, desc.SysDisk.StorageType),
|
||||
"labels": labels,
|
||||
}
|
||||
setDiskProvisionedPerformance(sysDiskParams, desc.SysDisk.StorageType, desc.SysDisk.SizeGB, desc.SysDisk.Iops, desc.SysDisk.Throughput)
|
||||
disks = append(disks, map[string]interface{}{
|
||||
"boot": true,
|
||||
"initializeParams": map[string]interface{}{
|
||||
"diskName": normalizeString(desc.SysDisk.Name),
|
||||
"sourceImage": desc.ExternalImageId,
|
||||
"diskSizeGb": desc.SysDisk.SizeGB,
|
||||
"diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, desc.SysDisk.StorageType),
|
||||
"labels": labels,
|
||||
},
|
||||
"autoDelete": true,
|
||||
"boot": true,
|
||||
"initializeParams": sysDiskParams,
|
||||
"autoDelete": true,
|
||||
})
|
||||
for _, disk := range desc.DataDisks {
|
||||
if len(disk.Name) == 0 {
|
||||
disk.Name = fmt.Sprintf("vdisk-%s-%d", desc.Name, time.Now().UnixNano())
|
||||
for i, disk := range desc.DataDisks {
|
||||
name := disk.Name
|
||||
if len(name) == 0 {
|
||||
name = fmt.Sprintf("vdisk-%s-%d-%d", desc.Name, i, time.Now().UnixNano())
|
||||
}
|
||||
dataDiskParams := map[string]interface{}{
|
||||
"diskName": normalizeString(name),
|
||||
"diskSizeGb": disk.SizeGB,
|
||||
"diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, disk.StorageType),
|
||||
"labels": labels,
|
||||
}
|
||||
setDiskProvisionedPerformance(dataDiskParams, disk.StorageType, disk.SizeGB, disk.Iops, disk.Throughput)
|
||||
disks = append(disks, map[string]interface{}{
|
||||
"boot": false,
|
||||
"initializeParams": map[string]interface{}{
|
||||
"diskName": normalizeString(disk.Name),
|
||||
"diskSizeGb": disk.SizeGB,
|
||||
"diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, disk.StorageType),
|
||||
"labels": labels,
|
||||
},
|
||||
"autoDelete": true,
|
||||
"boot": false,
|
||||
"initializeParams": dataDiskParams,
|
||||
"autoDelete": true,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
15
vendor/yunion.io/x/cloudmux/pkg/multicloud/google/securitygroup.go
generated
vendored
15
vendor/yunion.io/x/cloudmux/pkg/multicloud/google/securitygroup.go
generated
vendored
@@ -15,6 +15,7 @@
|
||||
package google
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -202,14 +203,16 @@ func normalizeString(input string) string {
|
||||
cleaned = regexp.MustCompile(`-$`).ReplaceAllString(cleaned, "")
|
||||
|
||||
// 5. 限制长度为63个字符(域名标签的最大长度)
|
||||
// 超长时保留前缀并追加原文 hash,避免仅截断前缀导致不同名称冲突
|
||||
if len(cleaned) > 63 {
|
||||
cleaned = cleaned[:63]
|
||||
// 确保截断后不以连字符结尾
|
||||
cleaned = regexp.MustCompile(`-$`).ReplaceAllString(cleaned, "")
|
||||
// 如果截断后为空,添加一个数字1
|
||||
if len(cleaned) == 0 {
|
||||
cleaned = "1"
|
||||
sum := sha1.Sum([]byte(cleaned))
|
||||
suffix := fmt.Sprintf("%x", sum[:4]) // 8 hex chars
|
||||
prefixLen := 63 - 1 - len(suffix)
|
||||
prefix := strings.TrimRight(cleaned[:prefixLen], "-")
|
||||
if len(prefix) == 0 {
|
||||
prefix = "a"
|
||||
}
|
||||
cleaned = prefix + "-" + suffix
|
||||
}
|
||||
|
||||
return cleaned
|
||||
|
||||
2
vendor/yunion.io/x/jsonutils/Makefile
generated
vendored
2
vendor/yunion.io/x/jsonutils/Makefile
generated
vendored
@@ -6,4 +6,4 @@ GOPROXY ?= direct
|
||||
mod:
|
||||
GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x \
|
||||
go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p'))
|
||||
go mod tidy
|
||||
GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x go mod tidy
|
||||
|
||||
16
vendor/yunion.io/x/jsonutils/access.go
generated
vendored
16
vendor/yunion.io/x/jsonutils/access.go
generated
vendored
@@ -20,10 +20,18 @@ import (
|
||||
"time"
|
||||
|
||||
"yunion.io/x/pkg/errors"
|
||||
"yunion.io/x/pkg/gotypes"
|
||||
"yunion.io/x/pkg/sortedmap"
|
||||
"yunion.io/x/pkg/util/timeutils"
|
||||
)
|
||||
|
||||
func normalizeJSONObject(obj JSONObject) JSONObject {
|
||||
if gotypes.IsNil(obj) {
|
||||
return JSONNull
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
type JSONPair struct {
|
||||
key string
|
||||
val JSONObject
|
||||
@@ -40,7 +48,7 @@ func NewDict(objs ...JSONPair) *JSONDict {
|
||||
func NewArray(objs ...JSONObject) *JSONArray {
|
||||
arr := JSONArray{data: make([]JSONObject, 0, len(objs))}
|
||||
for _, o := range objs {
|
||||
arr.data = append(arr.data, o)
|
||||
arr.data = append(arr.data, normalizeJSONObject(o))
|
||||
}
|
||||
return &arr
|
||||
}
|
||||
@@ -74,7 +82,7 @@ func NewBool(val bool) *JSONBool {
|
||||
}
|
||||
|
||||
func (this *JSONDict) Set(key string, value JSONObject) {
|
||||
this.data = sortedmap.Add(this.data, key, value)
|
||||
this.data = sortedmap.Add(this.data, key, normalizeJSONObject(value))
|
||||
}
|
||||
|
||||
func (this *JSONDict) Remove(key string) bool {
|
||||
@@ -130,12 +138,12 @@ func (this *JSONDict) Add(o JSONObject, keys ...string) error {
|
||||
}
|
||||
|
||||
func (this *JSONArray) SetAt(idx int, obj JSONObject) {
|
||||
this.data[idx] = obj
|
||||
this.data[idx] = normalizeJSONObject(obj)
|
||||
}
|
||||
|
||||
func (this *JSONArray) Add(objs ...JSONObject) {
|
||||
for _, o := range objs {
|
||||
this.data = append(this.data, o)
|
||||
this.data = append(this.data, normalizeJSONObject(o))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
3
vendor/yunion.io/x/jsonutils/consts.go
generated
vendored
3
vendor/yunion.io/x/jsonutils/consts.go
generated
vendored
@@ -20,4 +20,7 @@ import (
|
||||
|
||||
const (
|
||||
TAG_DEPRECATED_BY = reflectutils.TAG_DEPRECATED_BY
|
||||
|
||||
// maxParseDepth is the maximum nesting level accepted by the parser
|
||||
maxParseDepth = 10000
|
||||
)
|
||||
|
||||
10
vendor/yunion.io/x/jsonutils/equals.go
generated
vendored
10
vendor/yunion.io/x/jsonutils/equals.go
generated
vendored
@@ -15,6 +15,7 @@
|
||||
package jsonutils
|
||||
|
||||
import (
|
||||
"yunion.io/x/pkg/gotypes"
|
||||
"yunion.io/x/pkg/sortedmap"
|
||||
)
|
||||
|
||||
@@ -51,7 +52,14 @@ func (arr *JSONArray) Equals(json JSONObject) bool {
|
||||
return false
|
||||
}
|
||||
for i, v := range arr.data {
|
||||
if !v.Equals(arr2.data[i]) {
|
||||
oth := arr2.data[i]
|
||||
if gotypes.IsNil(v) || gotypes.IsNil(oth) {
|
||||
if gotypes.IsNil(v) != gotypes.IsNil(oth) {
|
||||
return false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !v.Equals(oth) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
6
vendor/yunion.io/x/jsonutils/errors.go
generated
vendored
6
vendor/yunion.io/x/jsonutils/errors.go
generated
vendored
@@ -26,10 +26,16 @@ const (
|
||||
ErrInvalidHex = errors.Error("invalid hex")
|
||||
ErrInvalidRune = errors.Error("invalid 4 byte rune")
|
||||
|
||||
ErrInvalidNodeId = errors.Error("invalid node id")
|
||||
ErrDuplicateNodeId = errors.Error("duplicate node id")
|
||||
ErrNestedTooDeep = errors.Error("json nested too deep")
|
||||
ErrNodeReferenceDisabled = errors.Error("node reference is not enabled, see ParseTrusted")
|
||||
|
||||
ErrTypeMismatch = errors.Error("unmarshal type mismatch")
|
||||
ErrArrayLengthMismatch = errors.Error("unmarshal array length mismatch")
|
||||
ErrInterfaceUnsupported = errors.Error("do not known how to deserialize json into this interface type")
|
||||
ErrMapKeyMustString = errors.Error("map key must be string")
|
||||
ErrNodeNotFound = errors.Error("no node found for the reference")
|
||||
|
||||
ErrMissingInputField = errors.Error("missing input field")
|
||||
ErrNilInputField = errors.Error("nil input field")
|
||||
|
||||
12
vendor/yunion.io/x/jsonutils/interface.go
generated
vendored
12
vendor/yunion.io/x/jsonutils/interface.go
generated
vendored
@@ -15,6 +15,7 @@
|
||||
package jsonutils
|
||||
|
||||
import (
|
||||
"yunion.io/x/pkg/gotypes"
|
||||
"yunion.io/x/pkg/sortedmap"
|
||||
)
|
||||
|
||||
@@ -41,6 +42,10 @@ func (self *JSONString) Interface() interface{} {
|
||||
func (self *JSONArray) Interface() interface{} {
|
||||
ret := make([]interface{}, len(self.data))
|
||||
for i := 0; i < len(self.data); i += 1 {
|
||||
if gotypes.IsNil(self.data[i]) {
|
||||
ret[i] = nil
|
||||
continue
|
||||
}
|
||||
ret[i] = self.data[i].Interface()
|
||||
}
|
||||
return ret
|
||||
@@ -51,7 +56,12 @@ func (self *JSONDict) Interface() interface{} {
|
||||
|
||||
for iter := sortedmap.NewIterator(self.data); iter.HasMore(); iter.Next() {
|
||||
k, v := iter.Get()
|
||||
mapping[k] = v.(JSONObject).Interface()
|
||||
jo, _ := v.(JSONObject)
|
||||
if gotypes.IsNil(jo) {
|
||||
mapping[k] = nil
|
||||
continue
|
||||
}
|
||||
mapping[k] = jo.Interface()
|
||||
}
|
||||
|
||||
return mapping
|
||||
|
||||
175
vendor/yunion.io/x/jsonutils/jsonutils.go
generated
vendored
175
vendor/yunion.io/x/jsonutils/jsonutils.go
generated
vendored
@@ -16,10 +16,13 @@ package jsonutils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf16"
|
||||
"unicode/utf8"
|
||||
|
||||
"yunion.io/x/log"
|
||||
@@ -117,6 +120,11 @@ func skipEmpty(str []byte, offset int) int {
|
||||
return i
|
||||
}
|
||||
|
||||
// isFiniteFloat reports whether the value has a json representation
|
||||
func isFiniteFloat(val float64) bool {
|
||||
return !math.IsNaN(val) && !math.IsInf(val, 0)
|
||||
}
|
||||
|
||||
func hexchar2num(v byte) (byte, error) {
|
||||
switch {
|
||||
case v >= '0' && v <= '9':
|
||||
@@ -183,9 +191,22 @@ ret:
|
||||
if e != nil {
|
||||
return "", i, NewJSONError(str, i, e.Error())
|
||||
}
|
||||
i += 4
|
||||
if utf16.IsSurrogate(r) {
|
||||
// a character outside the BMP is written as a
|
||||
// surrogate pair, e.g. 😀
|
||||
if i+6 <= len(str) && str[i] == '\\' && str[i+1] == 'u' {
|
||||
r2, e2 := hexstr2rune(str[i+2 : i+6])
|
||||
if e2 == nil {
|
||||
if combined := utf16.DecodeRune(r, r2); combined != utf8.RuneError {
|
||||
r = combined
|
||||
i += 6
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
runen = utf8.EncodeRune(runebytes, r)
|
||||
buffer = append(buffer, runebytes[0:runen]...)
|
||||
i += 4
|
||||
case 'x':
|
||||
i++
|
||||
if i+2 >= len(str) {
|
||||
@@ -253,13 +274,23 @@ ret2:
|
||||
return string(str[offset:i]), false, i, nil
|
||||
}
|
||||
|
||||
// isNodeReference reports whether the token has the form of a node
|
||||
// reference, that is a bare <N> with an integer N
|
||||
func isNodeReference(val string) bool {
|
||||
if len(val) < 3 || val[0] != '<' || val[len(val)-1] != '>' {
|
||||
return false
|
||||
}
|
||||
_, err := strconv.ParseInt(val[1:len(val)-1], 10, 64)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject, int, error) {
|
||||
val, quote, i, e := parseString(str, offset)
|
||||
if e != nil {
|
||||
return nil, i, errors.Wrap(e, "parseString")
|
||||
} else if quote {
|
||||
return &JSONString{data: val}, i, nil
|
||||
} else if val[0] == '<' && val[len(val)-1] == '>' {
|
||||
} else if s.allowNodeReference && len(val) > 1 && val[0] == '<' && val[len(val)-1] == '>' {
|
||||
// Pointer <nnnn>
|
||||
val = val[1 : len(val)-1]
|
||||
ival, err := strconv.ParseInt(val, 10, 64)
|
||||
@@ -272,6 +303,10 @@ func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject,
|
||||
}
|
||||
s.saveReferer(nodeId, ptr)
|
||||
return ptr, i, nil
|
||||
} else if !s.allowNodeReference && isNodeReference(val) {
|
||||
// a node reference can not be left unresolved and kept as a plain
|
||||
// value: a caller could not tell it apart from a real string
|
||||
return nil, i, errors.Wrap(ErrNodeReferenceDisabled, val)
|
||||
} else {
|
||||
lval := strings.ToLower(val)
|
||||
if len(lval) == 0 || lval == "null" || lval == "none" {
|
||||
@@ -288,9 +323,10 @@ func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject,
|
||||
return &JSONInt{data: ival}, i, nil
|
||||
}
|
||||
fval, err := strconv.ParseFloat(val, 64)
|
||||
if err == nil {
|
||||
if err == nil && isFiniteFloat(fval) {
|
||||
return &JSONFloat{data: fval}, i, nil
|
||||
}
|
||||
// nan and +-inf have no json representation, keep them as strings
|
||||
return &JSONString{data: val}, i, nil
|
||||
}
|
||||
}
|
||||
@@ -357,12 +393,33 @@ func escapeJsonChar(sb *strings.Builder, ch byte) {
|
||||
}
|
||||
}
|
||||
|
||||
// escapeJsonByte writes a byte that is not part of a valid utf-8 sequence
|
||||
// as a \xXX escape, which parseQuoteString reads back unchanged
|
||||
func escapeJsonByte(sb *strings.Builder, ch byte) {
|
||||
const hexdigits = "0123456789abcdef"
|
||||
sb.Write([]byte{'\\', 'x', hexdigits[ch>>4], hexdigits[ch&0xf]})
|
||||
}
|
||||
|
||||
func quoteString(str string) string {
|
||||
sb := &strings.Builder{}
|
||||
sb.Grow(len(str) + 2)
|
||||
sb.WriteByte('"')
|
||||
for i := 0; i < len(str); i += 1 {
|
||||
escapeJsonChar(sb, str[i])
|
||||
for i := 0; i < len(str); {
|
||||
ch := str[i]
|
||||
if ch < utf8.RuneSelf {
|
||||
escapeJsonChar(sb, ch)
|
||||
i++
|
||||
continue
|
||||
}
|
||||
r, size := utf8.DecodeRuneInString(str[i:])
|
||||
if r == utf8.RuneError && size <= 1 {
|
||||
// keep a non utf-8 byte reversible instead of writing it out
|
||||
escapeJsonByte(sb, ch)
|
||||
i++
|
||||
continue
|
||||
}
|
||||
sb.WriteString(str[i : i+size])
|
||||
i += size
|
||||
}
|
||||
sb.WriteByte('"')
|
||||
return sb.String()
|
||||
@@ -431,6 +488,11 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted
|
||||
var e error = nil
|
||||
var key string
|
||||
var stop = false
|
||||
// collect the keys first so that the sorted map can be built in key
|
||||
// order: adding to a sorted map out of order shifts the whole tail
|
||||
// on every insert
|
||||
values := make(map[string]JSONObject)
|
||||
keys := make([]string, 0)
|
||||
for !stop && i < len(str) {
|
||||
i = skipEmpty(str, i)
|
||||
if i >= len(str) {
|
||||
@@ -474,11 +536,18 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted
|
||||
if e != nil {
|
||||
return smap, i, nodeId, errors.Wrap(e, "parse misc")
|
||||
}
|
||||
if key == jsonPointerKey {
|
||||
if s.allowNodeReference && key == jsonPointerKey {
|
||||
// node id
|
||||
nodeId = int(val.(*JSONInt).data)
|
||||
jval, ok := val.(*JSONInt)
|
||||
if !ok {
|
||||
return smap, i, nodeId, errors.Wrap(ErrInvalidNodeId, jsonPointerKey)
|
||||
}
|
||||
nodeId = int(jval.data)
|
||||
} else {
|
||||
smap = sortedmap.Add(smap, key, val)
|
||||
if _, ok := values[key]; !ok {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
values[key] = val
|
||||
}
|
||||
i = skipEmpty(str, i)
|
||||
if i >= len(str) {
|
||||
@@ -494,6 +563,10 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted
|
||||
return smap, i, nodeId, NewJSONError(str, i, "Unexpected char")
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
smap = sortedmap.Add(smap, key, values[key])
|
||||
}
|
||||
return smap, i, nodeId, nil
|
||||
}
|
||||
|
||||
@@ -552,12 +625,20 @@ func (s *sJsonParseSession) parseArray(str []byte, offset int) ([]JSONObject, in
|
||||
}
|
||||
|
||||
func (this *JSONDict) parse(s *sJsonParseSession, str []byte, offset int) (int, error) {
|
||||
e := s.enter()
|
||||
if e != nil {
|
||||
return offset, errors.Wrap(e, "enter")
|
||||
}
|
||||
defer s.leave()
|
||||
smap, i, nodeId, e := s.parseDict(str, offset)
|
||||
if e == nil {
|
||||
this.nodeId = nodeId
|
||||
this.data = smap
|
||||
if this.nodeId > 0 {
|
||||
s.saveNode(nodeId, this)
|
||||
e = s.saveNode(nodeId, this)
|
||||
if e != nil {
|
||||
return i, errors.Wrap(e, "saveNode")
|
||||
}
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
@@ -591,17 +672,21 @@ func (this *JSONDict) prettyString(level int) string {
|
||||
buffer.WriteByte('\n')
|
||||
buffer.WriteString(tab)
|
||||
buffer.WriteString(" ")
|
||||
buffer.WriteByte('"')
|
||||
buffer.WriteString(k)
|
||||
buffer.WriteString("\":")
|
||||
_, okdict := v.(*JSONDict)
|
||||
_, okarray := v.(*JSONArray)
|
||||
if okdict || okarray {
|
||||
buffer.WriteByte('\n')
|
||||
buffer.WriteString(v.prettyString(level + 2))
|
||||
} else {
|
||||
buffer.WriteString(quoteString(k))
|
||||
buffer.WriteByte(':')
|
||||
if gotypes.IsNil(v) {
|
||||
buffer.WriteByte(' ')
|
||||
buffer.WriteString(v.String())
|
||||
buffer.WriteString("null")
|
||||
} else {
|
||||
_, okdict := v.(*JSONDict)
|
||||
_, okarray := v.(*JSONArray)
|
||||
if okdict || okarray {
|
||||
buffer.WriteByte('\n')
|
||||
buffer.WriteString(v.prettyString(level + 2))
|
||||
} else {
|
||||
buffer.WriteByte(' ')
|
||||
buffer.WriteString(v.String())
|
||||
}
|
||||
}
|
||||
idx++
|
||||
}
|
||||
@@ -614,6 +699,11 @@ func (this *JSONDict) prettyString(level int) string {
|
||||
}
|
||||
|
||||
func (this *JSONArray) parse(s *sJsonParseSession, str []byte, offset int) (int, error) {
|
||||
e := s.enter()
|
||||
if e != nil {
|
||||
return offset, errors.Wrap(e, "enter")
|
||||
}
|
||||
defer s.leave()
|
||||
val, i, e := s.parseArray(str, offset)
|
||||
if e == nil {
|
||||
this.data = val
|
||||
@@ -639,7 +729,12 @@ func (this *JSONArray) prettyString(level int) string {
|
||||
buffer.WriteString(",")
|
||||
}
|
||||
buffer.WriteByte('\n')
|
||||
buffer.WriteString(v.prettyString(level + 1))
|
||||
if gotypes.IsNil(v) {
|
||||
buffer.WriteString(tab)
|
||||
buffer.WriteString(" null")
|
||||
} else {
|
||||
buffer.WriteString(v.prettyString(level + 1))
|
||||
}
|
||||
}
|
||||
if len(this.data) > 0 {
|
||||
buffer.WriteByte('\n')
|
||||
@@ -654,12 +749,46 @@ func ParseString(str string) (JSONObject, error) {
|
||||
}
|
||||
|
||||
func Parse(str []byte) (JSONObject, error) {
|
||||
json, _, err := ParseStream(str, 0)
|
||||
json, offset, err := ParseStream(str, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if i := skipEmpty(str, offset); i < len(str) {
|
||||
return nil, NewJSONError(str, i, "Unexpected content after the value")
|
||||
}
|
||||
return json, nil
|
||||
}
|
||||
|
||||
// ParseStream parses one value starting at offset, it returns the value and
|
||||
// the offset just after it, so that a stream of concatenated values can be
|
||||
// walked. Node references are not resolved, see ParseTrusted.
|
||||
func ParseStream(str []byte, offset int) (JSONObject, int, error) {
|
||||
return parseStream(str, offset, false)
|
||||
}
|
||||
|
||||
// ParseTrusted parses a document from a trusted source, resolving the node
|
||||
// references that Marshal writes for a cyclic object: the ___jnid_ key inside
|
||||
// an object and a bare <N> value referring to it.
|
||||
//
|
||||
// A document from an untrusted source must be parsed with Parse instead. A
|
||||
// resolved reference makes two fields of the target struct point at the same
|
||||
// object, which is what the round trip of a cyclic object needs, but it also
|
||||
// lets a forged document do the same.
|
||||
//
|
||||
// Note that Marshal needs this syntax to terminate on a cyclic object, so it
|
||||
// keeps writing it either way.
|
||||
func ParseTrusted(str []byte) (JSONObject, error) {
|
||||
json, _, err := parseStream(str, 0, true)
|
||||
return json, err
|
||||
}
|
||||
|
||||
func ParseStream(str []byte, offset int) (JSONObject, int, error) {
|
||||
s := newJsonParseSession()
|
||||
// ParseTrustedString is ParseTrusted for a string
|
||||
func ParseTrustedString(str string) (JSONObject, error) {
|
||||
return ParseTrusted([]byte(str))
|
||||
}
|
||||
|
||||
func parseStream(str []byte, offset int, allowNodeReference bool) (JSONObject, int, error) {
|
||||
s := newJsonParseSession(allowNodeReference)
|
||||
i := offset
|
||||
i = skipEmpty(str, i)
|
||||
var val JSONObject = nil
|
||||
|
||||
21
vendor/yunion.io/x/jsonutils/marshal.go
generated
vendored
21
vendor/yunion.io/x/jsonutils/marshal.go
generated
vendored
@@ -48,7 +48,7 @@ func (s *sJsonMarshalSession) marshalSlice(val reflect.Value, info *reflectutils
|
||||
objs := make([]JSONObject, 0)
|
||||
for i := 0; i < val.Len(); i += 1 {
|
||||
val := s.marshalValue(val.Index(i), nil, omitEmpty)
|
||||
if val != nil {
|
||||
if !gotypes.IsNil(val) {
|
||||
objs = append(objs, val)
|
||||
}
|
||||
}
|
||||
@@ -84,7 +84,7 @@ func (s *sJsonMarshalSession) marshalMap(val reflect.Value, info *reflectutils.S
|
||||
for i := 0; i < len(keys); i += 1 {
|
||||
key := keys[i]
|
||||
val := s.marshalValue(val.MapIndex(key), nil, omitEmpty)
|
||||
if val != nil {
|
||||
if !gotypes.IsNil(val) {
|
||||
objPairs = append(objPairs, JSONPair{key: fmt.Sprintf("%s", key), val: val})
|
||||
}
|
||||
}
|
||||
@@ -128,12 +128,12 @@ func (s *sJsonMarshalSession) struct2JSONPairs(val reflect.Value, omitEmpty bool
|
||||
continue
|
||||
}
|
||||
key := jsonInfo.MarshalName()
|
||||
if deprecatedBy, ok := fields[i].Info.Tags[TAG_DEPRECATED_BY]; ok {
|
||||
if deprecatedBy, ok := fields[i].Info.Tag(TAG_DEPRECATED_BY); ok {
|
||||
depFields[key] = deprecatedBy
|
||||
continue
|
||||
}
|
||||
val := s.marshalValue(fields[i].Value, jsonInfo, omitEmpty)
|
||||
if val != nil {
|
||||
if !gotypes.IsNil(val) {
|
||||
objPair := JSONPair{key: key, val: val}
|
||||
objPairs = append(objPairs, objPair)
|
||||
}
|
||||
@@ -181,6 +181,9 @@ func marshalFloat64(val float64, info *reflectutils.SStructFieldInfo, bit int, o
|
||||
return nil
|
||||
} else if info != nil && info.ForceString {
|
||||
return NewString(fmt.Sprintf("%f", val))
|
||||
} else if !isFiniteFloat(val) {
|
||||
// nan and +-inf have no json representation
|
||||
return JSONNull
|
||||
} else {
|
||||
return NewFloat64(val)
|
||||
}
|
||||
@@ -191,6 +194,9 @@ func marshalFloat32(val float32, info *reflectutils.SStructFieldInfo, bit int, o
|
||||
return nil
|
||||
} else if info != nil && info.ForceString {
|
||||
return NewString(fmt.Sprintf("%f", val))
|
||||
} else if !isFiniteFloat(float64(val)) {
|
||||
// nan and +-inf have no json representation
|
||||
return JSONNull
|
||||
} else {
|
||||
return NewFloat32(val)
|
||||
}
|
||||
@@ -243,6 +249,13 @@ func marshalTime(val time.Time, info *reflectutils.SStructFieldInfo, omitEmpty b
|
||||
}
|
||||
}
|
||||
|
||||
// Marshal converts an object to a JSONObject, which can be written out with
|
||||
// String.
|
||||
//
|
||||
// A cyclic object is written with the node reference syntax, that is the
|
||||
// ___jnid_ key inside an object plus a bare <N> value referring to it. That
|
||||
// syntax is not standard json and its result is not a valid json document,
|
||||
// and Parse does not resolve it: read such a document back with ParseTrusted.
|
||||
func Marshal(obj interface{}) JSONObject {
|
||||
if obj == nil {
|
||||
return JSONNull
|
||||
|
||||
33
vendor/yunion.io/x/jsonutils/parse_session.go
generated
vendored
33
vendor/yunion.io/x/jsonutils/parse_session.go
generated
vendored
@@ -15,7 +15,7 @@
|
||||
package jsonutils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"yunion.io/x/pkg/errors"
|
||||
)
|
||||
|
||||
type sNodeReferer struct {
|
||||
@@ -25,14 +25,36 @@ type sNodeReferer struct {
|
||||
|
||||
type sJsonParseSession struct {
|
||||
objectMap map[int]*sNodeReferer
|
||||
|
||||
// depth is the current nesting level of the object/array being parsed
|
||||
depth int
|
||||
|
||||
// allowNodeReference tells whether the ___jnid_ key and a bare <N>
|
||||
// value are read as node references, see ParseTrusted
|
||||
allowNodeReference bool
|
||||
}
|
||||
|
||||
func newJsonParseSession() *sJsonParseSession {
|
||||
func newJsonParseSession(allowNodeReference bool) *sJsonParseSession {
|
||||
return &sJsonParseSession{
|
||||
objectMap: make(map[int]*sNodeReferer),
|
||||
objectMap: make(map[int]*sNodeReferer),
|
||||
allowNodeReference: allowNodeReference,
|
||||
}
|
||||
}
|
||||
|
||||
// enter records entering one more nesting level, it fails if the nesting
|
||||
// level exceeds maxParseDepth
|
||||
func (s *sJsonParseSession) enter() error {
|
||||
if s.depth >= maxParseDepth {
|
||||
return ErrNestedTooDeep
|
||||
}
|
||||
s.depth++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sJsonParseSession) leave() {
|
||||
s.depth--
|
||||
}
|
||||
|
||||
func (s *sJsonParseSession) saveReferer(nodeId int, ptr *sJSONPointer) {
|
||||
if nr, ok := s.objectMap[nodeId]; ok {
|
||||
nr.pointers = append(nr.pointers, ptr)
|
||||
@@ -43,10 +65,10 @@ func (s *sJsonParseSession) saveReferer(nodeId int, ptr *sJSONPointer) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) {
|
||||
func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) error {
|
||||
if nr, ok := s.objectMap[nodeId]; ok {
|
||||
if nr.node != nil {
|
||||
panic(fmt.Sprintf("nodeId %d alreayd exists: %s != %s", nodeId, nr.node, node))
|
||||
return errors.Wrapf(ErrDuplicateNodeId, "node id %d", nodeId)
|
||||
} else {
|
||||
nr.node = node
|
||||
}
|
||||
@@ -56,4 +78,5 @@ func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) {
|
||||
pointers: nil,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
10
vendor/yunion.io/x/jsonutils/querystring.go
generated
vendored
10
vendor/yunion.io/x/jsonutils/querystring.go
generated
vendored
@@ -55,7 +55,10 @@ func addQueryStringSeg(body JSONObject, segs []sTextNumber, val []string) (JSONO
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "get jsondict %s with %s fail", body, key)
|
||||
}
|
||||
addQueryStringSeg(next, segs[1:], val)
|
||||
_, err = addQueryStringSeg(next, segs[1:], val)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "addQueryStringSeg %s with %s fail", body, key)
|
||||
}
|
||||
}
|
||||
return jbody, nil
|
||||
case *JSONArray:
|
||||
@@ -66,7 +69,10 @@ func addQueryStringSeg(body JSONObject, segs []sTextNumber, val []string) (JSONO
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "get jsonarray %s at %d fail", body, index)
|
||||
}
|
||||
addQueryStringSeg(next, segs[1:], val)
|
||||
_, err = addQueryStringSeg(next, segs[1:], val)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "addQueryStringSeg %s at %d fail", body, index)
|
||||
}
|
||||
} else if arrSize == index {
|
||||
// new
|
||||
next, err := addQueryStringSeg(nil, segs[1:], val)
|
||||
|
||||
8
vendor/yunion.io/x/jsonutils/std_json.go
generated
vendored
8
vendor/yunion.io/x/jsonutils/std_json.go
generated
vendored
@@ -16,9 +16,9 @@ package jsonutils
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"yunion.io/x/log"
|
||||
"yunion.io/x/pkg/gotypes"
|
||||
)
|
||||
|
||||
@@ -110,11 +110,13 @@ func tryStdMarshal(v reflect.Value, marshalFunc func(v reflect.Value) JSONObject
|
||||
if m != nil {
|
||||
data, err := m.MarshalJSON()
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("MarshalJSON of %q error: %v", v.String(), err))
|
||||
log.Errorf("MarshalJSON of %s error: %v", v.Type(), err)
|
||||
return JSONNull
|
||||
}
|
||||
jo, err := Parse(data)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Parse data %q to json of %q error: %v", data, v.String(), err))
|
||||
log.Errorf("Parse data %s to json of %s error: %v", data, v.Type(), err)
|
||||
return JSONNull
|
||||
}
|
||||
return jo
|
||||
}
|
||||
|
||||
4
vendor/yunion.io/x/jsonutils/string.go
generated
vendored
4
vendor/yunion.io/x/jsonutils/string.go
generated
vendored
@@ -33,6 +33,10 @@ func (this *JSONInt) String() string {
|
||||
}
|
||||
|
||||
func (this *JSONFloat) String() string {
|
||||
if !isFiniteFloat(this.data) {
|
||||
// nan and +-inf have no json representation
|
||||
return "null"
|
||||
}
|
||||
if this.bit != 32 && this.bit != 64 {
|
||||
this.bit = 64
|
||||
}
|
||||
|
||||
36
vendor/yunion.io/x/jsonutils/unmarshal.go
generated
vendored
36
vendor/yunion.io/x/jsonutils/unmarshal.go
generated
vendored
@@ -62,12 +62,26 @@ func jsonUnmarshal(jo JSONObject, o interface{}, keys []string) error {
|
||||
return errors.Wrap(err, "Get")
|
||||
}
|
||||
}
|
||||
s := newJsonUnmarshalSession()
|
||||
value := reflect.ValueOf(o)
|
||||
err := jo.unmarshalValue(s, reflect.Indirect(value))
|
||||
if value.IsValid() && value.Kind() == reflect.Ptr && value.IsNil() {
|
||||
return errors.Wrapf(ErrTypeMismatch, "unmarshal into a nil pointer of type %s", value.Type())
|
||||
}
|
||||
target := reflect.Indirect(value)
|
||||
// a non addressable value can not be written to, unless it is a
|
||||
// non nil map, which is a reference type
|
||||
if target.IsValid() && !target.CanAddr() &&
|
||||
(target.Kind() != reflect.Map || target.IsNil()) {
|
||||
return errors.Wrapf(ErrTypeMismatch, "unmarshal into a value of type %s, a pointer is required", target.Type())
|
||||
}
|
||||
s := newJsonUnmarshalSession()
|
||||
err := jo.unmarshalValue(s, target)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "jo.unmarshalValue")
|
||||
}
|
||||
err = s.checkUnboundNodes()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "checkUnboundNodes")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -385,6 +399,11 @@ func (this *JSONString) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Va
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !isFiniteFloat(floatVal) {
|
||||
// nan and +-inf have no json representation, they would
|
||||
// silently break any comparison the field takes part in
|
||||
return errors.Wrap(ErrInvalidJsonFloat, "not a finite number")
|
||||
}
|
||||
val.SetFloat(floatVal)
|
||||
}
|
||||
case reflect.Bool:
|
||||
@@ -476,7 +495,10 @@ func (this *JSONArray) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Val
|
||||
|
||||
func (this *JSONDict) unmarshalValue(s *sJsonUnmarshalSession, val reflect.Value) error {
|
||||
if this.nodeId > 0 && val.CanAddr() {
|
||||
s.saveNodeValue(this.nodeId, val.Addr())
|
||||
err := s.saveNodeValue(this.nodeId, val.Addr())
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "saveNodeValue")
|
||||
}
|
||||
}
|
||||
return tryStdUnmarshal(s, this, val, this._unmarshalValue)
|
||||
}
|
||||
@@ -510,7 +532,11 @@ func (this *JSONDict) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Valu
|
||||
return err
|
||||
}
|
||||
if objPtr == nil {
|
||||
val.Set(reflect.ValueOf(this.data)) // ???
|
||||
dataVal := reflect.ValueOf(this.data)
|
||||
if !dataVal.Type().AssignableTo(val.Type()) {
|
||||
return errors.Wrapf(ErrInterfaceUnsupported, "JSONDict.unmarshalValue: %s", val.Type())
|
||||
}
|
||||
val.Set(dataVal)
|
||||
return nil
|
||||
}
|
||||
err = this.unmarshalValue(s, reflect.ValueOf(objPtr))
|
||||
@@ -600,7 +626,7 @@ func setStructFieldAt(s *sJsonUnmarshalSession, key string, v JSONObject, fieldV
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "JSONDict.unmarshalStruct")
|
||||
}
|
||||
depInfo, ok := fieldValues[index].Info.Tags[TAG_DEPRECATED_BY]
|
||||
depInfo, ok := fieldValues[index].Info.Tag(TAG_DEPRECATED_BY)
|
||||
if ok {
|
||||
err := setStructFieldAt(s, depInfo, v, fieldValues, keyIndexMap, visited)
|
||||
if err != nil {
|
||||
|
||||
39
vendor/yunion.io/x/jsonutils/unmarshal_session.go
generated
vendored
39
vendor/yunion.io/x/jsonutils/unmarshal_session.go
generated
vendored
@@ -16,6 +16,9 @@ package jsonutils
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sort"
|
||||
|
||||
"yunion.io/x/pkg/errors"
|
||||
)
|
||||
|
||||
type sJsonNodeValues struct {
|
||||
@@ -34,13 +37,27 @@ func newJsonUnmarshalSession() *sJsonUnmarshalSession {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) {
|
||||
// assignable reports whether the node value can be assigned to the value
|
||||
// referring to that node
|
||||
func assignable(nodeValue, refValue reflect.Value) bool {
|
||||
if !nodeValue.IsValid() || !refValue.IsValid() {
|
||||
return false
|
||||
}
|
||||
return nodeValue.Type().AssignableTo(refValue.Type())
|
||||
}
|
||||
|
||||
func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) error {
|
||||
if nv, ok := s.objectMap[nodeId]; !ok {
|
||||
s.objectMap[nodeId] = &sJsonNodeValues{
|
||||
nodeValue: val,
|
||||
nodeValueSet: true,
|
||||
}
|
||||
} else {
|
||||
for i := range nv.targetValues {
|
||||
if !assignable(val, nv.targetValues[i]) {
|
||||
return errors.Wrapf(ErrTypeMismatch, "node id %d vs %s", nodeId, nv.targetValues[i].Type())
|
||||
}
|
||||
}
|
||||
nv.nodeValue = val
|
||||
nv.nodeValueSet = true
|
||||
for i := range nv.targetValues {
|
||||
@@ -48,10 +65,30 @@ func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) {
|
||||
}
|
||||
nv.targetValues = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkUnboundNodes reports the node ids that were referred to but never
|
||||
// resolved, which would otherwise leave the referring field untouched
|
||||
func (s *sJsonUnmarshalSession) checkUnboundNodes() error {
|
||||
ids := make([]int, 0)
|
||||
for nodeId, nv := range s.objectMap {
|
||||
if !nv.nodeValueSet {
|
||||
ids = append(ids, nodeId)
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Ints(ids)
|
||||
return errors.Wrapf(ErrNodeNotFound, "node ids %v", ids)
|
||||
}
|
||||
|
||||
func (s *sJsonUnmarshalSession) setPointerValue(nodeId int, val reflect.Value) error {
|
||||
if nv, ok := s.objectMap[nodeId]; ok && nv.nodeValueSet {
|
||||
if !assignable(nv.nodeValue, val) {
|
||||
return errors.Wrapf(ErrTypeMismatch, "node id %d vs %s", nodeId, val.Type())
|
||||
}
|
||||
val.Set(nv.nodeValue)
|
||||
} else if ok && !nv.nodeValueSet {
|
||||
nv.targetValues = append(nv.targetValues, val)
|
||||
|
||||
21
vendor/yunion.io/x/jsonutils/write.go
generated
vendored
21
vendor/yunion.io/x/jsonutils/write.go
generated
vendored
@@ -18,9 +18,18 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/pkg/gotypes"
|
||||
"yunion.io/x/pkg/sortedmap"
|
||||
)
|
||||
|
||||
func writeJSONObject(sb *strings.Builder, v JSONObject) {
|
||||
if gotypes.IsNil(v) {
|
||||
sb.WriteString("null")
|
||||
return
|
||||
}
|
||||
v.buildString(sb)
|
||||
}
|
||||
|
||||
type writeSource interface {
|
||||
buildString(sb *strings.Builder)
|
||||
}
|
||||
@@ -46,6 +55,10 @@ func (this *JSONBool) buildString(sb *strings.Builder) {
|
||||
}
|
||||
|
||||
func (this *JSONDict) buildString(sb *strings.Builder) {
|
||||
if this == nil {
|
||||
sb.WriteString("null")
|
||||
return
|
||||
}
|
||||
sb.WriteByte('{')
|
||||
var idx = 0
|
||||
if this.nodeId > 0 {
|
||||
@@ -63,19 +76,23 @@ func (this *JSONDict) buildString(sb *strings.Builder) {
|
||||
sb.WriteString(quoteString(k))
|
||||
sb.WriteByte(':')
|
||||
|
||||
v.buildString(sb)
|
||||
writeJSONObject(sb, v)
|
||||
idx++
|
||||
}
|
||||
sb.WriteByte('}')
|
||||
}
|
||||
|
||||
func (this *JSONArray) buildString(sb *strings.Builder) {
|
||||
if this == nil {
|
||||
sb.WriteString("null")
|
||||
return
|
||||
}
|
||||
sb.WriteByte('[')
|
||||
for idx, v := range this.data {
|
||||
if idx > 0 {
|
||||
sb.WriteByte(',')
|
||||
}
|
||||
v.buildString(sb)
|
||||
writeJSONObject(sb, v)
|
||||
}
|
||||
sb.WriteByte(']')
|
||||
}
|
||||
|
||||
164
vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go
generated
vendored
164
vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go
generated
vendored
@@ -18,6 +18,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
@@ -99,32 +100,127 @@ func NewWriteFile(path string, content string, perm string, owner string, isBase
|
||||
return f
|
||||
}
|
||||
|
||||
// shellQuote wraps s so that a POSIX shell treats it as one literal word.
|
||||
// A single quote inside s is closed, escaped and reopened.
|
||||
func shellQuote(s string) string {
|
||||
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||
}
|
||||
|
||||
// validUserName reports whether name can be used as a user name.
|
||||
//
|
||||
// The set is deliberately narrow. A name becomes a directory under /home, a
|
||||
// file name under /etc/sudoers.d and an argument to useradd, so a slash or a
|
||||
// ".." would move the files that are written, and a control character would
|
||||
// split a generated line. Leading "-" is refused so the name cannot be read as
|
||||
// an option.
|
||||
func validUserName(name string) bool {
|
||||
if len(name) == 0 || name == "." || name == ".." || name[0] == '-' {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(name); i++ {
|
||||
c := name[i]
|
||||
switch {
|
||||
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
|
||||
case c == '_' || c == '-' || c == '.':
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// validWritePath reports whether path may be written by a generated script.
|
||||
//
|
||||
// Quoting keeps a path from running as a command, but a path that walks up out
|
||||
// of its directory would still write somewhere else. A path carrying a ".."
|
||||
// element is refused rather than normalised, so that what is written is what
|
||||
// was asked for.
|
||||
func validWritePath(filePath string) bool {
|
||||
if len(filePath) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, elem := range strings.Split(filePath, "/") {
|
||||
if elem == ".." {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// escapePowerShell escapes a value for use inside a PowerShell double quoted
|
||||
// string, where a backtick introduces an escape, $ introduces a variable, and
|
||||
// a control character written literally would break the line it sits on.
|
||||
func escapePowerShell(s string) string {
|
||||
return strings.NewReplacer(
|
||||
"`", "``",
|
||||
`"`, "`\"",
|
||||
"$", "`$",
|
||||
"\r", "`r",
|
||||
"\n", "`n",
|
||||
"\t", "`t",
|
||||
"\x00", "`0",
|
||||
).Replace(s)
|
||||
}
|
||||
|
||||
const heredocPrefix = "_YUNION_EOF_"
|
||||
|
||||
// heredocTerminator returns a terminator that does not occur in content, so
|
||||
// that a line of the content cannot end the heredoc early. Quote it at the use
|
||||
// site so the shell does not expand the content.
|
||||
//
|
||||
// The generated value is checked against the content rather than assumed to be
|
||||
// unique: utils.GenRequestId returns the empty string if the random source is
|
||||
// unavailable, which would otherwise leave a fixed, guessable terminator.
|
||||
func heredocTerminator(content string) string {
|
||||
return uniqueTerminator(heredocPrefix+utils.GenRequestId(8), content)
|
||||
}
|
||||
|
||||
// uniqueTerminator returns term with underscores appended until it does not
|
||||
// occur in content. Each pass lengthens it, so this ends once it is longer
|
||||
// than the content and can no longer occur in it.
|
||||
func uniqueTerminator(term, content string) string {
|
||||
for strings.Contains(content, term) {
|
||||
term += "_"
|
||||
}
|
||||
return term
|
||||
}
|
||||
|
||||
func setFilePermission(path, permission, owner string) []string {
|
||||
cmds := []string{}
|
||||
if len(permission) > 0 {
|
||||
cmds = append(cmds, fmt.Sprintf("chmod %s %s", permission, path))
|
||||
cmds = append(cmds, fmt.Sprintf("chmod %s %s", shellQuote(permission), shellQuote(path)))
|
||||
}
|
||||
if len(owner) > 0 {
|
||||
cmds = append(cmds, fmt.Sprintf("chown %s:%s %s", owner, owner, path))
|
||||
cmds = append(cmds, fmt.Sprintf("chown %s:%s %s", shellQuote(owner), shellQuote(owner), shellQuote(path)))
|
||||
}
|
||||
return cmds
|
||||
}
|
||||
|
||||
// mkWriteFileCmd builds the commands that create path with content. The
|
||||
// redirection and the path are quoted, and the heredoc is quoted with a
|
||||
// terminator that is not expected to occur in the content.
|
||||
func mkWriteFileCmd(redirect, filePath, content, permission, owner string) []string {
|
||||
terminator := heredocTerminator(content)
|
||||
cmds := []string{
|
||||
fmt.Sprintf("mkdir -p %s", shellQuote(path.Dir(filePath))),
|
||||
fmt.Sprintf("cat %s %s <<'%s'\n%s\n%s", redirect, shellQuote(filePath), terminator, content, terminator),
|
||||
}
|
||||
return append(cmds, setFilePermission(filePath, permission, owner)...)
|
||||
}
|
||||
|
||||
func mkPutFileCmd(path string, content string, permission string, owner string) []string {
|
||||
cmds := []string{}
|
||||
cmds = append(cmds, fmt.Sprintf("mkdir -p $(dirname %s)", path))
|
||||
cmds = append(cmds, fmt.Sprintf("cat > %s <<_END\n%s\n_END", path, content))
|
||||
return append(cmds, setFilePermission(path, permission, owner)...)
|
||||
return mkWriteFileCmd(">", path, content, permission, owner)
|
||||
}
|
||||
|
||||
func mkAppendFileCmd(path string, content string, permission string, owner string) []string {
|
||||
cmds := []string{}
|
||||
cmds = append(cmds, fmt.Sprintf("mkdir -p $(dirname %s)", path))
|
||||
cmds = append(cmds, fmt.Sprintf("cat >> %s <<_END\n%s\n_END", path, content))
|
||||
return append(cmds, setFilePermission(path, permission, owner)...)
|
||||
return mkWriteFileCmd(">>", path, content, permission, owner)
|
||||
}
|
||||
|
||||
func (wf *SWriteFile) ShellScripts() []string {
|
||||
if !validWritePath(wf.Path) {
|
||||
log.Errorf("cloudinit: skipping write_file with a path that walks up out of its directory: %q", wf.Path)
|
||||
return nil
|
||||
}
|
||||
content := wf.Content
|
||||
if wf.Encoding == "b64" {
|
||||
_content, _ := base64.StdEncoding.DecodeString(wf.Content)
|
||||
@@ -176,23 +272,36 @@ func (u *SUser) Password(passwd string) *SUser {
|
||||
}
|
||||
|
||||
func (u *SUser) PowerShellScripts() []string {
|
||||
if !validUserName(u.Name) {
|
||||
log.Errorf("cloudinit: skipping scripts for unusable user name %q", u.Name)
|
||||
return nil
|
||||
}
|
||||
// Every line below is parsed by PowerShell first, so the name and the
|
||||
// password are escaped the same way. Using the unescaped name for one of
|
||||
// the lines would have them name different accounts.
|
||||
name := escapePowerShell(u.Name)
|
||||
shells := []string{}
|
||||
shells = append(shells, fmt.Sprintf(`New-LocalUser -Name "%s" -Description "A New Local Account Created By PowerShell" -NoPassword`, u.Name))
|
||||
shells = append(shells, fmt.Sprintf(`Add-LocalGroupMember -Group "Administrators" -Member "%s"`, u.Name))
|
||||
shells = append(shells, fmt.Sprintf(`New-LocalUser -Name "%s" -Description "A New Local Account Created By PowerShell" -NoPassword`, name))
|
||||
shells = append(shells, fmt.Sprintf(`Add-LocalGroupMember -Group "Administrators" -Member "%s"`, name))
|
||||
if len(u.PlainTextPasswd) > 0 {
|
||||
shells = append(shells, fmt.Sprintf(`net user "%s" "%s"`, u.Name, u.PlainTextPasswd))
|
||||
shells = append(shells, fmt.Sprintf(`net user "%s" "%s"`, name, escapePowerShell(u.PlainTextPasswd)))
|
||||
}
|
||||
// enable需要再设置密码之后,否则会出现Enable-LocalUser : Unable to update the password. The value provided for the new password does not meet the length, complexity, or history requirements of the domain
|
||||
shells = append(shells, fmt.Sprintf(`Enable-LocalUser "%s"`, u.Name))
|
||||
shells = append(shells, fmt.Sprintf(`Enable-LocalUser "%s"`, name))
|
||||
return shells
|
||||
}
|
||||
|
||||
func (u *SUser) ShellScripts() []string {
|
||||
if !validUserName(u.Name) {
|
||||
log.Errorf("cloudinit: skipping scripts for unusable user name %q", u.Name)
|
||||
return nil
|
||||
}
|
||||
name := shellQuote(u.Name)
|
||||
shells := []string{}
|
||||
|
||||
shells = append(shells, fmt.Sprintf("useradd -m %s || true", u.Name))
|
||||
shells = append(shells, fmt.Sprintf("useradd -m %s || true", name))
|
||||
if len(u.HashedPasswd) > 0 {
|
||||
shells = append(shells, fmt.Sprintf("usermod -p '%s' %s", u.HashedPasswd, u.Name))
|
||||
shells = append(shells, fmt.Sprintf("usermod -p %s %s", shellQuote(u.HashedPasswd), name))
|
||||
}
|
||||
|
||||
home := "/" + u.Name
|
||||
@@ -202,7 +311,7 @@ func (u *SUser) ShellScripts() []string {
|
||||
|
||||
keyPath := fmt.Sprintf("%s/.ssh/authorized_keys", home)
|
||||
shells = append(shells, mkAppendFileCmd(keyPath, strings.Join(u.SshAuthorizedKeys, "\n"), "600", u.Name)...)
|
||||
shells = append(shells, fmt.Sprintf("chown -R %s:%s %s/.ssh", u.Name, u.Name, home))
|
||||
shells = append(shells, fmt.Sprintf("chown -R %s:%s %s", name, name, shellQuote(home+"/.ssh")))
|
||||
|
||||
if !utils.IsInStringArray(u.Sudo, []string{"", "False"}) {
|
||||
shells = append(shells, mkPutFileCmd("/etc/sudoers.d/"+u.Name, fmt.Sprintf("%s %s", u.Name, u.Sudo), "", "")...)
|
||||
@@ -212,6 +321,9 @@ func (u *SUser) ShellScripts() []string {
|
||||
}
|
||||
|
||||
func (conf *SCloudConfig) UserData() string {
|
||||
if conf == nil {
|
||||
return ""
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
jsonConf := jsonutils.Marshal(conf).(*jsonutils.JSONDict)
|
||||
if jsonConf.Contains("users") {
|
||||
@@ -228,6 +340,9 @@ func (conf *SCloudConfig) UserData() string {
|
||||
}
|
||||
|
||||
func (conf *SCloudConfig) UserDataScript() string {
|
||||
if conf == nil {
|
||||
return ""
|
||||
}
|
||||
shells := []string{}
|
||||
for _, u := range conf.Users {
|
||||
shells = append(shells, u.ShellScripts()...)
|
||||
@@ -248,8 +363,11 @@ func (conf *SCloudConfig) UserDataScript() string {
|
||||
}
|
||||
|
||||
for _, pkg := range conf.Packages {
|
||||
shells = append(shells, "which yum &>/dev/null && yum install -y "+pkg)
|
||||
shells = append(shells, "which apt-get &>/dev/null && apt-get install -y "+pkg)
|
||||
// A package name is an argument, not part of the command line, so it
|
||||
// is quoted rather than pasted in.
|
||||
quoted := shellQuote(pkg)
|
||||
shells = append(shells, "which yum &>/dev/null && yum install -y "+quoted)
|
||||
shells = append(shells, "which apt-get &>/dev/null && apt-get install -y "+quoted)
|
||||
}
|
||||
for _, wf := range conf.WriteFiles {
|
||||
shells = append(shells, wf.ShellScripts()...)
|
||||
@@ -317,7 +435,13 @@ func ParseUserData(data string) (*SCloudConfig, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "ParseYAML")
|
||||
}
|
||||
jsonDict := jsonConf.(*jsonutils.JSONDict)
|
||||
jsonDict, ok := jsonConf.(*jsonutils.JSONDict)
|
||||
if !ok {
|
||||
// Anything that is valid YAML but not a mapping, e.g. a list or a
|
||||
// scalar, is not a usable cloud-config document.
|
||||
return nil, errors.Wrapf(errors.ErrInvalidFormat,
|
||||
"cloud-config must be a YAML mapping, got %s", jsonConf.String())
|
||||
}
|
||||
if jsonDict.Contains("users") {
|
||||
userArray := jsonutils.NewArray()
|
||||
users, _ := jsonConf.GetArray("users")
|
||||
|
||||
7
vendor/yunion.io/x/pkg/util/encode/encode.go
generated
vendored
7
vendor/yunion.io/x/pkg/util/encode/encode.go
generated
vendored
@@ -40,8 +40,11 @@ func unhex(c rune) byte {
|
||||
return 0
|
||||
}
|
||||
|
||||
// shouldEncode reports whether c must be escaped by EncodeGoogleLabel.
|
||||
// Only runes in [0, 0xff] can be represented as "_" followed by two hex
|
||||
// digits, so anything above that range is written through unchanged.
|
||||
func shouldEncode(c rune) bool {
|
||||
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c > 256 {
|
||||
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c > 0xff {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
@@ -69,7 +72,7 @@ func DecodeGoogleLable(label string) string {
|
||||
var t strings.Builder
|
||||
for j := 0; j < len(s); {
|
||||
c := s[j]
|
||||
if c == rune('_') && j+2 <= len(s) && ishex(s[j+1]) && ishex(s[j+2]) {
|
||||
if c == rune('_') && j+2 < len(s) && ishex(s[j+1]) && ishex(s[j+2]) {
|
||||
t.WriteByte(unhex(s[j+1])<<4 | unhex(s[j+2]))
|
||||
j += 3
|
||||
} else {
|
||||
|
||||
11
vendor/yunion.io/x/pkg/util/httputils/fs.go
generated
vendored
11
vendor/yunion.io/x/pkg/util/httputils/fs.go
generated
vendored
@@ -35,11 +35,20 @@ func (fs FileSystem) Open(path string) (http.File, error) {
|
||||
}
|
||||
|
||||
s, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
if s.IsDir() {
|
||||
index := strings.TrimSuffix(path, "/") + "/index.html"
|
||||
if _, err := fs.fs.Open(index); err != nil {
|
||||
indexFile, err := fs.fs.Open(index)
|
||||
if err != nil {
|
||||
// Without an index file a directory must not be listed.
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
// The index file is only checked for; it is not the file returned.
|
||||
indexFile.Close()
|
||||
}
|
||||
|
||||
return f, nil
|
||||
|
||||
233
vendor/yunion.io/x/pkg/util/httputils/httputils.go
generated
vendored
233
vendor/yunion.io/x/pkg/util/httputils/httputils.go
generated
vendored
@@ -28,6 +28,7 @@ import (
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
@@ -93,6 +94,173 @@ type sClient interface {
|
||||
Do(req *http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
// Field and parameter names that carry a credential on their own, and
|
||||
// fragments that mark a name as carrying one. The same lists are used for the
|
||||
// fields of a request body and the parameters of a request URL, so that the
|
||||
// two cannot drift apart.
|
||||
var sensitiveBodyKeyNames = []string{
|
||||
"key", "code", "sig", "pwd", "auth", "ticket", "session", "sessionid",
|
||||
// The plain name is a credential; "access_key_id" is an identifier and
|
||||
// is deliberately not matched, so it stays readable in the message.
|
||||
"access_key",
|
||||
}
|
||||
|
||||
var sensitiveBodyKeyParts = []string{
|
||||
"token", "secret", "password", "passwd", "signature", "credential",
|
||||
"apikey", "api_key", "private_key",
|
||||
}
|
||||
|
||||
func isSensitiveBodyKey(name string) bool {
|
||||
lower := strings.ToLower(name)
|
||||
if utils.IsInStringArray(lower, sensitiveBodyKeyNames) {
|
||||
return true
|
||||
}
|
||||
for _, part := range sensitiveBodyKeyParts {
|
||||
if strings.Contains(lower, part) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// redactURL masks the values of query parameters that commonly carry a
|
||||
// credential, so that an error message can name the request without
|
||||
// reproducing the secret. A URL with nothing to mask is returned unchanged,
|
||||
// byte for byte, so no existing message is reformatted.
|
||||
func redactURL(urlStr string) string {
|
||||
parsed, err := url.Parse(urlStr)
|
||||
if err != nil || len(parsed.RawQuery) == 0 {
|
||||
return urlStr
|
||||
}
|
||||
query := parsed.Query()
|
||||
changed := false
|
||||
for name := range query {
|
||||
if isSensitiveBodyKey(name) {
|
||||
query.Set(name, "*")
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return urlStr
|
||||
}
|
||||
parsed.RawQuery = query.Encode()
|
||||
return parsed.String()
|
||||
}
|
||||
|
||||
// redactJSONObject returns obj with the values of credential carrying fields
|
||||
// replaced, at any depth.
|
||||
func redactJSONObject(obj jsonutils.JSONObject) jsonutils.JSONObject {
|
||||
switch v := obj.(type) {
|
||||
case *jsonutils.JSONDict:
|
||||
redacted := jsonutils.NewDict()
|
||||
for name, value := range v.Value() {
|
||||
if isSensitiveBodyKey(name) {
|
||||
redacted.Set(name, jsonutils.NewString("*"))
|
||||
} else {
|
||||
redacted.Set(name, redactJSONObject(value))
|
||||
}
|
||||
}
|
||||
return redacted
|
||||
case *jsonutils.JSONArray:
|
||||
redacted := jsonutils.NewArray()
|
||||
for _, value := range v.Value() {
|
||||
redacted.Add(redactJSONObject(value))
|
||||
}
|
||||
return redacted
|
||||
default:
|
||||
return obj
|
||||
}
|
||||
}
|
||||
|
||||
// redactFormBody masks the credential carrying fields of an urlencoded body.
|
||||
//
|
||||
// The body is walked pair by pair rather than through url.ParseQuery, which
|
||||
// stops at the first pair it cannot decode: the pairs it did decode are still
|
||||
// returned, but taking only the value it returns would either drop the rest of
|
||||
// the body or, if the raw body were used instead, leave the credential in it.
|
||||
func redactFormBody(body string) string {
|
||||
parts := strings.Split(body, "&")
|
||||
changed := false
|
||||
for i, part := range parts {
|
||||
eq := strings.IndexByte(part, '=')
|
||||
if eq < 0 {
|
||||
continue
|
||||
}
|
||||
name := part[:eq]
|
||||
if decoded, err := url.QueryUnescape(name); err == nil {
|
||||
name = decoded
|
||||
}
|
||||
if isSensitiveBodyKey(name) {
|
||||
parts[i] = part[:eq+1] + "*"
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return body
|
||||
}
|
||||
return strings.Join(parts, "&")
|
||||
}
|
||||
|
||||
// A credential carrying name as it appears in a body that cannot be walked
|
||||
// field by field, e.g. XML, or a JSON document that did not parse.
|
||||
const sensitiveBodyNamePattern = `(?i)[A-Za-z0-9_.:-]*(?:token|secret|password|passwd|signature|credential|apikey|api_key|private_key)[A-Za-z0-9_.:-]*`
|
||||
|
||||
var (
|
||||
// <name>value</name>, where the value may be a CDATA section
|
||||
xmlElementSensitiveRe = regexp.MustCompile(
|
||||
`(?is)(<` + sensitiveBodyNamePattern + `\b[^>]*>)(?:\s*<!\[CDATA\[.*?\]\]>|[^<]*)(</)`)
|
||||
// name="value" or name='value'
|
||||
xmlAttrSensitiveRe = regexp.MustCompile(
|
||||
`(` + sensitiveBodyNamePattern + `)(\s*=\s*)("[^"]*"|'[^']*')`)
|
||||
// "name": "value", and the same without quotes
|
||||
jsonFieldSensitiveRe = regexp.MustCompile(
|
||||
`(?i)("` + sensitiveBodyNamePattern + `"\s*:\s*)("[^"]*"|[^,}\s]+)`)
|
||||
)
|
||||
|
||||
// redactBodyText masks credential carrying elements, attributes and fields of
|
||||
// a body that is not a structure this package can walk.
|
||||
func redactBodyText(body string) string {
|
||||
// The value alternative is not a capturing group, so the closing tag is
|
||||
// group 2.
|
||||
body = xmlElementSensitiveRe.ReplaceAllString(body, "${1}*${2}")
|
||||
body = xmlAttrSensitiveRe.ReplaceAllString(body, `${1}${2}"*"`)
|
||||
body = jsonFieldSensitiveRe.ReplaceAllString(body, "${1}*")
|
||||
return body
|
||||
}
|
||||
|
||||
// maxRedactedBodyBytes bounds how large a body put into an error message can
|
||||
// be. A larger body is left out rather than cut down: a cut keeps whichever
|
||||
// part of the body happened to fall inside it.
|
||||
const maxRedactedBodyBytes = 4096
|
||||
|
||||
// redactRequestBody masks the credential carrying fields of a request body
|
||||
// before it is put into an error message.
|
||||
//
|
||||
// A body whose content type is not one this package can walk field by field is
|
||||
// left out entirely. There is no way to tell which part of it is a credential,
|
||||
// and an excerpt would keep whichever part happened to fall inside the excerpt
|
||||
// — a multipart body, for instance, names its fields in a header line, so
|
||||
// masking the names would leave the values readable.
|
||||
func redactRequestBody(body, contType string) jsonutils.JSONObject {
|
||||
if len(body) == 0 || len(body) > maxRedactedBodyBytes {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(contType, "json"):
|
||||
if parsed, err := jsonutils.ParseString(body); err == nil {
|
||||
return redactJSONObject(parsed)
|
||||
}
|
||||
// A JSON body that does not parse is still masked by field name,
|
||||
// since the names survive in the text.
|
||||
return jsonutils.NewString(redactBodyText(body))
|
||||
case strings.Contains(contType, "x-www-form-urlencoded"):
|
||||
return jsonutils.NewString(redactFormBody(body))
|
||||
case strings.Contains(contType, "xml"):
|
||||
return jsonutils.NewString(redactBodyText(body))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// body might have been consumed, so body is provided separately
|
||||
func newJsonClientErrorFromRequest(req *http.Request, body string) *JSONClientError {
|
||||
return newJsonClientErrorFromRequest2(req.Method, req.URL.String(), req.Header, body)
|
||||
@@ -102,7 +270,7 @@ func newJsonClientErrorFromRequest2(method string, urlStr string, hdrs http.Head
|
||||
jce := &JSONClientError{}
|
||||
|
||||
jce.Request.Method = strings.ToUpper(method)
|
||||
jce.Request.Url = urlStr
|
||||
jce.Request.Url = redactURL(urlStr)
|
||||
jce.Request.Headers = make(map[string]string)
|
||||
excludeHdrs := []string{
|
||||
"Accept",
|
||||
@@ -112,22 +280,13 @@ func newJsonClientErrorFromRequest2(method string, urlStr string, hdrs http.Head
|
||||
http.CanonicalHeaderKey("authorization"),
|
||||
http.CanonicalHeaderKey("x-auth-token"),
|
||||
http.CanonicalHeaderKey("x-subject-token"),
|
||||
http.CanonicalHeaderKey("cookie"),
|
||||
http.CanonicalHeaderKey("set-cookie"),
|
||||
}
|
||||
const (
|
||||
MAX_BODY = 128
|
||||
FIRST_PART = 100
|
||||
)
|
||||
switch jce.Request.Method {
|
||||
case "PUT", "POST", "PATCH":
|
||||
contType := hdrs.Get(http.CanonicalHeaderKey("content-type"))
|
||||
if len(body) > MAX_BODY {
|
||||
jce.Request.Body = jsonutils.NewString(body[:FIRST_PART] + "..." + body[len(body)-MAX_BODY+FIRST_PART+3:])
|
||||
} else if strings.Contains(contType, "json") {
|
||||
jce.Request.Body, _ = jsonutils.ParseString(body)
|
||||
} else if strings.Contains(contType, "xml") ||
|
||||
strings.Contains(contType, "x-www-form-urlencoded") {
|
||||
jce.Request.Body = jsonutils.NewString(body)
|
||||
}
|
||||
jce.Request.Body = redactRequestBody(body, contType)
|
||||
default:
|
||||
excludeHdrs = append(excludeHdrs, http.CanonicalHeaderKey("content-type"), http.CanonicalHeaderKey("content-length"))
|
||||
}
|
||||
@@ -233,13 +392,10 @@ func NewJsonClient(client sClient) *JsonClient {
|
||||
return &JsonClient{client: client}
|
||||
}
|
||||
|
||||
// Error renders the error as JSON. The request it carries is already masked
|
||||
// when the error is built, so this has no side effects and repeating it gives
|
||||
// the same message.
|
||||
func (e *JSONClientError) Error() string {
|
||||
if !gotypes.IsNil(e.Request.Body) {
|
||||
if body, ok := e.Request.Body.(*jsonutils.JSONDict); ok && body.Contains("password") {
|
||||
body.Set("password", jsonutils.NewString("***"))
|
||||
e.Request.Body = body
|
||||
}
|
||||
}
|
||||
errMsg := JSONClientErrorMsg{Error: e}
|
||||
return jsonutils.Marshal(errMsg).String()
|
||||
}
|
||||
@@ -427,6 +583,42 @@ func GetClient(insecure bool, timeout time.Duration) *http.Client {
|
||||
return getClient(insecure, false, timeout)
|
||||
}
|
||||
|
||||
// Session headers that a caller of this package sets, and that must not
|
||||
// travel to a host the request was not made to. net/http already drops
|
||||
// Authorization, Cookie and the proxy headers in that case, but does not know
|
||||
// about these.
|
||||
var redirectSensitiveHeaders = []string{
|
||||
"X-Auth-Token",
|
||||
"X-Subject-Token",
|
||||
}
|
||||
|
||||
// isSameOrSubdomain reports whether dest is initial, or a subdomain of it.
|
||||
// A redirect to a parent domain is not covered, matching the rule net/http
|
||||
// uses for the headers it strips.
|
||||
func isSameOrSubdomain(initial, dest *url.URL) bool {
|
||||
i := strings.ToLower(initial.Hostname())
|
||||
d := strings.ToLower(dest.Hostname())
|
||||
if i == d {
|
||||
return true
|
||||
}
|
||||
return strings.HasSuffix(d, "."+i)
|
||||
}
|
||||
|
||||
// checkRedirect drops the session headers above when a redirect leaves the
|
||||
// host the request was made to, and keeps the usual limit on how many
|
||||
// redirects are followed.
|
||||
func checkRedirect(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= 10 {
|
||||
return errors.Errorf("stopped after 10 redirects")
|
||||
}
|
||||
if len(via) > 0 && !isSameOrSubdomain(via[0].URL, req.URL) {
|
||||
for _, h := range redirectSensitiveHeaders {
|
||||
req.Header.Del(h)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getClient(insecure bool, legacyTLS bool, timeout time.Duration) *http.Client {
|
||||
adaptive := false
|
||||
if timeout == 0 {
|
||||
@@ -434,7 +626,8 @@ func getClient(insecure bool, legacyTLS bool, timeout time.Duration) *http.Clien
|
||||
}
|
||||
tr := getTransport(insecure, legacyTLS, adaptive, timeout)
|
||||
return &http.Client{
|
||||
Transport: tr,
|
||||
Transport: tr,
|
||||
CheckRedirect: checkRedirect,
|
||||
// 一个完整http request的超时时间
|
||||
// Timeout specifies a time limit for requests made by this
|
||||
// Client. The timeout includes connection time, any
|
||||
|
||||
47
vendor/yunion.io/x/pkg/util/netutils/http.go
generated
vendored
47
vendor/yunion.io/x/pkg/util/netutils/http.go
generated
vendored
@@ -15,26 +15,47 @@
|
||||
package netutils
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// parseIPLiteral returns s as a plain address if it is one, and "" otherwise.
|
||||
func parseIPLiteral(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) == 0 {
|
||||
return ""
|
||||
}
|
||||
if ip := net.ParseIP(s); ip != nil {
|
||||
return ip.String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetHttpRequestIp returns the address of the client that made the request.
|
||||
//
|
||||
// The forwarded headers are consulted first, since a proxy in front of the
|
||||
// service rewrites them. When no proxy is in place they are whatever the
|
||||
// client sent, so each candidate is parsed and only an actual address is
|
||||
// returned; a value that is not an address is skipped rather than passed on
|
||||
// as though it were one.
|
||||
//
|
||||
// The connection address is used as the last resort, split with
|
||||
// net.SplitHostPort so that an IPv6 peer is returned without its brackets and
|
||||
// without its port.
|
||||
func GetHttpRequestIp(r *http.Request) string {
|
||||
ipStr := r.Header.Get("X-Forwarded-For")
|
||||
if len(ipStr) > 0 {
|
||||
ipList := strings.Split(ipStr, ",")
|
||||
if len(ipList) > 0 {
|
||||
return ipList[0]
|
||||
if forwarded := r.Header.Get("X-Forwarded-For"); len(forwarded) > 0 {
|
||||
for _, part := range strings.Split(forwarded, ",") {
|
||||
if ip := parseIPLiteral(part); len(ip) > 0 {
|
||||
return ip
|
||||
}
|
||||
}
|
||||
}
|
||||
ipStr = r.Header.Get("X-Real-Ip")
|
||||
if len(ipStr) > 0 {
|
||||
return ipStr
|
||||
if ip := parseIPLiteral(r.Header.Get("X-Real-Ip")); len(ip) > 0 {
|
||||
return ip
|
||||
}
|
||||
ipStr = r.RemoteAddr
|
||||
colonPos := strings.Index(ipStr, ":")
|
||||
if colonPos > 0 {
|
||||
ipStr = ipStr[:colonPos]
|
||||
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
|
||||
return host
|
||||
}
|
||||
return ipStr
|
||||
return r.RemoteAddr
|
||||
}
|
||||
|
||||
4
vendor/yunion.io/x/pkg/util/netutils/ipv6.go
generated
vendored
4
vendor/yunion.io/x/pkg/util/netutils/ipv6.go
generated
vendored
@@ -154,6 +154,10 @@ func normalizeIpv6Addr(addrStr string) ([8]uint16, error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if len(parts) > expectPartCnt {
|
||||
// More groups than the address can hold. A "::" run can only reduce
|
||||
// the number of groups, so there is no valid reading of this input.
|
||||
return addr, errors.Wrapf(errors.ErrInvalidFormat, "%s too many groups", addrStr)
|
||||
} else {
|
||||
for i := 0; i < len(parts); i++ {
|
||||
partStr := strings.TrimSpace(parts[i])
|
||||
|
||||
95
vendor/yunion.io/x/pkg/util/netutils/netutils.go
generated
vendored
95
vendor/yunion.io/x/pkg/util/netutils/netutils.go
generated
vendored
@@ -181,12 +181,21 @@ func (ar IPV4AddrRange) ContainsRange(ar2 IPV4AddrRange) bool {
|
||||
return ar.start <= ar2.start && ar.end >= ar2.end
|
||||
}
|
||||
|
||||
// Random returns an address from the range, excluding end. A range that
|
||||
// covers a single address returns that address.
|
||||
func (ar IPV4AddrRange) Random() IPV4Addr {
|
||||
return IPV4Addr(uint32(ar.start) + uint32(rand.Intn(int(uint32(ar.end)-uint32(ar.start)))))
|
||||
if ar.start >= ar.end {
|
||||
return ar.start
|
||||
}
|
||||
// int64 so the span of a very large range cannot overflow the argument
|
||||
// to the random source.
|
||||
span := int64(uint32(ar.end) - uint32(ar.start))
|
||||
return IPV4Addr(uint32(ar.start) + uint32(rand.Int63n(span)))
|
||||
}
|
||||
|
||||
func (ar IPV4AddrRange) AddressCount() int {
|
||||
return int(uint32(ar.end) - uint32(ar.start) + 1)
|
||||
// 64 bit arithmetic so a full range does not wrap around to zero.
|
||||
return int(uint64(uint32(ar.end)) - uint64(uint32(ar.start)) + 1)
|
||||
}
|
||||
|
||||
func (ar IPV4AddrRange) String() string {
|
||||
@@ -360,10 +369,18 @@ func (ar IPV4AddrRange) equals(ar2 IPV4AddrRange) bool {
|
||||
return ar.start == ar2.start && ar.end == ar2.end
|
||||
}
|
||||
|
||||
// Masklen2Mask returns the network mask for a prefix length.
|
||||
//
|
||||
// A length above 32 cannot be represented and is treated as 32, i.e. a single
|
||||
// host, rather than shifting past the width of the mask and wrapping around to
|
||||
// a match-all mask.
|
||||
func Masklen2Mask(maskLen int8) IPV4Addr {
|
||||
if maskLen < 0 {
|
||||
panic("negative masklen")
|
||||
}
|
||||
if maskLen > 32 {
|
||||
maskLen = 32
|
||||
}
|
||||
return IPV4Addr(^(uint32(1<<(32-uint8(maskLen))) - 1))
|
||||
}
|
||||
|
||||
@@ -392,6 +409,25 @@ func Mask2Len(mask IPV4Addr) int8 {
|
||||
return int8(bits.LeadingZeros32(^uint32(mask)))
|
||||
}
|
||||
|
||||
// ParsePrefix parses an IPv4 prefix written as "address/masklen", or as a
|
||||
// bare address, which is taken to be a /32. An empty string parses as
|
||||
// 0.0.0.0/32 rather than being rejected.
|
||||
//
|
||||
// The prefix length may also be written as a dotted-decimal mask, e.g.
|
||||
// "10.0.0.0/255.0.0.0". A dotted mask is converted by counting its leading
|
||||
// one bits, so the result always has a contiguous mask:
|
||||
// "1.2.3.4/255.0.255.0" yields 1.0.0.0/8 rather than being rejected.
|
||||
//
|
||||
// The numeric form is read with strconv.Atoi, so it also accepts a leading
|
||||
// sign and leading zeros: "+8" and "024" parse as 8 and 24.
|
||||
//
|
||||
// These accepted forms are more permissive than net.ParseCIDR. Note that
|
||||
// regutils.MatchCIDR and regutils.MatchIP4Addr reject the dotted-mask and
|
||||
// leading-zero spellings that this function accepts, so a value checked with
|
||||
// one of those and then normalised here can end up covering something other
|
||||
// than what was checked. Prefer net.ParseCIDR where the stricter grammar is
|
||||
// wanted; the lenient spellings are kept for compatibility with prefixes
|
||||
// already stored by callers.
|
||||
func ParsePrefix(prefix string) (IPV4Addr, int8, error) {
|
||||
slash := strings.IndexByte(prefix, '/')
|
||||
if slash > 0 {
|
||||
@@ -438,7 +474,15 @@ func NewIPV4Prefix(prefix string) (IPV4Prefix, error) {
|
||||
return pref, nil
|
||||
}
|
||||
|
||||
// NewIPV4PrefixFromAddr builds a prefix from an address and a prefix length.
|
||||
//
|
||||
// A length above 32 is clamped to 32 so that the stored MaskLen always agrees
|
||||
// with the address and range it describes. A negative length keeps its
|
||||
// existing behaviour of panicking.
|
||||
func NewIPV4PrefixFromAddr(addr IPV4Addr, masklen int8) IPV4Prefix {
|
||||
if masklen > 32 {
|
||||
masklen = 32
|
||||
}
|
||||
pref := IPV4Prefix{
|
||||
Address: addr.NetAddr(masklen),
|
||||
MaskLen: masklen,
|
||||
@@ -465,14 +509,28 @@ const (
|
||||
multicastPrefix = "224.0.0.0/4"
|
||||
)
|
||||
|
||||
// Ranges that cannot be a public address but are not part of the private
|
||||
// ranges used to classify guest addresses. They are consulted by
|
||||
// IsExitAddress only, so that IsPrivate keeps its current meaning for the
|
||||
// callers that use it for address allocation and DHCP.
|
||||
var reservedPrefixes = []string{
|
||||
"0.0.0.0/8", // "this network"
|
||||
"192.0.2.0/24", // TEST-NET-1
|
||||
"198.51.100.0/24", // TEST-NET-2
|
||||
"203.0.113.0/24", // TEST-NET-3
|
||||
"240.0.0.0/4", // reserved, includes the limited broadcast address
|
||||
}
|
||||
|
||||
var privateIPRanges []IPV4AddrRange
|
||||
var customizedPrivateIPRanges []IPV4AddrRange
|
||||
var reservedIPRanges []IPV4AddrRange
|
||||
var hostLocalIPRange IPV4AddrRange
|
||||
var linkLocalIPRange IPV4AddrRange
|
||||
var multicastIPRange IPV4AddrRange
|
||||
|
||||
func init() {
|
||||
initPrivateIPRanges()
|
||||
initReservedIPRanges()
|
||||
|
||||
prefix, _ := NewIPV4Prefix(hostlocalPrefix)
|
||||
hostLocalIPRange = prefix.ToIPRange()
|
||||
@@ -502,6 +560,22 @@ func initPrivateIPRanges() {
|
||||
}
|
||||
}
|
||||
|
||||
func initReservedIPRanges() {
|
||||
reservedIPRanges = make([]IPV4AddrRange, 0, len(reservedPrefixes))
|
||||
for _, prefix := range reservedPrefixes {
|
||||
p, err := NewIPV4Prefix(prefix)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reservedIPRanges = append(reservedIPRanges, p.ToIPRange())
|
||||
}
|
||||
}
|
||||
|
||||
// GetReservedIPRanges returns the ranges that IsReserved consults.
|
||||
func GetReservedIPRanges() []IPV4AddrRange {
|
||||
return append([]IPV4AddrRange(nil), reservedIPRanges...)
|
||||
}
|
||||
|
||||
func SetPrivatePrefixes(pref []string) {
|
||||
customizedPrivateIPRanges = make([]IPV4AddrRange, 0)
|
||||
for _, prefix := range pref {
|
||||
@@ -538,8 +612,23 @@ func IsMulticast(addr IPV4Addr) bool {
|
||||
return multicastIPRange.Contains(addr)
|
||||
}
|
||||
|
||||
// IsReserved reports whether addr is in a range that is reserved and can
|
||||
// never be a public address.
|
||||
func IsReserved(addr IPV4Addr) bool {
|
||||
for _, ipRange := range reservedIPRanges {
|
||||
if ipRange.Contains(addr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsExitAddress reports whether addr can be a public address, i.e. one that a
|
||||
// guest can reach directly. Reserved ranges are excluded along with the
|
||||
// private, host local, link local and multicast ones.
|
||||
func IsExitAddress(addr IPV4Addr) bool {
|
||||
return !IsPrivate(addr) && !IsHostLocal(addr) && !IsLinkLocal(addr) && !IsMulticast(addr)
|
||||
return !IsPrivate(addr) && !IsReserved(addr) &&
|
||||
!IsHostLocal(addr) && !IsLinkLocal(addr) && !IsMulticast(addr)
|
||||
}
|
||||
|
||||
func MacUnpackHex(mac string) string {
|
||||
|
||||
97
vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go
generated
vendored
97
vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go
generated
vendored
@@ -24,34 +24,89 @@ const (
|
||||
TAG_OLD_DEPRECATED_BY = "deprecated-by"
|
||||
)
|
||||
|
||||
// expandAmbiguousPrefix prepends the prefix given by the
|
||||
// TAG_AMBIGUOUS_PREFIX tag to the fields sharing a name with another field.
|
||||
//
|
||||
// Expanding a prefix can itself collide with a name that is already in use, so
|
||||
// the expansion is repeated until the names settle. A field whose expanded
|
||||
// name would take over a name owned by a field outside its own ambiguous
|
||||
// group keeps its original name, so that the expansion never introduces a new
|
||||
// ambiguity.
|
||||
func expandAmbiguousPrefix(fields SStructFieldValueSet) SStructFieldValueSet {
|
||||
keyIndexMap := make(map[string][]int)
|
||||
for i := range fields {
|
||||
if fields[i].Info.Ignore {
|
||||
continue
|
||||
prefixed := make(map[int]bool)
|
||||
for {
|
||||
keyIndexMap := make(map[string][]int)
|
||||
for i := range fields {
|
||||
if fields[i].Info.Ignore {
|
||||
continue
|
||||
}
|
||||
key := fields[i].Info.MarshalName()
|
||||
values, ok := keyIndexMap[key]
|
||||
if !ok {
|
||||
values = make([]int, 0, 2)
|
||||
}
|
||||
keyIndexMap[key] = append(values, i)
|
||||
}
|
||||
key := fields[i].Info.MarshalName()
|
||||
values, ok := keyIndexMap[key]
|
||||
if !ok {
|
||||
values = make([]int, 0, 2)
|
||||
}
|
||||
keyIndexMap[key] = append(values, i)
|
||||
}
|
||||
for _, indexes := range keyIndexMap {
|
||||
if len(indexes) > 1 {
|
||||
changed := false
|
||||
for _, indexes := range keyIndexMap {
|
||||
if len(indexes) < 2 {
|
||||
continue
|
||||
}
|
||||
// ambiguous found
|
||||
for _, idx := range indexes {
|
||||
if amPrefix, ok := fields[idx].Info.Tags[TAG_AMBIGUOUS_PREFIX]; ok {
|
||||
fields[idx].Info.Name = fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Name)
|
||||
if depBy, ok := fields[idx].Info.Tags[TAG_DEPRECATED_BY]; ok {
|
||||
fields[idx].Info.Tags[TAG_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy)
|
||||
}
|
||||
if depBy, ok := fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY]; ok {
|
||||
fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy)
|
||||
}
|
||||
if prefixed[idx] {
|
||||
continue
|
||||
}
|
||||
amPrefix, ok := fields[idx].Info.Tags[TAG_AMBIGUOUS_PREFIX]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
expanded := fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Name)
|
||||
if takenByOther(fields, expanded, indexes) {
|
||||
continue
|
||||
}
|
||||
fields[idx].Info.Name = expanded
|
||||
if depBy, ok := fields[idx].Info.Tags[TAG_DEPRECATED_BY]; ok {
|
||||
fields[idx].Info.Tags[TAG_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy)
|
||||
}
|
||||
if depBy, ok := fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY]; ok {
|
||||
fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy)
|
||||
}
|
||||
for i := range fields[idx].Info.Aliases {
|
||||
fields[idx].Info.Aliases[i] = fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Aliases[i])
|
||||
}
|
||||
prefixed[idx] = true
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
break
|
||||
}
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// takenByOther reports whether name is already used by a field outside group
|
||||
func takenByOther(fields SStructFieldValueSet, name string, group []int) bool {
|
||||
for i := range fields {
|
||||
if fields[i].Info.Ignore {
|
||||
continue
|
||||
}
|
||||
if fields[i].Info.MarshalName() != name {
|
||||
continue
|
||||
}
|
||||
if !containsIndex(group, i) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsIndex(indexes []int, idx int) bool {
|
||||
for _, i := range indexes {
|
||||
if i == idx {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
24
vendor/yunion.io/x/pkg/util/reflectutils/doc.go
generated
vendored
24
vendor/yunion.io/x/pkg/util/reflectutils/doc.go
generated
vendored
@@ -12,4 +12,28 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package reflectutils walks struct values by reflection, mainly to give the
|
||||
// json encoder and decoder in yunion.io/x/jsonutils the field names and the
|
||||
// tags to honour.
|
||||
//
|
||||
// # Field naming
|
||||
//
|
||||
// Fields are collected in declaration order, with the fields of an embedded
|
||||
// struct expanded in place. A name is taken from the "name" tag, then from
|
||||
// the "json" tag, and finally from the kebab form of the field name, so it can
|
||||
// be looked up by any of those.
|
||||
//
|
||||
// Two collected fields can carry the same name. An embedded struct may bring
|
||||
// in a field that another embedded struct also contributes, and unlike
|
||||
// encoding/json an outer field does not hide a field of an embedded struct.
|
||||
// All the fields sharing a name are reported, and a value decoded into the
|
||||
// struct is written to every one of them, so that code reaching the field
|
||||
// through any of the embedded paths sees it. Shadowing is deliberately not
|
||||
// implemented: the list inputs of the resource APIs rely on this, e.g.
|
||||
// ServerListInput carries several fields named "domain" coming from different
|
||||
// embedded structs.
|
||||
//
|
||||
// To set such fields apart, tag an embedded struct with
|
||||
// TAG_AMBIGUOUS_PREFIX, whose value is prepended to the names of the fields
|
||||
// that embedded struct contributes.
|
||||
package reflectutils // import "yunion.io/x/pkg/util/reflectutils"
|
||||
|
||||
119
vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go
generated
vendored
119
vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go
generated
vendored
@@ -29,12 +29,31 @@ import (
|
||||
// This struct has unexported fields initialized by exported functions in this
|
||||
// package. Do not construct a literal or modify the exported fields in an
|
||||
// unmanaged way
|
||||
//
|
||||
// The tags of a field are meant to be read through Tag and TagMap rather than
|
||||
// off the Tags map, which is not handed out to stay writable by whoever read
|
||||
// it; a caller may modify what TagMap returns.
|
||||
type SStructFieldInfo struct {
|
||||
// True if the field has json tag `json:"-"`
|
||||
Ignore bool
|
||||
Ignore bool
|
||||
|
||||
// True if an empty string, slice, map, struct or dict is left out of
|
||||
// the json object. A number and a boolean are not governed by this,
|
||||
// see OmitZero and OmitFalse.
|
||||
//
|
||||
// An empty value is left out by default, whether or not the json tag
|
||||
// asks for it; `allowempty` turns that off and `omitempty` states it.
|
||||
OmitEmpty bool
|
||||
|
||||
// True if a false boolean is left out of the json object. A false is
|
||||
// written by default; `omitfalse` turns that off and `allowfalse`
|
||||
// states it.
|
||||
OmitFalse bool
|
||||
OmitZero bool
|
||||
|
||||
// True if a zero number is left out of the json object. A zero is
|
||||
// written by default; `omitzero` turns that off and `allowzero` states
|
||||
// it.
|
||||
OmitZero bool
|
||||
|
||||
// Name can take the following values, in descreasing preference
|
||||
//
|
||||
@@ -42,12 +61,24 @@ type SStructFieldInfo struct {
|
||||
// 2. name of "json" tag, when it's not for ignoration
|
||||
// 3. kebab form of FieldName concatenated with "_" when Ignore is false
|
||||
// 4. empty string
|
||||
Name string
|
||||
FieldName string
|
||||
kebabFieldName string
|
||||
ForceString bool
|
||||
Tags map[string]string
|
||||
Name string
|
||||
|
||||
// FieldName is the name of the go struct field
|
||||
FieldName string
|
||||
|
||||
kebabFieldName string
|
||||
|
||||
// True if the field has the "string" json tag option, which writes the
|
||||
// value as a json string
|
||||
ForceString bool
|
||||
|
||||
// Tags holds the tags of the field keyed by tag name, a tag without a
|
||||
// value being mapped to the empty string. Read it through Tag or
|
||||
// TagMap rather than off here.
|
||||
Tags map[string]string
|
||||
|
||||
// Aliases are the other names the field is looked up by, taken from
|
||||
// the "alias" tag
|
||||
Aliases []string
|
||||
}
|
||||
|
||||
@@ -145,6 +176,23 @@ func (info *SStructFieldInfo) MarshalName() string {
|
||||
return info.kebabFieldName
|
||||
}
|
||||
|
||||
// Tag returns the value of the tag named name and whether the field has it.
|
||||
// A tag without a value is reported as present with an empty value.
|
||||
func (info *SStructFieldInfo) Tag(name string) (string, bool) {
|
||||
val, ok := info.Tags[name]
|
||||
return val, ok
|
||||
}
|
||||
|
||||
// TagMap returns a copy of the tags of the field, which the caller owns and
|
||||
// is free to modify.
|
||||
func (info *SStructFieldInfo) TagMap() map[string]string {
|
||||
tags := make(map[string]string, len(info.Tags))
|
||||
for k, v := range info.Tags {
|
||||
tags[k] = v
|
||||
}
|
||||
return tags
|
||||
}
|
||||
|
||||
type SStructFieldValue struct {
|
||||
Info *SStructFieldInfo
|
||||
Value reflect.Value
|
||||
@@ -152,9 +200,41 @@ type SStructFieldValue struct {
|
||||
Parent *SEmbedStructFieldValue
|
||||
}
|
||||
|
||||
// SEmbedStructFieldValue ties the fields enumerated for a nil embedded struct
|
||||
// to the struct they belong to. Field is the embedded pointer on the struct,
|
||||
// Value is the value the fields were enumerated out of, and Parent is the
|
||||
// entry of the enclosing embedded struct, if any.
|
||||
type SEmbedStructFieldValue struct {
|
||||
Field reflect.Value
|
||||
Value reflect.Value
|
||||
|
||||
Parent *SEmbedStructFieldValue
|
||||
}
|
||||
|
||||
// adoptEmbeddedStruct assigns Value to Field for every nil embedded struct
|
||||
// the field was enumerated through, which puts the enumerated fields back on
|
||||
// the struct. The assignments are made from the outermost embedded struct
|
||||
// inwards, so that each one lands on a struct that is already part of the
|
||||
// real one. It reports whether the field is backed by the struct afterwards,
|
||||
// which is what writing to it requires.
|
||||
func (v *SStructFieldValue) adoptEmbeddedStruct() bool {
|
||||
chain := make([]*SEmbedStructFieldValue, 0, 2)
|
||||
for p := v.Parent; p != nil; p = p.Parent {
|
||||
chain = append(chain, p)
|
||||
}
|
||||
for i := len(chain) - 1; i >= 0; i -= 1 {
|
||||
p := chain[i]
|
||||
if !p.Field.IsValid() || p.Field.Kind() != reflect.Ptr || !p.Field.IsNil() {
|
||||
// nothing to adopt, the pointer is already there
|
||||
continue
|
||||
}
|
||||
if !p.Field.CanSet() {
|
||||
// the struct can not hold the embedded pointer
|
||||
return false
|
||||
}
|
||||
p.Field.Set(p.Value)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type SStructFieldValueSet []SStructFieldValue
|
||||
@@ -225,6 +305,19 @@ func fetchStructFieldValueSet2(dataValue reflect.Value, allocatePtr bool, tags m
|
||||
}
|
||||
|
||||
func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags map[string]string, includeIgnore bool, parent *SEmbedStructFieldValue) SStructFieldValueSet {
|
||||
if !dataValue.IsValid() || dataValue.Kind() != reflect.Struct {
|
||||
// A zero Value, a nil or a non struct value has no field to
|
||||
// enumerate. Report it as such rather than through a panic.
|
||||
return SStructFieldValueSet{}
|
||||
}
|
||||
if allocatePtr && !dataValue.CanAddr() {
|
||||
// The value can not be modified in place, so a nil embedded
|
||||
// pointer can not be allocated on it. The fields of such a
|
||||
// pointer are enumerated out of a value allocated on the side
|
||||
// instead, the same way as when the caller does not ask for
|
||||
// allocation; see the note below for how to write to them.
|
||||
allocatePtr = false
|
||||
}
|
||||
fields := SStructFieldValueSet{}
|
||||
dataType := dataValue.Type()
|
||||
fieldInfos := fetchCacheStructFieldInfos(dataType)
|
||||
@@ -252,9 +345,17 @@ func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags m
|
||||
if fv.Kind() == reflect.Ptr && allocatePtr {
|
||||
fv.Set(reflect.New(fv.Type().Elem()))
|
||||
} else if fv.Kind() == reflect.Ptr && !allocatePtr {
|
||||
// The embedded pointer is nil, so the fields are
|
||||
// enumerated out of a value allocated on the side.
|
||||
// Value is a pointer to that value and the fields
|
||||
// below are the fields of the value it points at,
|
||||
// so assigning Value to Field puts them back on
|
||||
// the struct. Callers writing to those fields
|
||||
// must do so first.
|
||||
efv = &SEmbedStructFieldValue{
|
||||
Field: fv,
|
||||
Value: reflect.New(fv.Type().Elem()),
|
||||
Field: fv,
|
||||
Value: reflect.New(fv.Type().Elem()),
|
||||
Parent: parent,
|
||||
}
|
||||
fv = efv.Value
|
||||
} else {
|
||||
|
||||
105
vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go
generated
vendored
105
vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go
generated
vendored
@@ -19,6 +19,7 @@ import (
|
||||
"reflect"
|
||||
|
||||
"yunion.io/x/log"
|
||||
"yunion.io/x/pkg/gotypes"
|
||||
)
|
||||
|
||||
/*
|
||||
@@ -96,13 +97,19 @@ func fetchStructFieldNameValues(dataType reflect.Type, dataValue reflect.Value,
|
||||
}
|
||||
*/
|
||||
|
||||
// FindStructFieldValue returns the field of dataValue named name. The field
|
||||
// has to be writable, so a field reached through a nil embedded pointer is
|
||||
// only returned once that pointer has been put in place, which this does.
|
||||
func FindStructFieldValue(dataValue reflect.Value, name string) (reflect.Value, bool) {
|
||||
set := FetchStructFieldValueSet(dataValue)
|
||||
val, find := set.GetValue(name)
|
||||
if find && val.CanSet() {
|
||||
return val, true
|
||||
idx := set.GetStructFieldIndex(name)
|
||||
if idx < 0 {
|
||||
return reflect.Value{}, false
|
||||
}
|
||||
return reflect.Value{}, false
|
||||
if !set[idx].adoptEmbeddedStruct() || !set[idx].Value.CanSet() {
|
||||
return reflect.Value{}, false
|
||||
}
|
||||
return set[idx].Value, true
|
||||
}
|
||||
|
||||
func FindStructFieldInterface(dataValue reflect.Value, name string) (interface{}, bool) {
|
||||
@@ -111,35 +118,53 @@ func FindStructFieldInterface(dataValue reflect.Value, name string) (interface{}
|
||||
}
|
||||
|
||||
func FillEmbededStructValue(container reflect.Value, embed reflect.Value) bool {
|
||||
if !container.IsValid() || container.Kind() != reflect.Struct || !embed.IsValid() {
|
||||
return false
|
||||
}
|
||||
containerType := container.Type()
|
||||
embedType := embed.Type()
|
||||
for i := 0; i < containerType.NumField(); i += 1 {
|
||||
fieldType := containerType.Field(i)
|
||||
fieldValue := container.Field(i)
|
||||
if fieldType.Type.Kind() == reflect.Struct && fieldType.Anonymous {
|
||||
if fieldType.Type == embed.Type() {
|
||||
fieldValue.Set(embed)
|
||||
return true
|
||||
} else {
|
||||
filled := FillEmbededStructValue(fieldValue, embed)
|
||||
if filled {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if fieldType.Type.Kind() != reflect.Struct || !fieldType.Anonymous {
|
||||
continue
|
||||
}
|
||||
fieldValue := container.Field(i)
|
||||
if !fieldValue.CanSet() {
|
||||
// an unexported embedded struct can not be assigned to, and
|
||||
// neither can anything inside it
|
||||
continue
|
||||
}
|
||||
if fieldType.Type == embedType {
|
||||
fieldValue.Set(embed)
|
||||
return true
|
||||
}
|
||||
if FillEmbededStructValue(fieldValue, embed) {
|
||||
return true
|
||||
}
|
||||
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SetStructFieldValue sets the field of structValue named fieldName to val.
|
||||
// A field reached through a nil embedded pointer is written only once that
|
||||
// pointer has been put in place, which this does.
|
||||
func SetStructFieldValue(structValue reflect.Value, fieldName string, val reflect.Value) bool {
|
||||
set := FetchStructFieldValueSet(structValue)
|
||||
target, find := set.GetValue(fieldName)
|
||||
if !find {
|
||||
idx := set.GetStructFieldIndex(fieldName)
|
||||
if idx < 0 {
|
||||
return false
|
||||
}
|
||||
if !set[idx].adoptEmbeddedStruct() {
|
||||
return false
|
||||
}
|
||||
target := set[idx].Value
|
||||
if !target.CanSet() {
|
||||
return false
|
||||
}
|
||||
if !val.IsValid() || !val.Type().AssignableTo(target.Type()) {
|
||||
// report a failure instead of letting reflect.Value.Set panic
|
||||
return false
|
||||
}
|
||||
target.Set(val)
|
||||
return true
|
||||
}
|
||||
@@ -161,15 +186,34 @@ func ExpandInterface(val interface{}) []interface{} {
|
||||
func getAnonymouStructPointer(structValue reflect.Value, targetType reflect.Type) interface{} {
|
||||
structType := structValue.Type()
|
||||
if structType == targetType {
|
||||
if !structValue.CanInterface() {
|
||||
// the value was reached through an unexported field
|
||||
return nil
|
||||
}
|
||||
return structValue.Addr().Interface()
|
||||
}
|
||||
for i := 0; i < structValue.NumField(); i += 1 {
|
||||
fieldType := structType.Field(i)
|
||||
if fieldType.Anonymous && fieldType.Type.Kind() == reflect.Struct {
|
||||
ptr := getAnonymouStructPointer(structValue.Field(i), targetType)
|
||||
if ptr != nil {
|
||||
return ptr
|
||||
if !fieldType.Anonymous || !gotypes.IsFieldExportable(fieldType.Name) {
|
||||
// an unexported embedded struct can not be pointed at
|
||||
continue
|
||||
}
|
||||
fieldValue := structValue.Field(i)
|
||||
fieldT := fieldType.Type
|
||||
if fieldT.Kind() == reflect.Ptr {
|
||||
// an embedded pointer that is nil has nothing to point at
|
||||
if fieldValue.IsNil() {
|
||||
continue
|
||||
}
|
||||
fieldValue = fieldValue.Elem()
|
||||
fieldT = fieldT.Elem()
|
||||
}
|
||||
if fieldT.Kind() != reflect.Struct {
|
||||
continue
|
||||
}
|
||||
ptr := getAnonymouStructPointer(fieldValue, targetType)
|
||||
if ptr != nil {
|
||||
return ptr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
@@ -214,11 +258,18 @@ func StructContains(type1 reflect.Type, type2 reflect.Type) bool {
|
||||
}
|
||||
for i := 0; i < type1.NumField(); i += 1 {
|
||||
field := type1.Field(i)
|
||||
if field.Anonymous && field.Type.Kind() == reflect.Struct {
|
||||
contains := StructContains(field.Type, type2)
|
||||
if contains {
|
||||
return true
|
||||
}
|
||||
if !field.Anonymous {
|
||||
continue
|
||||
}
|
||||
fieldType := field.Type
|
||||
if fieldType.Kind() == reflect.Ptr {
|
||||
fieldType = fieldType.Elem()
|
||||
}
|
||||
if fieldType.Kind() != reflect.Struct {
|
||||
continue
|
||||
}
|
||||
if StructContains(fieldType, type2) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
37
vendor/yunion.io/x/pkg/util/samlutils/encrypt.go
generated
vendored
37
vendor/yunion.io/x/pkg/util/samlutils/encrypt.go
generated
vendored
@@ -76,6 +76,9 @@ func (key EncryptedKey) decryptKey(privateKey *rsa.PrivateKey) ([]byte, error) {
|
||||
encAlg := key.EncryptionMethod.Algorithm
|
||||
switch encAlg {
|
||||
case "http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p":
|
||||
if key.EncryptionMethod.DigestMethod == nil {
|
||||
return nil, errors.Wrap(errors.ErrInvalidFormat, "missing DigestMethod")
|
||||
}
|
||||
var shaAlg hash.Hash
|
||||
hashAlg := key.EncryptionMethod.DigestMethod.Algorithm
|
||||
switch hashAlg {
|
||||
@@ -99,6 +102,9 @@ func (data EncryptedData) decryptData(privateKey *rsa.PrivateKey) ([]byte, error
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "base64.StdEncoding.DecodeString")
|
||||
}
|
||||
if data.KeyInfo.EncryptedKey == nil {
|
||||
return nil, errors.Wrap(errors.ErrInvalidFormat, "missing KeyInfo.EncryptedKey")
|
||||
}
|
||||
key, err := data.KeyInfo.EncryptedKey.decryptKey(privateKey)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "KeyInfo.EncryptedKey.decryptKey")
|
||||
@@ -112,7 +118,36 @@ func (data EncryptedData) decryptData(privateKey *rsa.PrivateKey) ([]byte, error
|
||||
}
|
||||
}
|
||||
|
||||
// stripPKCS7Padding removes the padding that XML Encryption appends to the
|
||||
// last block of a CBC payload.
|
||||
//
|
||||
// Data that does not carry valid padding is returned unchanged rather than
|
||||
// reported, so that this cannot be used to tell one payload from another.
|
||||
func stripPKCS7Padding(data []byte, blockSize int) []byte {
|
||||
if len(data) == 0 || len(data)%blockSize != 0 {
|
||||
return data
|
||||
}
|
||||
n := int(data[len(data)-1])
|
||||
if n == 0 || n > blockSize || n > len(data) {
|
||||
return data
|
||||
}
|
||||
for _, b := range data[len(data)-n:] {
|
||||
if int(b) != n {
|
||||
return data
|
||||
}
|
||||
}
|
||||
return data[:len(data)-n]
|
||||
}
|
||||
|
||||
func decryptAesCbc(key []byte, secret []byte) ([]byte, error) {
|
||||
// The payload is an IV followed by whole ciphertext blocks. Anything
|
||||
// shorter, or not a whole number of blocks, cannot be decrypted.
|
||||
if len(secret) < 2*aes.BlockSize || len(secret)%aes.BlockSize != 0 {
|
||||
return nil, errors.Wrapf(errors.ErrInvalidFormat,
|
||||
"ciphertext of %d bytes is not a whole number of %d byte blocks preceded by an IV",
|
||||
len(secret), aes.BlockSize)
|
||||
}
|
||||
|
||||
c, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "aes.NewCipher")
|
||||
@@ -125,5 +160,5 @@ func decryptAesCbc(key []byte, secret []byte) ([]byte, error) {
|
||||
|
||||
decrypter.CryptBlocks(data, data)
|
||||
|
||||
return data, nil
|
||||
return stripPKCS7Padding(data, aes.BlockSize), nil
|
||||
}
|
||||
|
||||
17
vendor/yunion.io/x/pkg/util/samlutils/util.go
generated
vendored
17
vendor/yunion.io/x/pkg/util/samlutils/util.go
generated
vendored
@@ -20,7 +20,9 @@ import (
|
||||
"crypto/rsa"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/ma314smith/signedxml"
|
||||
@@ -92,14 +94,23 @@ func SAMLEncode(input []byte) (string, error) {
|
||||
return base64.StdEncoding.EncodeToString(comp), nil
|
||||
}
|
||||
|
||||
// SAMLForm renders an auto-submitting HTML form carrying the given
|
||||
// parameters. Both the action and the values are escaped, and the keys are
|
||||
// sorted so the output is stable for a given input.
|
||||
func SAMLForm(action string, attrs map[string]string) string {
|
||||
form := strings.Builder{}
|
||||
// form.WriteString(`<!DOCTYPE html><html lang="en-US"><body>`)
|
||||
form.WriteString(`<form id="saml_submit_form" method="POST" action="`)
|
||||
form.WriteString(action)
|
||||
form.WriteString(html.EscapeString(action))
|
||||
form.WriteString(`">`)
|
||||
for k, v := range attrs {
|
||||
form.WriteString(fmt.Sprintf("<input type=\"hidden\" name=\"%s\" value=\"%s\" />", k, v))
|
||||
keys := make([]string, 0, len(attrs))
|
||||
for k := range attrs {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
form.WriteString(fmt.Sprintf("<input type=\"hidden\" name=\"%s\" value=\"%s\" />",
|
||||
html.EscapeString(k), html.EscapeString(attrs[k])))
|
||||
}
|
||||
form.WriteString(`<input type="submit" value="Submit" />`)
|
||||
form.WriteString("</form><script><!--\n")
|
||||
|
||||
12
vendor/yunion.io/x/pkg/util/seclib/certs.go
generated
vendored
12
vendor/yunion.io/x/pkg/util/seclib/certs.go
generated
vendored
@@ -32,11 +32,21 @@ func CleanCertificate(cert string) string {
|
||||
return cert
|
||||
}
|
||||
|
||||
// DecodePrivateKey parses a PEM encoded RSA private key. Both PKCS#8 and
|
||||
// PKCS#1 encodings are accepted. An error is returned for input that is not
|
||||
// PEM encoded, or that holds a key which is not RSA.
|
||||
func DecodePrivateKey(keyString []byte) (*rsa.PrivateKey, error) {
|
||||
block, _ := pem.Decode(keyString)
|
||||
if block == nil {
|
||||
return nil, errors.Wrap(errors.ErrInvalidFormat, "not a valid PEM block")
|
||||
}
|
||||
privKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
|
||||
if err == nil {
|
||||
return privKey.(*rsa.PrivateKey), nil
|
||||
rsaKey, ok := privKey.(*rsa.PrivateKey)
|
||||
if !ok {
|
||||
return nil, errors.Wrapf(errors.ErrInvalidFormat, "private key is %T, not RSA", privKey)
|
||||
}
|
||||
return rsaKey, nil
|
||||
}
|
||||
priv, err := x509.ParsePKCS1PrivateKey(block.Bytes)
|
||||
if err == nil {
|
||||
|
||||
38
vendor/yunion.io/x/pkg/util/seclib/seclib.go
generated
vendored
38
vendor/yunion.io/x/pkg/util/seclib/seclib.go
generated
vendored
@@ -15,9 +15,10 @@
|
||||
package seclib
|
||||
|
||||
import (
|
||||
crand "crypto/rand"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/tredoe/osutil/user/crypt/sha512_crypt"
|
||||
@@ -25,6 +26,33 @@ import (
|
||||
"yunion.io/x/pkg/utils"
|
||||
)
|
||||
|
||||
// randomIndex returns a uniformly random index in [0, n), read from the
|
||||
// system random source.
|
||||
//
|
||||
// A failure of that source cannot be recovered from here: falling back to a
|
||||
// deterministic value would hand out a predictable password, so the caller is
|
||||
// stopped rather than served a weaker one.
|
||||
func randomIndex(n int) int {
|
||||
if n <= 0 {
|
||||
panic(fmt.Sprintf("seclib: invalid random range %d", n))
|
||||
}
|
||||
if n == 1 {
|
||||
return 0
|
||||
}
|
||||
// Values at or above limit would make the modulo below uneven, so they
|
||||
// are drawn again instead.
|
||||
limit := 256 - (256 % n)
|
||||
buf := make([]byte, 1)
|
||||
for {
|
||||
if _, err := io.ReadFull(crand.Reader, buf); err != nil {
|
||||
panic(fmt.Sprintf("seclib: random source failed: %v", err))
|
||||
}
|
||||
if int(buf[0]) < limit {
|
||||
return int(buf[0]) % n
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DIGITS = "23456789"
|
||||
LETTERS = "abcdefghjkmnpqrstuvwxyz"
|
||||
@@ -58,7 +86,7 @@ func RandomPassword(width int) string {
|
||||
} else {
|
||||
candidates = CHARS
|
||||
}
|
||||
index := rand.Intn(len(candidates))
|
||||
index := randomIndex(len(candidates))
|
||||
ch = candidates[index]
|
||||
}
|
||||
if strings.IndexByte(DIGITS, ch) >= 0 {
|
||||
@@ -84,7 +112,7 @@ func GeneratePassword(passwd string) (string, error) {
|
||||
}
|
||||
|
||||
var WEAK_PASSWORDS []string = []string{
|
||||
"Huawei@", "huawei@", "Admin@", "admin@", "Root@", "root@", "ABC@", "abc@", "ABCD@", "abcd@", "Huawei123@", "huawei123@", "Admin123@", "admin123@", "Root123@", "root123@", "Huawei#", "huawei#", "Admin#", "admin#", "Root#", "root#", "ABC#", "abc#", "ABCD#", "abcd#", "Huawei123#", "huawei123#", "Admin123#", "admin123#", "Root123#", "root123#,Huawei!", "huawei!", "Admin!", "admin!", "Root!", "root!", "ABC!", "abc!", "ABCD!", "abcd!", "Huawei123!", "huawei123!", "Admin123!", "admin123!", "Root123!", "root123!", "ABC123!", "abc123!", "Huawei@123", "huawei@123", "Admin@123", "admin@123", "Root@123", "root@123", "ABC@123", "abc@123", "123@Huawei", "123@Root", "123@abc", "Huawei123", "huawei123", "Admin123", "admin123", "Root123", "root123", "abc123", "Huawei_123", "huawei_123", "Admin_123", "admin_123", "Root_123", "root_123", "ABC_123", "abc_123", "123abc", "123abcd", "1234abc", "1234abcd", "abcd123", "abc1234", "abcd1234", "abcd@1234", "abcd1234!", "abcd_1234", "a123456", "123.com", "123@com", "123_com", "Huawei!@#", "huawei!@#", "Admin!@#", "admin!@#", "Root!@#", "root!@#", "Huawei!@", "huawei!@", "Admin!@", "admin!@", "Root!@", "root!@", "Huaweiroot", "HuaweiRoot", "huaweiroot", "huaweiRoot", "Huaweiadmin", "HuaweiAdmin", "huaweiadmin", "huaweiAdmin", "Adminroot", "AdminRoot", "adminRoot", "adminroot", "Rootadmin", "RootAdmin", "rootAdmin", "rootadmin", "Rootroot", "RootRoot", "rootroot", "Administrator", "Password", "Password123", "Password@123", "Password_123", "Password123!", "DDM@123", "ddM@123", "dDm@123",
|
||||
"Huawei@", "huawei@", "Admin@", "admin@", "Root@", "root@", "ABC@", "abc@", "ABCD@", "abcd@", "Huawei123@", "huawei123@", "Admin123@", "admin123@", "Root123@", "root123@", "Huawei#", "huawei#", "Admin#", "admin#", "Root#", "root#", "ABC#", "abc#", "ABCD#", "abcd#", "Huawei123#", "huawei123#", "Admin123#", "admin123#", "Root123#", "root123#", "huawei!", "Admin!", "admin!", "Root!", "root!", "ABC!", "abc!", "ABCD!", "abcd!", "Huawei123!", "huawei123!", "Admin123!", "admin123!", "Root123!", "root123!", "ABC123!", "abc123!", "Huawei@123", "huawei@123", "Admin@123", "admin@123", "Root@123", "root@123", "ABC@123", "abc@123", "123@Huawei", "123@Root", "123@abc", "Huawei123", "huawei123", "Admin123", "admin123", "Root123", "root123", "abc123", "Huawei_123", "huawei_123", "Admin_123", "admin_123", "Root_123", "root_123", "ABC_123", "abc_123", "123abc", "123abcd", "1234abc", "1234abcd", "abcd123", "abc1234", "abcd1234", "abcd@1234", "abcd1234!", "abcd_1234", "a123456", "123.com", "123@com", "123_com", "Huawei!@#", "huawei!@#", "Admin!@#", "admin!@#", "Root!@#", "root!@#", "Huawei!@", "huawei!@", "Admin!@", "admin!@", "Root!@", "root!@", "Huaweiroot", "HuaweiRoot", "huaweiroot", "huaweiRoot", "Huaweiadmin", "HuaweiAdmin", "huaweiadmin", "huaweiAdmin", "Adminroot", "AdminRoot", "adminRoot", "adminroot", "Rootadmin", "RootAdmin", "rootAdmin", "rootadmin", "Rootroot", "RootRoot", "rootroot", "Administrator", "Password", "Password123", "Password@123", "Password_123", "Password123!", "DDM@123", "ddM@123", "dDm@123",
|
||||
}
|
||||
|
||||
var CHARS2 = fmt.Sprintf("%s%s%s%s", DIGITS, LETTERS, UPPERS, PUNC)
|
||||
@@ -122,8 +150,8 @@ func randomPassword2(width int) string {
|
||||
} else {
|
||||
candidates = CHARS2
|
||||
}
|
||||
index := rand.Intn(len(candidates))
|
||||
ch = CHARS2[index]
|
||||
index := randomIndex(len(candidates))
|
||||
ch = candidates[index]
|
||||
}
|
||||
if strings.IndexByte(DIGITS, ch) >= 0 {
|
||||
ps.Digits += 1
|
||||
|
||||
10
vendor/yunion.io/x/pkg/util/secrules/cut.go
generated
vendored
10
vendor/yunion.io/x/pkg/util/secrules/cut.go
generated
vendored
@@ -319,12 +319,20 @@ func (srcs securityRuleCuts) cutOutPorts(protocol string, ps1 []uint16) security
|
||||
src_.r.PortEnd = int(e)
|
||||
r = append(r, src_)
|
||||
}
|
||||
// ps1 is sorted. Walk the ports that are left over between the
|
||||
// cut ports, advancing past each one even when it leaves no room
|
||||
// for a range of its own (p == 1), so that it is not handed back
|
||||
// by the trailing range below.
|
||||
s := uint16(1)
|
||||
for _, p := range ps1 {
|
||||
if p < s {
|
||||
continue
|
||||
}
|
||||
if s <= p-1 {
|
||||
add(s, p-1)
|
||||
s = p + 1
|
||||
}
|
||||
// Wraps to 0 for p == 65535, which the check below rejects.
|
||||
s = p + 1
|
||||
}
|
||||
if s != 0 && s <= 65535 {
|
||||
add(s, 65535)
|
||||
|
||||
18
vendor/yunion.io/x/pkg/util/secrules/secrules.go
generated
vendored
18
vendor/yunion.io/x/pkg/util/secrules/secrules.go
generated
vendored
@@ -402,6 +402,20 @@ func (rule *SecurityRule) GetPortsString() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// cidrMaskLen returns the prefix length and the size in bits of the address
|
||||
// family that a CIDR string normalises to.
|
||||
//
|
||||
// The mask of the IPNet a rule holds cannot be used for this directly: a
|
||||
// v4-mapped IPv6 CIDR stringifies as IPv4, so its raw mask is expressed in the
|
||||
// 128 bit space and does not describe the string it is printed as.
|
||||
func cidrMaskLen(cidr string) (int, int) {
|
||||
_, n, err := net.ParseCIDR(cidr)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
return n.Mask.Size()
|
||||
}
|
||||
|
||||
func (rule *SecurityRule) String() (result string) {
|
||||
s := []string{}
|
||||
s = append(s, string(rule.Direction)+":"+string(rule.Action))
|
||||
@@ -409,13 +423,13 @@ func (rule *SecurityRule) String() (result string) {
|
||||
if rule.IPNet != nil {
|
||||
cidr := rule.IPNet.String()
|
||||
if regutils.MatchCIDR(cidr) {
|
||||
if ones, _ := rule.IPNet.Mask.Size(); ones < 32 {
|
||||
if ones, bits := cidrMaskLen(cidr); ones < bits {
|
||||
s = append(s, cidr)
|
||||
} else {
|
||||
s = append(s, rule.IPNet.IP.String())
|
||||
}
|
||||
} else if regutils.MatchCIDR6(cidr) {
|
||||
if ones, _ := rule.IPNet.Mask.Size(); ones < 128 {
|
||||
if ones, bits := cidrMaskLen(cidr); ones < bits {
|
||||
s = append(s, cidr)
|
||||
} else {
|
||||
s = append(s, rule.IPNet.IP.String())
|
||||
|
||||
38
vendor/yunion.io/x/pkg/util/streamutils/streamutils.go
generated
vendored
38
vendor/yunion.io/x/pkg/util/streamutils/streamutils.go
generated
vendored
@@ -39,19 +39,19 @@ type sXZReadAheadReader struct {
|
||||
|
||||
func newXZReadAheadReader(stream io.Reader) (*sXZReadAheadReader, error) {
|
||||
xzHdr := make([]byte, xz.HeaderLen)
|
||||
n, err := stream.Read(xzHdr)
|
||||
// io.ReadFull so that a stream handing its data over in more than one
|
||||
// piece is not mistaken for a stream that has already finished.
|
||||
n, err := io.ReadFull(stream, xzHdr)
|
||||
hdrEof := false
|
||||
if err != nil {
|
||||
if errors.Cause(err) == io.EOF {
|
||||
cause := errors.Cause(err)
|
||||
if cause == io.EOF || cause == io.ErrUnexpectedEOF {
|
||||
// delay the EOF
|
||||
hdrEof = true
|
||||
xzHdr = xzHdr[:n]
|
||||
} else {
|
||||
return nil, errors.Wrap(err, "Read XZ header")
|
||||
}
|
||||
} else if n != len(xzHdr) {
|
||||
hdrEof = true
|
||||
xzHdr = xzHdr[:n]
|
||||
}
|
||||
return &sXZReadAheadReader{
|
||||
offset: 0,
|
||||
@@ -90,17 +90,35 @@ func (s *sXZReadAheadReader) Read(buf []byte) (int, error) {
|
||||
return n + bufOffset, err
|
||||
}
|
||||
|
||||
func StreamPipe(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64)) (*SStreamProperty, error) {
|
||||
// ErrSizeLimitExceeded is returned when the stream produces more bytes than
|
||||
// the limit passed to StreamPipe or StreamPipe2 allows.
|
||||
const ErrSizeLimitExceeded = errors.Error("stream size limit exceeded")
|
||||
|
||||
// StreamPipe streams upstream into writer. See StreamPipe2 for the optional
|
||||
// size limit.
|
||||
func StreamPipe(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64), maxSize ...int64) (*SStreamProperty, error) {
|
||||
return StreamPipe2(upstream, writer, CalChecksum, func(savedTotal int64, savedOnce int64) {
|
||||
if callback != nil {
|
||||
callback(savedTotal)
|
||||
}
|
||||
})
|
||||
}, maxSize...)
|
||||
}
|
||||
|
||||
func StreamPipe2(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64, savedOnce int64)) (*SStreamProperty, error) {
|
||||
// StreamPipe2 streams upstream into writer, decompressing when the input is an
|
||||
// xz stream.
|
||||
//
|
||||
// An optional maxSize stops the transfer once that many bytes have been
|
||||
// produced, so that an input which expands to far more than it looks like it
|
||||
// should cannot fill the writer. Omit it, or pass a value of zero or less, for
|
||||
// no limit.
|
||||
func StreamPipe2(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64, savedOnce int64), maxSize ...int64) (*SStreamProperty, error) {
|
||||
sp := SStreamProperty{}
|
||||
|
||||
var limit int64
|
||||
if len(maxSize) > 0 {
|
||||
limit = maxSize[0]
|
||||
}
|
||||
|
||||
var md5sum hash.Hash
|
||||
if CalChecksum {
|
||||
md5sum = md5.New()
|
||||
@@ -128,6 +146,10 @@ func StreamPipe2(upstream io.Reader, writer io.Writer, CalChecksum bool, callbac
|
||||
n, err := reader.Read(buf)
|
||||
if n > 0 {
|
||||
sp.Size += int64(n)
|
||||
if limit > 0 && sp.Size > limit {
|
||||
// Stop before the excess reaches the writer.
|
||||
return nil, errors.Wrapf(ErrSizeLimitExceeded, "produced more than %d bytes", limit)
|
||||
}
|
||||
if callback != nil {
|
||||
callback(sp.Size, int64(n))
|
||||
}
|
||||
|
||||
156
vendor/yunion.io/x/pkg/utils/crypt.go
generated
vendored
156
vendor/yunion.io/x/pkg/utils/crypt.go
generated
vendored
@@ -172,6 +172,162 @@ func EncryptAESBase64Url(key, msg string) (string, error) {
|
||||
return base64.URLEncoding.EncodeToString(result), nil
|
||||
}
|
||||
|
||||
// Authenticated encryption with AES-256-GCM.
|
||||
//
|
||||
// Payloads produced by EncryptAESBase64GCM / EncryptAESBase64UrlGCM are self
|
||||
// describing: a fixed header, then the nonce, then the sealed ciphertext.
|
||||
// DescryptAESBase64Auto / DescryptAESBase64UrlAuto accept both these payloads
|
||||
// and the legacy ones produced by EncryptAESBase64 / EncryptAESBase64Url, so
|
||||
// stored values can be re-encrypted and migrated without a flag day.
|
||||
//
|
||||
// EncryptAESBase64 / EncryptAESBase64Url are deliberately left unchanged: the
|
||||
// legacy form has no header, so a reader built from an older revision would
|
||||
// treat a GCM payload as ciphertext and return garbage without reporting an
|
||||
// error. Roll out readers that call the Auto functions before switching any
|
||||
// writer to the GCM functions.
|
||||
const aesGCMVersion = byte(0x02)
|
||||
|
||||
var aesGCMHeader = []byte{'Y', 'N', 'P', 'K', aesGCMVersion}
|
||||
|
||||
func hasAESGCMHeader(secret []byte) bool {
|
||||
if len(secret) < len(aesGCMHeader) {
|
||||
return false
|
||||
}
|
||||
for i := range aesGCMHeader {
|
||||
if secret[i] != aesGCMHeader[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func encryptAESGCM(k, msg []byte) ([]byte, error) {
|
||||
block, err := aes.NewCipher(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aead, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nonce := make([]byte, aead.NonceSize())
|
||||
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]byte, 0, len(aesGCMHeader)+len(nonce)+len(msg)+aead.Overhead())
|
||||
out = append(out, aesGCMHeader...)
|
||||
out = append(out, nonce...)
|
||||
return aead.Seal(out, nonce, msg, nil), nil
|
||||
}
|
||||
|
||||
func descryptAESGCM(k, secret []byte) ([]byte, error) {
|
||||
if !hasAESGCMHeader(secret) {
|
||||
return nil, fmt.Errorf("payload is not in the authenticated format")
|
||||
}
|
||||
body := secret[len(aesGCMHeader):]
|
||||
block, err := aes.NewCipher(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aead, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(body) < aead.NonceSize()+aead.Overhead() {
|
||||
return nil, fmt.Errorf("payload is truncated")
|
||||
}
|
||||
nonce, cipherText := body[:aead.NonceSize()], body[aead.NonceSize():]
|
||||
return aead.Open(nil, nonce, cipherText, nil)
|
||||
}
|
||||
|
||||
// EncryptAESBase64GCM encrypts msg and returns the standard-base64 encoding of
|
||||
// the authenticated payload.
|
||||
func EncryptAESBase64GCM(key, msg string) (string, error) {
|
||||
result, err := encryptAESGCM(toAESKey(key), []byte(msg))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(result), nil
|
||||
}
|
||||
|
||||
// EncryptAESBase64UrlGCM is EncryptAESBase64GCM using the URL-safe alphabet.
|
||||
func EncryptAESBase64UrlGCM(key, msg string) (string, error) {
|
||||
result, err := encryptAESGCM(toAESKey(key), []byte(msg))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return base64.URLEncoding.EncodeToString(result), nil
|
||||
}
|
||||
|
||||
// DescryptAESBase64GCM decrypts a payload produced by EncryptAESBase64GCM.
|
||||
// A payload that was modified after it was written is reported as an error.
|
||||
func DescryptAESBase64GCM(key, secret string) (string, error) {
|
||||
s, err := base64.StdEncoding.DecodeString(secret)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
result, err := descryptAESGCM(toAESKey(key), s)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(result), nil
|
||||
}
|
||||
|
||||
// DescryptAESBase64UrlGCM decrypts a payload produced by
|
||||
// EncryptAESBase64UrlGCM.
|
||||
func DescryptAESBase64UrlGCM(key, secret string) (string, error) {
|
||||
s, err := base64.URLEncoding.DecodeString(secret)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
result, err := descryptAESGCM(toAESKey(key), s)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(result), nil
|
||||
}
|
||||
|
||||
// DescryptAESBase64Auto decrypts an authenticated payload
|
||||
// (EncryptAESBase64GCM) as well as a legacy payload (EncryptAESBase64).
|
||||
func DescryptAESBase64Auto(key, secret string) (string, error) {
|
||||
s, err := base64.StdEncoding.DecodeString(secret)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
k := toAESKey(key)
|
||||
var result []byte
|
||||
if hasAESGCMHeader(s) {
|
||||
result, err = descryptAESGCM(k, s)
|
||||
} else {
|
||||
result, err = descryptAES(k, s)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(result), nil
|
||||
}
|
||||
|
||||
// DescryptAESBase64UrlAuto decrypts an authenticated payload
|
||||
// (EncryptAESBase64UrlGCM) as well as a legacy payload
|
||||
// (EncryptAESBase64Url).
|
||||
func DescryptAESBase64UrlAuto(key, secret string) (string, error) {
|
||||
s, err := base64.URLEncoding.DecodeString(secret)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
k := toAESKey(key)
|
||||
var result []byte
|
||||
if hasAESGCMHeader(s) {
|
||||
result, err = descryptAESGCM(k, s)
|
||||
} else {
|
||||
result, err = descryptAES(k, s)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(result), nil
|
||||
}
|
||||
|
||||
// RSA 加密
|
||||
func rsaEncrypt(publicKey, origData []byte) ([]byte, error) {
|
||||
block, _ := pem.Decode(publicKey)
|
||||
|
||||
27
vendor/yunion.io/x/pkg/utils/misc.go
generated
vendored
27
vendor/yunion.io/x/pkg/utils/misc.go
generated
vendored
@@ -471,6 +471,9 @@ func GetSize(sizeStr, defaultSize string, base int64) (size int64, err error) {
|
||||
if IsMatchInteger(sizeStr) {
|
||||
sizeStr += defaultSize
|
||||
}
|
||||
if len(sizeStr) == 0 {
|
||||
return 0, fmt.Errorf("empty size string")
|
||||
}
|
||||
|
||||
sizeNumStr := sizeStr[0 : len(sizeStr)-1]
|
||||
size, err = strconv.ParseInt(sizeNumStr, 10, 64)
|
||||
@@ -481,16 +484,16 @@ func GetSize(sizeStr, defaultSize string, base int64) (size int64, err error) {
|
||||
switch u := sizeStr[len(sizeStr)-1]; u {
|
||||
|
||||
case 't', 'T':
|
||||
size = size * base * base * base * base
|
||||
size, err = scaleSize(size, base, 4)
|
||||
|
||||
case 'g', 'G':
|
||||
size = size * base * base * base
|
||||
size, err = scaleSize(size, base, 3)
|
||||
|
||||
case 'm', 'M':
|
||||
size = size * base * base
|
||||
size, err = scaleSize(size, base, 2)
|
||||
|
||||
case 'k', 'K':
|
||||
size = size * base
|
||||
size, err = scaleSize(size, base, 1)
|
||||
|
||||
case 'b', 'B':
|
||||
size = size
|
||||
@@ -502,6 +505,22 @@ func GetSize(sizeStr, defaultSize string, base int64) (size int64, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
// scaleSize multiplies size by base the given number of times, reporting a
|
||||
// value that does not fit in an int64 instead of silently wrapping around.
|
||||
func scaleSize(size, base int64, times int) (int64, error) {
|
||||
const maxInt64 = int64(^uint64(0) >> 1)
|
||||
if base <= 0 {
|
||||
return 0, fmt.Errorf("invalid base %d", base)
|
||||
}
|
||||
for i := 0; i < times; i++ {
|
||||
if size > maxInt64/base {
|
||||
return 0, fmt.Errorf("size value is too large")
|
||||
}
|
||||
size *= base
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
func GetSizeBytes(sizeStr, defaultSize string) (int64, error) {
|
||||
return GetSize(sizeStr, defaultSize, 1024)
|
||||
}
|
||||
|
||||
17
vendor/yunion.io/x/pkg/utils/net.go
generated
vendored
17
vendor/yunion.io/x/pkg/utils/net.go
generated
vendored
@@ -17,7 +17,6 @@ package utils
|
||||
import (
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func GetOutboundIP() net.IP {
|
||||
@@ -30,8 +29,18 @@ func GetOutboundIP() net.IP {
|
||||
return localAddr.IP
|
||||
}
|
||||
|
||||
// GetAddrPort splits a "host:port" address into its host and port parts.
|
||||
// IPv6 addresses must be bracketed, e.g. "[::1]:8080", which also yields the
|
||||
// address without the brackets. If addrPort cannot be split, or the port is
|
||||
// not a number, the host is returned with a port of 0.
|
||||
func GetAddrPort(addrPort string) (string, int) {
|
||||
parts := strings.Split(addrPort, ":")
|
||||
port, _ := strconv.Atoi(parts[1])
|
||||
return parts[0], port
|
||||
host, portStr, err := net.SplitHostPort(addrPort)
|
||||
if err != nil {
|
||||
return addrPort, 0
|
||||
}
|
||||
port, err := strconv.Atoi(portStr)
|
||||
if err != nil {
|
||||
return host, 0
|
||||
}
|
||||
return host, port
|
||||
}
|
||||
|
||||
4
vendor/yunion.io/x/pkg/utils/pidfile.go
generated
vendored
4
vendor/yunion.io/x/pkg/utils/pidfile.go
generated
vendored
@@ -25,7 +25,9 @@ func SavePid(pidfile string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(pidfile, os.O_RDWR|os.O_CREATE, 0644)
|
||||
// O_TRUNC so that a shorter pid cannot leave the tail of a longer
|
||||
// previously written pid behind in the file.
|
||||
file, err := os.OpenFile(pidfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening pidfile %s: %s", pidfile, err)
|
||||
}
|
||||
|
||||
34
vendor/yunion.io/x/pkg/utils/tags.go
generated
vendored
34
vendor/yunion.io/x/pkg/utils/tags.go
generated
vendored
@@ -101,10 +101,27 @@ func _findWord(str []byte, offset int, sepChars string, isQuoteChar func(ch byte
|
||||
return buffer.String(), i
|
||||
}
|
||||
|
||||
// FindWords splits str into words separated by a space, a colon, a comma, a
|
||||
// tab, a newline or a closing bracket. A quoted fragment counts as one word.
|
||||
//
|
||||
// A fragment that is neither a word nor one of the separators ends the scan,
|
||||
// so the caller gets the words found up to that point rather than nothing at
|
||||
// all. Use FindWords2 to be told about such a fragment instead.
|
||||
func FindWords(str []byte, offset int) []string {
|
||||
words, err := FindWords2(str, offset, " :,\t\n}]", isQuoteCharInternal)
|
||||
if err != nil {
|
||||
panic(err.Error())
|
||||
const sepChars = " :,\t\n}]"
|
||||
words := make([]string, 0)
|
||||
for offset < len(str) {
|
||||
word, i := _findWord(str, offset, sepChars, isQuoteCharInternal)
|
||||
if i <= offset {
|
||||
// Nothing was consumed; stop rather than loop forever.
|
||||
break
|
||||
}
|
||||
words = append(words, word)
|
||||
i = skipEmpty(str, i)
|
||||
if i < len(str) && strings.IndexByte(sepChars, str[i]) >= 0 {
|
||||
i++
|
||||
}
|
||||
offset = i
|
||||
}
|
||||
return words
|
||||
}
|
||||
@@ -150,13 +167,14 @@ func TagMap(tag reflect.StructTag) map[string]string {
|
||||
break
|
||||
}
|
||||
i = skipEmpty(str, i)
|
||||
if i >= len(str) || strings.IndexByte(EMPTYSTR, str[i]) >= 0 {
|
||||
val = ""
|
||||
} else if str[i] != ':' {
|
||||
panic(fmt.Sprintf("Invalid structTag: %s", tag))
|
||||
} else {
|
||||
if i < len(str) && str[i] == ':' {
|
||||
i++
|
||||
val, i = findWord(str, i)
|
||||
} else {
|
||||
// A fragment that is not followed by a value takes no value.
|
||||
// It is kept as a key so that the scan carries on with the
|
||||
// fragments that follow it, rather than dropping them.
|
||||
val = ""
|
||||
}
|
||||
ret[k] = val
|
||||
i = skipEmpty(str, i)
|
||||
|
||||
2
vendor/yunion.io/x/sqlchemy/parser.go
generated
vendored
2
vendor/yunion.io/x/sqlchemy/parser.go
generated
vendored
@@ -23,7 +23,7 @@ import (
|
||||
|
||||
func (table *STableSpec) structField2ColumnSpec(field *reflectutils.SStructFieldValue) IColumnSpec {
|
||||
fieldname := field.Info.MarshalName()
|
||||
tagmap := field.Info.Tags
|
||||
tagmap := field.Info.TagMap()
|
||||
if _, ok := tagmap[TAG_IGNORE]; ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
4
vendor/yunion.io/x/structarg/structarg.go
generated
vendored
4
vendor/yunion.io/x/structarg/structarg.go
generated
vendored
@@ -289,7 +289,7 @@ func (this *ArgumentParser) addStructArgument(prefix string, tpVal reflect.Value
|
||||
sets := reflectutils.FetchAllStructFieldValueSetForWrite(tpVal)
|
||||
for i := range sets {
|
||||
if sets[i].Value.Kind() == reflect.Struct && sets[i].Value.Type() != gotypes.TimeType {
|
||||
tagMap := sets[i].Info.Tags
|
||||
tagMap := sets[i].Info.TagMap()
|
||||
if _, ok := tagMap[reflectutils.TAG_DEPRECATED_BY]; ok {
|
||||
// deprecated field, ignore
|
||||
return nil
|
||||
@@ -314,7 +314,7 @@ func (this *ArgumentParser) addStructArgument(prefix string, tpVal reflect.Value
|
||||
}
|
||||
|
||||
func (this *ArgumentParser) addArgument(prefix string, fv reflect.Value, info *reflectutils.SStructFieldInfo) error {
|
||||
tagMap := info.Tags
|
||||
tagMap := info.TagMap()
|
||||
if _, ok := tagMap[reflectutils.TAG_DEPRECATED_BY]; ok {
|
||||
// deprecated field, ignore
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user