fix(mcp-server): read the field tags through the reflectutils accessor (#25691)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>
This commit is contained in:
Jian Qiu
2026-09-17 14:15:39 +08:00
committed by GitHub
parent eb97ca9b10
commit 8af361db41
49 changed files with 1814 additions and 279 deletions

10
go.mod
View File

@@ -117,15 +117,15 @@ require (
k8s.io/cri-api v0.28.15
k8s.io/klog/v2 v2.90.1
moul.io/http2curl/v2 v2.3.0
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120446-39657d0a072d
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101522-bf8649bcb146
yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005
yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c
yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f
yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12
yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac
)
require (

26
go.sum
View File

@@ -1682,6 +1682,7 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1999,28 +2000,29 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120446-39657d0a072d h1:z7q3rvsp8iTju0FuIKPBSrEU4peDulMNR1d7DwxEy1g=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120446-39657d0a072d/go.mod h1:FLNUcF+R7ep5AmH7cVHRw1xFfxEKahDzyDRWcV3naOc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101522-bf8649bcb146 h1:DIGcOQoFp0McIOysABzOjcLJBFsyMTH/8qwBUECiMPE=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101522-bf8649bcb146/go.mod h1:FLNUcF+R7ep5AmH7cVHRw1xFfxEKahDzyDRWcV3naOc=
yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005 h1:3sWwcjGXGjG9mLBWa7AyLq+QSi0udTAx21pfVQRFMBE=
yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c h1:FthZM1upg4/yF4vmYheo/J7a738sGPjWPKk5kY88sH0=
yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
yunion.io/x/jsonutils v0.0.0-20220106020632-953b71a4c3a8/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea h1:lB7Kwanf7REFaA19YmDc7p0HVkssVRoNQejMlqtpecw=
yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea/go.mod h1:8ctyIiHT+Af1Ldr0xhvZZvuf+kyghlBUuvTy6pOKwbo=
yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
yunion.io/x/log v1.0.0/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 h1:inY5o3LDa/zgsIZuPN0HmpzKIsu/lLgsBmMttuDPGj4=
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 h1:Hu/4ERvoWaN6aiFs4h4/yvVBMeVzb3eDaYTxkVx9Kdo=
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f h1:eRWHOwgnEK4dMoq7TQ61qZxly3LF0s7tH6nlcahvZiA=
yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758 h1:3p/GCOPvDfvlSfovOh9GKxsSa7PPqp//DUjGRX8Wyus=
yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758/go.mod h1:Crp6aUYB+Yq16DFjZPHhHq7ha5HL2DfyPvzCo2LJ+LU=
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4=
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc h1:di0CJ1Fu/aeY0Lfvr52sQx8VTOm0JyZXJz0aC+7p6No=
yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc/go.mod h1:vCIZpqhZ5Jzaq3tFyrti/vv8BijQKtkzSgNT/uH4H5A=
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ=
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12 h1:xU8Z/Zg+zc199Y58eB3fA8mvOHsA28UCDfd7OuK3QKA=
yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12/go.mod h1:RNiIM7ZFF7VFrk18DbwAa8PYR00YP7Y23XjqKIlfr3I=
yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac h1:hJAthswj6YfWd3GCOBl9eM7mv0+4EUmjnD0b9jdBYi0=
yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac/go.mod h1:DyMCdRyfV5PdEe/BDz2vBbemnmALUF3Rj1JmUUlu3KY=
zombiezen.com/go/sqlite v0.13.1 h1:qDzxyWWmMtSSEH5qxamqBFmqA2BLSSbtODi3ojaE02o=
zombiezen.com/go/sqlite v0.13.1/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4=

View File

@@ -87,17 +87,16 @@ func collectMcpFields(optionsProto interface{}) map[string]mcpFieldMeta {
if info == nil {
continue
}
tagMap := info.Tags
mcpVal, ok := tagMap[TagMCP]
mcpVal, ok := info.Tag(TagMCP)
if !ok || mcpVal == "" || mcpVal == "false" {
continue
}
// 即便 json:"-"Ignore也保留climc 仍可能用字段名作为 CLI token如 MemSpec -> mem-spec
token, tokOK := tagMap["token"]
token, tokOK := info.Tag("token")
if !tokOK {
if jsonName := tagMap["json"]; jsonName != "" && jsonName != "-" {
if jsonName, _ := info.Tag("json"); jsonName != "" && jsonName != "-" {
token = info.MarshalName()
} else if alias := tagMap["alias"]; alias != "" {
} else if alias, _ := info.Tag("alias"); alias != "" {
token = alias
} else {
// Ignore 字段的 info.Name 为空,必须用 FieldName
@@ -115,7 +114,7 @@ func collectMcpFields(optionsProto interface{}) map[string]mcpFieldMeta {
out[fieldTok] = meta
}
// json 名与字段名不一致时(如 Region / prefer_region两边都登记
if jsonName := tagMap["json"]; jsonName != "" && jsonName != "-" {
if jsonName, _ := info.Tag("json"); jsonName != "" && jsonName != "-" {
if jsonTok := utils.CamelSplit(info.MarshalName(), "-"); jsonTok != "" && jsonTok != cliToken {
out[jsonTok] = meta
}

10
vendor/modules.txt vendored
View File

@@ -2709,7 +2709,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
# sigs.k8s.io/yaml v1.3.0
## explicit; go 1.12
sigs.k8s.io/yaml
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120446-39657d0a072d
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101522-bf8649bcb146
## explicit; go 1.24
yunion.io/x/cloudmux/pkg/apis
yunion.io/x/cloudmux/pkg/apis/billing
@@ -2809,7 +2809,7 @@ yunion.io/x/cloudmux/pkg/multicloud/zstack/provider
yunion.io/x/executor/apis
yunion.io/x/executor/client
yunion.io/x/executor/server
# yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c
# yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea
## explicit; go 1.18
yunion.io/x/jsonutils
# yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
@@ -2821,7 +2821,7 @@ yunion.io/x/log/hooks
yunion.io/x/ovsdb/cli_util
yunion.io/x/ovsdb/schema/ovn_nb
yunion.io/x/ovsdb/types
# yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f
# yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758
## explicit; go 1.18
yunion.io/x/pkg/appctx
yunion.io/x/pkg/errors
@@ -2875,7 +2875,7 @@ yunion.io/x/pkg/utils
# yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
## explicit; go 1.12
yunion.io/x/s3cli
# yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc
# yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12
## explicit; go 1.17
yunion.io/x/sqlchemy
yunion.io/x/sqlchemy/backends
@@ -2883,7 +2883,7 @@ yunion.io/x/sqlchemy/backends/clickhouse
yunion.io/x/sqlchemy/backends/dameng
yunion.io/x/sqlchemy/backends/mysql
yunion.io/x/sqlchemy/backends/sqlite
# yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
# yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac
## explicit; go 1.12
yunion.io/x/structarg
# zombiezen.com/go/sqlite v0.13.1

View File

@@ -33,6 +33,7 @@ type SDiskConfig struct {
UnitNumber int32
Key int32
ImagePath string
DestPath string
IsRoot bool
Datastore *SDatastore
Preallocation string

View File

@@ -285,17 +285,105 @@ func (svm *SVirtualMachine) DoRebuildRoot(ctx context.Context, imagePath string,
return svm.rebuildDisk(ctx, &svm.vdisks[0], imagePath, uefi)
}
func rebuildRootDiskBackupPath(filename string) string {
const suffix = ".vmdk"
if strings.HasSuffix(strings.ToLower(filename), suffix) {
return filename[:len(filename)-len(suffix)] + ".rebuild-bak.vmdk"
}
return filename + ".rebuild-bak.vmdk"
}
func (svm *SVirtualMachine) moveVirtualDiskFile(ctx context.Context, ds *SDatastore, src, dst string) error {
obj, err := ds.getDatastoreObj(ctx)
if err != nil {
return errors.Wrapf(err, "getDatastoreObj")
}
fm := obj.NewFileManager(ds.datacenter.getObjectDatacenter(), true)
err = fm.Move(ctx, src, dst)
if err != nil {
return errors.Wrapf(err, "move %s -> %s", src, dst)
}
return nil
}
func (svm *SVirtualMachine) reattachNonRootDisks(ctx context.Context) error {
for i := 1; i < len(svm.vdisks); i++ {
err := svm.doDetachDisk(ctx, &svm.vdisks[i], false)
if err != nil {
return errors.Wrapf(err, "doDetachDisk %d", i)
}
err = svm.doAttachDisk(ctx, &svm.vdisks[i])
if err != nil {
return errors.Wrapf(err, "doAttachDisk: %d", i)
}
}
return nil
}
func (svm *SVirtualMachine) attachDisksInOriginalOrder(ctx context.Context, root *SVirtualDisk) error {
for i := 1; i < len(svm.vdisks); i++ {
err := svm.doDetachDisk(ctx, &svm.vdisks[i], false)
if err != nil {
return errors.Wrapf(err, "doDetachDisk %d", i)
}
}
err := svm.doAttachDisk(ctx, root)
if err != nil {
for i := 1; i < len(svm.vdisks); i++ {
if attachErr := svm.doAttachDisk(ctx, &svm.vdisks[i]); attachErr != nil {
log.Errorf("reattach disk %d after root attach failed: %s", i, attachErr)
}
}
return errors.Wrap(err, "doAttachDisk root")
}
for i := 1; i < len(svm.vdisks); i++ {
err = svm.doAttachDisk(ctx, &svm.vdisks[i])
if err != nil {
return errors.Wrapf(err, "doAttachDisk: %d", i)
}
}
return nil
}
func (svm *SVirtualMachine) restoreRootDiskAfterRebuild(ctx context.Context, disk *SVirtualDisk, ds *SDatastore, origPath, backupPath string) error {
if err := ds.Delete2(ctx, origPath, false, true); err != nil {
log.Errorf("delete rebuilt root disk %s before restore: %s", origPath, err)
}
if err := svm.moveVirtualDiskFile(ctx, ds, backupPath, origPath); err != nil {
return errors.Wrap(err, "rename backup root disk")
}
return svm.attachDisksInOriginalOrder(ctx, disk)
}
func (svm *SVirtualMachine) rebuildDisk(ctx context.Context, disk *SVirtualDisk, imagePath string, uefi bool) error {
uuid := disk.GetId()
sizeMb := disk.GetDiskSizeMB()
diskKey := disk.getKey()
ctlKey := disk.getControllerKey()
unitNumber := *disk.dev.GetVirtualDevice().UnitNumber
origPath := disk.GetFilename()
backupPath := rebuildRootDiskBackupPath(origPath)
err := svm.doDetachAndDeleteDisk(ctx, disk)
istorage, err := disk.GetIStorage()
if err != nil {
return errors.Wrap(err, "GetIStorage")
}
ds := istorage.(*SDatastore)
err = svm.doDetachDisk(ctx, disk, false)
if err != nil {
return err
}
err = svm.moveVirtualDiskFile(ctx, ds, origPath, backupPath)
if err != nil {
if attachErr := svm.attachDisksInOriginalOrder(ctx, disk); attachErr != nil {
log.Errorf("reattach disks in original order after backup failed: %s", attachErr)
}
return errors.Wrapf(err, "backup root disk %s", origPath)
}
log.Infof("backup root disk %s -> %s", origPath, backupPath)
err = svm.createDiskInternal(ctx, SDiskConfig{
Uefi: uefi,
SizeMb: int64(sizeMb),
@@ -304,20 +392,21 @@ func (svm *SVirtualMachine) rebuildDisk(ctx context.Context, disk *SVirtualDisk,
UnitNumber: unitNumber,
Key: diskKey,
ImagePath: imagePath,
DestPath: origPath,
IsRoot: len(imagePath) > 0,
Datastore: ds,
}, false)
if err != nil {
if restoreErr := svm.restoreRootDiskAfterRebuild(ctx, disk, ds, origPath, backupPath); restoreErr != nil {
log.Errorf("restore root disk %s from %s failed: %s", origPath, backupPath, restoreErr)
}
return errors.Wrapf(err, "createDiskInternal")
}
for i := 1; i < len(svm.vdisks); i++ {
err = svm.doDetachDisk(ctx, &svm.vdisks[i], false)
if err != nil {
return errors.Wrapf(err, "doDetachDisk %d", i)
}
err = svm.doAttachDisk(ctx, &svm.vdisks[i])
if err != nil {
return errors.Wrapf(err, "doAttachDisk: %d", i)
if err = svm.reattachNonRootDisks(ctx); err != nil {
return err
}
if err = ds.Delete2(ctx, backupPath, false, true); err != nil {
log.Errorf("delete backup root disk %s: %s", backupPath, err)
}
return nil
}
@@ -1256,11 +1345,25 @@ func (svm *SVirtualMachine) GetRootImagePath() (string, error) {
return path, nil
}
func (svm *SVirtualMachine) CopyRootDisk(ctx context.Context, imagePath string) (string, error) {
newImagePath, datastore, err := svm.getDatastoreAndRootImagePath(false)
func (svm *SVirtualMachine) CopyRootDisk(ctx context.Context, imagePath, destPath string, datastore *SDatastore) (string, error) {
var (
newImagePath string
err error
)
if len(destPath) > 0 {
newImagePath = destPath
if datastore == nil {
_, datastore, err = svm.getDatastoreAndRootImagePath(false)
if err != nil {
return "", errors.Wrapf(err, "GetRootImagePath")
}
}
} else {
newImagePath, datastore, err = svm.getDatastoreAndRootImagePath(false)
if err != nil {
return "", errors.Wrapf(err, "GetRootImagePath")
}
}
ds, err := datastore.getDatastoreObj(ctx)
if err != nil {
return "", errors.Wrapf(err, "getDatastoreObj")
@@ -1278,7 +1381,7 @@ func (svm *SVirtualMachine) createDiskWithDeviceChange(ctx context.Context, devi
// copy disk
if len(config.ImagePath) > 0 {
config.IsRoot = true
config.ImagePath, err = svm.CopyRootDisk(ctx, config.ImagePath)
config.ImagePath, err = svm.CopyRootDisk(ctx, config.ImagePath, config.DestPath, config.Datastore)
if err != nil {
return errors.Wrap(err, "unable to copyRootDisk")
}
@@ -1327,7 +1430,6 @@ func (svm *SVirtualMachine) createDiskWithDeviceChange(ctx context.Context, devi
}
func (svm *SVirtualMachine) createDiskInternal(ctx context.Context, config SDiskConfig, check bool) error {
return svm.createDiskWithDeviceChange(ctx, nil, config, check)
}

View File

@@ -220,6 +220,130 @@ func (disk *SDisk) GetProjectId() string {
return disk.storage.zone.region.GetProjectId()
}
func maxInt(a, b int) int {
if a > b {
return a
}
return b
}
func minInt(a, b int) int {
if a < b {
return a
}
return b
}
func diskTypeName(storageType string) string {
if idx := strings.LastIndex(storageType, "/"); idx >= 0 {
return storageType[idx+1:]
}
return storageType
}
func inRange(val, minVal, maxVal int) bool {
return val >= minVal && val <= maxVal
}
// hyperdiskBalancedIopsRange 计算 balanced / balanced-ha 的 IOPS 区间
func hyperdiskBalancedIopsRange(sizeGb, maxIopsCap int) (minIops, maxIops int, ok bool) {
switch {
case sizeGb == 4:
return 2000, 2000, true
case sizeGb == 5:
return 2500, 2500, true
case sizeGb >= 6:
return 3000, minInt(500*sizeGb, maxIopsCap), true
default:
return 0, 0, false
}
}
func hyperdiskBalancedDefaultIops(sizeGb, maxIopsCap int) int {
if sizeGb <= 6 {
return 500 * sizeGb
}
// 6 GiB ~ 26.666667 TiB ≈ 27307 GiB
if sizeGb <= 27307 {
return minInt(6*sizeGb+3000, maxIopsCap)
}
return maxIopsCap
}
func hyperdiskBalancedThroughputRange(iops int) (minTp, maxTp int) {
return maxInt(140, iops/256), minInt(2400, iops/4)
}
func setHyperdiskBalancedPerformance(params map[string]interface{}, sizeGb, iops, throughput, maxIopsCap int) {
minIops, maxIops, ok := hyperdiskBalancedIopsRange(sizeGb, maxIopsCap)
if !ok {
return
}
effectiveIops := iops
if iops > 0 && inRange(iops, minIops, maxIops) {
params["provisionedIops"] = iops
} else if throughput > 0 {
effectiveIops = hyperdiskBalancedDefaultIops(sizeGb, maxIopsCap)
} else {
return
}
if throughput > 0 {
minTp, maxTp := hyperdiskBalancedThroughputRange(effectiveIops)
if inRange(throughput, minTp, maxTp) {
params["provisionedThroughput"] = throughput
}
}
}
// setDiskProvisionedPerformance 按 GCP 盘型区间+容量公式写入 IOPS/吞吐,超范围或不支持则丢弃该参数
func setDiskProvisionedPerformance(params map[string]interface{}, storageType string, sizeGb, iops, throughput int) {
if iops <= 0 && throughput <= 0 {
return
}
diskType := diskTypeName(storageType)
switch diskType {
case api.STORAGE_GOOGLE_PD_EXTREME:
if iops > 0 && inRange(iops, 2500, 120000) {
params["provisionedIops"] = iops
}
case api.STORAGE_GOOGLE_HYPERDISK_EXTREME:
if sizeGb < 64 || iops <= 0 {
return
}
minIops := 2 * sizeGb
maxIops := 350000
if sizeGb <= 291 {
maxIops = 1200 * sizeGb
}
if inRange(iops, minIops, maxIops) {
params["provisionedIops"] = iops
}
case api.STORAGE_GOOGLE_HYPERDISK_BALANCED:
setHyperdiskBalancedPerformance(params, sizeGb, iops, throughput, 160000)
case "hyperdisk-balanced-high-availability":
setHyperdiskBalancedPerformance(params, sizeGb, iops, throughput, 100000)
case api.STORAGE_GOOGLE_HYPERDISK_THROUGHPUT:
if throughput <= 0 || sizeGb < 2048 || sizeGb > 32768 {
return
}
sizeTiB := float64(sizeGb) / 1024.0
minTp := maxInt(20, int(5*sizeTiB))
maxTp := minInt(int(90*sizeTiB), 2400)
if inRange(throughput, minTp, maxTp) {
params["provisionedThroughput"] = throughput
}
case api.STORAGE_GOOGLE_HYPERDISK_ML:
if throughput <= 0 {
return
}
minTp := maxInt(400, int(0.12*float64(sizeGb)))
maxTp := minInt(2097152, 1600*sizeGb)
if inRange(throughput, minTp, maxTp) {
params["provisionedThroughput"] = throughput
}
}
}
func (region *SRegion) CreateDisk(zone string, storageType string, opts *cloudprovider.DiskCreateConfig) (*SDisk, error) {
if !strings.HasPrefix(storageType, GOOGLE_COMPUTE_DOMAIN) {
storageType = fmt.Sprintf("projects/%s/zones/%s/diskTypes/%s", region.GetProjectId(), zone, storageType)
@@ -240,6 +364,7 @@ func (region *SRegion) CreateDisk(zone string, storageType string, opts *cloudpr
if len(opts.ImageId) > 0 {
body["sourceImage"] = opts.ImageId
}
setDiskProvisionedPerformance(body, storageType, opts.SizeGb, opts.Iops, opts.Throughput)
disk := &SDisk{}
resource := fmt.Sprintf("zones/%s/disks", zone)
err := region.Insert(resource, jsonutils.Marshal(body), disk)

View File

@@ -581,29 +581,34 @@ func (region *SRegion) _createVM(zone string, desc *cloudprovider.SManagedVMCrea
labels[encode.EncodeGoogleLabel(k)] = encode.EncodeGoogleLabel(v)
}
disks = append(disks, map[string]interface{}{
"boot": true,
"initializeParams": map[string]interface{}{
sysDiskParams := map[string]interface{}{
"diskName": normalizeString(desc.SysDisk.Name),
"sourceImage": desc.ExternalImageId,
"diskSizeGb": desc.SysDisk.SizeGB,
"diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, desc.SysDisk.StorageType),
"labels": labels,
},
}
setDiskProvisionedPerformance(sysDiskParams, desc.SysDisk.StorageType, desc.SysDisk.SizeGB, desc.SysDisk.Iops, desc.SysDisk.Throughput)
disks = append(disks, map[string]interface{}{
"boot": true,
"initializeParams": sysDiskParams,
"autoDelete": true,
})
for _, disk := range desc.DataDisks {
if len(disk.Name) == 0 {
disk.Name = fmt.Sprintf("vdisk-%s-%d", desc.Name, time.Now().UnixNano())
for i, disk := range desc.DataDisks {
name := disk.Name
if len(name) == 0 {
name = fmt.Sprintf("vdisk-%s-%d-%d", desc.Name, i, time.Now().UnixNano())
}
disks = append(disks, map[string]interface{}{
"boot": false,
"initializeParams": map[string]interface{}{
"diskName": normalizeString(disk.Name),
dataDiskParams := map[string]interface{}{
"diskName": normalizeString(name),
"diskSizeGb": disk.SizeGB,
"diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, disk.StorageType),
"labels": labels,
},
}
setDiskProvisionedPerformance(dataDiskParams, disk.StorageType, disk.SizeGB, disk.Iops, disk.Throughput)
disks = append(disks, map[string]interface{}{
"boot": false,
"initializeParams": dataDiskParams,
"autoDelete": true,
})
}

View File

@@ -15,6 +15,7 @@
package google
import (
"crypto/sha1"
"fmt"
"regexp"
"strings"
@@ -202,14 +203,16 @@ func normalizeString(input string) string {
cleaned = regexp.MustCompile(`-$`).ReplaceAllString(cleaned, "")
// 5. 限制长度为63个字符域名标签的最大长度
// 超长时保留前缀并追加原文 hash避免仅截断前缀导致不同名称冲突
if len(cleaned) > 63 {
cleaned = cleaned[:63]
// 确保截断后不以连字符结尾
cleaned = regexp.MustCompile(`-$`).ReplaceAllString(cleaned, "")
// 如果截断后为空添加一个数字1
if len(cleaned) == 0 {
cleaned = "1"
sum := sha1.Sum([]byte(cleaned))
suffix := fmt.Sprintf("%x", sum[:4]) // 8 hex chars
prefixLen := 63 - 1 - len(suffix)
prefix := strings.TrimRight(cleaned[:prefixLen], "-")
if len(prefix) == 0 {
prefix = "a"
}
cleaned = prefix + "-" + suffix
}
return cleaned

View File

@@ -6,4 +6,4 @@ GOPROXY ?= direct
mod:
GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x \
go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p'))
go mod tidy
GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x go mod tidy

View File

@@ -20,10 +20,18 @@ import (
"time"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/gotypes"
"yunion.io/x/pkg/sortedmap"
"yunion.io/x/pkg/util/timeutils"
)
func normalizeJSONObject(obj JSONObject) JSONObject {
if gotypes.IsNil(obj) {
return JSONNull
}
return obj
}
type JSONPair struct {
key string
val JSONObject
@@ -40,7 +48,7 @@ func NewDict(objs ...JSONPair) *JSONDict {
func NewArray(objs ...JSONObject) *JSONArray {
arr := JSONArray{data: make([]JSONObject, 0, len(objs))}
for _, o := range objs {
arr.data = append(arr.data, o)
arr.data = append(arr.data, normalizeJSONObject(o))
}
return &arr
}
@@ -74,7 +82,7 @@ func NewBool(val bool) *JSONBool {
}
func (this *JSONDict) Set(key string, value JSONObject) {
this.data = sortedmap.Add(this.data, key, value)
this.data = sortedmap.Add(this.data, key, normalizeJSONObject(value))
}
func (this *JSONDict) Remove(key string) bool {
@@ -130,12 +138,12 @@ func (this *JSONDict) Add(o JSONObject, keys ...string) error {
}
func (this *JSONArray) SetAt(idx int, obj JSONObject) {
this.data[idx] = obj
this.data[idx] = normalizeJSONObject(obj)
}
func (this *JSONArray) Add(objs ...JSONObject) {
for _, o := range objs {
this.data = append(this.data, o)
this.data = append(this.data, normalizeJSONObject(o))
}
}

View File

@@ -20,4 +20,7 @@ import (
const (
TAG_DEPRECATED_BY = reflectutils.TAG_DEPRECATED_BY
// maxParseDepth is the maximum nesting level accepted by the parser
maxParseDepth = 10000
)

View File

@@ -15,6 +15,7 @@
package jsonutils
import (
"yunion.io/x/pkg/gotypes"
"yunion.io/x/pkg/sortedmap"
)
@@ -51,7 +52,14 @@ func (arr *JSONArray) Equals(json JSONObject) bool {
return false
}
for i, v := range arr.data {
if !v.Equals(arr2.data[i]) {
oth := arr2.data[i]
if gotypes.IsNil(v) || gotypes.IsNil(oth) {
if gotypes.IsNil(v) != gotypes.IsNil(oth) {
return false
}
continue
}
if !v.Equals(oth) {
return false
}
}

View File

@@ -26,10 +26,16 @@ const (
ErrInvalidHex = errors.Error("invalid hex")
ErrInvalidRune = errors.Error("invalid 4 byte rune")
ErrInvalidNodeId = errors.Error("invalid node id")
ErrDuplicateNodeId = errors.Error("duplicate node id")
ErrNestedTooDeep = errors.Error("json nested too deep")
ErrNodeReferenceDisabled = errors.Error("node reference is not enabled, see ParseTrusted")
ErrTypeMismatch = errors.Error("unmarshal type mismatch")
ErrArrayLengthMismatch = errors.Error("unmarshal array length mismatch")
ErrInterfaceUnsupported = errors.Error("do not known how to deserialize json into this interface type")
ErrMapKeyMustString = errors.Error("map key must be string")
ErrNodeNotFound = errors.Error("no node found for the reference")
ErrMissingInputField = errors.Error("missing input field")
ErrNilInputField = errors.Error("nil input field")

View File

@@ -15,6 +15,7 @@
package jsonutils
import (
"yunion.io/x/pkg/gotypes"
"yunion.io/x/pkg/sortedmap"
)
@@ -41,6 +42,10 @@ func (self *JSONString) Interface() interface{} {
func (self *JSONArray) Interface() interface{} {
ret := make([]interface{}, len(self.data))
for i := 0; i < len(self.data); i += 1 {
if gotypes.IsNil(self.data[i]) {
ret[i] = nil
continue
}
ret[i] = self.data[i].Interface()
}
return ret
@@ -51,7 +56,12 @@ func (self *JSONDict) Interface() interface{} {
for iter := sortedmap.NewIterator(self.data); iter.HasMore(); iter.Next() {
k, v := iter.Get()
mapping[k] = v.(JSONObject).Interface()
jo, _ := v.(JSONObject)
if gotypes.IsNil(jo) {
mapping[k] = nil
continue
}
mapping[k] = jo.Interface()
}
return mapping

View File

@@ -16,10 +16,13 @@ package jsonutils
import (
"bytes"
"math"
"reflect"
"sort"
"strconv"
"strings"
"time"
"unicode/utf16"
"unicode/utf8"
"yunion.io/x/log"
@@ -117,6 +120,11 @@ func skipEmpty(str []byte, offset int) int {
return i
}
// isFiniteFloat reports whether the value has a json representation
func isFiniteFloat(val float64) bool {
return !math.IsNaN(val) && !math.IsInf(val, 0)
}
func hexchar2num(v byte) (byte, error) {
switch {
case v >= '0' && v <= '9':
@@ -183,9 +191,22 @@ ret:
if e != nil {
return "", i, NewJSONError(str, i, e.Error())
}
i += 4
if utf16.IsSurrogate(r) {
// a character outside the BMP is written as a
// surrogate pair, e.g. 😀
if i+6 <= len(str) && str[i] == '\\' && str[i+1] == 'u' {
r2, e2 := hexstr2rune(str[i+2 : i+6])
if e2 == nil {
if combined := utf16.DecodeRune(r, r2); combined != utf8.RuneError {
r = combined
i += 6
}
}
}
}
runen = utf8.EncodeRune(runebytes, r)
buffer = append(buffer, runebytes[0:runen]...)
i += 4
case 'x':
i++
if i+2 >= len(str) {
@@ -253,13 +274,23 @@ ret2:
return string(str[offset:i]), false, i, nil
}
// isNodeReference reports whether the token has the form of a node
// reference, that is a bare <N> with an integer N
func isNodeReference(val string) bool {
if len(val) < 3 || val[0] != '<' || val[len(val)-1] != '>' {
return false
}
_, err := strconv.ParseInt(val[1:len(val)-1], 10, 64)
return err == nil
}
func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject, int, error) {
val, quote, i, e := parseString(str, offset)
if e != nil {
return nil, i, errors.Wrap(e, "parseString")
} else if quote {
return &JSONString{data: val}, i, nil
} else if val[0] == '<' && val[len(val)-1] == '>' {
} else if s.allowNodeReference && len(val) > 1 && val[0] == '<' && val[len(val)-1] == '>' {
// Pointer <nnnn>
val = val[1 : len(val)-1]
ival, err := strconv.ParseInt(val, 10, 64)
@@ -272,6 +303,10 @@ func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject,
}
s.saveReferer(nodeId, ptr)
return ptr, i, nil
} else if !s.allowNodeReference && isNodeReference(val) {
// a node reference can not be left unresolved and kept as a plain
// value: a caller could not tell it apart from a real string
return nil, i, errors.Wrap(ErrNodeReferenceDisabled, val)
} else {
lval := strings.ToLower(val)
if len(lval) == 0 || lval == "null" || lval == "none" {
@@ -288,9 +323,10 @@ func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject,
return &JSONInt{data: ival}, i, nil
}
fval, err := strconv.ParseFloat(val, 64)
if err == nil {
if err == nil && isFiniteFloat(fval) {
return &JSONFloat{data: fval}, i, nil
}
// nan and +-inf have no json representation, keep them as strings
return &JSONString{data: val}, i, nil
}
}
@@ -357,12 +393,33 @@ func escapeJsonChar(sb *strings.Builder, ch byte) {
}
}
// escapeJsonByte writes a byte that is not part of a valid utf-8 sequence
// as a \xXX escape, which parseQuoteString reads back unchanged
func escapeJsonByte(sb *strings.Builder, ch byte) {
const hexdigits = "0123456789abcdef"
sb.Write([]byte{'\\', 'x', hexdigits[ch>>4], hexdigits[ch&0xf]})
}
func quoteString(str string) string {
sb := &strings.Builder{}
sb.Grow(len(str) + 2)
sb.WriteByte('"')
for i := 0; i < len(str); i += 1 {
escapeJsonChar(sb, str[i])
for i := 0; i < len(str); {
ch := str[i]
if ch < utf8.RuneSelf {
escapeJsonChar(sb, ch)
i++
continue
}
r, size := utf8.DecodeRuneInString(str[i:])
if r == utf8.RuneError && size <= 1 {
// keep a non utf-8 byte reversible instead of writing it out
escapeJsonByte(sb, ch)
i++
continue
}
sb.WriteString(str[i : i+size])
i += size
}
sb.WriteByte('"')
return sb.String()
@@ -431,6 +488,11 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted
var e error = nil
var key string
var stop = false
// collect the keys first so that the sorted map can be built in key
// order: adding to a sorted map out of order shifts the whole tail
// on every insert
values := make(map[string]JSONObject)
keys := make([]string, 0)
for !stop && i < len(str) {
i = skipEmpty(str, i)
if i >= len(str) {
@@ -474,11 +536,18 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted
if e != nil {
return smap, i, nodeId, errors.Wrap(e, "parse misc")
}
if key == jsonPointerKey {
if s.allowNodeReference && key == jsonPointerKey {
// node id
nodeId = int(val.(*JSONInt).data)
jval, ok := val.(*JSONInt)
if !ok {
return smap, i, nodeId, errors.Wrap(ErrInvalidNodeId, jsonPointerKey)
}
nodeId = int(jval.data)
} else {
smap = sortedmap.Add(smap, key, val)
if _, ok := values[key]; !ok {
keys = append(keys, key)
}
values[key] = val
}
i = skipEmpty(str, i)
if i >= len(str) {
@@ -494,6 +563,10 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted
return smap, i, nodeId, NewJSONError(str, i, "Unexpected char")
}
}
sort.Strings(keys)
for _, key := range keys {
smap = sortedmap.Add(smap, key, values[key])
}
return smap, i, nodeId, nil
}
@@ -552,12 +625,20 @@ func (s *sJsonParseSession) parseArray(str []byte, offset int) ([]JSONObject, in
}
func (this *JSONDict) parse(s *sJsonParseSession, str []byte, offset int) (int, error) {
e := s.enter()
if e != nil {
return offset, errors.Wrap(e, "enter")
}
defer s.leave()
smap, i, nodeId, e := s.parseDict(str, offset)
if e == nil {
this.nodeId = nodeId
this.data = smap
if this.nodeId > 0 {
s.saveNode(nodeId, this)
e = s.saveNode(nodeId, this)
if e != nil {
return i, errors.Wrap(e, "saveNode")
}
}
return i, nil
}
@@ -591,9 +672,12 @@ func (this *JSONDict) prettyString(level int) string {
buffer.WriteByte('\n')
buffer.WriteString(tab)
buffer.WriteString(" ")
buffer.WriteByte('"')
buffer.WriteString(k)
buffer.WriteString("\":")
buffer.WriteString(quoteString(k))
buffer.WriteByte(':')
if gotypes.IsNil(v) {
buffer.WriteByte(' ')
buffer.WriteString("null")
} else {
_, okdict := v.(*JSONDict)
_, okarray := v.(*JSONArray)
if okdict || okarray {
@@ -603,6 +687,7 @@ func (this *JSONDict) prettyString(level int) string {
buffer.WriteByte(' ')
buffer.WriteString(v.String())
}
}
idx++
}
if len(this.data) > 0 {
@@ -614,6 +699,11 @@ func (this *JSONDict) prettyString(level int) string {
}
func (this *JSONArray) parse(s *sJsonParseSession, str []byte, offset int) (int, error) {
e := s.enter()
if e != nil {
return offset, errors.Wrap(e, "enter")
}
defer s.leave()
val, i, e := s.parseArray(str, offset)
if e == nil {
this.data = val
@@ -639,8 +729,13 @@ func (this *JSONArray) prettyString(level int) string {
buffer.WriteString(",")
}
buffer.WriteByte('\n')
if gotypes.IsNil(v) {
buffer.WriteString(tab)
buffer.WriteString(" null")
} else {
buffer.WriteString(v.prettyString(level + 1))
}
}
if len(this.data) > 0 {
buffer.WriteByte('\n')
buffer.WriteString(tab)
@@ -654,12 +749,46 @@ func ParseString(str string) (JSONObject, error) {
}
func Parse(str []byte) (JSONObject, error) {
json, _, err := ParseStream(str, 0)
json, offset, err := ParseStream(str, 0)
if err != nil {
return nil, err
}
if i := skipEmpty(str, offset); i < len(str) {
return nil, NewJSONError(str, i, "Unexpected content after the value")
}
return json, nil
}
// ParseStream parses one value starting at offset, it returns the value and
// the offset just after it, so that a stream of concatenated values can be
// walked. Node references are not resolved, see ParseTrusted.
func ParseStream(str []byte, offset int) (JSONObject, int, error) {
return parseStream(str, offset, false)
}
// ParseTrusted parses a document from a trusted source, resolving the node
// references that Marshal writes for a cyclic object: the ___jnid_ key inside
// an object and a bare <N> value referring to it.
//
// A document from an untrusted source must be parsed with Parse instead. A
// resolved reference makes two fields of the target struct point at the same
// object, which is what the round trip of a cyclic object needs, but it also
// lets a forged document do the same.
//
// Note that Marshal needs this syntax to terminate on a cyclic object, so it
// keeps writing it either way.
func ParseTrusted(str []byte) (JSONObject, error) {
json, _, err := parseStream(str, 0, true)
return json, err
}
func ParseStream(str []byte, offset int) (JSONObject, int, error) {
s := newJsonParseSession()
// ParseTrustedString is ParseTrusted for a string
func ParseTrustedString(str string) (JSONObject, error) {
return ParseTrusted([]byte(str))
}
func parseStream(str []byte, offset int, allowNodeReference bool) (JSONObject, int, error) {
s := newJsonParseSession(allowNodeReference)
i := offset
i = skipEmpty(str, i)
var val JSONObject = nil

View File

@@ -48,7 +48,7 @@ func (s *sJsonMarshalSession) marshalSlice(val reflect.Value, info *reflectutils
objs := make([]JSONObject, 0)
for i := 0; i < val.Len(); i += 1 {
val := s.marshalValue(val.Index(i), nil, omitEmpty)
if val != nil {
if !gotypes.IsNil(val) {
objs = append(objs, val)
}
}
@@ -84,7 +84,7 @@ func (s *sJsonMarshalSession) marshalMap(val reflect.Value, info *reflectutils.S
for i := 0; i < len(keys); i += 1 {
key := keys[i]
val := s.marshalValue(val.MapIndex(key), nil, omitEmpty)
if val != nil {
if !gotypes.IsNil(val) {
objPairs = append(objPairs, JSONPair{key: fmt.Sprintf("%s", key), val: val})
}
}
@@ -128,12 +128,12 @@ func (s *sJsonMarshalSession) struct2JSONPairs(val reflect.Value, omitEmpty bool
continue
}
key := jsonInfo.MarshalName()
if deprecatedBy, ok := fields[i].Info.Tags[TAG_DEPRECATED_BY]; ok {
if deprecatedBy, ok := fields[i].Info.Tag(TAG_DEPRECATED_BY); ok {
depFields[key] = deprecatedBy
continue
}
val := s.marshalValue(fields[i].Value, jsonInfo, omitEmpty)
if val != nil {
if !gotypes.IsNil(val) {
objPair := JSONPair{key: key, val: val}
objPairs = append(objPairs, objPair)
}
@@ -181,6 +181,9 @@ func marshalFloat64(val float64, info *reflectutils.SStructFieldInfo, bit int, o
return nil
} else if info != nil && info.ForceString {
return NewString(fmt.Sprintf("%f", val))
} else if !isFiniteFloat(val) {
// nan and +-inf have no json representation
return JSONNull
} else {
return NewFloat64(val)
}
@@ -191,6 +194,9 @@ func marshalFloat32(val float32, info *reflectutils.SStructFieldInfo, bit int, o
return nil
} else if info != nil && info.ForceString {
return NewString(fmt.Sprintf("%f", val))
} else if !isFiniteFloat(float64(val)) {
// nan and +-inf have no json representation
return JSONNull
} else {
return NewFloat32(val)
}
@@ -243,6 +249,13 @@ func marshalTime(val time.Time, info *reflectutils.SStructFieldInfo, omitEmpty b
}
}
// Marshal converts an object to a JSONObject, which can be written out with
// String.
//
// A cyclic object is written with the node reference syntax, that is the
// ___jnid_ key inside an object plus a bare <N> value referring to it. That
// syntax is not standard json and its result is not a valid json document,
// and Parse does not resolve it: read such a document back with ParseTrusted.
func Marshal(obj interface{}) JSONObject {
if obj == nil {
return JSONNull

View File

@@ -15,7 +15,7 @@
package jsonutils
import (
"fmt"
"yunion.io/x/pkg/errors"
)
type sNodeReferer struct {
@@ -25,14 +25,36 @@ type sNodeReferer struct {
type sJsonParseSession struct {
objectMap map[int]*sNodeReferer
// depth is the current nesting level of the object/array being parsed
depth int
// allowNodeReference tells whether the ___jnid_ key and a bare <N>
// value are read as node references, see ParseTrusted
allowNodeReference bool
}
func newJsonParseSession() *sJsonParseSession {
func newJsonParseSession(allowNodeReference bool) *sJsonParseSession {
return &sJsonParseSession{
objectMap: make(map[int]*sNodeReferer),
allowNodeReference: allowNodeReference,
}
}
// enter records entering one more nesting level, it fails if the nesting
// level exceeds maxParseDepth
func (s *sJsonParseSession) enter() error {
if s.depth >= maxParseDepth {
return ErrNestedTooDeep
}
s.depth++
return nil
}
func (s *sJsonParseSession) leave() {
s.depth--
}
func (s *sJsonParseSession) saveReferer(nodeId int, ptr *sJSONPointer) {
if nr, ok := s.objectMap[nodeId]; ok {
nr.pointers = append(nr.pointers, ptr)
@@ -43,10 +65,10 @@ func (s *sJsonParseSession) saveReferer(nodeId int, ptr *sJSONPointer) {
}
}
func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) {
func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) error {
if nr, ok := s.objectMap[nodeId]; ok {
if nr.node != nil {
panic(fmt.Sprintf("nodeId %d alreayd exists: %s != %s", nodeId, nr.node, node))
return errors.Wrapf(ErrDuplicateNodeId, "node id %d", nodeId)
} else {
nr.node = node
}
@@ -56,4 +78,5 @@ func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) {
pointers: nil,
}
}
return nil
}

View File

@@ -55,7 +55,10 @@ func addQueryStringSeg(body JSONObject, segs []sTextNumber, val []string) (JSONO
if err != nil {
return nil, errors.Wrapf(err, "get jsondict %s with %s fail", body, key)
}
addQueryStringSeg(next, segs[1:], val)
_, err = addQueryStringSeg(next, segs[1:], val)
if err != nil {
return nil, errors.Wrapf(err, "addQueryStringSeg %s with %s fail", body, key)
}
}
return jbody, nil
case *JSONArray:
@@ -66,7 +69,10 @@ func addQueryStringSeg(body JSONObject, segs []sTextNumber, val []string) (JSONO
if err != nil {
return nil, errors.Wrapf(err, "get jsonarray %s at %d fail", body, index)
}
addQueryStringSeg(next, segs[1:], val)
_, err = addQueryStringSeg(next, segs[1:], val)
if err != nil {
return nil, errors.Wrapf(err, "addQueryStringSeg %s at %d fail", body, index)
}
} else if arrSize == index {
// new
next, err := addQueryStringSeg(nil, segs[1:], val)

View File

@@ -16,9 +16,9 @@ package jsonutils
import (
"encoding/json"
"fmt"
"reflect"
"yunion.io/x/log"
"yunion.io/x/pkg/gotypes"
)
@@ -110,11 +110,13 @@ func tryStdMarshal(v reflect.Value, marshalFunc func(v reflect.Value) JSONObject
if m != nil {
data, err := m.MarshalJSON()
if err != nil {
panic(fmt.Sprintf("MarshalJSON of %q error: %v", v.String(), err))
log.Errorf("MarshalJSON of %s error: %v", v.Type(), err)
return JSONNull
}
jo, err := Parse(data)
if err != nil {
panic(fmt.Sprintf("Parse data %q to json of %q error: %v", data, v.String(), err))
log.Errorf("Parse data %s to json of %s error: %v", data, v.Type(), err)
return JSONNull
}
return jo
}

View File

@@ -33,6 +33,10 @@ func (this *JSONInt) String() string {
}
func (this *JSONFloat) String() string {
if !isFiniteFloat(this.data) {
// nan and +-inf have no json representation
return "null"
}
if this.bit != 32 && this.bit != 64 {
this.bit = 64
}

View File

@@ -62,12 +62,26 @@ func jsonUnmarshal(jo JSONObject, o interface{}, keys []string) error {
return errors.Wrap(err, "Get")
}
}
s := newJsonUnmarshalSession()
value := reflect.ValueOf(o)
err := jo.unmarshalValue(s, reflect.Indirect(value))
if value.IsValid() && value.Kind() == reflect.Ptr && value.IsNil() {
return errors.Wrapf(ErrTypeMismatch, "unmarshal into a nil pointer of type %s", value.Type())
}
target := reflect.Indirect(value)
// a non addressable value can not be written to, unless it is a
// non nil map, which is a reference type
if target.IsValid() && !target.CanAddr() &&
(target.Kind() != reflect.Map || target.IsNil()) {
return errors.Wrapf(ErrTypeMismatch, "unmarshal into a value of type %s, a pointer is required", target.Type())
}
s := newJsonUnmarshalSession()
err := jo.unmarshalValue(s, target)
if err != nil {
return errors.Wrap(err, "jo.unmarshalValue")
}
err = s.checkUnboundNodes()
if err != nil {
return errors.Wrap(err, "checkUnboundNodes")
}
return nil
}
@@ -385,6 +399,11 @@ func (this *JSONString) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Va
if err != nil {
return err
}
if !isFiniteFloat(floatVal) {
// nan and +-inf have no json representation, they would
// silently break any comparison the field takes part in
return errors.Wrap(ErrInvalidJsonFloat, "not a finite number")
}
val.SetFloat(floatVal)
}
case reflect.Bool:
@@ -476,7 +495,10 @@ func (this *JSONArray) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Val
func (this *JSONDict) unmarshalValue(s *sJsonUnmarshalSession, val reflect.Value) error {
if this.nodeId > 0 && val.CanAddr() {
s.saveNodeValue(this.nodeId, val.Addr())
err := s.saveNodeValue(this.nodeId, val.Addr())
if err != nil {
return errors.Wrap(err, "saveNodeValue")
}
}
return tryStdUnmarshal(s, this, val, this._unmarshalValue)
}
@@ -510,7 +532,11 @@ func (this *JSONDict) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Valu
return err
}
if objPtr == nil {
val.Set(reflect.ValueOf(this.data)) // ???
dataVal := reflect.ValueOf(this.data)
if !dataVal.Type().AssignableTo(val.Type()) {
return errors.Wrapf(ErrInterfaceUnsupported, "JSONDict.unmarshalValue: %s", val.Type())
}
val.Set(dataVal)
return nil
}
err = this.unmarshalValue(s, reflect.ValueOf(objPtr))
@@ -600,7 +626,7 @@ func setStructFieldAt(s *sJsonUnmarshalSession, key string, v JSONObject, fieldV
if err != nil {
return errors.Wrap(err, "JSONDict.unmarshalStruct")
}
depInfo, ok := fieldValues[index].Info.Tags[TAG_DEPRECATED_BY]
depInfo, ok := fieldValues[index].Info.Tag(TAG_DEPRECATED_BY)
if ok {
err := setStructFieldAt(s, depInfo, v, fieldValues, keyIndexMap, visited)
if err != nil {

View File

@@ -16,6 +16,9 @@ package jsonutils
import (
"reflect"
"sort"
"yunion.io/x/pkg/errors"
)
type sJsonNodeValues struct {
@@ -34,13 +37,27 @@ func newJsonUnmarshalSession() *sJsonUnmarshalSession {
}
}
func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) {
// assignable reports whether the node value can be assigned to the value
// referring to that node
func assignable(nodeValue, refValue reflect.Value) bool {
if !nodeValue.IsValid() || !refValue.IsValid() {
return false
}
return nodeValue.Type().AssignableTo(refValue.Type())
}
func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) error {
if nv, ok := s.objectMap[nodeId]; !ok {
s.objectMap[nodeId] = &sJsonNodeValues{
nodeValue: val,
nodeValueSet: true,
}
} else {
for i := range nv.targetValues {
if !assignable(val, nv.targetValues[i]) {
return errors.Wrapf(ErrTypeMismatch, "node id %d vs %s", nodeId, nv.targetValues[i].Type())
}
}
nv.nodeValue = val
nv.nodeValueSet = true
for i := range nv.targetValues {
@@ -48,10 +65,30 @@ func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) {
}
nv.targetValues = nil
}
return nil
}
// checkUnboundNodes reports the node ids that were referred to but never
// resolved, which would otherwise leave the referring field untouched
func (s *sJsonUnmarshalSession) checkUnboundNodes() error {
ids := make([]int, 0)
for nodeId, nv := range s.objectMap {
if !nv.nodeValueSet {
ids = append(ids, nodeId)
}
}
if len(ids) == 0 {
return nil
}
sort.Ints(ids)
return errors.Wrapf(ErrNodeNotFound, "node ids %v", ids)
}
func (s *sJsonUnmarshalSession) setPointerValue(nodeId int, val reflect.Value) error {
if nv, ok := s.objectMap[nodeId]; ok && nv.nodeValueSet {
if !assignable(nv.nodeValue, val) {
return errors.Wrapf(ErrTypeMismatch, "node id %d vs %s", nodeId, val.Type())
}
val.Set(nv.nodeValue)
} else if ok && !nv.nodeValueSet {
nv.targetValues = append(nv.targetValues, val)

View File

@@ -18,9 +18,18 @@ import (
"fmt"
"strings"
"yunion.io/x/pkg/gotypes"
"yunion.io/x/pkg/sortedmap"
)
func writeJSONObject(sb *strings.Builder, v JSONObject) {
if gotypes.IsNil(v) {
sb.WriteString("null")
return
}
v.buildString(sb)
}
type writeSource interface {
buildString(sb *strings.Builder)
}
@@ -46,6 +55,10 @@ func (this *JSONBool) buildString(sb *strings.Builder) {
}
func (this *JSONDict) buildString(sb *strings.Builder) {
if this == nil {
sb.WriteString("null")
return
}
sb.WriteByte('{')
var idx = 0
if this.nodeId > 0 {
@@ -63,19 +76,23 @@ func (this *JSONDict) buildString(sb *strings.Builder) {
sb.WriteString(quoteString(k))
sb.WriteByte(':')
v.buildString(sb)
writeJSONObject(sb, v)
idx++
}
sb.WriteByte('}')
}
func (this *JSONArray) buildString(sb *strings.Builder) {
if this == nil {
sb.WriteString("null")
return
}
sb.WriteByte('[')
for idx, v := range this.data {
if idx > 0 {
sb.WriteByte(',')
}
v.buildString(sb)
writeJSONObject(sb, v)
}
sb.WriteByte(']')
}

View File

@@ -18,6 +18,7 @@ import (
"bytes"
"encoding/base64"
"fmt"
"path"
"strings"
"yunion.io/x/jsonutils"
@@ -99,32 +100,127 @@ func NewWriteFile(path string, content string, perm string, owner string, isBase
return f
}
// shellQuote wraps s so that a POSIX shell treats it as one literal word.
// A single quote inside s is closed, escaped and reopened.
func shellQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}
// validUserName reports whether name can be used as a user name.
//
// The set is deliberately narrow. A name becomes a directory under /home, a
// file name under /etc/sudoers.d and an argument to useradd, so a slash or a
// ".." would move the files that are written, and a control character would
// split a generated line. Leading "-" is refused so the name cannot be read as
// an option.
func validUserName(name string) bool {
if len(name) == 0 || name == "." || name == ".." || name[0] == '-' {
return false
}
for i := 0; i < len(name); i++ {
c := name[i]
switch {
case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
case c == '_' || c == '-' || c == '.':
default:
return false
}
}
return true
}
// validWritePath reports whether path may be written by a generated script.
//
// Quoting keeps a path from running as a command, but a path that walks up out
// of its directory would still write somewhere else. A path carrying a ".."
// element is refused rather than normalised, so that what is written is what
// was asked for.
func validWritePath(filePath string) bool {
if len(filePath) == 0 {
return false
}
for _, elem := range strings.Split(filePath, "/") {
if elem == ".." {
return false
}
}
return true
}
// escapePowerShell escapes a value for use inside a PowerShell double quoted
// string, where a backtick introduces an escape, $ introduces a variable, and
// a control character written literally would break the line it sits on.
func escapePowerShell(s string) string {
return strings.NewReplacer(
"`", "``",
`"`, "`\"",
"$", "`$",
"\r", "`r",
"\n", "`n",
"\t", "`t",
"\x00", "`0",
).Replace(s)
}
const heredocPrefix = "_YUNION_EOF_"
// heredocTerminator returns a terminator that does not occur in content, so
// that a line of the content cannot end the heredoc early. Quote it at the use
// site so the shell does not expand the content.
//
// The generated value is checked against the content rather than assumed to be
// unique: utils.GenRequestId returns the empty string if the random source is
// unavailable, which would otherwise leave a fixed, guessable terminator.
func heredocTerminator(content string) string {
return uniqueTerminator(heredocPrefix+utils.GenRequestId(8), content)
}
// uniqueTerminator returns term with underscores appended until it does not
// occur in content. Each pass lengthens it, so this ends once it is longer
// than the content and can no longer occur in it.
func uniqueTerminator(term, content string) string {
for strings.Contains(content, term) {
term += "_"
}
return term
}
func setFilePermission(path, permission, owner string) []string {
cmds := []string{}
if len(permission) > 0 {
cmds = append(cmds, fmt.Sprintf("chmod %s %s", permission, path))
cmds = append(cmds, fmt.Sprintf("chmod %s %s", shellQuote(permission), shellQuote(path)))
}
if len(owner) > 0 {
cmds = append(cmds, fmt.Sprintf("chown %s:%s %s", owner, owner, path))
cmds = append(cmds, fmt.Sprintf("chown %s:%s %s", shellQuote(owner), shellQuote(owner), shellQuote(path)))
}
return cmds
}
// mkWriteFileCmd builds the commands that create path with content. The
// redirection and the path are quoted, and the heredoc is quoted with a
// terminator that is not expected to occur in the content.
func mkWriteFileCmd(redirect, filePath, content, permission, owner string) []string {
terminator := heredocTerminator(content)
cmds := []string{
fmt.Sprintf("mkdir -p %s", shellQuote(path.Dir(filePath))),
fmt.Sprintf("cat %s %s <<'%s'\n%s\n%s", redirect, shellQuote(filePath), terminator, content, terminator),
}
return append(cmds, setFilePermission(filePath, permission, owner)...)
}
func mkPutFileCmd(path string, content string, permission string, owner string) []string {
cmds := []string{}
cmds = append(cmds, fmt.Sprintf("mkdir -p $(dirname %s)", path))
cmds = append(cmds, fmt.Sprintf("cat > %s <<_END\n%s\n_END", path, content))
return append(cmds, setFilePermission(path, permission, owner)...)
return mkWriteFileCmd(">", path, content, permission, owner)
}
func mkAppendFileCmd(path string, content string, permission string, owner string) []string {
cmds := []string{}
cmds = append(cmds, fmt.Sprintf("mkdir -p $(dirname %s)", path))
cmds = append(cmds, fmt.Sprintf("cat >> %s <<_END\n%s\n_END", path, content))
return append(cmds, setFilePermission(path, permission, owner)...)
return mkWriteFileCmd(">>", path, content, permission, owner)
}
func (wf *SWriteFile) ShellScripts() []string {
if !validWritePath(wf.Path) {
log.Errorf("cloudinit: skipping write_file with a path that walks up out of its directory: %q", wf.Path)
return nil
}
content := wf.Content
if wf.Encoding == "b64" {
_content, _ := base64.StdEncoding.DecodeString(wf.Content)
@@ -176,23 +272,36 @@ func (u *SUser) Password(passwd string) *SUser {
}
func (u *SUser) PowerShellScripts() []string {
if !validUserName(u.Name) {
log.Errorf("cloudinit: skipping scripts for unusable user name %q", u.Name)
return nil
}
// Every line below is parsed by PowerShell first, so the name and the
// password are escaped the same way. Using the unescaped name for one of
// the lines would have them name different accounts.
name := escapePowerShell(u.Name)
shells := []string{}
shells = append(shells, fmt.Sprintf(`New-LocalUser -Name "%s" -Description "A New Local Account Created By PowerShell" -NoPassword`, u.Name))
shells = append(shells, fmt.Sprintf(`Add-LocalGroupMember -Group "Administrators" -Member "%s"`, u.Name))
shells = append(shells, fmt.Sprintf(`New-LocalUser -Name "%s" -Description "A New Local Account Created By PowerShell" -NoPassword`, name))
shells = append(shells, fmt.Sprintf(`Add-LocalGroupMember -Group "Administrators" -Member "%s"`, name))
if len(u.PlainTextPasswd) > 0 {
shells = append(shells, fmt.Sprintf(`net user "%s" "%s"`, u.Name, u.PlainTextPasswd))
shells = append(shells, fmt.Sprintf(`net user "%s" "%s"`, name, escapePowerShell(u.PlainTextPasswd)))
}
// enable需要再设置密码之后否则会出现Enable-LocalUser : Unable to update the password. The value provided for the new password does not meet the length, complexity, or history requirements of the domain
shells = append(shells, fmt.Sprintf(`Enable-LocalUser "%s"`, u.Name))
shells = append(shells, fmt.Sprintf(`Enable-LocalUser "%s"`, name))
return shells
}
func (u *SUser) ShellScripts() []string {
if !validUserName(u.Name) {
log.Errorf("cloudinit: skipping scripts for unusable user name %q", u.Name)
return nil
}
name := shellQuote(u.Name)
shells := []string{}
shells = append(shells, fmt.Sprintf("useradd -m %s || true", u.Name))
shells = append(shells, fmt.Sprintf("useradd -m %s || true", name))
if len(u.HashedPasswd) > 0 {
shells = append(shells, fmt.Sprintf("usermod -p '%s' %s", u.HashedPasswd, u.Name))
shells = append(shells, fmt.Sprintf("usermod -p %s %s", shellQuote(u.HashedPasswd), name))
}
home := "/" + u.Name
@@ -202,7 +311,7 @@ func (u *SUser) ShellScripts() []string {
keyPath := fmt.Sprintf("%s/.ssh/authorized_keys", home)
shells = append(shells, mkAppendFileCmd(keyPath, strings.Join(u.SshAuthorizedKeys, "\n"), "600", u.Name)...)
shells = append(shells, fmt.Sprintf("chown -R %s:%s %s/.ssh", u.Name, u.Name, home))
shells = append(shells, fmt.Sprintf("chown -R %s:%s %s", name, name, shellQuote(home+"/.ssh")))
if !utils.IsInStringArray(u.Sudo, []string{"", "False"}) {
shells = append(shells, mkPutFileCmd("/etc/sudoers.d/"+u.Name, fmt.Sprintf("%s %s", u.Name, u.Sudo), "", "")...)
@@ -212,6 +321,9 @@ func (u *SUser) ShellScripts() []string {
}
func (conf *SCloudConfig) UserData() string {
if conf == nil {
return ""
}
var buf bytes.Buffer
jsonConf := jsonutils.Marshal(conf).(*jsonutils.JSONDict)
if jsonConf.Contains("users") {
@@ -228,6 +340,9 @@ func (conf *SCloudConfig) UserData() string {
}
func (conf *SCloudConfig) UserDataScript() string {
if conf == nil {
return ""
}
shells := []string{}
for _, u := range conf.Users {
shells = append(shells, u.ShellScripts()...)
@@ -248,8 +363,11 @@ func (conf *SCloudConfig) UserDataScript() string {
}
for _, pkg := range conf.Packages {
shells = append(shells, "which yum &>/dev/null && yum install -y "+pkg)
shells = append(shells, "which apt-get &>/dev/null && apt-get install -y "+pkg)
// A package name is an argument, not part of the command line, so it
// is quoted rather than pasted in.
quoted := shellQuote(pkg)
shells = append(shells, "which yum &>/dev/null && yum install -y "+quoted)
shells = append(shells, "which apt-get &>/dev/null && apt-get install -y "+quoted)
}
for _, wf := range conf.WriteFiles {
shells = append(shells, wf.ShellScripts()...)
@@ -317,7 +435,13 @@ func ParseUserData(data string) (*SCloudConfig, error) {
if err != nil {
return nil, errors.Wrapf(err, "ParseYAML")
}
jsonDict := jsonConf.(*jsonutils.JSONDict)
jsonDict, ok := jsonConf.(*jsonutils.JSONDict)
if !ok {
// Anything that is valid YAML but not a mapping, e.g. a list or a
// scalar, is not a usable cloud-config document.
return nil, errors.Wrapf(errors.ErrInvalidFormat,
"cloud-config must be a YAML mapping, got %s", jsonConf.String())
}
if jsonDict.Contains("users") {
userArray := jsonutils.NewArray()
users, _ := jsonConf.GetArray("users")

View File

@@ -40,8 +40,11 @@ func unhex(c rune) byte {
return 0
}
// shouldEncode reports whether c must be escaped by EncodeGoogleLabel.
// Only runes in [0, 0xff] can be represented as "_" followed by two hex
// digits, so anything above that range is written through unchanged.
func shouldEncode(c rune) bool {
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c > 256 {
if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c > 0xff {
return false
}
return true
@@ -69,7 +72,7 @@ func DecodeGoogleLable(label string) string {
var t strings.Builder
for j := 0; j < len(s); {
c := s[j]
if c == rune('_') && j+2 <= len(s) && ishex(s[j+1]) && ishex(s[j+2]) {
if c == rune('_') && j+2 < len(s) && ishex(s[j+1]) && ishex(s[j+2]) {
t.WriteByte(unhex(s[j+1])<<4 | unhex(s[j+2]))
j += 3
} else {

View File

@@ -35,11 +35,20 @@ func (fs FileSystem) Open(path string) (http.File, error) {
}
s, err := f.Stat()
if s.IsDir() {
index := strings.TrimSuffix(path, "/") + "/index.html"
if _, err := fs.fs.Open(index); err != nil {
if err != nil {
f.Close()
return nil, err
}
if s.IsDir() {
index := strings.TrimSuffix(path, "/") + "/index.html"
indexFile, err := fs.fs.Open(index)
if err != nil {
// Without an index file a directory must not be listed.
f.Close()
return nil, err
}
// The index file is only checked for; it is not the file returned.
indexFile.Close()
}
return f, nil

View File

@@ -28,6 +28,7 @@ import (
"net/http/httputil"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"syscall"
@@ -93,6 +94,173 @@ type sClient interface {
Do(req *http.Request) (*http.Response, error)
}
// Field and parameter names that carry a credential on their own, and
// fragments that mark a name as carrying one. The same lists are used for the
// fields of a request body and the parameters of a request URL, so that the
// two cannot drift apart.
var sensitiveBodyKeyNames = []string{
"key", "code", "sig", "pwd", "auth", "ticket", "session", "sessionid",
// The plain name is a credential; "access_key_id" is an identifier and
// is deliberately not matched, so it stays readable in the message.
"access_key",
}
var sensitiveBodyKeyParts = []string{
"token", "secret", "password", "passwd", "signature", "credential",
"apikey", "api_key", "private_key",
}
func isSensitiveBodyKey(name string) bool {
lower := strings.ToLower(name)
if utils.IsInStringArray(lower, sensitiveBodyKeyNames) {
return true
}
for _, part := range sensitiveBodyKeyParts {
if strings.Contains(lower, part) {
return true
}
}
return false
}
// redactURL masks the values of query parameters that commonly carry a
// credential, so that an error message can name the request without
// reproducing the secret. A URL with nothing to mask is returned unchanged,
// byte for byte, so no existing message is reformatted.
func redactURL(urlStr string) string {
parsed, err := url.Parse(urlStr)
if err != nil || len(parsed.RawQuery) == 0 {
return urlStr
}
query := parsed.Query()
changed := false
for name := range query {
if isSensitiveBodyKey(name) {
query.Set(name, "*")
changed = true
}
}
if !changed {
return urlStr
}
parsed.RawQuery = query.Encode()
return parsed.String()
}
// redactJSONObject returns obj with the values of credential carrying fields
// replaced, at any depth.
func redactJSONObject(obj jsonutils.JSONObject) jsonutils.JSONObject {
switch v := obj.(type) {
case *jsonutils.JSONDict:
redacted := jsonutils.NewDict()
for name, value := range v.Value() {
if isSensitiveBodyKey(name) {
redacted.Set(name, jsonutils.NewString("*"))
} else {
redacted.Set(name, redactJSONObject(value))
}
}
return redacted
case *jsonutils.JSONArray:
redacted := jsonutils.NewArray()
for _, value := range v.Value() {
redacted.Add(redactJSONObject(value))
}
return redacted
default:
return obj
}
}
// redactFormBody masks the credential carrying fields of an urlencoded body.
//
// The body is walked pair by pair rather than through url.ParseQuery, which
// stops at the first pair it cannot decode: the pairs it did decode are still
// returned, but taking only the value it returns would either drop the rest of
// the body or, if the raw body were used instead, leave the credential in it.
func redactFormBody(body string) string {
parts := strings.Split(body, "&")
changed := false
for i, part := range parts {
eq := strings.IndexByte(part, '=')
if eq < 0 {
continue
}
name := part[:eq]
if decoded, err := url.QueryUnescape(name); err == nil {
name = decoded
}
if isSensitiveBodyKey(name) {
parts[i] = part[:eq+1] + "*"
changed = true
}
}
if !changed {
return body
}
return strings.Join(parts, "&")
}
// A credential carrying name as it appears in a body that cannot be walked
// field by field, e.g. XML, or a JSON document that did not parse.
const sensitiveBodyNamePattern = `(?i)[A-Za-z0-9_.:-]*(?:token|secret|password|passwd|signature|credential|apikey|api_key|private_key)[A-Za-z0-9_.:-]*`
var (
// <name>value</name>, where the value may be a CDATA section
xmlElementSensitiveRe = regexp.MustCompile(
`(?is)(<` + sensitiveBodyNamePattern + `\b[^>]*>)(?:\s*<!\[CDATA\[.*?\]\]>|[^<]*)(</)`)
// name="value" or name='value'
xmlAttrSensitiveRe = regexp.MustCompile(
`(` + sensitiveBodyNamePattern + `)(\s*=\s*)("[^"]*"|'[^']*')`)
// "name": "value", and the same without quotes
jsonFieldSensitiveRe = regexp.MustCompile(
`(?i)("` + sensitiveBodyNamePattern + `"\s*:\s*)("[^"]*"|[^,}\s]+)`)
)
// redactBodyText masks credential carrying elements, attributes and fields of
// a body that is not a structure this package can walk.
func redactBodyText(body string) string {
// The value alternative is not a capturing group, so the closing tag is
// group 2.
body = xmlElementSensitiveRe.ReplaceAllString(body, "${1}*${2}")
body = xmlAttrSensitiveRe.ReplaceAllString(body, `${1}${2}"*"`)
body = jsonFieldSensitiveRe.ReplaceAllString(body, "${1}*")
return body
}
// maxRedactedBodyBytes bounds how large a body put into an error message can
// be. A larger body is left out rather than cut down: a cut keeps whichever
// part of the body happened to fall inside it.
const maxRedactedBodyBytes = 4096
// redactRequestBody masks the credential carrying fields of a request body
// before it is put into an error message.
//
// A body whose content type is not one this package can walk field by field is
// left out entirely. There is no way to tell which part of it is a credential,
// and an excerpt would keep whichever part happened to fall inside the excerpt
// — a multipart body, for instance, names its fields in a header line, so
// masking the names would leave the values readable.
func redactRequestBody(body, contType string) jsonutils.JSONObject {
if len(body) == 0 || len(body) > maxRedactedBodyBytes {
return nil
}
switch {
case strings.Contains(contType, "json"):
if parsed, err := jsonutils.ParseString(body); err == nil {
return redactJSONObject(parsed)
}
// A JSON body that does not parse is still masked by field name,
// since the names survive in the text.
return jsonutils.NewString(redactBodyText(body))
case strings.Contains(contType, "x-www-form-urlencoded"):
return jsonutils.NewString(redactFormBody(body))
case strings.Contains(contType, "xml"):
return jsonutils.NewString(redactBodyText(body))
}
return nil
}
// body might have been consumed, so body is provided separately
func newJsonClientErrorFromRequest(req *http.Request, body string) *JSONClientError {
return newJsonClientErrorFromRequest2(req.Method, req.URL.String(), req.Header, body)
@@ -102,7 +270,7 @@ func newJsonClientErrorFromRequest2(method string, urlStr string, hdrs http.Head
jce := &JSONClientError{}
jce.Request.Method = strings.ToUpper(method)
jce.Request.Url = urlStr
jce.Request.Url = redactURL(urlStr)
jce.Request.Headers = make(map[string]string)
excludeHdrs := []string{
"Accept",
@@ -112,22 +280,13 @@ func newJsonClientErrorFromRequest2(method string, urlStr string, hdrs http.Head
http.CanonicalHeaderKey("authorization"),
http.CanonicalHeaderKey("x-auth-token"),
http.CanonicalHeaderKey("x-subject-token"),
http.CanonicalHeaderKey("cookie"),
http.CanonicalHeaderKey("set-cookie"),
}
const (
MAX_BODY = 128
FIRST_PART = 100
)
switch jce.Request.Method {
case "PUT", "POST", "PATCH":
contType := hdrs.Get(http.CanonicalHeaderKey("content-type"))
if len(body) > MAX_BODY {
jce.Request.Body = jsonutils.NewString(body[:FIRST_PART] + "..." + body[len(body)-MAX_BODY+FIRST_PART+3:])
} else if strings.Contains(contType, "json") {
jce.Request.Body, _ = jsonutils.ParseString(body)
} else if strings.Contains(contType, "xml") ||
strings.Contains(contType, "x-www-form-urlencoded") {
jce.Request.Body = jsonutils.NewString(body)
}
jce.Request.Body = redactRequestBody(body, contType)
default:
excludeHdrs = append(excludeHdrs, http.CanonicalHeaderKey("content-type"), http.CanonicalHeaderKey("content-length"))
}
@@ -233,13 +392,10 @@ func NewJsonClient(client sClient) *JsonClient {
return &JsonClient{client: client}
}
// Error renders the error as JSON. The request it carries is already masked
// when the error is built, so this has no side effects and repeating it gives
// the same message.
func (e *JSONClientError) Error() string {
if !gotypes.IsNil(e.Request.Body) {
if body, ok := e.Request.Body.(*jsonutils.JSONDict); ok && body.Contains("password") {
body.Set("password", jsonutils.NewString("***"))
e.Request.Body = body
}
}
errMsg := JSONClientErrorMsg{Error: e}
return jsonutils.Marshal(errMsg).String()
}
@@ -427,6 +583,42 @@ func GetClient(insecure bool, timeout time.Duration) *http.Client {
return getClient(insecure, false, timeout)
}
// Session headers that a caller of this package sets, and that must not
// travel to a host the request was not made to. net/http already drops
// Authorization, Cookie and the proxy headers in that case, but does not know
// about these.
var redirectSensitiveHeaders = []string{
"X-Auth-Token",
"X-Subject-Token",
}
// isSameOrSubdomain reports whether dest is initial, or a subdomain of it.
// A redirect to a parent domain is not covered, matching the rule net/http
// uses for the headers it strips.
func isSameOrSubdomain(initial, dest *url.URL) bool {
i := strings.ToLower(initial.Hostname())
d := strings.ToLower(dest.Hostname())
if i == d {
return true
}
return strings.HasSuffix(d, "."+i)
}
// checkRedirect drops the session headers above when a redirect leaves the
// host the request was made to, and keeps the usual limit on how many
// redirects are followed.
func checkRedirect(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return errors.Errorf("stopped after 10 redirects")
}
if len(via) > 0 && !isSameOrSubdomain(via[0].URL, req.URL) {
for _, h := range redirectSensitiveHeaders {
req.Header.Del(h)
}
}
return nil
}
func getClient(insecure bool, legacyTLS bool, timeout time.Duration) *http.Client {
adaptive := false
if timeout == 0 {
@@ -435,6 +627,7 @@ func getClient(insecure bool, legacyTLS bool, timeout time.Duration) *http.Clien
tr := getTransport(insecure, legacyTLS, adaptive, timeout)
return &http.Client{
Transport: tr,
CheckRedirect: checkRedirect,
// 一个完整http request的超时时间
// Timeout specifies a time limit for requests made by this
// Client. The timeout includes connection time, any

View File

@@ -15,26 +15,47 @@
package netutils
import (
"net"
"net/http"
"strings"
)
func GetHttpRequestIp(r *http.Request) string {
ipStr := r.Header.Get("X-Forwarded-For")
if len(ipStr) > 0 {
ipList := strings.Split(ipStr, ",")
if len(ipList) > 0 {
return ipList[0]
// parseIPLiteral returns s as a plain address if it is one, and "" otherwise.
func parseIPLiteral(s string) string {
s = strings.TrimSpace(s)
if len(s) == 0 {
return ""
}
if ip := net.ParseIP(s); ip != nil {
return ip.String()
}
ipStr = r.Header.Get("X-Real-Ip")
if len(ipStr) > 0 {
return ipStr
}
ipStr = r.RemoteAddr
colonPos := strings.Index(ipStr, ":")
if colonPos > 0 {
ipStr = ipStr[:colonPos]
}
return ipStr
return ""
}
// GetHttpRequestIp returns the address of the client that made the request.
//
// The forwarded headers are consulted first, since a proxy in front of the
// service rewrites them. When no proxy is in place they are whatever the
// client sent, so each candidate is parsed and only an actual address is
// returned; a value that is not an address is skipped rather than passed on
// as though it were one.
//
// The connection address is used as the last resort, split with
// net.SplitHostPort so that an IPv6 peer is returned without its brackets and
// without its port.
func GetHttpRequestIp(r *http.Request) string {
if forwarded := r.Header.Get("X-Forwarded-For"); len(forwarded) > 0 {
for _, part := range strings.Split(forwarded, ",") {
if ip := parseIPLiteral(part); len(ip) > 0 {
return ip
}
}
}
if ip := parseIPLiteral(r.Header.Get("X-Real-Ip")); len(ip) > 0 {
return ip
}
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
return host
}
return r.RemoteAddr
}

View File

@@ -154,6 +154,10 @@ func normalizeIpv6Addr(addrStr string) ([8]uint16, error) {
}
}
}
} else if len(parts) > expectPartCnt {
// More groups than the address can hold. A "::" run can only reduce
// the number of groups, so there is no valid reading of this input.
return addr, errors.Wrapf(errors.ErrInvalidFormat, "%s too many groups", addrStr)
} else {
for i := 0; i < len(parts); i++ {
partStr := strings.TrimSpace(parts[i])

View File

@@ -181,12 +181,21 @@ func (ar IPV4AddrRange) ContainsRange(ar2 IPV4AddrRange) bool {
return ar.start <= ar2.start && ar.end >= ar2.end
}
// Random returns an address from the range, excluding end. A range that
// covers a single address returns that address.
func (ar IPV4AddrRange) Random() IPV4Addr {
return IPV4Addr(uint32(ar.start) + uint32(rand.Intn(int(uint32(ar.end)-uint32(ar.start)))))
if ar.start >= ar.end {
return ar.start
}
// int64 so the span of a very large range cannot overflow the argument
// to the random source.
span := int64(uint32(ar.end) - uint32(ar.start))
return IPV4Addr(uint32(ar.start) + uint32(rand.Int63n(span)))
}
func (ar IPV4AddrRange) AddressCount() int {
return int(uint32(ar.end) - uint32(ar.start) + 1)
// 64 bit arithmetic so a full range does not wrap around to zero.
return int(uint64(uint32(ar.end)) - uint64(uint32(ar.start)) + 1)
}
func (ar IPV4AddrRange) String() string {
@@ -360,10 +369,18 @@ func (ar IPV4AddrRange) equals(ar2 IPV4AddrRange) bool {
return ar.start == ar2.start && ar.end == ar2.end
}
// Masklen2Mask returns the network mask for a prefix length.
//
// A length above 32 cannot be represented and is treated as 32, i.e. a single
// host, rather than shifting past the width of the mask and wrapping around to
// a match-all mask.
func Masklen2Mask(maskLen int8) IPV4Addr {
if maskLen < 0 {
panic("negative masklen")
}
if maskLen > 32 {
maskLen = 32
}
return IPV4Addr(^(uint32(1<<(32-uint8(maskLen))) - 1))
}
@@ -392,6 +409,25 @@ func Mask2Len(mask IPV4Addr) int8 {
return int8(bits.LeadingZeros32(^uint32(mask)))
}
// ParsePrefix parses an IPv4 prefix written as "address/masklen", or as a
// bare address, which is taken to be a /32. An empty string parses as
// 0.0.0.0/32 rather than being rejected.
//
// The prefix length may also be written as a dotted-decimal mask, e.g.
// "10.0.0.0/255.0.0.0". A dotted mask is converted by counting its leading
// one bits, so the result always has a contiguous mask:
// "1.2.3.4/255.0.255.0" yields 1.0.0.0/8 rather than being rejected.
//
// The numeric form is read with strconv.Atoi, so it also accepts a leading
// sign and leading zeros: "+8" and "024" parse as 8 and 24.
//
// These accepted forms are more permissive than net.ParseCIDR. Note that
// regutils.MatchCIDR and regutils.MatchIP4Addr reject the dotted-mask and
// leading-zero spellings that this function accepts, so a value checked with
// one of those and then normalised here can end up covering something other
// than what was checked. Prefer net.ParseCIDR where the stricter grammar is
// wanted; the lenient spellings are kept for compatibility with prefixes
// already stored by callers.
func ParsePrefix(prefix string) (IPV4Addr, int8, error) {
slash := strings.IndexByte(prefix, '/')
if slash > 0 {
@@ -438,7 +474,15 @@ func NewIPV4Prefix(prefix string) (IPV4Prefix, error) {
return pref, nil
}
// NewIPV4PrefixFromAddr builds a prefix from an address and a prefix length.
//
// A length above 32 is clamped to 32 so that the stored MaskLen always agrees
// with the address and range it describes. A negative length keeps its
// existing behaviour of panicking.
func NewIPV4PrefixFromAddr(addr IPV4Addr, masklen int8) IPV4Prefix {
if masklen > 32 {
masklen = 32
}
pref := IPV4Prefix{
Address: addr.NetAddr(masklen),
MaskLen: masklen,
@@ -465,14 +509,28 @@ const (
multicastPrefix = "224.0.0.0/4"
)
// Ranges that cannot be a public address but are not part of the private
// ranges used to classify guest addresses. They are consulted by
// IsExitAddress only, so that IsPrivate keeps its current meaning for the
// callers that use it for address allocation and DHCP.
var reservedPrefixes = []string{
"0.0.0.0/8", // "this network"
"192.0.2.0/24", // TEST-NET-1
"198.51.100.0/24", // TEST-NET-2
"203.0.113.0/24", // TEST-NET-3
"240.0.0.0/4", // reserved, includes the limited broadcast address
}
var privateIPRanges []IPV4AddrRange
var customizedPrivateIPRanges []IPV4AddrRange
var reservedIPRanges []IPV4AddrRange
var hostLocalIPRange IPV4AddrRange
var linkLocalIPRange IPV4AddrRange
var multicastIPRange IPV4AddrRange
func init() {
initPrivateIPRanges()
initReservedIPRanges()
prefix, _ := NewIPV4Prefix(hostlocalPrefix)
hostLocalIPRange = prefix.ToIPRange()
@@ -502,6 +560,22 @@ func initPrivateIPRanges() {
}
}
func initReservedIPRanges() {
reservedIPRanges = make([]IPV4AddrRange, 0, len(reservedPrefixes))
for _, prefix := range reservedPrefixes {
p, err := NewIPV4Prefix(prefix)
if err != nil {
continue
}
reservedIPRanges = append(reservedIPRanges, p.ToIPRange())
}
}
// GetReservedIPRanges returns the ranges that IsReserved consults.
func GetReservedIPRanges() []IPV4AddrRange {
return append([]IPV4AddrRange(nil), reservedIPRanges...)
}
func SetPrivatePrefixes(pref []string) {
customizedPrivateIPRanges = make([]IPV4AddrRange, 0)
for _, prefix := range pref {
@@ -538,8 +612,23 @@ func IsMulticast(addr IPV4Addr) bool {
return multicastIPRange.Contains(addr)
}
// IsReserved reports whether addr is in a range that is reserved and can
// never be a public address.
func IsReserved(addr IPV4Addr) bool {
for _, ipRange := range reservedIPRanges {
if ipRange.Contains(addr) {
return true
}
}
return false
}
// IsExitAddress reports whether addr can be a public address, i.e. one that a
// guest can reach directly. Reserved ranges are excluded along with the
// private, host local, link local and multicast ones.
func IsExitAddress(addr IPV4Addr) bool {
return !IsPrivate(addr) && !IsHostLocal(addr) && !IsLinkLocal(addr) && !IsMulticast(addr)
return !IsPrivate(addr) && !IsReserved(addr) &&
!IsHostLocal(addr) && !IsLinkLocal(addr) && !IsMulticast(addr)
}
func MacUnpackHex(mac string) string {

View File

@@ -24,7 +24,17 @@ const (
TAG_OLD_DEPRECATED_BY = "deprecated-by"
)
// expandAmbiguousPrefix prepends the prefix given by the
// TAG_AMBIGUOUS_PREFIX tag to the fields sharing a name with another field.
//
// Expanding a prefix can itself collide with a name that is already in use, so
// the expansion is repeated until the names settle. A field whose expanded
// name would take over a name owned by a field outside its own ambiguous
// group keeps its original name, so that the expansion never introduces a new
// ambiguity.
func expandAmbiguousPrefix(fields SStructFieldValueSet) SStructFieldValueSet {
prefixed := make(map[int]bool)
for {
keyIndexMap := make(map[string][]int)
for i := range fields {
if fields[i].Info.Ignore {
@@ -37,21 +47,66 @@ func expandAmbiguousPrefix(fields SStructFieldValueSet) SStructFieldValueSet {
}
keyIndexMap[key] = append(values, i)
}
changed := false
for _, indexes := range keyIndexMap {
if len(indexes) > 1 {
if len(indexes) < 2 {
continue
}
// ambiguous found
for _, idx := range indexes {
if amPrefix, ok := fields[idx].Info.Tags[TAG_AMBIGUOUS_PREFIX]; ok {
fields[idx].Info.Name = fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Name)
if prefixed[idx] {
continue
}
amPrefix, ok := fields[idx].Info.Tags[TAG_AMBIGUOUS_PREFIX]
if !ok {
continue
}
expanded := fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Name)
if takenByOther(fields, expanded, indexes) {
continue
}
fields[idx].Info.Name = expanded
if depBy, ok := fields[idx].Info.Tags[TAG_DEPRECATED_BY]; ok {
fields[idx].Info.Tags[TAG_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy)
}
if depBy, ok := fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY]; ok {
fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy)
}
for i := range fields[idx].Info.Aliases {
fields[idx].Info.Aliases[i] = fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Aliases[i])
}
prefixed[idx] = true
changed = true
}
}
if !changed {
break
}
}
return fields
}
// takenByOther reports whether name is already used by a field outside group
func takenByOther(fields SStructFieldValueSet, name string, group []int) bool {
for i := range fields {
if fields[i].Info.Ignore {
continue
}
if fields[i].Info.MarshalName() != name {
continue
}
if !containsIndex(group, i) {
return true
}
}
return false
}
func containsIndex(indexes []int, idx int) bool {
for _, i := range indexes {
if i == idx {
return true
}
}
return false
}

View File

@@ -12,4 +12,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package reflectutils walks struct values by reflection, mainly to give the
// json encoder and decoder in yunion.io/x/jsonutils the field names and the
// tags to honour.
//
// # Field naming
//
// Fields are collected in declaration order, with the fields of an embedded
// struct expanded in place. A name is taken from the "name" tag, then from
// the "json" tag, and finally from the kebab form of the field name, so it can
// be looked up by any of those.
//
// Two collected fields can carry the same name. An embedded struct may bring
// in a field that another embedded struct also contributes, and unlike
// encoding/json an outer field does not hide a field of an embedded struct.
// All the fields sharing a name are reported, and a value decoded into the
// struct is written to every one of them, so that code reaching the field
// through any of the embedded paths sees it. Shadowing is deliberately not
// implemented: the list inputs of the resource APIs rely on this, e.g.
// ServerListInput carries several fields named "domain" coming from different
// embedded structs.
//
// To set such fields apart, tag an embedded struct with
// TAG_AMBIGUOUS_PREFIX, whose value is prepended to the names of the fields
// that embedded struct contributes.
package reflectutils // import "yunion.io/x/pkg/util/reflectutils"

View File

@@ -29,11 +29,30 @@ import (
// This struct has unexported fields initialized by exported functions in this
// package. Do not construct a literal or modify the exported fields in an
// unmanaged way
//
// The tags of a field are meant to be read through Tag and TagMap rather than
// off the Tags map, which is not handed out to stay writable by whoever read
// it; a caller may modify what TagMap returns.
type SStructFieldInfo struct {
// True if the field has json tag `json:"-"`
Ignore bool
// True if an empty string, slice, map, struct or dict is left out of
// the json object. A number and a boolean are not governed by this,
// see OmitZero and OmitFalse.
//
// An empty value is left out by default, whether or not the json tag
// asks for it; `allowempty` turns that off and `omitempty` states it.
OmitEmpty bool
// True if a false boolean is left out of the json object. A false is
// written by default; `omitfalse` turns that off and `allowfalse`
// states it.
OmitFalse bool
// True if a zero number is left out of the json object. A zero is
// written by default; `omitzero` turns that off and `allowzero` states
// it.
OmitZero bool
// Name can take the following values, in descreasing preference
@@ -43,11 +62,23 @@ type SStructFieldInfo struct {
// 3. kebab form of FieldName concatenated with "_" when Ignore is false
// 4. empty string
Name string
// FieldName is the name of the go struct field
FieldName string
kebabFieldName string
// True if the field has the "string" json tag option, which writes the
// value as a json string
ForceString bool
// Tags holds the tags of the field keyed by tag name, a tag without a
// value being mapped to the empty string. Read it through Tag or
// TagMap rather than off here.
Tags map[string]string
// Aliases are the other names the field is looked up by, taken from
// the "alias" tag
Aliases []string
}
@@ -145,6 +176,23 @@ func (info *SStructFieldInfo) MarshalName() string {
return info.kebabFieldName
}
// Tag returns the value of the tag named name and whether the field has it.
// A tag without a value is reported as present with an empty value.
func (info *SStructFieldInfo) Tag(name string) (string, bool) {
val, ok := info.Tags[name]
return val, ok
}
// TagMap returns a copy of the tags of the field, which the caller owns and
// is free to modify.
func (info *SStructFieldInfo) TagMap() map[string]string {
tags := make(map[string]string, len(info.Tags))
for k, v := range info.Tags {
tags[k] = v
}
return tags
}
type SStructFieldValue struct {
Info *SStructFieldInfo
Value reflect.Value
@@ -152,9 +200,41 @@ type SStructFieldValue struct {
Parent *SEmbedStructFieldValue
}
// SEmbedStructFieldValue ties the fields enumerated for a nil embedded struct
// to the struct they belong to. Field is the embedded pointer on the struct,
// Value is the value the fields were enumerated out of, and Parent is the
// entry of the enclosing embedded struct, if any.
type SEmbedStructFieldValue struct {
Field reflect.Value
Value reflect.Value
Parent *SEmbedStructFieldValue
}
// adoptEmbeddedStruct assigns Value to Field for every nil embedded struct
// the field was enumerated through, which puts the enumerated fields back on
// the struct. The assignments are made from the outermost embedded struct
// inwards, so that each one lands on a struct that is already part of the
// real one. It reports whether the field is backed by the struct afterwards,
// which is what writing to it requires.
func (v *SStructFieldValue) adoptEmbeddedStruct() bool {
chain := make([]*SEmbedStructFieldValue, 0, 2)
for p := v.Parent; p != nil; p = p.Parent {
chain = append(chain, p)
}
for i := len(chain) - 1; i >= 0; i -= 1 {
p := chain[i]
if !p.Field.IsValid() || p.Field.Kind() != reflect.Ptr || !p.Field.IsNil() {
// nothing to adopt, the pointer is already there
continue
}
if !p.Field.CanSet() {
// the struct can not hold the embedded pointer
return false
}
p.Field.Set(p.Value)
}
return true
}
type SStructFieldValueSet []SStructFieldValue
@@ -225,6 +305,19 @@ func fetchStructFieldValueSet2(dataValue reflect.Value, allocatePtr bool, tags m
}
func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags map[string]string, includeIgnore bool, parent *SEmbedStructFieldValue) SStructFieldValueSet {
if !dataValue.IsValid() || dataValue.Kind() != reflect.Struct {
// A zero Value, a nil or a non struct value has no field to
// enumerate. Report it as such rather than through a panic.
return SStructFieldValueSet{}
}
if allocatePtr && !dataValue.CanAddr() {
// The value can not be modified in place, so a nil embedded
// pointer can not be allocated on it. The fields of such a
// pointer are enumerated out of a value allocated on the side
// instead, the same way as when the caller does not ask for
// allocation; see the note below for how to write to them.
allocatePtr = false
}
fields := SStructFieldValueSet{}
dataType := dataValue.Type()
fieldInfos := fetchCacheStructFieldInfos(dataType)
@@ -252,9 +345,17 @@ func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags m
if fv.Kind() == reflect.Ptr && allocatePtr {
fv.Set(reflect.New(fv.Type().Elem()))
} else if fv.Kind() == reflect.Ptr && !allocatePtr {
// The embedded pointer is nil, so the fields are
// enumerated out of a value allocated on the side.
// Value is a pointer to that value and the fields
// below are the fields of the value it points at,
// so assigning Value to Field puts them back on
// the struct. Callers writing to those fields
// must do so first.
efv = &SEmbedStructFieldValue{
Field: fv,
Value: reflect.New(fv.Type().Elem()),
Parent: parent,
}
fv = efv.Value
} else {

View File

@@ -19,6 +19,7 @@ import (
"reflect"
"yunion.io/x/log"
"yunion.io/x/pkg/gotypes"
)
/*
@@ -96,13 +97,19 @@ func fetchStructFieldNameValues(dataType reflect.Type, dataValue reflect.Value,
}
*/
// FindStructFieldValue returns the field of dataValue named name. The field
// has to be writable, so a field reached through a nil embedded pointer is
// only returned once that pointer has been put in place, which this does.
func FindStructFieldValue(dataValue reflect.Value, name string) (reflect.Value, bool) {
set := FetchStructFieldValueSet(dataValue)
val, find := set.GetValue(name)
if find && val.CanSet() {
return val, true
}
idx := set.GetStructFieldIndex(name)
if idx < 0 {
return reflect.Value{}, false
}
if !set[idx].adoptEmbeddedStruct() || !set[idx].Value.CanSet() {
return reflect.Value{}, false
}
return set[idx].Value, true
}
func FindStructFieldInterface(dataValue reflect.Value, name string) (interface{}, bool) {
@@ -111,35 +118,53 @@ func FindStructFieldInterface(dataValue reflect.Value, name string) (interface{}
}
func FillEmbededStructValue(container reflect.Value, embed reflect.Value) bool {
if !container.IsValid() || container.Kind() != reflect.Struct || !embed.IsValid() {
return false
}
containerType := container.Type()
embedType := embed.Type()
for i := 0; i < containerType.NumField(); i += 1 {
fieldType := containerType.Field(i)
if fieldType.Type.Kind() != reflect.Struct || !fieldType.Anonymous {
continue
}
fieldValue := container.Field(i)
if fieldType.Type.Kind() == reflect.Struct && fieldType.Anonymous {
if fieldType.Type == embed.Type() {
if !fieldValue.CanSet() {
// an unexported embedded struct can not be assigned to, and
// neither can anything inside it
continue
}
if fieldType.Type == embedType {
fieldValue.Set(embed)
return true
} else {
filled := FillEmbededStructValue(fieldValue, embed)
if filled {
}
if FillEmbededStructValue(fieldValue, embed) {
return true
}
}
}
}
return false
}
// SetStructFieldValue sets the field of structValue named fieldName to val.
// A field reached through a nil embedded pointer is written only once that
// pointer has been put in place, which this does.
func SetStructFieldValue(structValue reflect.Value, fieldName string, val reflect.Value) bool {
set := FetchStructFieldValueSet(structValue)
target, find := set.GetValue(fieldName)
if !find {
idx := set.GetStructFieldIndex(fieldName)
if idx < 0 {
return false
}
if !set[idx].adoptEmbeddedStruct() {
return false
}
target := set[idx].Value
if !target.CanSet() {
return false
}
if !val.IsValid() || !val.Type().AssignableTo(target.Type()) {
// report a failure instead of letting reflect.Value.Set panic
return false
}
target.Set(val)
return true
}
@@ -161,17 +186,36 @@ func ExpandInterface(val interface{}) []interface{} {
func getAnonymouStructPointer(structValue reflect.Value, targetType reflect.Type) interface{} {
structType := structValue.Type()
if structType == targetType {
if !structValue.CanInterface() {
// the value was reached through an unexported field
return nil
}
return structValue.Addr().Interface()
}
for i := 0; i < structValue.NumField(); i += 1 {
fieldType := structType.Field(i)
if fieldType.Anonymous && fieldType.Type.Kind() == reflect.Struct {
ptr := getAnonymouStructPointer(structValue.Field(i), targetType)
if !fieldType.Anonymous || !gotypes.IsFieldExportable(fieldType.Name) {
// an unexported embedded struct can not be pointed at
continue
}
fieldValue := structValue.Field(i)
fieldT := fieldType.Type
if fieldT.Kind() == reflect.Ptr {
// an embedded pointer that is nil has nothing to point at
if fieldValue.IsNil() {
continue
}
fieldValue = fieldValue.Elem()
fieldT = fieldT.Elem()
}
if fieldT.Kind() != reflect.Struct {
continue
}
ptr := getAnonymouStructPointer(fieldValue, targetType)
if ptr != nil {
return ptr
}
}
}
return nil
}
@@ -214,11 +258,18 @@ func StructContains(type1 reflect.Type, type2 reflect.Type) bool {
}
for i := 0; i < type1.NumField(); i += 1 {
field := type1.Field(i)
if field.Anonymous && field.Type.Kind() == reflect.Struct {
contains := StructContains(field.Type, type2)
if contains {
return true
if !field.Anonymous {
continue
}
fieldType := field.Type
if fieldType.Kind() == reflect.Ptr {
fieldType = fieldType.Elem()
}
if fieldType.Kind() != reflect.Struct {
continue
}
if StructContains(fieldType, type2) {
return true
}
}
return false

View File

@@ -76,6 +76,9 @@ func (key EncryptedKey) decryptKey(privateKey *rsa.PrivateKey) ([]byte, error) {
encAlg := key.EncryptionMethod.Algorithm
switch encAlg {
case "http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p":
if key.EncryptionMethod.DigestMethod == nil {
return nil, errors.Wrap(errors.ErrInvalidFormat, "missing DigestMethod")
}
var shaAlg hash.Hash
hashAlg := key.EncryptionMethod.DigestMethod.Algorithm
switch hashAlg {
@@ -99,6 +102,9 @@ func (data EncryptedData) decryptData(privateKey *rsa.PrivateKey) ([]byte, error
if err != nil {
return nil, errors.Wrap(err, "base64.StdEncoding.DecodeString")
}
if data.KeyInfo.EncryptedKey == nil {
return nil, errors.Wrap(errors.ErrInvalidFormat, "missing KeyInfo.EncryptedKey")
}
key, err := data.KeyInfo.EncryptedKey.decryptKey(privateKey)
if err != nil {
return nil, errors.Wrap(err, "KeyInfo.EncryptedKey.decryptKey")
@@ -112,7 +118,36 @@ func (data EncryptedData) decryptData(privateKey *rsa.PrivateKey) ([]byte, error
}
}
// stripPKCS7Padding removes the padding that XML Encryption appends to the
// last block of a CBC payload.
//
// Data that does not carry valid padding is returned unchanged rather than
// reported, so that this cannot be used to tell one payload from another.
func stripPKCS7Padding(data []byte, blockSize int) []byte {
if len(data) == 0 || len(data)%blockSize != 0 {
return data
}
n := int(data[len(data)-1])
if n == 0 || n > blockSize || n > len(data) {
return data
}
for _, b := range data[len(data)-n:] {
if int(b) != n {
return data
}
}
return data[:len(data)-n]
}
func decryptAesCbc(key []byte, secret []byte) ([]byte, error) {
// The payload is an IV followed by whole ciphertext blocks. Anything
// shorter, or not a whole number of blocks, cannot be decrypted.
if len(secret) < 2*aes.BlockSize || len(secret)%aes.BlockSize != 0 {
return nil, errors.Wrapf(errors.ErrInvalidFormat,
"ciphertext of %d bytes is not a whole number of %d byte blocks preceded by an IV",
len(secret), aes.BlockSize)
}
c, err := aes.NewCipher(key)
if err != nil {
return nil, errors.Wrap(err, "aes.NewCipher")
@@ -125,5 +160,5 @@ func decryptAesCbc(key []byte, secret []byte) ([]byte, error) {
decrypter.CryptBlocks(data, data)
return data, nil
return stripPKCS7Padding(data, aes.BlockSize), nil
}

View File

@@ -20,7 +20,9 @@ import (
"crypto/rsa"
"encoding/base64"
"fmt"
"html"
"io"
"sort"
"strings"
"github.com/ma314smith/signedxml"
@@ -92,14 +94,23 @@ func SAMLEncode(input []byte) (string, error) {
return base64.StdEncoding.EncodeToString(comp), nil
}
// SAMLForm renders an auto-submitting HTML form carrying the given
// parameters. Both the action and the values are escaped, and the keys are
// sorted so the output is stable for a given input.
func SAMLForm(action string, attrs map[string]string) string {
form := strings.Builder{}
// form.WriteString(`<!DOCTYPE html><html lang="en-US"><body>`)
form.WriteString(`<form id="saml_submit_form" method="POST" action="`)
form.WriteString(action)
form.WriteString(html.EscapeString(action))
form.WriteString(`">`)
for k, v := range attrs {
form.WriteString(fmt.Sprintf("<input type=\"hidden\" name=\"%s\" value=\"%s\" />", k, v))
keys := make([]string, 0, len(attrs))
for k := range attrs {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
form.WriteString(fmt.Sprintf("<input type=\"hidden\" name=\"%s\" value=\"%s\" />",
html.EscapeString(k), html.EscapeString(attrs[k])))
}
form.WriteString(`<input type="submit" value="Submit" />`)
form.WriteString("</form><script><!--\n")

View File

@@ -32,11 +32,21 @@ func CleanCertificate(cert string) string {
return cert
}
// DecodePrivateKey parses a PEM encoded RSA private key. Both PKCS#8 and
// PKCS#1 encodings are accepted. An error is returned for input that is not
// PEM encoded, or that holds a key which is not RSA.
func DecodePrivateKey(keyString []byte) (*rsa.PrivateKey, error) {
block, _ := pem.Decode(keyString)
if block == nil {
return nil, errors.Wrap(errors.ErrInvalidFormat, "not a valid PEM block")
}
privKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
if err == nil {
return privKey.(*rsa.PrivateKey), nil
rsaKey, ok := privKey.(*rsa.PrivateKey)
if !ok {
return nil, errors.Wrapf(errors.ErrInvalidFormat, "private key is %T, not RSA", privKey)
}
return rsaKey, nil
}
priv, err := x509.ParsePKCS1PrivateKey(block.Bytes)
if err == nil {

View File

@@ -15,9 +15,10 @@
package seclib
import (
crand "crypto/rand"
"bytes"
"fmt"
"math/rand"
"io"
"strings"
"github.com/tredoe/osutil/user/crypt/sha512_crypt"
@@ -25,6 +26,33 @@ import (
"yunion.io/x/pkg/utils"
)
// randomIndex returns a uniformly random index in [0, n), read from the
// system random source.
//
// A failure of that source cannot be recovered from here: falling back to a
// deterministic value would hand out a predictable password, so the caller is
// stopped rather than served a weaker one.
func randomIndex(n int) int {
if n <= 0 {
panic(fmt.Sprintf("seclib: invalid random range %d", n))
}
if n == 1 {
return 0
}
// Values at or above limit would make the modulo below uneven, so they
// are drawn again instead.
limit := 256 - (256 % n)
buf := make([]byte, 1)
for {
if _, err := io.ReadFull(crand.Reader, buf); err != nil {
panic(fmt.Sprintf("seclib: random source failed: %v", err))
}
if int(buf[0]) < limit {
return int(buf[0]) % n
}
}
}
const (
DIGITS = "23456789"
LETTERS = "abcdefghjkmnpqrstuvwxyz"
@@ -58,7 +86,7 @@ func RandomPassword(width int) string {
} else {
candidates = CHARS
}
index := rand.Intn(len(candidates))
index := randomIndex(len(candidates))
ch = candidates[index]
}
if strings.IndexByte(DIGITS, ch) >= 0 {
@@ -84,7 +112,7 @@ func GeneratePassword(passwd string) (string, error) {
}
var WEAK_PASSWORDS []string = []string{
"Huawei@", "huawei@", "Admin@", "admin@", "Root@", "root@", "ABC@", "abc@", "ABCD@", "abcd@", "Huawei123@", "huawei123@", "Admin123@", "admin123@", "Root123@", "root123@", "Huawei#", "huawei#", "Admin#", "admin#", "Root#", "root#", "ABC#", "abc#", "ABCD#", "abcd#", "Huawei123#", "huawei123#", "Admin123#", "admin123#", "Root123#", "root123#Huawei!", "huawei!", "Admin!", "admin!", "Root!", "root!", "ABC!", "abc!", "ABCD!", "abcd!", "Huawei123!", "huawei123!", "Admin123!", "admin123!", "Root123!", "root123!", "ABC123!", "abc123!", "Huawei@123", "huawei@123", "Admin@123", "admin@123", "Root@123", "root@123", "ABC@123", "abc@123", "123@Huawei", "123@Root", "123@abc", "Huawei123", "huawei123", "Admin123", "admin123", "Root123", "root123", "abc123", "Huawei_123", "huawei_123", "Admin_123", "admin_123", "Root_123", "root_123", "ABC_123", "abc_123", "123abc", "123abcd", "1234abc", "1234abcd", "abcd123", "abc1234", "abcd1234", "abcd@1234", "abcd1234!", "abcd_1234", "a123456", "123.com", "123@com", "123_com", "Huawei!@#", "huawei!@#", "Admin!@#", "admin!@#", "Root!@#", "root!@#", "Huawei!@", "huawei!@", "Admin!@", "admin!@", "Root!@", "root!@", "Huaweiroot", "HuaweiRoot", "huaweiroot", "huaweiRoot", "Huaweiadmin", "HuaweiAdmin", "huaweiadmin", "huaweiAdmin", "Adminroot", "AdminRoot", "adminRoot", "adminroot", "Rootadmin", "RootAdmin", "rootAdmin", "rootadmin", "Rootroot", "RootRoot", "rootroot", "Administrator", "Password", "Password123", "Password@123", "Password_123", "Password123!", "DDM@123", "ddM@123", "dDm@123",
"Huawei@", "huawei@", "Admin@", "admin@", "Root@", "root@", "ABC@", "abc@", "ABCD@", "abcd@", "Huawei123@", "huawei123@", "Admin123@", "admin123@", "Root123@", "root123@", "Huawei#", "huawei#", "Admin#", "admin#", "Root#", "root#", "ABC#", "abc#", "ABCD#", "abcd#", "Huawei123#", "huawei123#", "Admin123#", "admin123#", "Root123#", "root123#", "huawei!", "Admin!", "admin!", "Root!", "root!", "ABC!", "abc!", "ABCD!", "abcd!", "Huawei123!", "huawei123!", "Admin123!", "admin123!", "Root123!", "root123!", "ABC123!", "abc123!", "Huawei@123", "huawei@123", "Admin@123", "admin@123", "Root@123", "root@123", "ABC@123", "abc@123", "123@Huawei", "123@Root", "123@abc", "Huawei123", "huawei123", "Admin123", "admin123", "Root123", "root123", "abc123", "Huawei_123", "huawei_123", "Admin_123", "admin_123", "Root_123", "root_123", "ABC_123", "abc_123", "123abc", "123abcd", "1234abc", "1234abcd", "abcd123", "abc1234", "abcd1234", "abcd@1234", "abcd1234!", "abcd_1234", "a123456", "123.com", "123@com", "123_com", "Huawei!@#", "huawei!@#", "Admin!@#", "admin!@#", "Root!@#", "root!@#", "Huawei!@", "huawei!@", "Admin!@", "admin!@", "Root!@", "root!@", "Huaweiroot", "HuaweiRoot", "huaweiroot", "huaweiRoot", "Huaweiadmin", "HuaweiAdmin", "huaweiadmin", "huaweiAdmin", "Adminroot", "AdminRoot", "adminRoot", "adminroot", "Rootadmin", "RootAdmin", "rootAdmin", "rootadmin", "Rootroot", "RootRoot", "rootroot", "Administrator", "Password", "Password123", "Password@123", "Password_123", "Password123!", "DDM@123", "ddM@123", "dDm@123",
}
var CHARS2 = fmt.Sprintf("%s%s%s%s", DIGITS, LETTERS, UPPERS, PUNC)
@@ -122,8 +150,8 @@ func randomPassword2(width int) string {
} else {
candidates = CHARS2
}
index := rand.Intn(len(candidates))
ch = CHARS2[index]
index := randomIndex(len(candidates))
ch = candidates[index]
}
if strings.IndexByte(DIGITS, ch) >= 0 {
ps.Digits += 1

View File

@@ -319,12 +319,20 @@ func (srcs securityRuleCuts) cutOutPorts(protocol string, ps1 []uint16) security
src_.r.PortEnd = int(e)
r = append(r, src_)
}
// ps1 is sorted. Walk the ports that are left over between the
// cut ports, advancing past each one even when it leaves no room
// for a range of its own (p == 1), so that it is not handed back
// by the trailing range below.
s := uint16(1)
for _, p := range ps1 {
if p < s {
continue
}
if s <= p-1 {
add(s, p-1)
s = p + 1
}
// Wraps to 0 for p == 65535, which the check below rejects.
s = p + 1
}
if s != 0 && s <= 65535 {
add(s, 65535)

View File

@@ -402,6 +402,20 @@ func (rule *SecurityRule) GetPortsString() string {
return ""
}
// cidrMaskLen returns the prefix length and the size in bits of the address
// family that a CIDR string normalises to.
//
// The mask of the IPNet a rule holds cannot be used for this directly: a
// v4-mapped IPv6 CIDR stringifies as IPv4, so its raw mask is expressed in the
// 128 bit space and does not describe the string it is printed as.
func cidrMaskLen(cidr string) (int, int) {
_, n, err := net.ParseCIDR(cidr)
if err != nil {
return 0, 0
}
return n.Mask.Size()
}
func (rule *SecurityRule) String() (result string) {
s := []string{}
s = append(s, string(rule.Direction)+":"+string(rule.Action))
@@ -409,13 +423,13 @@ func (rule *SecurityRule) String() (result string) {
if rule.IPNet != nil {
cidr := rule.IPNet.String()
if regutils.MatchCIDR(cidr) {
if ones, _ := rule.IPNet.Mask.Size(); ones < 32 {
if ones, bits := cidrMaskLen(cidr); ones < bits {
s = append(s, cidr)
} else {
s = append(s, rule.IPNet.IP.String())
}
} else if regutils.MatchCIDR6(cidr) {
if ones, _ := rule.IPNet.Mask.Size(); ones < 128 {
if ones, bits := cidrMaskLen(cidr); ones < bits {
s = append(s, cidr)
} else {
s = append(s, rule.IPNet.IP.String())

View File

@@ -39,19 +39,19 @@ type sXZReadAheadReader struct {
func newXZReadAheadReader(stream io.Reader) (*sXZReadAheadReader, error) {
xzHdr := make([]byte, xz.HeaderLen)
n, err := stream.Read(xzHdr)
// io.ReadFull so that a stream handing its data over in more than one
// piece is not mistaken for a stream that has already finished.
n, err := io.ReadFull(stream, xzHdr)
hdrEof := false
if err != nil {
if errors.Cause(err) == io.EOF {
cause := errors.Cause(err)
if cause == io.EOF || cause == io.ErrUnexpectedEOF {
// delay the EOF
hdrEof = true
xzHdr = xzHdr[:n]
} else {
return nil, errors.Wrap(err, "Read XZ header")
}
} else if n != len(xzHdr) {
hdrEof = true
xzHdr = xzHdr[:n]
}
return &sXZReadAheadReader{
offset: 0,
@@ -90,17 +90,35 @@ func (s *sXZReadAheadReader) Read(buf []byte) (int, error) {
return n + bufOffset, err
}
func StreamPipe(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64)) (*SStreamProperty, error) {
// ErrSizeLimitExceeded is returned when the stream produces more bytes than
// the limit passed to StreamPipe or StreamPipe2 allows.
const ErrSizeLimitExceeded = errors.Error("stream size limit exceeded")
// StreamPipe streams upstream into writer. See StreamPipe2 for the optional
// size limit.
func StreamPipe(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64), maxSize ...int64) (*SStreamProperty, error) {
return StreamPipe2(upstream, writer, CalChecksum, func(savedTotal int64, savedOnce int64) {
if callback != nil {
callback(savedTotal)
}
})
}, maxSize...)
}
func StreamPipe2(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64, savedOnce int64)) (*SStreamProperty, error) {
// StreamPipe2 streams upstream into writer, decompressing when the input is an
// xz stream.
//
// An optional maxSize stops the transfer once that many bytes have been
// produced, so that an input which expands to far more than it looks like it
// should cannot fill the writer. Omit it, or pass a value of zero or less, for
// no limit.
func StreamPipe2(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64, savedOnce int64), maxSize ...int64) (*SStreamProperty, error) {
sp := SStreamProperty{}
var limit int64
if len(maxSize) > 0 {
limit = maxSize[0]
}
var md5sum hash.Hash
if CalChecksum {
md5sum = md5.New()
@@ -128,6 +146,10 @@ func StreamPipe2(upstream io.Reader, writer io.Writer, CalChecksum bool, callbac
n, err := reader.Read(buf)
if n > 0 {
sp.Size += int64(n)
if limit > 0 && sp.Size > limit {
// Stop before the excess reaches the writer.
return nil, errors.Wrapf(ErrSizeLimitExceeded, "produced more than %d bytes", limit)
}
if callback != nil {
callback(sp.Size, int64(n))
}

156
vendor/yunion.io/x/pkg/utils/crypt.go generated vendored
View File

@@ -172,6 +172,162 @@ func EncryptAESBase64Url(key, msg string) (string, error) {
return base64.URLEncoding.EncodeToString(result), nil
}
// Authenticated encryption with AES-256-GCM.
//
// Payloads produced by EncryptAESBase64GCM / EncryptAESBase64UrlGCM are self
// describing: a fixed header, then the nonce, then the sealed ciphertext.
// DescryptAESBase64Auto / DescryptAESBase64UrlAuto accept both these payloads
// and the legacy ones produced by EncryptAESBase64 / EncryptAESBase64Url, so
// stored values can be re-encrypted and migrated without a flag day.
//
// EncryptAESBase64 / EncryptAESBase64Url are deliberately left unchanged: the
// legacy form has no header, so a reader built from an older revision would
// treat a GCM payload as ciphertext and return garbage without reporting an
// error. Roll out readers that call the Auto functions before switching any
// writer to the GCM functions.
const aesGCMVersion = byte(0x02)
var aesGCMHeader = []byte{'Y', 'N', 'P', 'K', aesGCMVersion}
func hasAESGCMHeader(secret []byte) bool {
if len(secret) < len(aesGCMHeader) {
return false
}
for i := range aesGCMHeader {
if secret[i] != aesGCMHeader[i] {
return false
}
}
return true
}
func encryptAESGCM(k, msg []byte) ([]byte, error) {
block, err := aes.NewCipher(k)
if err != nil {
return nil, err
}
aead, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
nonce := make([]byte, aead.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, err
}
out := make([]byte, 0, len(aesGCMHeader)+len(nonce)+len(msg)+aead.Overhead())
out = append(out, aesGCMHeader...)
out = append(out, nonce...)
return aead.Seal(out, nonce, msg, nil), nil
}
func descryptAESGCM(k, secret []byte) ([]byte, error) {
if !hasAESGCMHeader(secret) {
return nil, fmt.Errorf("payload is not in the authenticated format")
}
body := secret[len(aesGCMHeader):]
block, err := aes.NewCipher(k)
if err != nil {
return nil, err
}
aead, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
if len(body) < aead.NonceSize()+aead.Overhead() {
return nil, fmt.Errorf("payload is truncated")
}
nonce, cipherText := body[:aead.NonceSize()], body[aead.NonceSize():]
return aead.Open(nil, nonce, cipherText, nil)
}
// EncryptAESBase64GCM encrypts msg and returns the standard-base64 encoding of
// the authenticated payload.
func EncryptAESBase64GCM(key, msg string) (string, error) {
result, err := encryptAESGCM(toAESKey(key), []byte(msg))
if err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(result), nil
}
// EncryptAESBase64UrlGCM is EncryptAESBase64GCM using the URL-safe alphabet.
func EncryptAESBase64UrlGCM(key, msg string) (string, error) {
result, err := encryptAESGCM(toAESKey(key), []byte(msg))
if err != nil {
return "", err
}
return base64.URLEncoding.EncodeToString(result), nil
}
// DescryptAESBase64GCM decrypts a payload produced by EncryptAESBase64GCM.
// A payload that was modified after it was written is reported as an error.
func DescryptAESBase64GCM(key, secret string) (string, error) {
s, err := base64.StdEncoding.DecodeString(secret)
if err != nil {
return "", err
}
result, err := descryptAESGCM(toAESKey(key), s)
if err != nil {
return "", err
}
return string(result), nil
}
// DescryptAESBase64UrlGCM decrypts a payload produced by
// EncryptAESBase64UrlGCM.
func DescryptAESBase64UrlGCM(key, secret string) (string, error) {
s, err := base64.URLEncoding.DecodeString(secret)
if err != nil {
return "", err
}
result, err := descryptAESGCM(toAESKey(key), s)
if err != nil {
return "", err
}
return string(result), nil
}
// DescryptAESBase64Auto decrypts an authenticated payload
// (EncryptAESBase64GCM) as well as a legacy payload (EncryptAESBase64).
func DescryptAESBase64Auto(key, secret string) (string, error) {
s, err := base64.StdEncoding.DecodeString(secret)
if err != nil {
return "", err
}
k := toAESKey(key)
var result []byte
if hasAESGCMHeader(s) {
result, err = descryptAESGCM(k, s)
} else {
result, err = descryptAES(k, s)
}
if err != nil {
return "", err
}
return string(result), nil
}
// DescryptAESBase64UrlAuto decrypts an authenticated payload
// (EncryptAESBase64UrlGCM) as well as a legacy payload
// (EncryptAESBase64Url).
func DescryptAESBase64UrlAuto(key, secret string) (string, error) {
s, err := base64.URLEncoding.DecodeString(secret)
if err != nil {
return "", err
}
k := toAESKey(key)
var result []byte
if hasAESGCMHeader(s) {
result, err = descryptAESGCM(k, s)
} else {
result, err = descryptAES(k, s)
}
if err != nil {
return "", err
}
return string(result), nil
}
// RSA 加密
func rsaEncrypt(publicKey, origData []byte) ([]byte, error) {
block, _ := pem.Decode(publicKey)

27
vendor/yunion.io/x/pkg/utils/misc.go generated vendored
View File

@@ -471,6 +471,9 @@ func GetSize(sizeStr, defaultSize string, base int64) (size int64, err error) {
if IsMatchInteger(sizeStr) {
sizeStr += defaultSize
}
if len(sizeStr) == 0 {
return 0, fmt.Errorf("empty size string")
}
sizeNumStr := sizeStr[0 : len(sizeStr)-1]
size, err = strconv.ParseInt(sizeNumStr, 10, 64)
@@ -481,16 +484,16 @@ func GetSize(sizeStr, defaultSize string, base int64) (size int64, err error) {
switch u := sizeStr[len(sizeStr)-1]; u {
case 't', 'T':
size = size * base * base * base * base
size, err = scaleSize(size, base, 4)
case 'g', 'G':
size = size * base * base * base
size, err = scaleSize(size, base, 3)
case 'm', 'M':
size = size * base * base
size, err = scaleSize(size, base, 2)
case 'k', 'K':
size = size * base
size, err = scaleSize(size, base, 1)
case 'b', 'B':
size = size
@@ -502,6 +505,22 @@ func GetSize(sizeStr, defaultSize string, base int64) (size int64, err error) {
return
}
// scaleSize multiplies size by base the given number of times, reporting a
// value that does not fit in an int64 instead of silently wrapping around.
func scaleSize(size, base int64, times int) (int64, error) {
const maxInt64 = int64(^uint64(0) >> 1)
if base <= 0 {
return 0, fmt.Errorf("invalid base %d", base)
}
for i := 0; i < times; i++ {
if size > maxInt64/base {
return 0, fmt.Errorf("size value is too large")
}
size *= base
}
return size, nil
}
func GetSizeBytes(sizeStr, defaultSize string) (int64, error) {
return GetSize(sizeStr, defaultSize, 1024)
}

17
vendor/yunion.io/x/pkg/utils/net.go generated vendored
View File

@@ -17,7 +17,6 @@ package utils
import (
"net"
"strconv"
"strings"
)
func GetOutboundIP() net.IP {
@@ -30,8 +29,18 @@ func GetOutboundIP() net.IP {
return localAddr.IP
}
// GetAddrPort splits a "host:port" address into its host and port parts.
// IPv6 addresses must be bracketed, e.g. "[::1]:8080", which also yields the
// address without the brackets. If addrPort cannot be split, or the port is
// not a number, the host is returned with a port of 0.
func GetAddrPort(addrPort string) (string, int) {
parts := strings.Split(addrPort, ":")
port, _ := strconv.Atoi(parts[1])
return parts[0], port
host, portStr, err := net.SplitHostPort(addrPort)
if err != nil {
return addrPort, 0
}
port, err := strconv.Atoi(portStr)
if err != nil {
return host, 0
}
return host, port
}

View File

@@ -25,7 +25,9 @@ func SavePid(pidfile string) error {
return err
}
file, err := os.OpenFile(pidfile, os.O_RDWR|os.O_CREATE, 0644)
// O_TRUNC so that a shorter pid cannot leave the tail of a longer
// previously written pid behind in the file.
file, err := os.OpenFile(pidfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return fmt.Errorf("error opening pidfile %s: %s", pidfile, err)
}

34
vendor/yunion.io/x/pkg/utils/tags.go generated vendored
View File

@@ -101,10 +101,27 @@ func _findWord(str []byte, offset int, sepChars string, isQuoteChar func(ch byte
return buffer.String(), i
}
// FindWords splits str into words separated by a space, a colon, a comma, a
// tab, a newline or a closing bracket. A quoted fragment counts as one word.
//
// A fragment that is neither a word nor one of the separators ends the scan,
// so the caller gets the words found up to that point rather than nothing at
// all. Use FindWords2 to be told about such a fragment instead.
func FindWords(str []byte, offset int) []string {
words, err := FindWords2(str, offset, " :,\t\n}]", isQuoteCharInternal)
if err != nil {
panic(err.Error())
const sepChars = " :,\t\n}]"
words := make([]string, 0)
for offset < len(str) {
word, i := _findWord(str, offset, sepChars, isQuoteCharInternal)
if i <= offset {
// Nothing was consumed; stop rather than loop forever.
break
}
words = append(words, word)
i = skipEmpty(str, i)
if i < len(str) && strings.IndexByte(sepChars, str[i]) >= 0 {
i++
}
offset = i
}
return words
}
@@ -150,13 +167,14 @@ func TagMap(tag reflect.StructTag) map[string]string {
break
}
i = skipEmpty(str, i)
if i >= len(str) || strings.IndexByte(EMPTYSTR, str[i]) >= 0 {
val = ""
} else if str[i] != ':' {
panic(fmt.Sprintf("Invalid structTag: %s", tag))
} else {
if i < len(str) && str[i] == ':' {
i++
val, i = findWord(str, i)
} else {
// A fragment that is not followed by a value takes no value.
// It is kept as a key so that the scan carries on with the
// fragments that follow it, rather than dropping them.
val = ""
}
ret[k] = val
i = skipEmpty(str, i)

View File

@@ -23,7 +23,7 @@ import (
func (table *STableSpec) structField2ColumnSpec(field *reflectutils.SStructFieldValue) IColumnSpec {
fieldname := field.Info.MarshalName()
tagmap := field.Info.Tags
tagmap := field.Info.TagMap()
if _, ok := tagmap[TAG_IGNORE]; ok {
return nil
}

View File

@@ -289,7 +289,7 @@ func (this *ArgumentParser) addStructArgument(prefix string, tpVal reflect.Value
sets := reflectutils.FetchAllStructFieldValueSetForWrite(tpVal)
for i := range sets {
if sets[i].Value.Kind() == reflect.Struct && sets[i].Value.Type() != gotypes.TimeType {
tagMap := sets[i].Info.Tags
tagMap := sets[i].Info.TagMap()
if _, ok := tagMap[reflectutils.TAG_DEPRECATED_BY]; ok {
// deprecated field, ignore
return nil
@@ -314,7 +314,7 @@ func (this *ArgumentParser) addStructArgument(prefix string, tpVal reflect.Value
}
func (this *ArgumentParser) addArgument(prefix string, fv reflect.Value, info *reflectutils.SStructFieldInfo) error {
tagMap := info.Tags
tagMap := info.TagMap()
if _, ok := tagMap[reflectutils.TAG_DEPRECATED_BY]; ok {
// deprecated field, ignore
return nil