feature: s3gateway

Makefile

@@ -55,6 +55,12 @@ export GO111MODULE:=on
 export CGO_CFLAGS = ${X_CGO_CFLAGS}
 export CGO_LDFLAGS = ${X_CGO_LDFLAGS}
 
+UNAME := $(shell uname)
+
+ifeq ($(UNAME), Linux)
+XARGS_FLAGS = --no-run-if-empty
+endif
+
 all: build
 
 
@@ -116,7 +122,7 @@ clean:
 
 fmt:
 	@$(if $(ONECLOUD_CI_BUILD),:,find) . -type f -name "*.go" -not -path "./_output/*" \
-		-not -path "./vendor/*" | xargs --no-run-if-empty gofmt -s -w
+		-not -path "./vendor/*" | xargs $(XARGS_FLAGS) gofmt -s -w
 
 define depDeprecated
 OneCloud now requires using go-mod for dependency management.  dep target,

cmd/aliyuncli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/aliyun"
@@ -71,7 +70,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/awscli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/aws"
@@ -71,7 +70,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/azurecli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/azure"
@@ -75,7 +74,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/climc/climc.go

@@ -114,7 +114,8 @@ func getSubcommandsParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	fmt.Fprintf(os.Stderr, "Error: %s\n", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/climc/shell/buckets.go

@@ -0,0 +1,87 @@
+package shell
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	type BucketListOptions struct {
+		options.BaseListOptions
+	}
+	R(&BucketListOptions{}, "bucket-list", "List all buckets", func(s *mcclient.ClientSession, args *BucketListOptions) error {
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.Buckets.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.Buckets.GetColumns(s))
+		return nil
+	})
+
+	type BucketShowOptions struct {
+		ID string `help:"ID or name of bucket"`
+	}
+	R(&BucketShowOptions{}, "bucket-show", "Show details of bucket", func(s *mcclient.ClientSession, args *BucketShowOptions) error {
+		result, err := modules.Buckets.Get(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type BucketUpdateOptions struct {
+		ID   string `help:"ID or name of bucket" json:"-"`
+		Name string `help:"new name of bucket" json:"name"`
+		Desc string `help:"Description of bucket" json:"description" token:"desc"`
+	}
+	R(&BucketUpdateOptions{}, "bucket-update", "update bucket", func(s *mcclient.ClientSession, args *BucketUpdateOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Buckets.Update(s, args.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type BucketDeleteOptions struct {
+		ID string `help:"ID or name of bucket" json:"-"`
+	}
+	R(&BucketDeleteOptions{}, "bucket-delete", "delete bucket", func(s *mcclient.ClientSession, args *BucketDeleteOptions) error {
+		result, err := modules.Buckets.Delete(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type BucketCreateOptions struct {
+		NAME        string `help:"name of bucket" json:"name"`
+		CLOUDREGION string `help:"location of bucket" json:"cloudregion"`
+		MANAGER     string `help:"cloud provider" json:"manager"`
+
+		StorageClass string `help:"bucket storage class"`
+		Acl          string `help:"bucket ACL"`
+	}
+	R(&BucketCreateOptions{}, "bucket-create", "Create a bucket", func(s *mcclient.ClientSession, args *BucketCreateOptions) error {
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.Buckets.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+}

cmd/cryptool/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -68,7 +67,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/esxicli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/esxi"
@@ -71,7 +70,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/huaweicli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/huawei"
@@ -73,7 +72,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/ipmicli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/baremetal/utils/ipmitool"
@@ -37,8 +36,9 @@ type BaseOptions struct {
 	SUBCOMMAND string `help:"ipmicli subcommand" subcommand:"true"`
 }
 
-func showErrorAndExit(err error) {
-	log.Errorf("%s", err)
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/ldapcli/main.go

@@ -73,7 +73,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/openstackcli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/openstack"
@@ -76,7 +75,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/qcloudcli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/qcloud"
@@ -72,7 +71,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/s3cli/main.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"yunion.io/x/structarg"
+
+	"yunion.io/x/onecloud/pkg/multicloud/objectstore"
+	_ "yunion.io/x/onecloud/pkg/multicloud/objectstore/shell"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+type BaseOptions struct {
+	Debug      bool   `help:"debug mode"`
+	Help       bool   `help:"Show help"`
+	AccessUrl  string `help:"Access url" default:"$S3_ACCESS_URL"`
+	AccessKey  string `help:"Access key" default:"$S3_ACCESS_KEY"`
+	Secret     string `help:"Secret" default:"$S3_SECRET"`
+	SUBCOMMAND string `help:"s3cli subcommand" subcommand:"true"`
+}
+
+func getSubcommandParser() (*structarg.ArgumentParser, error) {
+	parse, e := structarg.NewArgumentParser(&BaseOptions{},
+		"s3cli",
+		"Command-line interface to standard S3 API.",
+		`See "s3cli help COMMAND" for help on a specific command.`)
+
+	if e != nil {
+		return nil, e
+	}
+
+	subcmd := parse.GetSubcommand()
+	if subcmd == nil {
+		return nil, fmt.Errorf("No subcommand argument.")
+	}
+	type HelpOptions struct {
+		SUBCOMMAND string `help:"sub-command name"`
+	}
+	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
+		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
+		if e != nil {
+			return e
+		} else {
+			fmt.Print(helpstr)
+			return nil
+		}
+	})
+	for _, v := range shellutils.CommandTable {
+		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
+		if e != nil {
+			return nil, e
+		}
+	}
+	return parse, nil
+}
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClient(options *BaseOptions) (*objectstore.SObjectStoreClient, error) {
+	if len(options.AccessUrl) == 0 {
+		return nil, fmt.Errorf("Missing accessUrl")
+	}
+
+	if len(options.AccessKey) == 0 {
+		return nil, fmt.Errorf("Missing accessKey")
+	}
+
+	if len(options.Secret) == 0 {
+		return nil, fmt.Errorf("Missing secret")
+	}
+
+	return objectstore.NewObjectStoreClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug)
+}
+
+func main() {
+	parser, e := getSubcommandParser()
+	if e != nil {
+		showErrorAndExit(e)
+	}
+	e = parser.ParseArgs(os.Args[1:], false)
+	options := parser.Options().(*BaseOptions)
+
+	if options.Help {
+		fmt.Print(parser.HelpString())
+	} else {
+		subcmd := parser.GetSubcommand()
+		subparser := subcmd.GetSubParser()
+		if e != nil {
+			if subparser != nil {
+				fmt.Print(subparser.Usage())
+			} else {
+				fmt.Print(parser.Usage())
+			}
+			showErrorAndExit(e)
+		} else {
+			suboptions := subparser.Options()
+			if options.SUBCOMMAND == "help" {
+				e = subcmd.Invoke(suboptions)
+			} else {
+				var client *objectstore.SObjectStoreClient
+				client, e = newClient(options)
+				if e != nil {
+					showErrorAndExit(e)
+				}
+				e = subcmd.Invoke(client, suboptions)
+			}
+			if e != nil {
+				showErrorAndExit(e)
+			}
+		}
+	}
+}

cmd/s3gateway/main.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"yunion.io/x/onecloud/pkg/s3gateway/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
+
+func main() {
+	defer atexit.Handle()
+
+	service.StartService()
+}

cmd/ucloudcli/main.go

@@ -19,7 +19,6 @@ import (
 	"os"
 	"yunion.io/x/onecloud/pkg/util/ucloud"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -73,7 +72,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/zstackcli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -72,7 +71,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

go.mod

@@ -52,6 +52,7 @@ require (
 	github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 // indirect
 	github.com/gin-gonic/gin v1.3.0
 	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect
+	github.com/go-ini/ini v1.44.0 // indirect
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.4.1
@@ -95,6 +96,8 @@ require (
 	github.com/mdlayher/raw v0.0.0-20190606144222-a54781e5f38f
 	github.com/mholt/caddy v0.10.11
 	github.com/miekg/dns v1.1.1
+	github.com/minio/minio-go v6.0.14+incompatible
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
@@ -141,9 +144,7 @@ require (
 	google.golang.org/genproto v0.0.0-20181218023534-67d6565462c5 // indirect
 	google.golang.org/grpc v1.19.0
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
-	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/validator.v8 v8.18.2 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ldap.v3 v3.0.3
 	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0-20181004124137-fd83cbc87e76

go.sum

@@ -143,6 +143,8 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru
 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5iLFSdbEVA/idR6Q2WhCm+deb7ir2CcM=
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/go-ini/ini v1.44.0 h1:8+SRbfpRFlIunpSum4BEf1ClTtVjOgKzgBv9pHFkI6w=
+github.com/go-ini/ini v1.44.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-ole/go-ole v1.2.2 h1:QNWhweRd9D5Py2rRVboZ2L4SEoW/dyraWJCc8bgS8kE=
@@ -274,6 +276,10 @@ github.com/mholt/caddy v0.10.11 h1:s8X+R8DuBbrrMuUTcWSxlDe567B0s5EDmiDBKSYsioY=
 github.com/mholt/caddy v0.10.11/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
 github.com/miekg/dns v1.1.1 h1:DVkblRdiScEnEr0LR9nTnEQqHYycjkXW9bOjd+2EL2o=
 github.com/miekg/dns v1.1.1/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
+github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
@@ -447,12 +453,8 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
-gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
 gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ=
 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
-gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
-gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ldap.v3 v3.0.3 h1:YKRHW/2sIl05JsCtx/5ZuUueFuJyoj/6+DGXe3wp6ro=
 gopkg.in/ldap.v3 v3.0.3/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=

pkg/apis/compute/bucket.go

@@ -0,0 +1,12 @@
+package compute
+
+const (
+	BUCKET_STATUS_START_CREATE = "start_create"
+	BUCKET_STATUS_CREATING     = "creating"
+	BUCKET_STATUS_READY        = "ready"
+	BUCKET_STATUS_CREATE_FAIL  = "create_fail"
+	BUCKET_STATUS_START_DELETE = "start_delete"
+	BUCKET_STATUS_DELETING     = "deleting"
+	BUCKET_STATUS_DELETED      = "deleted"
+	BUCKET_STATUS_DELETE_FAIL  = "delete_fail"
+)

pkg/apis/compute/cloudaccount_const.go

@@ -40,6 +40,8 @@ const (
 	CLOUD_PROVIDER_UCLOUD    = "Ucloud"
 	CLOUD_PROVIDER_ZSTACK    = "ZStack"
 
+	CLOUD_PROVIDER_GENERICS3 = "S3"
+
 	CLOUD_PROVIDER_HEALTH_NORMAL        = "normal"        // 远端处于健康状态
 	CLOUD_PROVIDER_HEALTH_INSUFFICIENT  = "insufficient"  // 不足按需资源余额
 	CLOUD_PROVIDER_HEALTH_SUSPENDED     = "suspended"     // 远端处于冻结状态

pkg/apis/image/consts.go

@@ -17,6 +17,8 @@ package image
 type TImageType string
 
 const (
+	SERVICE_TYPE = "image"
+
 	// https://docs.openstack.org/glance/pike/user/statuses.html
 	//
 	IMAGE_STATUS_QUEUED     = "queued"

pkg/apis/s3gateway/consts.go

@@ -0,0 +1,5 @@
+package s3gateway
+
+const (
+	SERVICE_TYPE = "s3gateway"
+)

pkg/apis/s3gateway/doc.go

@@ -0,0 +1 @@
+package s3gateway // import "yunion.io/x/onecloud/pkg/apis/s3gateway"

pkg/cloudcommon/validators/errors.go

@@ -128,6 +128,8 @@ func newError(typ ErrType, errFmt string, params ...interface{}) error {
 			return nil
 		case ERR_GENERAL, ERR_MODEL_MANAGER:
 			return httperrors.NewInternalServerError(errFmt, params...)
+		case ERR_MODEL_NOT_FOUND:
+			return httperrors.NewResourceNotFoundError(errFmt, params...)
 		default:
 			return httperrors.NewInputParameterError(errFmt, params...)
 		}

pkg/cloudcommon/validators/validators.go

@@ -21,6 +21,7 @@ package validators
 // uri
 
 import (
+	"database/sql"
 	"math"
 	"net"
 	"reflect"
@@ -35,6 +36,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/choices"
 )
@@ -472,7 +474,11 @@ func (v *ValidatorModelIdOrName) validate(data *jsonutils.JSONDict) error {
 	v.ModelManager = modelManager
 	model, err := modelManager.FetchByIdOrName(v, modelIdOrName)
 	if err != nil {
-		return newModelNotFoundError(v.ModelKeyword, modelIdOrName, err)
+		if err == sql.ErrNoRows {
+			return newModelNotFoundError(v.ModelKeyword, modelIdOrName, err)
+		} else {
+			return httperrors.NewGeneralError(err)
+		}
 	}
 	if v.noPendingDeleted {
 		if pd, ok := model.(db.IPendingDeletable); ok && pd.GetPendingDeleted() {

pkg/cloudprovider/consts.go

@@ -15,7 +15,7 @@
 package cloudprovider
 
 import (
-	"errors"
+	"yunion.io/x/pkg/errors"
 )
 
 const (
@@ -25,13 +25,13 @@ const (
 	CloudVMStatusChangeFlavor = "change_flavor"
 	CloudVMStatusDeploying    = "deploying"
 	CloudVMStatusOther        = "other"
-)
 
-var ErrNotFound = errors.New("id not found")
-var ErrDuplicateId = errors.New("duplicate id")
-var ErrInvalidStatus = errors.New("invalid status")
-var ErrTimeout = errors.New("timeout")
-var ErrNotImplemented = errors.New("Not implemented")
-var ErrNotSupported = errors.New("Not supported")
-var ErrInvalidProvider = errors.New("Invalid provider")
-var ErrNoBalancePermission = errors.New("No balance permission")
+	ErrNotFound            = errors.Error("id not found")
+	ErrDuplicateId         = errors.Error("duplicate id")
+	ErrInvalidStatus       = errors.Error("invalid status")
+	ErrTimeout             = errors.Error("timeout")
+	ErrNotImplemented      = errors.Error("Not implemented")
+	ErrNotSupported        = errors.Error("Not supported")
+	ErrInvalidProvider     = errors.Error("Invalid provider")
+	ErrNoBalancePermission = errors.Error("No balance permission")
+)

pkg/cloudprovider/objectstore.go

@@ -0,0 +1,38 @@
+package cloudprovider
+
+import (
+	"time"
+
+	"yunion.io/x/pkg/errors"
+)
+
+type SBucketAccessUrl struct {
+	Url         string
+	Description string
+}
+
+type ICloudBucket interface {
+	IVirtualResource
+
+	GetGlobalId() string
+	GetName() string
+	GetAcl() string
+	GetLocation() string
+	GetIRegion() ICloudRegion
+	GetCreateAt() time.Time
+	GetStorageClass() string
+	GetAccessUrls() []SBucketAccessUrl
+}
+
+func GetIBucketByName(region ICloudRegion, name string) (ICloudBucket, error) {
+	buckets, err := region.GetIBuckets()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetIBuckets")
+	}
+	for i := range buckets {
+		if buckets[i].GetName() == name {
+			return buckets[i], nil
+		}
+	}
+	return nil, ErrNotFound
+}

pkg/cloudprovider/resources.go

@@ -110,6 +110,12 @@ type ICloudRegion interface {
 
 	GetINetworkInterfaces() ([]ICloudNetworkInterface, error)
 
+	GetIBuckets() ([]ICloudBucket, error)
+	CreateIBucket(name string, storageClassStr string, acl string) error
+	DeleteIBucket(name string) error
+	IBucketExist(name string) (bool, error)
+	GetIBucketByName(name string) (ICloudBucket, error)
+
 	GetProvider() string
 }
 

pkg/compute/models/buckets.go

@@ -0,0 +1,403 @@
+package models
+
+import (
+	"context"
+	"database/sql"
+	"regexp"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+type SBucketManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var BucketManager *SBucketManager
+
+func init() {
+	BucketManager = &SBucketManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SBucket{},
+			"buckets_tbl",
+			"bucket",
+			"buckets",
+		),
+	}
+	BucketManager.SetVirtualObject(BucketManager)
+}
+
+type SBucket struct {
+	db.SVirtualResourceBase
+	db.SExternalizedResourceBase
+
+	SManagedResourceBase
+
+	CloudregionId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"admin_required"`
+
+	StorageClass string `width:"36" charset:"ascii" nullable:"false" list:"user"`
+	Location     string `width:"36" charset:"ascii" nullable:"false" list:"user"`
+	Acl          string `width:"36" charset:"ascii" nullable:"false" list:"user"`
+}
+
+func (manager *SBucketManager) fetchBuckets(provider *SCloudprovider, region *SCloudregion) ([]SBucket, error) {
+	q := manager.Query()
+	if provider != nil {
+		q = q.Equals("manager_id", provider.GetId())
+	}
+	if region != nil {
+		q = q.Equals("cloudregion_id", region.GetId())
+	}
+	buckets := make([]SBucket, 0)
+	err := db.FetchModelObjects(manager, q, &buckets)
+	if err != nil && err != sql.ErrNoRows {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	return buckets, nil
+}
+
+func (manager *SBucketManager) syncBuckets(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, buckets []cloudprovider.ICloudBucket) compare.SyncResult {
+	lockman.LockClass(ctx, manager, "")
+	defer lockman.ReleaseClass(ctx, manager, "")
+
+	syncResult := compare.SyncResult{}
+
+	dbBuckets, err := manager.fetchBuckets(provider, region)
+	if err != nil {
+		syncResult.Error(err)
+		return syncResult
+	}
+
+	removed := make([]SBucket, 0)
+	commondb := make([]SBucket, 0)
+	commonext := make([]cloudprovider.ICloudBucket, 0)
+	added := make([]cloudprovider.ICloudBucket, 0)
+
+	err = compare.CompareSets(dbBuckets, buckets, &removed, &commondb, &commonext, &added)
+	if err != nil {
+		syncResult.Error(err)
+		return syncResult
+	}
+
+	for i := 0; i < len(removed); i += 1 {
+		err = removed[i].syncRemoveCloudBucket(ctx, userCred)
+		if err != nil {
+			syncResult.DeleteError(err)
+		} else {
+			syncResult.Delete()
+		}
+	}
+	for i := 0; i < len(commondb); i += 1 {
+		err = commondb[i].syncWithCloudBucket(ctx, userCred, commonext[i], provider)
+		if err != nil {
+			syncResult.UpdateError(err)
+		} else {
+			syncResult.Update()
+		}
+	}
+	for i := 0; i < len(added); i += 1 {
+		_, err := manager.newFromCloudBucket(ctx, userCred, added[i], provider, region)
+		if err != nil {
+			syncResult.AddError(err)
+		} else {
+			syncResult.Add()
+		}
+	}
+
+	return syncResult
+}
+
+func (manager *SBucketManager) newFromCloudBucket(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	extBucket cloudprovider.ICloudBucket,
+	provider *SCloudprovider,
+	region *SCloudregion,
+) (*SBucket, error) {
+	bucket := SBucket{}
+	bucket.SetModelManager(manager, &bucket)
+
+	bucket.ExternalId = extBucket.GetGlobalId()
+	bucket.ManagerId = provider.Id
+	bucket.CloudregionId = region.Id
+	bucket.Status = api.BUCKET_STATUS_READY
+
+	newName, err := db.GenerateName(manager, nil, extBucket.GetName())
+	if err != nil {
+		return nil, errors.Wrap(err, "db.GenerateName")
+	}
+
+	bucket.Name = newName
+
+	created := extBucket.GetCreateAt()
+	if !created.IsZero() {
+		bucket.CreatedAt = created
+	}
+
+	bucket.Location = extBucket.GetLocation()
+	bucket.StorageClass = extBucket.GetStorageClass()
+	// bucket.Acl = extBucket.GetAcl()
+
+	bucket.IsEmulated = false
+
+	err = manager.TableSpec().Insert(&bucket)
+	if err != nil {
+		return nil, errors.Wrap(err, "Insert")
+	}
+
+	SyncCloudProject(userCred, &bucket, provider.GetOwnerId(), extBucket, provider.Id)
+
+	db.OpsLog.LogEvent(&bucket, db.ACT_CREATE, bucket.GetShortDesc(ctx), userCred)
+
+	return &bucket, nil
+}
+
+func (bucket *SBucket) syncWithCloudBucket(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	extBucket cloudprovider.ICloudBucket,
+	provider *SCloudprovider,
+) error {
+	diff, err := db.UpdateWithLock(ctx, bucket, func() error {
+		// bucket.Acl = extBucket.GetAcl()
+		bucket.Location = extBucket.GetLocation()
+		bucket.StorageClass = extBucket.GetStorageClass()
+
+		bucket.Status = api.BUCKET_STATUS_READY
+		return nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "db.UpdateWithLock")
+	}
+
+	db.OpsLog.LogSyncUpdate(bucket, diff, userCred)
+
+	if provider != nil {
+		SyncCloudProject(userCred, bucket, provider.GetOwnerId(), extBucket, provider.Id)
+	}
+
+	return nil
+}
+
+func (bucket *SBucket) syncRemoveCloudBucket(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+) error {
+	lockman.LockObject(ctx, bucket)
+	defer lockman.ReleaseObject(ctx, bucket)
+
+	err := bucket.RealDelete(ctx, userCred)
+	if err != nil {
+		return errors.Wrap(err, "RealDelete")
+	}
+	return nil
+}
+
+func (bucket *SBucket) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	// override
+	log.Infof("bucket delete do nothing")
+	return nil
+}
+
+func (bucket *SBucket) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return bucket.SVirtualResourceBase.Delete(ctx, userCred)
+}
+
+func (bucket *SBucket) RemoteDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	iregion, err := bucket.GetIRegion()
+	if err != nil {
+		return errors.Wrap(err, "bucket.GetIRegion")
+	}
+	err = iregion.DeleteIBucket(bucket.ExternalId)
+	if err != nil {
+		return errors.Wrap(err, "iregion.DeleteIBucket")
+	}
+	err = bucket.RealDelete(ctx, userCred)
+	if err != nil {
+		return errors.Wrap(err, "bucket.RealDelete")
+	}
+	return nil
+}
+
+func (bucket *SBucket) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	return bucket.StartBucketDeleteTask(ctx, userCred, "")
+}
+
+func (bucket *SBucket) StartBucketDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
+	params := jsonutils.NewDict()
+	task, err := taskman.TaskManager.NewTask(ctx, "BucketDeleteTask", bucket, userCred, params, parentTaskId, "", nil)
+	if err != nil {
+		log.Errorf("%s", err)
+		return err
+	}
+	bucket.SetStatus(userCred, api.CLOUD_PROVIDER_START_DELETE, "StartBucketDeleteTask")
+	task.ScheduleRun(nil)
+	return nil
+}
+
+func (bucket *SBucket) GetRegion() (*SCloudregion, error) {
+	region, err := CloudregionManager.FetchById(bucket.CloudregionId)
+	if err != nil {
+		return nil, errors.Wrap(err, "CloudregionManager.FetchById")
+	}
+	return region.(*SCloudregion), nil
+}
+
+func (bucket *SBucket) GetIRegion() (cloudprovider.ICloudRegion, error) {
+	region, err := bucket.GetRegion()
+	if err != nil {
+		return nil, errors.Wrap(err, "bucket.GetRegion")
+	}
+	provider, err := bucket.GetDriver()
+	if err != nil {
+		return nil, err
+	}
+	return provider.GetIRegionById(region.GetExternalId())
+}
+
+var BUCKET_NAME_REG = regexp.MustCompile(`^[a-zA-Z0-9-]+$`)
+
+func isValidBucketName(name string) bool {
+	return BUCKET_NAME_REG.MatchString(name)
+}
+
+func (manager *SBucketManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	data *jsonutils.JSONDict,
+) (*jsonutils.JSONDict, error) {
+	for _, v := range []validators.IValidator{
+		validators.NewModelIdOrNameValidator("cloudregion", CloudregionManager.Keyword(), ownerId),
+		validators.NewModelIdOrNameValidator("manager", CloudproviderManager.Keyword(), ownerId),
+	} {
+		err := v.Validate(data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	nameStr, _ := data.GetString("name")
+	if len(nameStr) == 0 {
+		return nil, httperrors.NewInputParameterError("missing name")
+	}
+	if !isValidBucketName(nameStr) {
+		return nil, httperrors.NewInputParameterError("invalid name, only alphabets, digits and hyphen(-) allowed")
+	}
+	return manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data)
+}
+
+func (bucket *SBucket) PostCreate(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) {
+	bucket.SetStatus(userCred, api.BUCKET_STATUS_START_CREATE, "PostCreate")
+	task, err := taskman.TaskManager.NewTask(ctx, "BucketCreateTask", bucket, userCred, nil, "", "", nil)
+	if err != nil {
+		log.Errorf("BucketCreateTask newTask error %s", err)
+	} else {
+		task.ScheduleRun(nil)
+	}
+}
+
+func (bucket *SBucket) ValidateUpdateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data *jsonutils.JSONDict,
+) (*jsonutils.JSONDict, error) {
+	nameStr, _ := data.GetString("name")
+	if len(nameStr) > 0 {
+		if !isValidBucketName(nameStr) {
+			return nil, httperrors.NewInputParameterError("invalid name, only alphabets, digits and hyphen(-) allowed")
+		}
+	}
+	return bucket.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
+func (bucket *SBucket) RemoteCreate(ctx context.Context, userCred mcclient.TokenCredential) error {
+	iregion, err := bucket.GetIRegion()
+	if err != nil {
+		return errors.Wrap(err, "bucket.GetIRegion")
+	}
+	err = iregion.CreateIBucket(bucket.Name, bucket.StorageClass, bucket.Acl)
+	if err != nil {
+		return errors.Wrap(err, "iregion.CreateIBucket")
+	}
+	err = db.SetExternalId(bucket, userCred, bucket.Name)
+	if err != nil {
+		return errors.Wrap(err, "db.SetExternalId")
+	}
+	extBucket, err := iregion.GetIBucketByName(bucket.Name)
+	if err != nil {
+		return errors.Wrap(err, "iregion.GetIBucketByName")
+	}
+	err = bucket.syncWithCloudBucket(ctx, userCred, extBucket, nil)
+	if err != nil {
+		return errors.Wrap(err, "bucket.syncWithCloudBucket")
+	}
+	return nil
+}
+
+func (bucket *SBucket) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
+	extra := bucket.SVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query)
+	return bucket.getMoreDetails(extra)
+}
+
+func (bucket *SBucket) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*jsonutils.JSONDict, error) {
+	extra, err := bucket.SVirtualResourceBase.GetExtraDetails(ctx, userCred, query)
+	if err != nil {
+		return nil, err
+	}
+	return bucket.getMoreDetails(extra), nil
+}
+
+func (bucket *SBucket) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	info := bucket.getCloudProviderInfo()
+	extra.Update(jsonutils.Marshal(&info))
+
+	return extra
+}
+
+func (bucket *SBucket) getCloudProviderInfo() SCloudProviderInfo {
+	region, _ := bucket.GetRegion()
+	provider := bucket.GetCloudprovider()
+	return MakeCloudProviderInfo(region, nil, provider)
+}
+
+func (manager *SBucketManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	var err error
+
+	q, err = managedResourceFilterByAccount(q, query, "", nil)
+	if err != nil {
+		return nil, err
+	}
+	q = managedResourceFilterByCloudType(q, query, "", nil)
+
+	q, err = managedResourceFilterByDomain(q, query, "", nil)
+	if err != nil {
+		return nil, err
+	}
+
+	q, err = manager.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
+	if err != nil {
+		return nil, err
+	}
+
+	return q, nil
+}

pkg/compute/models/cloudproviders.go

@@ -1087,6 +1087,7 @@ func (self *SCloudprovider) RealDelete(ctx context.Context, userCred mcclient.To
 	var err error
 
 	for _, manager := range []IPurgeableManager{
+		BucketManager,
 		HostManager,
 		SnapshotManager,
 		SnapshotPolicyManager,

pkg/compute/models/cloudregions.go

@@ -654,3 +654,7 @@ func (self *SCloudregion) getMinDataDiskCount() int {
 func (self *SCloudregion) getMaxDataDiskCount() int {
 	return options.Options.MaxDataDiskCount
 }
+
+func (manager *SCloudregionManager) FetchDefaultRegion() *SCloudregion {
+	return manager.FetchRegionById(api.DEFAULT_REGION_ID)
+}

pkg/compute/models/cloudsync.go

@@ -169,6 +169,28 @@ func syncRegionEips(ctx context.Context, userCred mcclient.TokenCredential, sync
 	// db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred)
 }
 
+func syncRegionBuckets(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) {
+	buckets, err := remoteRegion.GetIBuckets()
+	if err != nil {
+		msg := fmt.Sprintf("GetIBuckets for region %s failed %s", remoteRegion.GetName(), err)
+		log.Errorf(msg)
+		return
+	}
+
+	result := BucketManager.syncBuckets(ctx, userCred, provider, localRegion, buckets)
+
+	syncResults.Add(BucketManager, result)
+
+	msg := result.Result()
+	notes := fmt.Sprintf("GetIBuckets for region %s result: %s", localRegion.Name, msg)
+	log.Infof(notes)
+	if result.IsError() {
+		return
+	}
+	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred)
+	// logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
+}
+
 func syncRegionVPCs(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) {
 	vpcs, err := remoteRegion.GetIVpcs()
 	if err != nil {
@@ -956,6 +978,8 @@ func syncPublicCloudProviderInfo(
 
 	// no need to lock public cloud region as cloud region for public cloud is readonly
 
+	syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, remoteRegion)
+
 	// 需要先同步vpc，避免私有云eip找不到network
 	syncRegionVPCs(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
 
@@ -1018,6 +1042,9 @@ func syncOnPremiseCloudProviderInfo(
 	syncRange *SSyncRange,
 ) error {
 	log.Debugf("Start sync on-premise provider %s(%s)", provider.Name, provider.Provider)
+
+	syncProjects(ctx, userCred, syncResults, driver, provider)
+
 	iregion, err := driver.GetOnPremiseIRegion()
 	if err != nil {
 		msg := fmt.Sprintf("GetOnPremiseIRegion for provider %s failed %s", provider.GetName(), err)
@@ -1025,6 +1052,9 @@ func syncOnPremiseCloudProviderInfo(
 		return err
 	}
 
+	localRegion := CloudregionManager.FetchDefaultRegion()
+	syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, iregion)
+
 	ihosts, err := iregion.GetIHosts()
 	if err != nil {
 		msg := fmt.Sprintf("GetIHosts for provider %s failed %s", provider.GetName(), err)

pkg/compute/models/purge.go

@@ -1039,3 +1039,30 @@ func (manager *SNetworkInterfaceManager) purgeAll(ctx context.Context, userCred
 	}
 	return nil
 }
+
+func (bucket *SBucket) purge(ctx context.Context, userCred mcclient.TokenCredential) error {
+	lockman.LockObject(ctx, bucket)
+	defer lockman.ReleaseObject(ctx, bucket)
+
+	err := bucket.ValidateDeleteCondition(ctx)
+	if err != nil {
+		return err
+	}
+
+	return bucket.RealDelete(ctx, userCred)
+}
+
+func (bucketManager *SBucketManager) purgeAll(ctx context.Context, userCred mcclient.TokenCredential, providerId string) error {
+	buckets := make([]SBucket, 0)
+	err := fetchByManagerId(bucketManager, providerId, &buckets)
+	if err != nil {
+		return err
+	}
+	for i := range buckets {
+		err := buckets[i].purge(ctx, userCred)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/compute/models/vpcs.go

@@ -476,22 +476,32 @@ func (manager *SVpcManager) InitializeData() error {
 }
 
 func (manager *SVpcManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	regionId, err := data.GetString("cloudregion_id")
-	if err != nil {
+	regionId := jsonutils.GetAnyString(data, []string{"region", "cloudregion", "cloudregion_id"})
+	if len(regionId) == 0 {
 		return nil, httperrors.NewMissingParameterError("cloudregion_id")
 	}
-	region := CloudregionManager.FetchRegionById(regionId)
-	if region == nil {
-		return nil, httperrors.NewInputParameterError("Invalid cloudregion_id")
+	regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionId)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, httperrors.NewResourceNotFoundError2(CloudregionManager.Keyword(), regionId)
+		} else {
+			return nil, httperrors.NewGeneralError(err)
+		}
 	}
+	region := regionObj.(*SCloudregion)
+	data.Add(jsonutils.NewString(region.GetId()), "cloudregion_id")
 	if region.isManaged() {
 		managerStr := jsonutils.GetAnyString(data, []string{"manager_id", "manager"})
 		if len(managerStr) == 0 {
 			return nil, httperrors.NewMissingParameterError("manager_id")
 		}
-		managerObj := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr)
+		managerObj, err := CloudproviderManager.FetchByIdOrName(userCred, managerStr)
 		if err != nil {
-			return nil, httperrors.NewResourceNotFoundError("Cloud provider/manager %s not found", managerStr)
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
+			} else {
+				return nil, httperrors.NewGeneralError(err)
+			}
 		}
 		data.Add(jsonutils.NewString(managerObj.GetId()), "manager_id")
 	} else {

pkg/compute/service/handlers.go

@@ -64,6 +64,7 @@ func InitHandlers(app *appsrv.Application) {
 	for _, manager := range []db.IModelManager{
 		db.OpsLog,
 		db.Metadata,
+		models.BucketManager,
 		models.CloudaccountManager,
 		models.CloudproviderManager,
 		models.CloudregionManager,

pkg/compute/tasks/bucket_create_task.go

@@ -0,0 +1,44 @@
+package tasks
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type BucketCreateTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(BucketCreateTask{})
+}
+
+func (task *BucketCreateTask) taskFailed(ctx context.Context, bucket *models.SBucket, err error) {
+	bucket.SetStatus(task.UserCred, api.BUCKET_STATUS_CREATE_FAIL, err.Error())
+	db.OpsLog.LogEvent(bucket, db.ACT_ALLOCATE_FAIL, err.Error(), task.UserCred)
+	logclient.AddActionLogWithStartable(task, bucket, logclient.ACT_ALLOCATE, err.Error(), task.UserCred, false)
+	task.SetStageFailed(ctx, err.Error())
+}
+
+func (task *BucketCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	bucket := obj.(*models.SBucket)
+
+	bucket.SetStatus(task.UserCred, api.BUCKET_STATUS_CREATING, "StartBucketCreateTask")
+
+	err := bucket.RemoteCreate(ctx, task.UserCred)
+	if err != nil {
+		task.taskFailed(ctx, bucket, err)
+		return
+	}
+
+	bucket.SetStatus(task.UserCred, api.BUCKET_STATUS_READY, "BucketCreateTask")
+	logclient.AddActionLogWithStartable(task, bucket, logclient.ACT_ALLOCATE, nil, task.UserCred, true)
+	task.SetStageComplete(ctx, nil)
+}

pkg/compute/tasks/bucket_delete_task.go

@@ -0,0 +1,43 @@
+package tasks
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type BucketDeleteTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(BucketDeleteTask{})
+}
+
+func (task *BucketDeleteTask) taskFailed(ctx context.Context, bucket *models.SBucket, err error) {
+	bucket.SetStatus(task.UserCred, api.VPC_STATUS_DELETE_FAILED, err.Error())
+	db.OpsLog.LogEvent(bucket, db.ACT_DELOCATE_FAIL, err.Error(), task.UserCred)
+	logclient.AddActionLogWithStartable(task, bucket, logclient.ACT_DELETE, err.Error(), task.UserCred, false)
+	task.SetStageFailed(ctx, err.Error())
+}
+
+func (task *BucketDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	bucket := obj.(*models.SBucket)
+
+	bucket.SetStatus(task.UserCred, api.BUCKET_STATUS_DELETING, "StartBucketDeleteTask")
+
+	err := bucket.RemoteDelete(ctx, task.UserCred)
+	if err != nil {
+		task.taskFailed(ctx, bucket, err)
+		return
+	}
+
+	logclient.AddActionLogWithStartable(task, bucket, logclient.ACT_DELETE, nil, task.UserCred, true)
+	task.SetStageComplete(ctx, nil)
+}

pkg/image/service/service.go

@@ -23,6 +23,7 @@ import (
 
 	"yunion.io/x/log"
 
+	api "yunion.io/x/onecloud/pkg/apis/image"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
@@ -37,16 +38,12 @@ import (
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
 
-const (
-	SERVICE_TYPE = "image"
-)
-
 func StartService() {
 	opts := &options.Options
 	commonOpts := &opts.CommonOptions
 	baseOpts := &opts.BaseOptions
 	dbOpts := &opts.DBOptions
-	common_options.ParseOptions(opts, os.Args, "glance-api.conf", SERVICE_TYPE)
+	common_options.ParseOptions(opts, os.Args, "glance-api.conf", api.SERVICE_TYPE)
 
 	isRoot := sysutils.IsRootPermission()
 	if !isRoot {

pkg/mcclient/modules/mod_buckets.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+var (
+	Buckets ResourceManager
+)
+
+func init() {
+	Buckets = NewComputeManager("bucket", "buckets",
+		[]string{"ID", "Name", "Storage_Class",
+			"Status", "location", "acl",
+			"region",
+		},
+		[]string{})
+
+	registerCompute(&Buckets)
+}

pkg/multicloud/no_storage_region.go

@@ -0,0 +1,29 @@
+package multicloud
+
+import "yunion.io/x/onecloud/pkg/cloudprovider"
+
+type SNoObjectStorageRegion struct{}
+
+///////////////// S3 ///////////////////
+
+func (cli *SNoObjectStorageRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SNoObjectStorageRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SNoObjectStorageRegion) DeleteIBucket(name string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SNoObjectStorageRegion) IBucketExist(name string) (bool, error) {
+	return false, cloudprovider.ErrNotSupported
+}
+
+func (cli *SNoObjectStorageRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+////////////////// END S3 fake API //////////

pkg/multicloud/objectstore/buckets.go

@@ -0,0 +1,60 @@
+package objectstore
+
+import (
+	"fmt"
+	"path"
+	"time"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SBucket struct {
+	client *SObjectStoreClient
+
+	Name         string
+	Location     string
+	CreatedAt    time.Time
+	StorageClass string
+	Acl          string
+}
+
+func (bucket *SBucket) GetProjectId() string {
+	return ""
+}
+
+func (bucket *SBucket) GetGlobalId() string {
+	return bucket.Name
+}
+
+func (bucket *SBucket) GetName() string {
+	return bucket.Name
+}
+
+func (bucket *SBucket) GetAcl() string {
+	return bucket.Acl
+}
+
+func (bucket *SBucket) GetLocation() string {
+	return bucket.Location
+}
+
+func (bucket *SBucket) GetIRegion() cloudprovider.ICloudRegion {
+	return bucket.client
+}
+
+func (bucket *SBucket) GetCreateAt() time.Time {
+	return bucket.CreatedAt
+}
+
+func (bucket *SBucket) GetStorageClass() string {
+	return bucket.StorageClass
+}
+
+func (bucket *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	return []cloudprovider.SBucketAccessUrl{
+		{
+			Url:         path.Join(bucket.client.endpoint, bucket.Name),
+			Description: fmt.Sprintf("%s", bucket.Location),
+		},
+	}
+}

pkg/multicloud/objectstore/doc.go

@@ -0,0 +1 @@
+package objectstore // import "yunion.io/x/onecloud/pkg/multicloud/objectstore"

pkg/multicloud/objectstore/objectstore.go

@@ -0,0 +1,324 @@
+package objectstore
+
+import (
+	"net/url"
+
+	"github.com/minio/minio-go"
+	"github.com/pkg/errors"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/util/secrules"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/object"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
+)
+
+type SObjectStoreClient struct {
+	object.SObject
+
+	cloudprovider.SFakeOnPremiseRegion
+	multicloud.SRegion
+
+	providerId   string
+	providerName string
+	endpoint     string
+	accessKey    string
+	secret       string
+
+	client *minio.Client
+
+	Debug bool
+}
+
+func NewObjectStoreClient(providerId string, providerName string, endpoint string, accessKey string, secret string, isDebug bool) (*SObjectStoreClient, error) {
+	client := SObjectStoreClient{
+		providerId:   providerId,
+		providerName: providerName,
+		endpoint:     endpoint,
+		accessKey:    accessKey,
+		secret:       secret,
+		Debug:        isDebug,
+	}
+	parts, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, errors.Wrap(err, "url.Parse endpoint")
+	}
+	useSsl := false
+	if parts.Scheme == "https" {
+		useSsl = true
+	}
+	cli, err := minio.New(parts.Host, accessKey, secret, useSsl)
+	if err != nil {
+		return nil, errors.Wrap(err, "minio.New")
+	}
+
+	client.client = cli
+
+	return &client, nil
+}
+
+func (cli *SObjectStoreClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
+	subAccount := cloudprovider.SSubAccount{
+		Account:      cli.accessKey,
+		Name:         cli.providerName,
+		HealthStatus: api.CLOUD_PROVIDER_HEALTH_NORMAL,
+	}
+	return []cloudprovider.SSubAccount{subAccount}, nil
+}
+
+func (cli *SObjectStoreClient) GetIRegion() cloudprovider.ICloudRegion {
+	return cli.GetVirtualObject().(cloudprovider.ICloudRegion)
+}
+
+func (cli *SObjectStoreClient) GetVersion() string {
+	return ""
+}
+
+func (cli *SObjectStoreClient) About() jsonutils.JSONObject {
+	about := jsonutils.NewDict()
+	return about
+}
+
+func (cli *SObjectStoreClient) GetProvider() string {
+	return api.CLOUD_PROVIDER_GENERICS3
+}
+
+///////////////////////////////// fake impletementations //////////////////////
+
+func (cli *SObjectStoreClient) GetIZones() ([]cloudprovider.ICloudZone, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIVpcs() ([]cloudprovider.ICloudVpc, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIEips() ([]cloudprovider.ICloudEIP, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIZoneById(id string) (cloudprovider.ICloudZone, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIEipById(id string) (cloudprovider.ICloudEIP, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIVMById(id string) (cloudprovider.ICloudVM, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIDiskById(id string) (cloudprovider.ICloudDisk, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) DeleteSecurityGroup(vpcId, secgroupId string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) {
+	return "", cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateIVpc(name string, desc string, cidr string) (cloudprovider.ICloudVpc, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateEIP(eip *cloudprovider.SEip) (cloudprovider.ICloudEIP, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateSnapshotPolicy(*cloudprovider.SnapshotPolicyInput) (string, error) {
+	return "", cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) DeleteSnapshotPolicy(string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskIds []string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CancelSnapshotPolicyToDisks(diskIds []string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISnapshotPolicies() ([]cloudprovider.ICloudSnapshotPolicy, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISnapshotPolicyById(snapshotPolicyId string) (cloudprovider.ICloudSnapshotPolicy, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIHosts() ([]cloudprovider.ICloudHost, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIHostById(id string) (cloudprovider.ICloudHost, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIStorages() ([]cloudprovider.ICloudStorage, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIStoragecaches() ([]cloudprovider.ICloudStoragecache, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancers() ([]cloudprovider.ICloudLoadbalancer, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerAcls() ([]cloudprovider.ICloudLoadbalancerAcl, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerCertificates() ([]cloudprovider.ICloudLoadbalancerCertificate, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerById(loadbalancerId string) (cloudprovider.ICloudLoadbalancer, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerAclById(aclId string) (cloudprovider.ICloudLoadbalancerAcl, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerCertificateById(certId string) (cloudprovider.ICloudLoadbalancerCertificate, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbalancer) (cloudprovider.ICloudLoadbalancer, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateILoadBalancerCertificate(cert *cloudprovider.SLoadbalancerCertificate) (cloudprovider.ICloudLoadbalancerCertificate, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISkuById(skuId string) (cloudprovider.ICloudSku, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISkus(zoneId string) ([]cloudprovider.ICloudSku, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateISku(sku *cloudprovider.SServerSku) (cloudprovider.ICloudSku, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+////////////////////////////////// S3 API ///////////////////////////////////
+
+func (cli *SObjectStoreClient) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	buckets, err := cli.client.ListBuckets()
+	if err != nil {
+		return nil, errors.Wrap(err, "client.ListBuckets")
+	}
+	ret := make([]cloudprovider.ICloudBucket, len(buckets))
+	for i := range buckets {
+		b := SBucket{
+			client:    cli,
+			Name:      buckets[i].Name,
+			CreatedAt: buckets[i].CreationDate,
+		}
+		ret[i] = &b
+	}
+	return ret, nil
+}
+
+func (cli *SObjectStoreClient) CreateIBucket(name string, storageClass string, acl string) error {
+	err := cli.client.MakeBucket(name, "")
+	if err != nil {
+		return errors.Wrap(err, "MakeBucket")
+	}
+	return nil
+}
+
+func minioErrCode(err error) int {
+	if srvErr, ok := err.(minio.ErrorResponse); ok {
+		return srvErr.StatusCode
+	}
+	if srvErr, ok := err.(*minio.ErrorResponse); ok {
+		return srvErr.StatusCode
+	}
+	return -1
+}
+
+func (cli *SObjectStoreClient) DeleteIBucket(name string) error {
+	err := cli.client.RemoveBucket(name)
+	if err != nil {
+		if minioErrCode(err) == 404 {
+			return nil
+		}
+		return errors.Wrap(err, "RemoveBucket")
+	}
+	return nil
+}
+
+func (cli *SObjectStoreClient) GetIBucketPolicy(name string) (string, error) {
+	policy, err := cli.client.GetBucketPolicy(name)
+	if err != nil {
+		return "", errors.Wrap(err, "GetBucketPolicy")
+	}
+	return policy, nil
+}
+
+func (cli *SObjectStoreClient) SetIBucketPolicy(name string, policy string) error {
+	err := cli.client.SetBucketPolicy(name, policy)
+	if err != nil {
+		return errors.Wrap(err, "SetBucketPolicy")
+	}
+	return nil
+}
+
+func (cli *SObjectStoreClient) GetIBucketLiftcycle(name string) (string, error) {
+	liftcycle, err := cli.client.GetBucketLifecycle(name)
+	if err != nil {
+		return "", errors.Wrap(err, "GetBucketLifecycle")
+	}
+	return liftcycle, nil
+}
+
+func (cli *SObjectStoreClient) IBucketExist(name string) (bool, error) {
+	exist, err := cli.client.BucketExists(name)
+	if err != nil {
+		return false, errors.Wrap(err, "BucketExists")
+	}
+	return exist, nil
+}
+
+func (cli *SObjectStoreClient) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
+	return cloudprovider.GetIBucketByName(cli, name)
+}

pkg/multicloud/objectstore/provider/doc.go

@@ -0,0 +1 @@
+package provider // import "yunion.io/x/onecloud/pkg/multicloud/objectstore/provider"

pkg/multicloud/objectstore/provider/provider.go

@@ -0,0 +1,136 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package provider
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/multicloud/objectstore"
+)
+
+type SObjectStoreProviderFactory struct {
+	cloudprovider.SPremiseBaseProviderFactory
+}
+
+func (self *SObjectStoreProviderFactory) GetId() string {
+	return api.CLOUD_PROVIDER_GENERICS3
+}
+
+func (self *SObjectStoreProviderFactory) GetName() string {
+	return api.CLOUD_PROVIDER_GENERICS3
+}
+
+func (self *SObjectStoreProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) error {
+	accessKeyID, _ := data.GetString("access_key")
+	if len(accessKeyID) == 0 {
+		return httperrors.NewMissingParameterError("access_key")
+	}
+	accessKeySecret, _ := data.GetString("secret_key")
+	if len(accessKeySecret) == 0 {
+		return httperrors.NewMissingParameterError("secret_key")
+	}
+	endpointURL, _ := data.GetString("endpoint")
+	if len(endpointURL) == 0 {
+		return httperrors.NewMissingParameterError("endpoint")
+	}
+	data.Set("account", jsonutils.NewString(accessKeyID))
+	data.Set("secret", jsonutils.NewString(accessKeySecret))
+	data.Set("url", jsonutils.NewString(endpointURL))
+	return nil
+}
+
+func (self *SObjectStoreProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+	accessKeyID, _ := data.GetString("access_key")
+	if len(accessKeyID) == 0 {
+		return nil, httperrors.NewMissingParameterError("access_key")
+	}
+	accessKeySecret, _ := data.GetString("secret_key")
+	if len(accessKeySecret) == 0 {
+		return nil, httperrors.NewMissingParameterError("secret_key")
+	}
+	account := &cloudprovider.SCloudaccount{
+		Account: accessKeyID,
+		Secret:  accessKeySecret,
+	}
+	return account, nil
+}
+
+func (self *SObjectStoreProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {
+	client, err := objectstore.NewObjectStoreClient(providerId, providerName, url, account, secret, false)
+	if err != nil {
+		return nil, err
+	}
+	client.SetVirtualObject(client)
+	return &SObjectStoreProvider{
+		SBaseProvider: cloudprovider.NewBaseProvider(self),
+		client:        client,
+	}, nil
+}
+
+func (self *SObjectStoreProviderFactory) GetClientRC(url, account, secret string) (map[string]string, error) {
+	return map[string]string{
+		"OBJECTSTORE_ACCESSKEY": account,
+		"OBJECTSTORE_SECRET":    secret,
+		"OBJECTSTORE_ENDPOINT":  url,
+	}, nil
+}
+
+func init() {
+	factory := SObjectStoreProviderFactory{}
+	cloudprovider.RegisterFactory(&factory)
+}
+
+type SObjectStoreProvider struct {
+	cloudprovider.SBaseProvider
+	client *objectstore.SObjectStoreClient
+}
+
+func (self *SObjectStoreProvider) GetIRegions() []cloudprovider.ICloudRegion {
+	return nil
+}
+
+func (self *SObjectStoreProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (self *SObjectStoreProvider) GetBalance() (float64, string, error) {
+	return 0.0, api.CLOUD_PROVIDER_HEALTH_NORMAL, cloudprovider.ErrNotSupported
+}
+
+func (self *SObjectStoreProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) {
+	return self.client, nil
+}
+
+func (self *SObjectStoreProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (self *SObjectStoreProvider) GetSysInfo() (jsonutils.JSONObject, error) {
+	return self.client.About(), nil
+}
+
+func (self *SObjectStoreProvider) GetVersion() string {
+	return self.client.GetVersion()
+}
+
+func (self *SObjectStoreProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
+	return self.client.GetSubAccounts()
+}

pkg/multicloud/objectstore/shell/bucket.go

@@ -0,0 +1,63 @@
+package shell
+
+import (
+	"fmt"
+	"yunion.io/x/onecloud/pkg/multicloud/objectstore"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type BucketListOptions struct {
+	}
+	shellutils.R(&BucketListOptions{}, "bucket-list", "List all bucket", func(cli *objectstore.SObjectStoreClient, args *BucketListOptions) error {
+		buckets, err := cli.GetIBuckets()
+		if err != nil {
+			return err
+		}
+		printList(buckets, nil)
+		return nil
+	})
+
+	type BucketCreateOptions struct {
+		NAME string `help:"name of bucket to create"`
+	}
+	shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli *objectstore.SObjectStoreClient, args *BucketCreateOptions) error {
+		err := cli.CreateIBucket(args.NAME, "", "")
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type BucketDeleteOptions struct {
+		NAME string `help:"name of bucket to delete"`
+	}
+	shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete bucket", func(cli *objectstore.SObjectStoreClient, args *BucketDeleteOptions) error {
+		err := cli.DeleteIBucket(args.NAME)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type BucketPolicyOptions struct {
+		NAME string `help:"name of bucket to get policy"`
+	}
+	shellutils.R(&BucketPolicyOptions{}, "bucket-policy", "Get bucket policy", func(cli *objectstore.SObjectStoreClient, args *BucketPolicyOptions) error {
+		policy, err := cli.GetIBucketPolicy(args.NAME)
+		if err != nil {
+			return err
+		}
+		fmt.Println(policy)
+		return nil
+	})
+
+	shellutils.R(&BucketPolicyOptions{}, "bucket-lifecycle", "Get bucket lifecycle", func(cli *objectstore.SObjectStoreClient, args *BucketPolicyOptions) error {
+		lifecycle, err := cli.GetIBucketLiftcycle(args.NAME)
+		if err != nil {
+			return err
+		}
+		fmt.Println(lifecycle)
+		return nil
+	})
+}

pkg/multicloud/objectstore/shell/doc.go

@@ -0,0 +1 @@
+package shell // import "yunion.io/x/onecloud/pkg/multicloud/objectstore/shell"

pkg/multicloud/objectstore/shell/printutils.go

@@ -0,0 +1,11 @@
+package shell
+
+import "yunion.io/x/onecloud/pkg/util/printutils"
+
+func printList(data interface{}, columns []string) {
+	printutils.PrintGetterList(data, columns)
+}
+
+func printObject(obj interface{}) {
+	printutils.PrintInterfaceObject(obj)
+}

pkg/s3gateway/models/doc.go

@@ -0,0 +1 @@
+package models // import "yunion.io/x/onecloud/pkg/s3gateway/models"

pkg/s3gateway/models/initdb.go

@@ -0,0 +1,23 @@
+package models
+
+import (
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+)
+
+func InitDB() error {
+	for _, manager := range []db.IModelManager{
+		/*
+		 * Important!!!
+		 * initialization order matters, do not change the order
+		 */
+	} {
+		err := manager.InitializeData()
+		if err != nil {
+			log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err)
+			// return err skip error table
+		}
+	}
+	return nil
+}

pkg/s3gateway/options/doc.go

@@ -0,0 +1 @@
+package options // import "yunion.io/x/onecloud/pkg/s3gateway/options"

pkg/s3gateway/options/options.go

@@ -0,0 +1,15 @@
+package options
+
+import (
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+)
+
+type SS3GatewayOptions struct {
+	common_options.CommonOptions
+
+	common_options.DBOptions
+}
+
+var (
+	Options SS3GatewayOptions
+)

pkg/s3gateway/service/doc.go

@@ -0,0 +1 @@
+package service // import "yunion.io/x/onecloud/pkg/s3gateway/service"

pkg/s3gateway/service/handlers.go

@@ -0,0 +1,34 @@
+package service
+
+import (
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	// "yunion.io/x/onecloud/pkg/s3gateway/models"
+)
+
+func initHandlers(app *appsrv.Application) {
+	db.InitAllManagers()
+
+	// quotas.AddQuotaHandler(models.QuotaManager, API_VERSION, app)
+	// usages.AddUsageHandler(API_VERSION, app)
+	taskman.AddTaskHandler("", app)
+
+	for _, manager := range []db.IModelManager{
+		taskman.TaskManager,
+		taskman.SubTaskManager,
+		taskman.TaskObjectManager,
+		db.Metadata,
+	} {
+		db.RegisterModelManager(manager)
+	}
+
+	for _, manager := range []db.IModelManager{
+		db.OpsLog,
+	} {
+		db.RegisterModelManager(manager)
+		handler := db.NewModelHandler(manager)
+		dispatcher.AddModelDispatcher("", app, handler)
+	}
+}

pkg/s3gateway/service/service.go

@@ -0,0 +1,59 @@
+package service
+
+import (
+	"os"
+
+	"yunion.io/x/log"
+
+	api "yunion.io/x/onecloud/pkg/apis/s3gateway"
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+	"yunion.io/x/onecloud/pkg/s3gateway/models"
+	"yunion.io/x/onecloud/pkg/s3gateway/options"
+)
+
+func StartService() {
+	opts := &options.Options
+	commonOpts := &opts.CommonOptions
+	baseOpts := &opts.BaseOptions
+	dbOpts := &opts.DBOptions
+	common_options.ParseOptions(opts, os.Args, "s3gateway.conf", api.SERVICE_TYPE)
+
+	app_common.InitAuth(commonOpts, func() {
+		log.Infof("Auth complete!!")
+	})
+
+	cloudcommon.InitDB(dbOpts)
+
+	app := app_common.InitApp(&opts.BaseOptions, true)
+	initHandlers(app)
+
+	cloudcommon.InitDB(&opts.DBOptions)
+
+	if !db.CheckSync(opts.AutoSyncTable) {
+		log.Fatalf("database schema not in sync!")
+	}
+
+	models.InitDB()
+
+	if opts.ExitAfterDBInit {
+		log.Infof("Exiting after db initialization ...")
+		os.Exit(0)
+	}
+
+	/*if !opts.IsSlaveNode {
+		cron := cronman.GetCronJobManager(true)
+		cron.AddJob1("CleanPendingDeleteImages", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.ImageManager.CleanPendingDeleteImages)
+		cron.AddJob1("CalculateQuotaUsages", time.Duration(opts.CalculateQuotaUsageIntervalSeconds)*time.Second, models.QuotaManager.CalculateQuotaUsages)
+
+		cron.Start()
+	}*/
+
+	cloudcommon.AppDBInit(app)
+	app_common.ServeForeverWithCleanup(app, baseOpts, func() {
+		cloudcommon.CloseDB()
+
+	})
+}

pkg/util/aliyun/bucket.go

@@ -0,0 +1,63 @@
+package aliyun
+
+import (
+	"fmt"
+	"time"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SBucket struct {
+	region *SRegion
+
+	Name         string
+	Location     string
+	CreationDate time.Time
+	StorageClass string
+	Acl          string
+}
+
+func (b *SBucket) GetProjectId() string {
+	return ""
+}
+
+func (b *SBucket) GetGlobalId() string {
+	return b.Name
+}
+
+func (b *SBucket) GetName() string {
+	return b.Name
+}
+
+func (b *SBucket) GetAcl() string {
+	return b.Acl
+}
+
+func (b *SBucket) GetLocation() string {
+	return b.Location
+}
+
+func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion {
+	return b.region
+}
+
+func (b *SBucket) GetCreateAt() time.Time {
+	return b.CreationDate
+}
+
+func (b *SBucket) GetStorageClass() string {
+	return b.StorageClass
+}
+
+func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	return []cloudprovider.SBucketAccessUrl{
+		{
+			Url:         fmt.Sprintf("https://%s.aliyuncs.com", b.Location),
+			Description: "ExtranetEndpoint",
+		},
+		{
+			Url:         fmt.Sprintf("https://%s-internal.aliyuncs.com", b.Location),
+			Description: "IntranetEndpoint",
+		},
+	}
+}

pkg/util/aliyun/region.go

@@ -24,6 +24,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/utils"
@@ -917,3 +918,136 @@ func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAc
 	}
 	return iAcl, region.AddAccessControlListEntry(aclId, acl.Entrys)
 }
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetOssClient")
+	}
+	result, err := osscli.ListBuckets()
+	if err != nil {
+		return nil, errors.Wrap(err, "oss.ListBuckets")
+	}
+
+	ret := make([]cloudprovider.ICloudBucket, 0)
+	for _, bInfo := range result.Buckets {
+		if bInfo.Location[4:] != region.GetId() {
+			continue
+		}
+		acl := string(oss.ACLPrivate)
+		aclResp, err := osscli.GetBucketACL(bInfo.Name)
+		if err == nil {
+			acl = aclResp.ACL
+		}
+		b := SBucket{
+			region: region,
+
+			Name:         bInfo.Name,
+			Location:     bInfo.Location,
+			CreationDate: bInfo.CreationDate,
+			StorageClass: bInfo.StorageClass,
+			Acl:          acl,
+		}
+		ret = append(ret, &b)
+	}
+	return ret, nil
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return errors.Wrap(err, "region.GetOssClient")
+	}
+	opts := make([]oss.Option, 0)
+	if len(storageClassStr) > 0 {
+		storageClass := oss.StorageStandard
+		if strings.EqualFold(storageClassStr, string(oss.StorageStandard)) {
+			//
+		} else if strings.EqualFold(storageClassStr, string(oss.StorageIA)) {
+			storageClass = oss.StorageIA
+		} else if strings.EqualFold(storageClassStr, string(oss.StorageArchive)) {
+			storageClass = oss.StorageArchive
+		} else {
+			return errors.Error("not supported storageClass")
+		}
+		opt := oss.StorageClass(storageClass)
+		opts = append(opts, opt)
+	}
+	if len(aclStr) > 0 {
+		acl := oss.ACLPrivate
+		if strings.EqualFold(aclStr, string(oss.ACLPrivate)) {
+			// private, default
+		} else if strings.EqualFold(aclStr, string(oss.ACLPublicRead)) {
+			acl = oss.ACLPublicRead
+		} else if strings.EqualFold(aclStr, string(oss.ACLPublicReadWrite)) {
+			acl = oss.ACLPublicReadWrite
+		} else {
+			return errors.Error("not supported acl")
+		}
+		opt := oss.ACL(acl)
+		opts = append(opts, opt)
+	}
+	err = osscli.CreateBucket(name, opts...)
+	if err != nil {
+		return errors.Wrap(err, "oss.CreateBucket")
+	}
+	return nil
+}
+
+func ossErrorCode(err error) int {
+	if srvErr, ok := err.(oss.ServiceError); ok {
+		return srvErr.StatusCode
+	}
+	if srvErr, ok := err.(*oss.ServiceError); ok {
+		return srvErr.StatusCode
+	}
+	return -1
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return errors.Wrap(err, "region.GetOssClient")
+	}
+	err = osscli.DeleteBucket(name)
+	if err != nil {
+		if ossErrorCode(err) == 404 {
+			return nil
+		}
+		return errors.Wrap(err, "DeleteBucket")
+	}
+	return nil
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return false, errors.Wrap(err, "region.GetOssClient")
+	}
+	exist, err := osscli.IsBucketExist(name)
+	if err != nil {
+		return false, errors.Wrap(err, "IsBucketExist")
+	}
+	return exist, nil
+}
+
+func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetOssClient")
+	}
+	bi, err := osscli.GetBucketInfo(name)
+	if err != nil {
+		return nil, errors.Wrap(err, "Bucket")
+	}
+	bInfo := bi.BucketInfo
+	b := SBucket{
+		region:       region,
+		Name:         bInfo.Name,
+		Location:     bInfo.Location,
+		CreationDate: bInfo.CreationDate,
+		StorageClass: bInfo.StorageClass,
+		Acl:          bInfo.ACL,
+	}
+	return &b, nil
+}

pkg/util/aliyun/shell/oss.go

@@ -59,15 +59,11 @@ func init() {
 	type OssListOptions struct {
 	}
 	shellutils.R(&OssListOptions{}, "oss-list", "List OSS buckets", func(cli *aliyun.SRegion, args *OssListOptions) error {
-		oss, err := cli.GetOssClient()
+		buckets, err := cli.GetIBuckets()
 		if err != nil {
 			return err
 		}
-		result, err := oss.ListBuckets()
-		if err != nil {
-			return err
-		}
-		printList(result.Buckets, len(result.Buckets), 0, 50, nil)
+		printList(buckets, len(buckets), 0, 50, nil)
 		return nil
 	})
 
@@ -92,12 +88,25 @@ func init() {
 		return nil
 	})
 
-	shellutils.R(&OssListBucketOptions{}, "oss-create-bucket", "Create a OSS bucket", func(cli *aliyun.SRegion, args *OssListBucketOptions) error {
-		oss, err := cli.GetOssClient()
+	type OssCreateBucketOptions struct {
+		BUCKET       string `help:"bucket name"`
+		StorageClass string `help:"storage class" choices:"Standard|IA|Archive"`
+
+		Acl string `help:"ACL" choices:"private|public-read|public-read-write"`
+	}
+	shellutils.R(&OssCreateBucketOptions{}, "oss-create-bucket", "Create a OSS bucket", func(cli *aliyun.SRegion, args *OssCreateBucketOptions) error {
+		err := cli.CreateIBucket(args.BUCKET, args.StorageClass, args.Acl)
 		if err != nil {
 			return err
 		}
-		err = oss.CreateBucket(args.BUCKET)
+		return nil
+	})
+
+	type OssDeleteBucketOptions struct {
+		BUCKET string `help:"bucket name"`
+	}
+	shellutils.R(&OssDeleteBucketOptions{}, "oss-delete-bucket", "Delete a OSS bucket", func(cli *aliyun.SRegion, args *OssDeleteBucketOptions) error {
+		err := cli.DeleteIBucket(args.BUCKET)
 		if err != nil {
 			return err
 		}

pkg/util/aws/aws.go

@@ -154,9 +154,9 @@ func (self *SAwsClient) GetRegion(regionId string) *SRegion {
 	if len(regionId) == 0 {
 		regionId = AWS_INTERNATIONAL_DEFAULT_REGION
 		switch self.accessUrl {
-		case "InternationalCloud":
+		case AWS_INTERNATIONAL_CLOUDENV:
 			regionId = AWS_INTERNATIONAL_DEFAULT_REGION
-		case "ChinaCloud":
+		case AWS_CHINA_CLOUDENV:
 			regionId = AWS_CHINA_DEFAULT_REGION
 		}
 	}

pkg/util/aws/bucket.go

@@ -0,0 +1,61 @@
+package aws
+
+import (
+	"fmt"
+	"time"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SBucket struct {
+	region *SRegion
+
+	Name         string
+	Location     string
+	CreationDate time.Time
+}
+
+func (b *SBucket) GetProjectId() string {
+	return ""
+}
+
+func (b *SBucket) GetGlobalId() string {
+	return b.Name
+}
+
+func (b *SBucket) GetName() string {
+	return b.Name
+}
+
+func (b *SBucket) GetLocation() string {
+	return b.Location
+}
+
+func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion {
+	return b.region
+}
+
+func (b *SBucket) GetCreateAt() time.Time {
+	return b.CreationDate
+}
+
+func (b *SBucket) GetStorageClass() string {
+	return ""
+}
+
+func (b *SBucket) GetAcl() string {
+	return ""
+}
+
+func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	return []cloudprovider.SBucketAccessUrl{
+		{
+			Url:         fmt.Sprintf("https://%s.%s", b.Name, b.region.getS3Endpoint()),
+			Description: "bucket domain",
+		},
+		{
+			Url:         fmt.Sprintf("https://%s/%s", b.region.getS3Endpoint(), b.Name),
+			Description: "s3 domain",
+		},
+	}
+}

pkg/util/aws/region.go

@@ -16,6 +16,7 @@ package aws
 
 import (
 	"fmt"
+	"strings"
 
 	sdk "github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -26,6 +27,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -524,3 +526,109 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala
 func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) {
 	return nil, cloudprovider.ErrNotImplemented
 }
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	s3cli, err := region.GetS3Client()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetS3Client")
+	}
+	output, err := s3cli.ListBuckets(&s3.ListBucketsInput{})
+	if err != nil {
+		return nil, errors.Wrap(err, "ListBuckets")
+	}
+	ret := make([]cloudprovider.ICloudBucket, 0)
+	for _, bInfo := range output.Buckets {
+		input := &s3.GetBucketLocationInput{}
+		input.Bucket = bInfo.Name
+		output, err := s3cli.GetBucketLocation(input)
+		if err != nil {
+			log.Errorf("s3cli.GetBucketLocation error %s", err)
+			continue
+		}
+		if *output.LocationConstraint != region.GetId() {
+			continue
+		}
+		b := SBucket{
+			region:       region,
+			Name:         *bInfo.Name,
+			Location:     region.GetId(),
+			CreationDate: *bInfo.CreationDate,
+		}
+		ret = append(ret, &b)
+	}
+	return ret, nil
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, acl string) error {
+	s3cli, err := region.GetS3Client()
+	if err != nil {
+		return errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.CreateBucketInput{}
+	input.Bucket = &name
+	input.CreateBucketConfiguration = &s3.CreateBucketConfiguration{}
+	location := region.GetId()
+	input.CreateBucketConfiguration.LocationConstraint = &location
+	_, err = s3cli.CreateBucket(input)
+	if err != nil {
+		return errors.Wrap(err, "CreateBucket")
+	}
+	// if *output.Location != region.GetId() {
+	// 	log.Warningf("Request location %s != got locaiton %s", region.GetId(), *output.Location)
+	// }
+	return nil
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	s3cli, err := region.GetS3Client()
+	if err != nil {
+		return errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.DeleteBucketInput{}
+	input.Bucket = &name
+	_, err = s3cli.DeleteBucket(input)
+	if err != nil {
+		if strings.Index(err.Error(), "NoSuchBucket") >= 0 {
+			return nil
+		}
+		return errors.Wrap(err, "DeleteBucket")
+	}
+	return nil
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	s3cli, err := region.GetS3Client()
+	if err != nil {
+		return false, errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.HeadBucketInput{}
+	input.Bucket = &name
+	_, err = s3cli.HeadBucket(input)
+	if err != nil {
+		return false, errors.Wrap(err, "IsBucketExist")
+	}
+	return true, nil
+}
+
+func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
+	return cloudprovider.GetIBucketByName(region, name)
+}
+
+func (region *SRegion) getBaseEndpoint() string {
+	if len(region.RegionEndpoint) > 4 {
+		return region.RegionEndpoint[4:]
+	}
+	return ""
+}
+
+func (region *SRegion) getS3Endpoint() string {
+	base := region.getBaseEndpoint()
+	if len(base) > 0 {
+		return "s3." + base
+	}
+	return ""
+}
+
+func (region *SRegion) getEc2Endpoint() string {
+	return region.RegionEndpoint
+}

pkg/util/aws/shell/s3.go

@@ -16,9 +16,12 @@ package shell
 
 import (
 	"fmt"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"os"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+
 	"yunion.io/x/onecloud/pkg/util/aws"
+	"yunion.io/x/onecloud/pkg/util/printutils"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 	"yunion.io/x/onecloud/pkg/util/streamutils"
 )
@@ -27,15 +30,34 @@ func init() {
 	type S3BucketListOptions struct {
 	}
 	shellutils.R(&S3BucketListOptions{}, "s3-list", "List all buckets", func(cli *aws.SRegion, args *S3BucketListOptions) error {
-		s3cli, err := cli.GetS3Client()
+		buckets, err := cli.GetIBuckets()
 		if err != nil {
 			return err
 		}
-		output, err := s3cli.ListBuckets(&s3.ListBucketsInput{})
+		printList(buckets, 0, 0, 0, nil)
+		printutils.PrintGetterList(buckets, nil)
+		return nil
+	})
+
+	type S3CreateBucketOptions struct {
+		BUCKET string `help:"bucket name"`
+	}
+	shellutils.R(&S3CreateBucketOptions{}, "s3-create-bucket", "Create a bucket", func(cli *aws.SRegion, args *S3CreateBucketOptions) error {
+		err := cli.CreateIBucket(args.BUCKET, "", "")
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type S3DeleteBucketOptions struct {
+		BUCKET string `help:"bucket name"`
+	}
+	shellutils.R(&S3DeleteBucketOptions{}, "s3-delete-bucket", "Delete a bucket", func(cli *aws.SRegion, args *S3DeleteBucketOptions) error {
+		err := cli.DeleteIBucket(args.BUCKET)
 		if err != nil {
 			return err
 		}
-		printList(output.Buckets, 0, 0, 0, nil)
 		return nil
 	})
 

pkg/util/azure/region.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/secrules"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -608,3 +609,48 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala
 func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) {
 	return nil, cloudprovider.ErrNotImplemented
 }
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	accounts, err := region.GetStorageAccounts()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetStorageAccounts")
+	}
+	ret := make([]cloudprovider.ICloudBucket, len(accounts))
+	for i := range accounts {
+		ret[i] = &accounts[i]
+	}
+	return ret, nil
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, acl string) error {
+	_, err := region.createStorageAccount(name, storageClassStr)
+	if err != nil {
+		return errors.Wrap(err, "region.createStorageAccount")
+	}
+	return nil
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	accounts, err := region.GetStorageAccounts()
+	if err != nil {
+		return errors.Wrap(err, "GetStorageAccounts")
+	}
+	for i := range accounts {
+		if accounts[i].Name == name {
+			err = region.client.Delete(accounts[i].ID)
+			if err != nil {
+				return errors.Wrap(err, "region.client.Delete")
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	return region.checkStorageAccountNameExist(name)
+}
+
+func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
+	return cloudprovider.GetIBucketByName(region, name)
+}

pkg/util/azure/shell/bucket.go

@@ -0,0 +1,55 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/util/azure"
+	"yunion.io/x/onecloud/pkg/util/printutils"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type BucketListOptions struct {
+	}
+	shellutils.R(&BucketListOptions{}, "bucket-list", "List buckets", func(cli *azure.SRegion, args *BucketListOptions) error {
+		buckets, err := cli.GetIBuckets()
+		if err != nil {
+			return err
+		}
+		printutils.PrintGetterList(buckets, nil)
+		return nil
+	})
+
+	type BucketCreateOptions struct {
+		BUCKET       string `help:"bucket name"`
+		STORAGECLASS string `help:"storage class"`
+	}
+	shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli *azure.SRegion, args *BucketCreateOptions) error {
+		err := cli.CreateIBucket(args.BUCKET, args.STORAGECLASS, "")
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type BucketDeleteOptions struct {
+		BUCKET string `help:"bucket name"`
+	}
+	shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete a bucket", func(cli *azure.SRegion, args *BucketDeleteOptions) error {
+		err := cli.DeleteIBucket(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type BucketShowOptions struct {
+		BUCKET string `help:"bucket name"`
+	}
+	shellutils.R(&BucketShowOptions{}, "bucket-show", "Show a bucket", func(cli *azure.SRegion, args *BucketShowOptions) error {
+		bucket, err := cli.GetIBucketByName(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		printObject(bucket)
+		return nil
+	})
+}

pkg/util/azure/shell/storageaccount.go

@@ -23,14 +23,12 @@ import (
 
 func init() {
 	type StorageAccountListOptions struct {
-		Limit  int `help:"page size"`
-		Offset int `help:"page offset"`
 	}
 	shellutils.R(&StorageAccountListOptions{}, "storage-account-list", "List storage account", func(cli *azure.SRegion, args *StorageAccountListOptions) error {
 		if accounts, err := cli.GetStorageAccounts(); err != nil {
 			return err
 		} else {
-			printList(accounts, len(accounts), args.Offset, args.Limit, []string{})
+			printList(accounts, len(accounts), 0, 0, []string{})
 			return nil
 		}
 	})
@@ -152,4 +150,14 @@ func init() {
 		return nil
 	})
 
+	type SStorageAccountSkuOptions struct {
+	}
+	shellutils.R(&SStorageAccountSkuOptions{}, "storage-account-skus", "List skus of storage account", func(cli *azure.SRegion, args *SStorageAccountSkuOptions) error {
+		skus, err := cli.GetStorageAccountSkus()
+		if err != nil {
+			return err
+		}
+		printList(skus, 0, 0, 0, nil)
+		return nil
+	})
 }

pkg/util/azure/storageaccount.go

@@ -26,8 +26,10 @@ import (
 	"github.com/Microsoft/azure-vhd-utils/vhdcore/diskstream"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/pkg/utils"
 )
 
 type SContainer struct {
@@ -35,7 +37,7 @@ type SContainer struct {
 	Name           string
 }
 
-type Sku struct {
+type SSku struct {
 	Name         string
 	Tier         string
 	Kind         string
@@ -48,7 +50,7 @@ type Identity struct {
 	Type        string
 }
 
-type PrimaryEndpoints struct {
+type SStorageEndpoints struct {
 	Blob  string
 	Queue string
 	Table string
@@ -60,10 +62,11 @@ type AccountProperties struct {
 	ClassicStorageProperties
 
 	//normal
-	PrimaryEndpoints  PrimaryEndpoints `json:"primaryEndpoints,omitempty"`
-	ProvisioningState string
-	PrimaryLocation   string
-	SecondaryLocation string
+	PrimaryEndpoints   SStorageEndpoints `json:"primaryEndpoints,omitempty"`
+	ProvisioningState  string
+	PrimaryLocation    string
+	SecondaryEndpoints SStorageEndpoints `json:"secondaryEndpoints,omitempty"`
+	SecondaryLocation  string
 	//CreationTime           time.Time
 	AccessTier               string `json:"accessTier,omitempty"`
 	EnableHTTPSTrafficOnly   *bool  `json:"supportsHttpsTrafficOnly,omitempty"`
@@ -74,7 +77,7 @@ type AccountProperties struct {
 type SStorageAccount struct {
 	region     *SRegion
 	accountKey string
-	Sku        Sku    `json:"sku,omitempty"`
+	Sku        SSku   `json:"sku,omitempty"`
 	Kind       string `json:"kind,omitempty"`
 	Identity   *Identity
 	Properties AccountProperties
@@ -125,6 +128,119 @@ func (self *SRegion) GetUniqStorageAccountName() string {
 	}
 }
 
+type sStorageAccountCheckNameAvailabilityInput struct {
+	Name string
+	Type string
+}
+
+type sStorageAccountCheckNameAvailabilityOutput struct {
+	NameAvailable bool   `json:"nameAvailable"`
+	Reason        string `json:"reason"`
+	Message       string `json:"message"`
+}
+
+func (self *SRegion) checkStorageAccountNameExist(name string) (bool, error) {
+	url := fmt.Sprintf("/subscriptions/%s/providers/Microsoft.Storage/checkNameAvailability?api-version=2019-04-01", self.client.subscriptionId)
+	body := jsonutils.Marshal(sStorageAccountCheckNameAvailabilityInput{
+		Name: name,
+		Type: "Microsoft.Storage/storageAccounts",
+	})
+	resp, err := self.client.jsonRequest("POST", url, body.String())
+	if err != nil {
+		return false, errors.Wrap(err, "jsonRequest")
+	}
+	output := sStorageAccountCheckNameAvailabilityOutput{}
+	err = resp.Unmarshal(&output)
+	if err != nil {
+		return false, errors.Wrap(err, "Unmarshal")
+	}
+	if output.NameAvailable {
+		return false, nil
+	} else {
+		if output.Reason == "AlreadyExists" {
+			return true, nil
+		} else {
+			return false, errors.Error(output.Reason)
+		}
+	}
+}
+
+type SStorageAccountSku struct {
+	ResourceType string   `json:"resourceType"`
+	Name         string   `json:"name"`
+	Tier         string   `json:"tier"`
+	Kind         string   `json:"kind"`
+	Locations    []string `json:"locations"`
+	Capabilities []struct {
+		Name  string `json:"name"`
+		Value string `json:"value"`
+	} `json:"capabilities"`
+	Restrictions []struct {
+		Type       string   `json:"type"`
+		Values     []string `json:"values"`
+		ReasonCode string   `json:"reasonCode"`
+	} `json:"restrictions"`
+}
+
+func (self *SRegion) GetStorageAccountSkus() ([]SStorageAccountSku, error) {
+	skus := make([]SStorageAccountSku, 0)
+	err := self.client.List("providers/Microsoft.Storage/skus?api-version=2019-04-01", &skus)
+	if err != nil {
+		return nil, errors.Wrap(err, "List")
+	}
+	ret := make([]SStorageAccountSku, 0)
+	for i := range skus {
+		if utils.IsInStringArray(self.GetId(), skus[i].Locations) {
+			ret = append(ret, skus[i])
+		}
+	}
+	return ret, nil
+}
+
+func (self *SRegion) getStorageAccountSkuByName(name string) (*SStorageAccountSku, error) {
+	skus, err := self.GetStorageAccountSkus()
+	if err != nil {
+		return nil, errors.Wrap(err, "getStorageAccountSkus")
+	}
+	for _, kind := range []string{
+		"StorageV2",
+		"Storage",
+	} {
+		for i := range skus {
+			if skus[i].Name == name && skus[i].Kind == kind {
+				return &skus[i], nil
+			}
+		}
+	}
+	return nil, cloudprovider.ErrNotFound
+}
+
+func (self *SRegion) createStorageAccount(name string, skuName string) (*SStorageAccount, error) {
+	sku, err := self.getStorageAccountSkuByName(skuName)
+	if err != nil {
+		return nil, errors.Wrap(err, "getStorageAccountSkuByName")
+	}
+	stoargeaccount := SStorageAccount{
+		region: self,
+		Sku: SSku{
+			Name: sku.Name,
+		},
+		Location: self.Name,
+		Kind:     "Storage",
+		Properties: AccountProperties{
+			IsHnsEnabled:             true,
+			AzureFilesAadIntegration: true,
+		},
+		Name: name,
+		Type: "Microsoft.Storage/storageAccounts",
+	}
+	err = self.client.Create(jsonutils.Marshal(stoargeaccount), &stoargeaccount)
+	if err != nil {
+		return nil, errors.Wrap(err, "Create")
+	}
+	return &stoargeaccount, nil
+}
+
 func (self *SRegion) CreateStorageAccount(storageAccount string) (*SStorageAccount, error) {
 	account, err := self.getStorageAccountID(storageAccount)
 	if err == nil {
@@ -134,7 +250,7 @@ func (self *SRegion) CreateStorageAccount(storageAccount string) (*SStorageAccou
 		uniqName := self.GetUniqStorageAccountName()
 		stoargeaccount := SStorageAccount{
 			region: self,
-			Sku: Sku{
+			Sku: SSku{
 				Name: "Standard_GRS",
 			},
 			Location: self.Name,
@@ -430,3 +546,81 @@ func (self *SStorageAccount) UploadFile(containerName string, filePath string) (
 	}
 	return container.UploadFile(filePath)
 }
+
+func (b *SStorageAccount) GetProjectId() string {
+	return ""
+}
+
+func (b *SStorageAccount) GetGlobalId() string {
+	return b.Name
+}
+
+func (b *SStorageAccount) GetName() string {
+	return b.Name
+}
+
+func (b *SStorageAccount) GetLocation() string {
+	return b.Location
+}
+
+func (b *SStorageAccount) GetIRegion() cloudprovider.ICloudRegion {
+	return b.region
+}
+
+func (b *SStorageAccount) GetCreateAt() time.Time {
+	return time.Time{}
+}
+
+func (b *SStorageAccount) GetStorageClass() string {
+	return b.Sku.Tier
+}
+
+func (b *SStorageAccount) GetAcl() string {
+	return ""
+}
+
+func getDesc(prefix, name string) string {
+	if len(prefix) > 0 {
+		return prefix + "-" + name
+	} else {
+		return name
+	}
+}
+
+func (ep SStorageEndpoints) getUrls(prefix string) []cloudprovider.SBucketAccessUrl {
+	ret := make([]cloudprovider.SBucketAccessUrl, 0)
+	if len(ep.Blob) > 0 {
+		ret = append(ret, cloudprovider.SBucketAccessUrl{
+			Url:         ep.Blob,
+			Description: getDesc(prefix, "blob"),
+		})
+	}
+	if len(ep.Queue) > 0 {
+		ret = append(ret, cloudprovider.SBucketAccessUrl{
+			Url:         ep.Queue,
+			Description: getDesc(prefix, "queue"),
+		})
+	}
+	if len(ep.Table) > 0 {
+		ret = append(ret, cloudprovider.SBucketAccessUrl{
+			Url:         ep.Table,
+			Description: getDesc(prefix, "table"),
+		})
+	}
+	if len(ep.File) > 0 {
+		ret = append(ret, cloudprovider.SBucketAccessUrl{
+			Url:         ep.File,
+			Description: getDesc(prefix, "file"),
+		})
+	}
+	return ret
+}
+
+func (b *SStorageAccount) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	primary := b.Properties.PrimaryEndpoints.getUrls("")
+	secondary := b.Properties.SecondaryEndpoints.getUrls("secondary")
+	if len(secondary) > 0 {
+		primary = append(primary, secondary...)
+	}
+	return primary
+}

pkg/util/esxi/manager.go

@@ -45,6 +45,7 @@ const (
 type SESXiClient struct {
 	cloudprovider.SFakeOnPremiseRegion
 	multicloud.SRegion
+	multicloud.SNoObjectStorageRegion
 
 	providerId   string
 	providerName string

pkg/util/httputils/httputils.go

@@ -132,9 +132,10 @@ func GetClient(insecure bool, timeout time.Duration) *http.Client {
 		DialContext: (&net.Dialer{
 			Timeout: 5 * time.Second,
 		}).DialContext,
-		IdleConnTimeout:     5 * time.Second,
-		TLSHandshakeTimeout: 10 * time.Second,
-		TLSClientConfig:     &tls.Config{InsecureSkipVerify: insecure},
+		IdleConnTimeout:       5 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       &tls.Config{InsecureSkipVerify: insecure},
 	}
 	return &http.Client{
 		Transport: tr,
@@ -223,6 +224,26 @@ func JSONRequest(client *http.Client, ctx context.Context, method THttpMethod, u
 	return ParseJSONResponse(resp, err, debug)
 }
 
+// closeResponse close non nil response with any response Body.
+// convenient wrapper to drain any remaining data on response body.
+//
+// Subsequently this allows golang http RoundTripper
+// to re-use the same connection for future requests.
+func closeResponse(resp *http.Response) {
+	// Callers should close resp.Body when done reading from it.
+	// If resp.Body is not closed, the Client's underlying RoundTripper
+	// (typically Transport) may not be able to re-use a persistent TCP
+	// connection to the server for a subsequent "keep-alive" request.
+	if resp != nil && resp.Body != nil {
+		// Drain any remaining Body and then close the connection.
+		// Without this closing connection would disallow re-using
+		// the same connection for future uses.
+		//  - http://stackoverflow.com/a/17961593/4465767
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}
+}
+
 func ParseJSONResponse(resp *http.Response, err error, debug bool) (http.Header, jsonutils.JSONObject, error) {
 	if err != nil {
 		ce := JSONClientError{}
@@ -230,7 +251,7 @@ func ParseJSONResponse(resp *http.Response, err error, debug bool) (http.Header,
 		ce.Details = err.Error()
 		return nil, nil, &ce
 	}
-	defer resp.Body.Close()
+	defer closeResponse(resp)
 	if debug {
 		if resp.StatusCode < 300 {
 			green("Status:", resp.StatusCode)

pkg/util/huawei/bucket.go

@@ -0,0 +1,75 @@
+package huawei
+
+import (
+	"fmt"
+	"time"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SBucket struct {
+	region *SRegion
+
+	Name         string
+	Location     string
+	CreationDate time.Time
+
+	StorageClass string
+	Acl          string
+
+	Size         int64
+	ObjectNumber int
+}
+
+func (b *SBucket) GetProjectId() string {
+	return ""
+}
+
+func (b *SBucket) GetGlobalId() string {
+	return b.Name
+}
+
+func (b *SBucket) GetName() string {
+	return b.Name
+}
+
+func (b *SBucket) GetLocation() string {
+	return b.Location
+}
+
+func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion {
+	return b.region
+}
+
+func (b *SBucket) GetCreateAt() time.Time {
+	return b.CreationDate
+}
+
+func (b *SBucket) GetStorageClass() string {
+	return b.StorageClass
+}
+
+func (b *SBucket) GetAcl() string {
+	return b.Acl
+}
+
+func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	return []cloudprovider.SBucketAccessUrl{
+		{
+			Url:         fmt.Sprintf("https://%s.%s", b.Name, b.region.getOBSEndpoint()),
+			Description: "bucket url",
+		},
+		{
+			Url:         fmt.Sprintf("https://%s/%s", b.region.getOBSEndpoint(), b.Name),
+			Description: "obs url",
+		},
+	}
+}
+
+func (b *SBucket) GetSizeByte() int64 {
+	return b.Size
+}
+
+func (b *SBucket) GetObjectNumber() int {
+	return b.ObjectNumber
+}

pkg/util/huawei/region.go

@@ -22,6 +22,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/secrules"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -86,9 +87,13 @@ func (self *SRegion) getECSClient() (*client.Client, error) {
 	return self.ecsClient, err
 }
 
+func (self *SRegion) getOBSEndpoint() string {
+	return fmt.Sprintf("obs.%s.myhuaweicloud.com", self.GetId())
+}
+
 func (self *SRegion) getOBSClient() (*obs.ObsClient, error) {
 	if self.obsClient == nil {
-		endpoint := fmt.Sprintf("obs.%s.myhuaweicloud.com", self.GetId())
+		endpoint := self.getOBSEndpoint()
 		obsClient, err := obs.New(self.client.accessKey, self.client.secret, endpoint)
 		if err != nil {
 			return nil, err
@@ -674,3 +679,111 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala
 func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) {
 	return nil, cloudprovider.ErrNotImplemented
 }
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	obsClient, err := region.getOBSClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.getOBSClient")
+	}
+	input := &obs.ListBucketsInput{}
+	input.QueryLocation = true
+	output, err := obsClient.ListBuckets(input)
+	if err != nil {
+		return nil, errors.Wrap(err, "ListBuckets")
+	}
+	ret := make([]cloudprovider.ICloudBucket, len(output.Buckets))
+	for i, bInfo := range output.Buckets {
+		b := SBucket{
+			region: region,
+
+			Name:         bInfo.Name,
+			Location:     bInfo.Location,
+			CreationDate: bInfo.CreationDate,
+		}
+		ret[i] = &b
+	}
+	return ret, nil
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
+	obsClient, err := region.getOBSClient()
+	if err != nil {
+		return errors.Wrap(err, "region.getOBSClient")
+	}
+	input := &obs.CreateBucketInput{}
+	input.Bucket = name
+	input.Location = region.GetId()
+	if len(aclStr) > 0 {
+		if strings.EqualFold(aclStr, string(obs.AclPrivate)) {
+			input.ACL = obs.AclPrivate
+		} else if strings.EqualFold(aclStr, string(obs.AclPublicRead)) {
+			input.ACL = obs.AclPublicRead
+		} else if strings.EqualFold(aclStr, string(obs.AclPublicReadWrite)) {
+			input.ACL = obs.AclPublicReadWrite
+		} else {
+			return errors.Error("unsupported acl")
+		}
+	}
+	if len(storageClassStr) > 0 {
+		if strings.EqualFold(storageClassStr, string(obs.StorageClassStandard)) {
+			input.StorageClass = obs.StorageClassStandard
+		} else if strings.EqualFold(storageClassStr, string(obs.StorageClassWarm)) {
+			input.StorageClass = obs.StorageClassWarm
+		} else if strings.EqualFold(storageClassStr, string(obs.StorageClassCold)) {
+			input.StorageClass = obs.StorageClassCold
+		} else {
+			return errors.Error("unsupported storageClass")
+		}
+	}
+	_, err = obsClient.CreateBucket(input)
+	if err != nil {
+		return errors.Wrap(err, "obsClient.CreateBucket")
+	}
+	return nil
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	obsClient, err := region.getOBSClient()
+	if err != nil {
+		return errors.Wrap(err, "region.getOBSClient")
+	}
+	_, err = obsClient.DeleteBucket(name)
+	if err != nil {
+		if strings.Index(err.Error(), "Code=NoSuchBucket") >= 0 {
+			return nil
+		}
+		return errors.Wrap(err, "DeleteBucket")
+	}
+	return nil
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	obsClient, err := region.getOBSClient()
+	if err != nil {
+		return false, errors.Wrap(err, "region.getOBSClient")
+	}
+	_, err = obsClient.HeadBucket(name)
+	if err != nil {
+		return false, errors.Wrap(err, "HeadBucket")
+	}
+	return true, nil
+}
+
+func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
+	obsClient, err := region.getOBSClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.getOBSClient")
+	}
+	info, err := obsClient.GetBucketStorageInfo(name)
+	if err != nil {
+		return nil, errors.Wrap(err, "obsClient.GetBucketStorageInfo")
+	}
+	b := SBucket{
+		region: region,
+
+		Name:         name,
+		Size:         info.Size,
+		ObjectNumber: info.ObjectNumber,
+	}
+	return &b, nil
+}

pkg/util/huawei/shell/obs.go

@@ -0,0 +1,61 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/util/huawei"
+	"yunion.io/x/onecloud/pkg/util/printutils"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	type ObsBucketListOptions struct {
+	}
+	shellutils.R(&ObsBucketListOptions{}, "obs-list", "List all buckets", func(cli *huawei.SRegion, args *ObsBucketListOptions) error {
+		buckets, err := cli.GetIBuckets()
+		if err != nil {
+			return err
+		}
+		printList(buckets, 0, 0, 0, nil)
+		return nil
+	})
+	shellutils.R(&ObsBucketListOptions{}, "bucket-list", "List all buckets", func(cli *huawei.SRegion, args *ObsBucketListOptions) error {
+		buckets, err := cli.GetIBuckets()
+		if err != nil {
+			return err
+		}
+		printutils.PrintGetterList(buckets, nil)
+		return nil
+	})
+
+	type ObsBucketShowOptions struct {
+		BUCKET string `help:"bucket name to show"`
+	}
+	shellutils.R(&ObsBucketShowOptions{}, "obs-show", "Show bucket detail", func(cli *huawei.SRegion, args *ObsBucketShowOptions) error {
+		bucket, err := cli.GetIBucketByName(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		printObject(bucket)
+		return nil
+	})
+
+	type ObsBucketCreateOptions struct {
+		BUCKET       string `help:"bucket name to show"`
+		StorageClass string `help:"storage class"`
+		Acl          string `help:"acl"`
+	}
+	shellutils.R(&ObsBucketCreateOptions{}, "obs-create", "Create new OBS bucket", func(cli *huawei.SRegion, args *ObsBucketCreateOptions) error {
+		err := cli.CreateIBucket(args.BUCKET, args.StorageClass, args.Acl)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	shellutils.R(&ObsBucketShowOptions{}, "obs-delete", "Delete OBS bucket", func(cli *huawei.SRegion, args *ObsBucketShowOptions) error {
+		err := cli.DeleteIBucket(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}

pkg/util/openstack/region.go

@@ -482,3 +482,23 @@ func (region *SRegion) GetISkus(zoneId string) ([]cloudprovider.ICloudSku, error
 func (region *SRegion) GetISkuById(skuId string) (cloudprovider.ICloudSku, error) {
 	return region.GetFlavor(skuId)
 }
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, acl string) error {
+	return cloudprovider.ErrNotImplemented
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	return cloudprovider.ErrNotImplemented
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	return false, cloudprovider.ErrNotImplemented
+}
+
+func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}

pkg/util/qcloud/bucket.go

@@ -0,0 +1,63 @@
+package qcloud
+
+import (
+	"fmt"
+	"time"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SBucket struct {
+	region *SRegion
+
+	Name       string
+	FullName   string
+	Location   string
+	CreateDate time.Time
+	Acl        string
+}
+
+func (b *SBucket) GetProjectId() string {
+	return ""
+}
+
+func (b *SBucket) GetGlobalId() string {
+	return b.Name
+}
+
+func (b *SBucket) GetName() string {
+	return b.Name
+}
+
+func (b *SBucket) GetLocation() string {
+	return b.Location
+}
+
+func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion {
+	return b.region
+}
+
+func (b *SBucket) GetCreateAt() time.Time {
+	return b.CreateDate
+}
+
+func (b *SBucket) GetStorageClass() string {
+	return ""
+}
+
+func (b *SBucket) GetAcl() string {
+	return b.Acl
+}
+
+func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	return []cloudprovider.SBucketAccessUrl{
+		{
+			Url:         fmt.Sprintf("https://%s.%s", b.FullName, b.region.getCosEndpoint()),
+			Description: "bucket domain",
+		},
+		{
+			Url:         fmt.Sprintf("https://%s/%s", b.region.getCosEndpoint(), b.FullName),
+			Description: "cos domain",
+		},
+	}
+}

pkg/util/qcloud/qcloud.go

@@ -20,16 +20,22 @@ import (
 	"strings"
 	"time"
 
+	"yunion.io/x/pkg/utils"
+
+	coslib "github.com/nelsonken/cos-go-sdk-v5/cos"
 	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
-	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors"
+	sdkerrors "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors"
 	tchttp "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http"
 	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/pkg/util/timeutils"
 )
 
 const (
@@ -179,7 +185,7 @@ func (r *vpc2017JsonResponse) ParseErrorFromHTTPResponse(body []byte) (err error
 		return
 	}
 	if resp.Code != 0 {
-		return errors.NewTencentCloudSDKError(resp.CodeDesc, resp.Message, "")
+		return sdkerrors.NewTencentCloudSDKError(resp.CodeDesc, resp.Message, "")
 	}
 
 	return nil
@@ -212,7 +218,7 @@ func (r *wssJsonResponse) ParseErrorFromHTTPResponse(body []byte) (err error) {
 		return
 	}
 	if resp.Code != 0 {
-		return errors.NewTencentCloudSDKError(resp.CodeDesc, resp.Message, "")
+		return sdkerrors.NewTencentCloudSDKError(resp.CodeDesc, resp.Message, "")
 	}
 
 	return nil
@@ -234,7 +240,7 @@ func (r *lbJsonResponse) ParseErrorFromHTTPResponse(body []byte) (err error) {
 		return
 	}
 	if resp.Code != 0 {
-		return errors.NewTencentCloudSDKError(resp.CodeDesc, resp.Message, "")
+		return sdkerrors.NewTencentCloudSDKError(resp.CodeDesc, resp.Message, "")
 	}
 
 	// hook 由于目前只能从这个方法中拿到原始的body.这里将原始body hook 到 Response
@@ -581,3 +587,103 @@ func (client *SQcloudClient) GetIProjects() ([]cloudprovider.ICloudProject, erro
 	}
 	return iprojects, nil
 }
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	ctx := appctx.Background
+	cos, err := region.GetCosClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetCosClient")
+	}
+	result, err := cos.GetBucketList(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetBucketList")
+	}
+	ret := make([]cloudprovider.ICloudBucket, 0)
+	for i := range result.Buckets.Bucket {
+		bInfo := result.Buckets.Bucket[i]
+		// ignore buckets not belong to this region
+		if bInfo.Location != region.GetId() {
+			continue
+		}
+		createAt, _ := timeutils.ParseTimeStr(bInfo.CreateDate)
+		name := bInfo.Name
+		// name = name[:len(name)-len(result.Owner.ID)-1]
+		name = name[:strings.LastIndexByte(name, '-')]
+		b := SBucket{
+			region: region,
+
+			Name:       name,
+			FullName:   bInfo.Name,
+			Location:   bInfo.Location,
+			CreateDate: createAt,
+		}
+		ret = append(ret, &b)
+	}
+	return ret, nil
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
+	ctx := appctx.Background
+	cos, err := region.GetCosClient()
+	if err != nil {
+		return errors.Wrap(err, "GetCosClient")
+	}
+	acl := &coslib.AccessControl{}
+	if len(aclStr) > 0 {
+		if utils.IsInStringArray(aclStr, []string{
+			"private", "public-read", "public-read-write", "authenticated-read",
+		}) {
+			acl.ACL = aclStr
+		} else {
+			return errors.Error("invalid acl")
+		}
+	}
+	err = cos.CreateBucket(ctx, name, acl)
+	if err != nil {
+		return errors.Wrap(err, "oss.CreateBucket")
+	}
+	return nil
+}
+
+func cosHttpCode(err error) int {
+	if httpErr, ok := err.(coslib.HTTPError); ok {
+		return httpErr.Code
+	}
+	if httpErr, ok := err.(*coslib.HTTPError); ok {
+		return httpErr.Code
+	}
+	return -1
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	ctx := appctx.Background
+	cos, err := region.GetCosClient()
+	if err != nil {
+		return errors.Wrap(err, "GetCosClient")
+	}
+	err = cos.DeleteBucket(ctx, name)
+	if err != nil {
+		if cosHttpCode(err) == 404 {
+			return nil
+		}
+		return errors.Wrap(err, "DeleteBucket")
+	}
+	return nil
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	ctx := appctx.Background
+	cos, err := region.GetCosClient()
+	if err != nil {
+		return false, errors.Wrap(err, "GetCosClient")
+	}
+	err = cos.BucketExists(ctx, name)
+	if err != nil {
+		return false, errors.Wrap(err, "BucketExists")
+	}
+	return true, nil
+}
+
+func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
+	return cloudprovider.GetIBucketByName(region, name)
+}

pkg/util/qcloud/region.go

@@ -816,3 +816,7 @@ func (self *SRegion) GetInstanceStatus(instanceId string) (string, error) {
 func (self *SRegion) QueryAccountBalance() (*SAccountBalance, error) {
 	return self.client.QueryAccountBalance()
 }
+
+func (self *SRegion) getCosEndpoint() string {
+	return fmt.Sprintf("cos.%s.myqcloud.com", self.GetId())
+}

pkg/util/qcloud/shell/cos.go

@@ -20,6 +20,7 @@ import (
 
 	coslib "github.com/nelsonken/cos-go-sdk-v5/cos"
 
+	"yunion.io/x/onecloud/pkg/util/printutils"
 	"yunion.io/x/onecloud/pkg/util/qcloud"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
@@ -28,15 +29,43 @@ func init() {
 	type CosListOptions struct {
 	}
 	shellutils.R(&CosListOptions{}, "cos-list", "List COS buckets", func(cli *qcloud.SRegion, args *CosListOptions) error {
-		cos, err := cli.GetCosClient()
+		buckets, err := cli.GetIBuckets()
 		if err != nil {
 			return err
 		}
-		result, err := cos.GetBucketList(context.Background())
+		printList(buckets, 0, 0, 0, nil)
+		return nil
+	})
+
+	shellutils.R(&CosListOptions{}, "bucket-list", "List COS buckets", func(cli *qcloud.SRegion, args *CosListOptions) error {
+		buckets, err := cli.GetIBuckets()
+		if err != nil {
+			return err
+		}
+		printutils.PrintGetterList(buckets, nil)
+		return nil
+	})
+
+	type CosCreateBucketOptions struct {
+		BUCKET string `help:"name of bucket to create"`
+		Acl    string `help:"Acl"`
+	}
+	shellutils.R(&CosCreateBucketOptions{}, "cos-create-bucket", "Create a COS bucket", func(cli *qcloud.SRegion, args *CosCreateBucketOptions) error {
+		err := cli.CreateIBucket(args.BUCKET, "", args.Acl)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type CosDeleteBucketOptions struct {
+		BUCKET string `help:"name of bucket to delete"`
+	}
+	shellutils.R(&CosDeleteBucketOptions{}, "cos-delete-bucket", "Delete a COS bucket", func(cli *qcloud.SRegion, args *CosDeleteBucketOptions) error {
+		err := cli.DeleteIBucket(args.BUCKET)
 		if err != nil {
 			return err
 		}
-		printList(result.Buckets.Bucket, len(result.Buckets.Bucket), 0, len(result.Buckets.Bucket), nil)
 		return nil
 	})
 

pkg/util/ucloud/client.go

@@ -151,6 +151,8 @@ func parseUcloudResponse(params SParams, resp jsonutils.JSONObject) (jsonutils.J
 		return nil, e
 	}
 
+	err.Action, _ = params.data.GetString("Action")
+
 	if err.RetCode > 0 {
 		log.Debugf("Ucloud json request err %s", params.PrettyString())
 		return nil, err

pkg/util/ucloud/region.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/secrules"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -634,3 +635,87 @@ func (self *SRegion) syncSecgroupRules(secgroupId string, rules []secrules.Secur
 func (self *SRegion) GetClient() *SUcloudClient {
 	return self.client
 }
+
+func (region *SRegion) listBuckets(name string, offset int, limit int) ([]SBucket, error) {
+	params := NewUcloudParams()
+	if len(name) > 0 {
+		params.Set("BucketName", name)
+	} else {
+		params.Set("Limit", limit)
+		params.Set("Offset", offset)
+	}
+	buckets := make([]SBucket, 0)
+	err := region.DoAction("DescribeBucket", params, &buckets)
+	if err != nil {
+		return nil, errors.Wrap(err, "DoAction DescribeBucket")
+	}
+	return buckets, nil
+}
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	buckets := make([]SBucket, 0)
+	offset := 0
+	limit := 50
+	for {
+		parts, err := region.listBuckets("", offset, limit)
+		if err != nil {
+			return nil, errors.Wrap(err, "region.listBuckets")
+		}
+		if len(parts) > 0 {
+			buckets = append(buckets, parts...)
+		}
+		if len(parts) < limit {
+			break
+		} else {
+			offset += limit
+		}
+	}
+	ret := make([]cloudprovider.ICloudBucket, len(buckets))
+	for i := range buckets {
+		buckets[i].region = region
+		ret[i] = &buckets[i]
+	}
+	return ret, nil
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
+	if aclStr != "private" && aclStr != "public" {
+		return errors.Error("invalid acl")
+	}
+	return region.CreateBucket(name, aclStr)
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	err := region.DeleteBucket(name)
+	if err != nil {
+		if strings.Index(err.Error(), "bucket not found") >= 0 {
+			return nil
+		}
+		return errors.Wrap(err, "region.DeleteBucket")
+	}
+	return nil
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	parts, err := region.listBuckets(name, 0, 1)
+	if err != nil {
+		return false, errors.Wrap(err, "region.listBuckets")
+	}
+	if len(parts) == 0 {
+		return false, cloudprovider.ErrNotFound
+	}
+	return true, nil
+}
+
+func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) {
+	parts, err := region.listBuckets(name, 0, 1)
+	if err != nil {
+		return nil, errors.Wrap(err, "region.listBuckets")
+	}
+	if len(parts) == 0 {
+		return nil, cloudprovider.ErrNotFound
+	}
+	bucket := parts[0]
+	bucket.region = region
+	return &bucket, nil
+}

pkg/util/ucloud/shell/ufile.go

@@ -0,0 +1,43 @@
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/util/printutils"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+	"yunion.io/x/onecloud/pkg/util/ucloud"
+)
+
+func init() {
+	type UFileBucketListOptions struct {
+	}
+	shellutils.R(&UFileBucketListOptions{}, "bucket-list", "List buckets", func(cli *ucloud.SRegion, args *UFileBucketListOptions) error {
+		buckets, err := cli.GetIBuckets()
+		if err != nil {
+			return err
+		}
+		printutils.PrintGetterList(buckets, nil)
+		return nil
+	})
+
+	type UFileBucketCreateOptions struct {
+		BUCKET string `help:"Name of bucket"`
+		Acl    string `help:"Acl" choices:"private|public"`
+	}
+	shellutils.R(&UFileBucketCreateOptions{}, "bucket-create", "create a bucket", func(cli *ucloud.SRegion, args *UFileBucketCreateOptions) error {
+		err := cli.CreateIBucket(args.BUCKET, "", args.Acl)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type UFileBucketDeleteOptions struct {
+		BUCKET string `help:"Name of bucket"`
+	}
+	shellutils.R(&UFileBucketDeleteOptions{}, "bucket-delete", "delete a bucket", func(cli *ucloud.SRegion, args *UFileBucketDeleteOptions) error {
+		err := cli.DeleteIBucket(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+}

pkg/util/ucloud/ufile.go

@@ -19,16 +19,21 @@ import (
 	"crypto/sha1"
 	"encoding/base64"
 	"fmt"
-	"github.com/coredns/coredns/plugin/pkg/log"
 	"io"
 	"net/http"
 	"net/url"
 	"strconv"
 	"time"
+
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 type SBucket struct {
+	region *SRegion
+
 	Domain        Domain   `json:"Domain"`
 	BucketID      string   `json:"BucketId"`
 	Region        string   `json:"Region"`
@@ -143,3 +148,53 @@ func (self *SFile) request(req *http.Request) error {
 
 	return nil
 }
+
+func (b *SBucket) GetProjectId() string {
+	return ""
+}
+
+func (b *SBucket) GetGlobalId() string {
+	return b.BucketName
+}
+
+func (b *SBucket) GetName() string {
+	return b.BucketName
+}
+
+func (b *SBucket) GetLocation() string {
+	return b.Region
+}
+
+func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion {
+	return b.region
+}
+
+func (b *SBucket) GetCreateAt() time.Time {
+	return time.Unix(b.CreateTime, 0)
+}
+
+func (b *SBucket) GetStorageClass() string {
+	return ""
+}
+
+func (b *SBucket) GetAcl() string {
+	return b.Type
+}
+
+func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	ret := make([]cloudprovider.SBucketAccessUrl, 0)
+	regionId := b.region.GetId()
+	// hack, remove trailing digits
+	for len(regionId) > 0 {
+		lastDigit := regionId[len(regionId)-1]
+		if lastDigit >= '0' && lastDigit <= '9' {
+			regionId = regionId[:len(regionId)-1]
+		} else {
+			break
+		}
+	}
+	ret = append(ret, cloudprovider.SBucketAccessUrl{
+		Url: fmt.Sprintf("https://%s.%s.ufileos.com", b.BucketName, regionId),
+	})
+	return ret
+}

pkg/util/ucloud/utils.go

@@ -50,10 +50,10 @@ func doListPart(client *SUcloudClient, action string, params SParams, resultKey
 		return 0, 0, err
 	}
 
-	total, err := ret.Int("TotalCount")
-	if err != nil {
-		log.Debugf("%s TotalCount %s", action, err.Error())
-	}
+	total, _ := ret.Int("TotalCount")
+	// if err != nil {
+	//	log.Debugf("%s TotalCount %s", action, err.Error())
+	//}
 
 	var lst []jsonutils.JSONObject
 	lst, err = ret.GetArray(resultKey)

pkg/util/zstack/region.go

@@ -29,6 +29,8 @@ import (
 
 type SRegion struct {
 	multicloud.SRegion
+	multicloud.SNoObjectStorageRegion
+
 	client *SZStackClient
 
 	Name string

vendor/github.com/go-ini/ini/.gitignore

@@ -0,0 +1,6 @@
+testdata/conf_out.ini
+ini.sublime-project
+ini.sublime-workspace
+testdata/conf_reflect.ini
+.idea
+/.vscode

vendor/github.com/go-ini/ini/.travis.yml

@@ -0,0 +1,18 @@
+sudo: false
+language: go
+go:
+  - 1.6.x
+  - 1.7.x
+  - 1.8.x
+  - 1.9.x
+  - 1.10.x
+  - 1.11.x
+  - 1.12.x
+
+script:
+  - go get golang.org/x/tools/cmd/cover
+  - go get github.com/smartystreets/goconvey
+  - mkdir -p $HOME/gopath/src/gopkg.in
+  - ln -s $HOME/gopath/src/github.com/go-ini/ini $HOME/gopath/src/gopkg.in/ini.v1
+  - cd $HOME/gopath/src/gopkg.in/ini.v1
+  - go test -v -cover -race

vendor/github.com/go-ini/ini/LICENSE

@@ -0,0 +1,191 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object code,
+generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made
+available under the License, as indicated by a copyright notice that is included
+in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative Works
+thereof, that is intentionally submitted to Licensor for inclusion in the Work
+by the copyright owner or by an individual or Legal Entity authorized to submit
+on behalf of the copyright owner. For the purposes of this definition,
+"submitted" means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor for
+the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.
+
+3. Grant of Patent License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable (except as stated in this section) patent license to make, have
+made, use, offer to sell, sell, import, and otherwise transfer the Work, where
+such license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by combination
+of their Contribution(s) with the Work to which such Contribution(s) was
+submitted. If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this License
+for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution.
+
+You may reproduce and distribute copies of the Work or Derivative Works thereof
+in any medium, with or without modifications, and in Source or Object form,
+provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of
+this License; and
+You must cause any modified files to carry prominent notices stating that You
+changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source form
+of the Work, excluding those notices that do not pertain to any part of the
+Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any
+Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents of
+the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a whole,
+provided Your use, reproduction, and distribution of the Work otherwise complies
+with the conditions stated in this License.
+
+5. Submission of Contributions.
+
+Unless You explicitly state otherwise, any Contribution intentionally submitted
+for inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the terms of
+any separate license agreement you may have executed with Licensor regarding
+such Contributions.
+
+6. Trademarks.
+
+This License does not grant permission to use the trade names, trademarks,
+service marks, or product names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty.
+
+Unless required by applicable law or agreed to in writing, Licensor provides the
+Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+including, without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
+solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise of
+permissions under this License.
+
+8. Limitation of Liability.
+
+In no event and under no legal theory, whether in tort (including negligence),
+contract, or otherwise, unless required by applicable law (such as deliberate
+and grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special, incidental,
+or consequential damages of any character arising as a result of this License or
+out of the use or inability to use the Work (including but not limited to
+damages for loss of goodwill, work stoppage, computer failure or malfunction, or
+any and all other commercial damages or losses), even if such Contributor has
+been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability.
+
+While redistributing the Work or Derivative Works thereof, You may choose to
+offer, and charge a fee for, acceptance of support, warranty, indemnity, or
+other liability obligations and/or rights consistent with this License. However,
+in accepting such obligations, You may act only on Your own behalf and on Your
+sole responsibility, not on behalf of any other Contributor, and only if You
+agree to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included on
+the same "printed page" as the copyright notice for easier identification within
+third-party archives.
+
+   Copyright 2014 Unknwon
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/go-ini/ini/Makefile

@@ -0,0 +1,15 @@
+.PHONY: build test bench vet coverage
+
+build: vet bench
+
+test:
+	go test -v -cover -race
+
+bench:
+	go test -v -cover -race -test.bench=. -test.benchmem
+
+vet:
+	go vet
+
+coverage:
+	go test -coverprofile=c.out && go tool cover -html=c.out && rm c.out

vendor/github.com/go-ini/ini/README.md

@@ -0,0 +1,46 @@
+INI [![Build Status](https://travis-ci.org/go-ini/ini.svg?branch=master)](https://travis-ci.org/go-ini/ini) [![Sourcegraph](https://img.shields.io/badge/view%20on-Sourcegraph-brightgreen.svg)](https://sourcegraph.com/github.com/go-ini/ini)
+===
+
+![](https://avatars0.githubusercontent.com/u/10216035?v=3&s=200)
+
+Package ini provides INI file read and write functionality in Go.
+
+## Features
+
+- Load from multiple data sources(`[]byte`, file and `io.ReadCloser`) with overwrites.
+- Read with recursion values.
+- Read with parent-child sections.
+- Read with auto-increment key names.
+- Read with multiple-line values.
+- Read with tons of helper methods.
+- Read and convert values to Go types.
+- Read and **WRITE** comments of sections and keys.
+- Manipulate sections, keys and comments with ease.
+- Keep sections and keys in order as you parse and save.
+
+## Installation
+
+The minimum requirement of Go is **1.6**.
+
+To use a tagged revision:
+
+```sh
+$ go get gopkg.in/ini.v1
+```
+
+To use with latest changes:
+
+```sh
+$ go get github.com/go-ini/ini
+```
+
+Please add `-u` flag to update in the future.
+
+## Getting Help
+
+- [Getting Started](https://ini.unknwon.io/docs/intro/getting_started)
+- [API Documentation](https://gowalker.org/gopkg.in/ini.v1)
+
+## License
+
+This project is under Apache v2 License. See the [LICENSE](LICENSE) file for the full license text.

vendor/github.com/go-ini/ini/error.go

@@ -0,0 +1,34 @@
+// Copyright 2016 Unknwon
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package ini
+
+import (
+	"fmt"
+)
+
+// ErrDelimiterNotFound indicates the error type of no delimiter is found which there should be one.
+type ErrDelimiterNotFound struct {
+	Line string
+}
+
+// IsErrDelimiterNotFound returns true if the given error is an instance of ErrDelimiterNotFound.
+func IsErrDelimiterNotFound(err error) bool {
+	_, ok := err.(ErrDelimiterNotFound)
+	return ok
+}
+
+func (err ErrDelimiterNotFound) Error() string {
+	return fmt.Sprintf("key-value delimiter not found: %s", err.Line)
+}

vendor/github.com/go-ini/ini/file.go

@@ -0,0 +1,418 @@
+// Copyright 2017 Unknwon
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package ini
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"sync"
+)
+
+// File represents a combination of a or more INI file(s) in memory.
+type File struct {
+	options     LoadOptions
+	dataSources []dataSource
+
+	// Should make things safe, but sometimes doesn't matter.
+	BlockMode bool
+	lock      sync.RWMutex
+
+	// To keep data in order.
+	sectionList []string
+	// Actual data is stored here.
+	sections map[string]*Section
+
+	NameMapper
+	ValueMapper
+}
+
+// newFile initializes File object with given data sources.
+func newFile(dataSources []dataSource, opts LoadOptions) *File {
+	if len(opts.KeyValueDelimiters) == 0 {
+		opts.KeyValueDelimiters = "=:"
+	}
+	return &File{
+		BlockMode:   true,
+		dataSources: dataSources,
+		sections:    make(map[string]*Section),
+		sectionList: make([]string, 0, 10),
+		options:     opts,
+	}
+}
+
+// Empty returns an empty file object.
+func Empty() *File {
+	// Ignore error here, we sure our data is good.
+	f, _ := Load([]byte(""))
+	return f
+}
+
+// NewSection creates a new section.
+func (f *File) NewSection(name string) (*Section, error) {
+	if len(name) == 0 {
+		return nil, errors.New("error creating new section: empty section name")
+	} else if f.options.Insensitive && name != DefaultSection {
+		name = strings.ToLower(name)
+	}
+
+	if f.BlockMode {
+		f.lock.Lock()
+		defer f.lock.Unlock()
+	}
+
+	if inSlice(name, f.sectionList) {
+		return f.sections[name], nil
+	}
+
+	f.sectionList = append(f.sectionList, name)
+	f.sections[name] = newSection(f, name)
+	return f.sections[name], nil
+}
+
+// NewRawSection creates a new section with an unparseable body.
+func (f *File) NewRawSection(name, body string) (*Section, error) {
+	section, err := f.NewSection(name)
+	if err != nil {
+		return nil, err
+	}
+
+	section.isRawSection = true
+	section.rawBody = body
+	return section, nil
+}
+
+// NewSections creates a list of sections.
+func (f *File) NewSections(names ...string) (err error) {
+	for _, name := range names {
+		if _, err = f.NewSection(name); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// GetSection returns section by given name.
+func (f *File) GetSection(name string) (*Section, error) {
+	if len(name) == 0 {
+		name = DefaultSection
+	}
+	if f.options.Insensitive {
+		name = strings.ToLower(name)
+	}
+
+	if f.BlockMode {
+		f.lock.RLock()
+		defer f.lock.RUnlock()
+	}
+
+	sec := f.sections[name]
+	if sec == nil {
+		return nil, fmt.Errorf("section '%s' does not exist", name)
+	}
+	return sec, nil
+}
+
+// Section assumes named section exists and returns a zero-value when not.
+func (f *File) Section(name string) *Section {
+	sec, err := f.GetSection(name)
+	if err != nil {
+		// Note: It's OK here because the only possible error is empty section name,
+		// but if it's empty, this piece of code won't be executed.
+		sec, _ = f.NewSection(name)
+		return sec
+	}
+	return sec
+}
+
+// Sections returns a list of Section stored in the current instance.
+func (f *File) Sections() []*Section {
+	if f.BlockMode {
+		f.lock.RLock()
+		defer f.lock.RUnlock()
+	}
+
+	sections := make([]*Section, len(f.sectionList))
+	for i, name := range f.sectionList {
+		sections[i] = f.sections[name]
+	}
+	return sections
+}
+
+// ChildSections returns a list of child sections of given section name.
+func (f *File) ChildSections(name string) []*Section {
+	return f.Section(name).ChildSections()
+}
+
+// SectionStrings returns list of section names.
+func (f *File) SectionStrings() []string {
+	list := make([]string, len(f.sectionList))
+	copy(list, f.sectionList)
+	return list
+}
+
+// DeleteSection deletes a section.
+func (f *File) DeleteSection(name string) {
+	if f.BlockMode {
+		f.lock.Lock()
+		defer f.lock.Unlock()
+	}
+
+	if len(name) == 0 {
+		name = DefaultSection
+	}
+
+	for i, s := range f.sectionList {
+		if s == name {
+			f.sectionList = append(f.sectionList[:i], f.sectionList[i+1:]...)
+			delete(f.sections, name)
+			return
+		}
+	}
+}
+
+func (f *File) reload(s dataSource) error {
+	r, err := s.ReadCloser()
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	return f.parse(r)
+}
+
+// Reload reloads and parses all data sources.
+func (f *File) Reload() (err error) {
+	for _, s := range f.dataSources {
+		if err = f.reload(s); err != nil {
+			// In loose mode, we create an empty default section for nonexistent files.
+			if os.IsNotExist(err) && f.options.Loose {
+				f.parse(bytes.NewBuffer(nil))
+				continue
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+// Append appends one or more data sources and reloads automatically.
+func (f *File) Append(source interface{}, others ...interface{}) error {
+	ds, err := parseDataSource(source)
+	if err != nil {
+		return err
+	}
+	f.dataSources = append(f.dataSources, ds)
+	for _, s := range others {
+		ds, err = parseDataSource(s)
+		if err != nil {
+			return err
+		}
+		f.dataSources = append(f.dataSources, ds)
+	}
+	return f.Reload()
+}
+
+func (f *File) writeToBuffer(indent string) (*bytes.Buffer, error) {
+	equalSign := DefaultFormatLeft + "=" + DefaultFormatRight
+
+	if PrettyFormat || PrettyEqual {
+		equalSign = " = "
+	}
+
+	// Use buffer to make sure target is safe until finish encoding.
+	buf := bytes.NewBuffer(nil)
+	for i, sname := range f.sectionList {
+		sec := f.Section(sname)
+		if len(sec.Comment) > 0 {
+			// Support multiline comments
+			lines := strings.Split(sec.Comment, LineBreak)
+			for i := range lines {
+				if lines[i][0] != '#' && lines[i][0] != ';' {
+					lines[i] = "; " + lines[i]
+				} else {
+					lines[i] = lines[i][:1] + " " + strings.TrimSpace(lines[i][1:])
+				}
+
+				if _, err := buf.WriteString(lines[i] + LineBreak); err != nil {
+					return nil, err
+				}
+			}
+		}
+
+		if i > 0 || DefaultHeader {
+			if _, err := buf.WriteString("[" + sname + "]" + LineBreak); err != nil {
+				return nil, err
+			}
+		} else {
+			// Write nothing if default section is empty
+			if len(sec.keyList) == 0 {
+				continue
+			}
+		}
+
+		if sec.isRawSection {
+			if _, err := buf.WriteString(sec.rawBody); err != nil {
+				return nil, err
+			}
+
+			if PrettySection {
+				// Put a line between sections
+				if _, err := buf.WriteString(LineBreak); err != nil {
+					return nil, err
+				}
+			}
+			continue
+		}
+
+		// Count and generate alignment length and buffer spaces using the
+		// longest key. Keys may be modifed if they contain certain characters so
+		// we need to take that into account in our calculation.
+		alignLength := 0
+		if PrettyFormat {
+			for _, kname := range sec.keyList {
+				keyLength := len(kname)
+				// First case will surround key by ` and second by """
+				if strings.Contains(kname, "\"") || strings.ContainsAny(kname, f.options.KeyValueDelimiters) {
+					keyLength += 2
+				} else if strings.Contains(kname, "`") {
+					keyLength += 6
+				}
+
+				if keyLength > alignLength {
+					alignLength = keyLength
+				}
+			}
+		}
+		alignSpaces := bytes.Repeat([]byte(" "), alignLength)
+
+	KEY_LIST:
+		for _, kname := range sec.keyList {
+			key := sec.Key(kname)
+			if len(key.Comment) > 0 {
+				if len(indent) > 0 && sname != DefaultSection {
+					buf.WriteString(indent)
+				}
+
+				// Support multiline comments
+				lines := strings.Split(key.Comment, LineBreak)
+				for i := range lines {
+					if lines[i][0] != '#' && lines[i][0] != ';' {
+						lines[i] = "; " + strings.TrimSpace(lines[i])
+					} else {
+						lines[i] = lines[i][:1] + " " + strings.TrimSpace(lines[i][1:])
+					}
+
+					if _, err := buf.WriteString(lines[i] + LineBreak); err != nil {
+						return nil, err
+					}
+				}
+			}
+
+			if len(indent) > 0 && sname != DefaultSection {
+				buf.WriteString(indent)
+			}
+
+			switch {
+			case key.isAutoIncrement:
+				kname = "-"
+			case strings.Contains(kname, "\"") || strings.ContainsAny(kname, f.options.KeyValueDelimiters):
+				kname = "`" + kname + "`"
+			case strings.Contains(kname, "`"):
+				kname = `"""` + kname + `"""`
+			}
+
+			for _, val := range key.ValueWithShadows() {
+				if _, err := buf.WriteString(kname); err != nil {
+					return nil, err
+				}
+
+				if key.isBooleanType {
+					if kname != sec.keyList[len(sec.keyList)-1] {
+						buf.WriteString(LineBreak)
+					}
+					continue KEY_LIST
+				}
+
+				// Write out alignment spaces before "=" sign
+				if PrettyFormat {
+					buf.Write(alignSpaces[:alignLength-len(kname)])
+				}
+
+				// In case key value contains "\n", "`", "\"", "#" or ";"
+				if strings.ContainsAny(val, "\n`") {
+					val = `"""` + val + `"""`
+				} else if !f.options.IgnoreInlineComment && strings.ContainsAny(val, "#;") {
+					val = "`" + val + "`"
+				}
+				if _, err := buf.WriteString(equalSign + val + LineBreak); err != nil {
+					return nil, err
+				}
+			}
+
+			for _, val := range key.nestedValues {
+				if _, err := buf.WriteString(indent + "  " + val + LineBreak); err != nil {
+					return nil, err
+				}
+			}
+		}
+
+		if PrettySection {
+			// Put a line between sections
+			if _, err := buf.WriteString(LineBreak); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return buf, nil
+}
+
+// WriteToIndent writes content into io.Writer with given indention.
+// If PrettyFormat has been set to be true,
+// it will align "=" sign with spaces under each section.
+func (f *File) WriteToIndent(w io.Writer, indent string) (int64, error) {
+	buf, err := f.writeToBuffer(indent)
+	if err != nil {
+		return 0, err
+	}
+	return buf.WriteTo(w)
+}
+
+// WriteTo writes file content into io.Writer.
+func (f *File) WriteTo(w io.Writer) (int64, error) {
+	return f.WriteToIndent(w, "")
+}
+
+// SaveToIndent writes content to file system with given value indention.
+func (f *File) SaveToIndent(filename, indent string) error {
+	// Note: Because we are truncating with os.Create,
+	// 	so it's safer to save to a temporary file location and rename afte done.
+	buf, err := f.writeToBuffer(indent)
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(filename, buf.Bytes(), 0666)
+}
+
+// SaveTo writes content to file system.
+func (f *File) SaveTo(filename string) error {
+	return f.SaveToIndent(filename, "")
+}

vendor/github.com/go-ini/ini/ini.go

@@ -0,0 +1,223 @@
+// +build go1.6
+
+// Copyright 2014 Unknwon
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+// Package ini provides INI file read and write functionality in Go.
+package ini
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"runtime"
+)
+
+const (
+	// DefaultSection is the name of default section. You can use this constant or the string literal.
+	// In most of cases, an empty string is all you need to access the section.
+	DefaultSection = "DEFAULT"
+	// Deprecated: Use "DefaultSection" instead.
+	DEFAULT_SECTION = DefaultSection
+
+	// Maximum allowed depth when recursively substituing variable names.
+	depthValues = 99
+	version     = "1.44.0"
+)
+
+// Version returns current package version literal.
+func Version() string {
+	return version
+}
+
+var (
+	// LineBreak is the delimiter to determine or compose a new line.
+	// This variable will be changed to "\r\n" automatically on Windows at package init time.
+	LineBreak = "\n"
+
+	// DefaultFormatLeft places custom spaces on the left when PrettyFormat and PrettyEqual are both disabled.
+	DefaultFormatLeft = ""
+	// DefaultFormatRight places custom spaces on the right when PrettyFormat and PrettyEqual are both disabled.
+	DefaultFormatRight = ""
+
+	// Variable regexp pattern: %(variable)s
+	varPattern = regexp.MustCompile(`%\(([^\)]+)\)s`)
+
+	// PrettyFormat indicates whether to align "=" sign with spaces to produce pretty output
+	// or reduce all possible spaces for compact format.
+	PrettyFormat = true
+
+	// PrettyEqual places spaces around "=" sign even when PrettyFormat is false.
+	PrettyEqual = false
+
+	// DefaultHeader explicitly writes default section header.
+	DefaultHeader = false
+
+	// PrettySection indicates whether to put a line between sections.
+	PrettySection = true
+)
+
+func init() {
+	if runtime.GOOS == "windows" {
+		LineBreak = "\r\n"
+	}
+}
+
+func inSlice(str string, s []string) bool {
+	for _, v := range s {
+		if str == v {
+			return true
+		}
+	}
+	return false
+}
+
+// dataSource is an interface that returns object which can be read and closed.
+type dataSource interface {
+	ReadCloser() (io.ReadCloser, error)
+}
+
+// sourceFile represents an object that contains content on the local file system.
+type sourceFile struct {
+	name string
+}
+
+func (s sourceFile) ReadCloser() (_ io.ReadCloser, err error) {
+	return os.Open(s.name)
+}
+
+// sourceData represents an object that contains content in memory.
+type sourceData struct {
+	data []byte
+}
+
+func (s *sourceData) ReadCloser() (io.ReadCloser, error) {
+	return ioutil.NopCloser(bytes.NewReader(s.data)), nil
+}
+
+// sourceReadCloser represents an input stream with Close method.
+type sourceReadCloser struct {
+	reader io.ReadCloser
+}
+
+func (s *sourceReadCloser) ReadCloser() (io.ReadCloser, error) {
+	return s.reader, nil
+}
+
+func parseDataSource(source interface{}) (dataSource, error) {
+	switch s := source.(type) {
+	case string:
+		return sourceFile{s}, nil
+	case []byte:
+		return &sourceData{s}, nil
+	case io.ReadCloser:
+		return &sourceReadCloser{s}, nil
+	default:
+		return nil, fmt.Errorf("error parsing data source: unknown type '%s'", s)
+	}
+}
+
+// LoadOptions contains all customized options used for load data source(s).
+type LoadOptions struct {
+	// Loose indicates whether the parser should ignore nonexistent files or return error.
+	Loose bool
+	// Insensitive indicates whether the parser forces all section and key names to lowercase.
+	Insensitive bool
+	// IgnoreContinuation indicates whether to ignore continuation lines while parsing.
+	IgnoreContinuation bool
+	// IgnoreInlineComment indicates whether to ignore comments at the end of value and treat it as part of value.
+	IgnoreInlineComment bool
+	// SkipUnrecognizableLines indicates whether to skip unrecognizable lines that do not conform to key/value pairs.
+	SkipUnrecognizableLines bool
+	// AllowBooleanKeys indicates whether to allow boolean type keys or treat as value is missing.
+	// This type of keys are mostly used in my.cnf.
+	AllowBooleanKeys bool
+	// AllowShadows indicates whether to keep track of keys with same name under same section.
+	AllowShadows bool
+	// AllowNestedValues indicates whether to allow AWS-like nested values.
+	// Docs: http://docs.aws.amazon.com/cli/latest/topic/config-vars.html#nested-values
+	AllowNestedValues bool
+	// AllowPythonMultilineValues indicates whether to allow Python-like multi-line values.
+	// Docs: https://docs.python.org/3/library/configparser.html#supported-ini-file-structure
+	// Relevant quote:  Values can also span multiple lines, as long as they are indented deeper
+	// than the first line of the value.
+	AllowPythonMultilineValues bool
+	// SpaceBeforeInlineComment indicates whether to allow comment symbols (\# and \;) inside value.
+	// Docs: https://docs.python.org/2/library/configparser.html
+	// Quote: Comments may appear on their own in an otherwise empty line, or may be entered in lines holding values or section names.
+	// In the latter case, they need to be preceded by a whitespace character to be recognized as a comment.
+	SpaceBeforeInlineComment bool
+	// UnescapeValueDoubleQuotes indicates whether to unescape double quotes inside value to regular format
+	// when value is surrounded by double quotes, e.g. key="a \"value\"" => key=a "value"
+	UnescapeValueDoubleQuotes bool
+	// UnescapeValueCommentSymbols indicates to unescape comment symbols (\# and \;) inside value to regular format
+	// when value is NOT surrounded by any quotes.
+	// Note: UNSTABLE, behavior might change to only unescape inside double quotes but may noy necessary at all.
+	UnescapeValueCommentSymbols bool
+	// UnparseableSections stores a list of blocks that are allowed with raw content which do not otherwise
+	// conform to key/value pairs. Specify the names of those blocks here.
+	UnparseableSections []string
+	// KeyValueDelimiters is the sequence of delimiters that are used to separate key and value. By default, it is "=:".
+	KeyValueDelimiters string
+	// PreserveSurroundedQuote indicates whether to preserve surrounded quote (single and double quotes).
+	PreserveSurroundedQuote bool
+}
+
+// LoadSources allows caller to apply customized options for loading from data source(s).
+func LoadSources(opts LoadOptions, source interface{}, others ...interface{}) (_ *File, err error) {
+	sources := make([]dataSource, len(others)+1)
+	sources[0], err = parseDataSource(source)
+	if err != nil {
+		return nil, err
+	}
+	for i := range others {
+		sources[i+1], err = parseDataSource(others[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+	f := newFile(sources, opts)
+	if err = f.Reload(); err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+// Load loads and parses from INI data sources.
+// Arguments can be mixed of file name with string type, or raw data in []byte.
+// It will return error if list contains nonexistent files.
+func Load(source interface{}, others ...interface{}) (*File, error) {
+	return LoadSources(LoadOptions{}, source, others...)
+}
+
+// LooseLoad has exactly same functionality as Load function
+// except it ignores nonexistent files instead of returning error.
+func LooseLoad(source interface{}, others ...interface{}) (*File, error) {
+	return LoadSources(LoadOptions{Loose: true}, source, others...)
+}
+
+// InsensitiveLoad has exactly same functionality as Load function
+// except it forces all section and key names to be lowercased.
+func InsensitiveLoad(source interface{}, others ...interface{}) (*File, error) {
+	return LoadSources(LoadOptions{Insensitive: true}, source, others...)
+}
+
+// ShadowLoad has exactly same functionality as Load function
+// except it allows have shadow keys.
+func ShadowLoad(source interface{}, others ...interface{}) (*File, error) {
+	return LoadSources(LoadOptions{AllowShadows: true}, source, others...)
+}

vendor/github.com/go-ini/ini/key.go

@@ -0,0 +1,753 @@
+// Copyright 2014 Unknwon
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package ini
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Key represents a key under a section.
+type Key struct {
+	s               *Section
+	Comment         string
+	name            string
+	value           string
+	isAutoIncrement bool
+	isBooleanType   bool
+
+	isShadow bool
+	shadows  []*Key
+
+	nestedValues []string
+}
+
+// newKey simply return a key object with given values.
+func newKey(s *Section, name, val string) *Key {
+	return &Key{
+		s:     s,
+		name:  name,
+		value: val,
+	}
+}
+
+func (k *Key) addShadow(val string) error {
+	if k.isShadow {
+		return errors.New("cannot add shadow to another shadow key")
+	} else if k.isAutoIncrement || k.isBooleanType {
+		return errors.New("cannot add shadow to auto-increment or boolean key")
+	}
+
+	shadow := newKey(k.s, k.name, val)
+	shadow.isShadow = true
+	k.shadows = append(k.shadows, shadow)
+	return nil
+}
+
+// AddShadow adds a new shadow key to itself.
+func (k *Key) AddShadow(val string) error {
+	if !k.s.f.options.AllowShadows {
+		return errors.New("shadow key is not allowed")
+	}
+	return k.addShadow(val)
+}
+
+func (k *Key) addNestedValue(val string) error {
+	if k.isAutoIncrement || k.isBooleanType {
+		return errors.New("cannot add nested value to auto-increment or boolean key")
+	}
+
+	k.nestedValues = append(k.nestedValues, val)
+	return nil
+}
+
+// AddNestedValue adds a nested value to the key.
+func (k *Key) AddNestedValue(val string) error {
+	if !k.s.f.options.AllowNestedValues {
+		return errors.New("nested value is not allowed")
+	}
+	return k.addNestedValue(val)
+}
+
+// ValueMapper represents a mapping function for values, e.g. os.ExpandEnv
+type ValueMapper func(string) string
+
+// Name returns name of key.
+func (k *Key) Name() string {
+	return k.name
+}
+
+// Value returns raw value of key for performance purpose.
+func (k *Key) Value() string {
+	return k.value
+}
+
+// ValueWithShadows returns raw values of key and its shadows if any.
+func (k *Key) ValueWithShadows() []string {
+	if len(k.shadows) == 0 {
+		return []string{k.value}
+	}
+	vals := make([]string, len(k.shadows)+1)
+	vals[0] = k.value
+	for i := range k.shadows {
+		vals[i+1] = k.shadows[i].value
+	}
+	return vals
+}
+
+// NestedValues returns nested values stored in the key.
+// It is possible returned value is nil if no nested values stored in the key.
+func (k *Key) NestedValues() []string {
+	return k.nestedValues
+}
+
+// transformValue takes a raw value and transforms to its final string.
+func (k *Key) transformValue(val string) string {
+	if k.s.f.ValueMapper != nil {
+		val = k.s.f.ValueMapper(val)
+	}
+
+	// Fail-fast if no indicate char found for recursive value
+	if !strings.Contains(val, "%") {
+		return val
+	}
+	for i := 0; i < depthValues; i++ {
+		vr := varPattern.FindString(val)
+		if len(vr) == 0 {
+			break
+		}
+
+		// Take off leading '%(' and trailing ')s'.
+		noption := vr[2 : len(vr)-2]
+
+		// Search in the same section.
+		nk, err := k.s.GetKey(noption)
+		if err != nil || k == nk {
+			// Search again in default section.
+			nk, _ = k.s.f.Section("").GetKey(noption)
+		}
+
+		// Substitute by new value and take off leading '%(' and trailing ')s'.
+		val = strings.Replace(val, vr, nk.value, -1)
+	}
+	return val
+}
+
+// String returns string representation of value.
+func (k *Key) String() string {
+	return k.transformValue(k.value)
+}
+
+// Validate accepts a validate function which can
+// return modifed result as key value.
+func (k *Key) Validate(fn func(string) string) string {
+	return fn(k.String())
+}
+
+// parseBool returns the boolean value represented by the string.
+//
+// It accepts 1, t, T, TRUE, true, True, YES, yes, Yes, y, ON, on, On,
+// 0, f, F, FALSE, false, False, NO, no, No, n, OFF, off, Off.
+// Any other value returns an error.
+func parseBool(str string) (value bool, err error) {
+	switch str {
+	case "1", "t", "T", "true", "TRUE", "True", "YES", "yes", "Yes", "y", "ON", "on", "On":
+		return true, nil
+	case "0", "f", "F", "false", "FALSE", "False", "NO", "no", "No", "n", "OFF", "off", "Off":
+		return false, nil
+	}
+	return false, fmt.Errorf("parsing \"%s\": invalid syntax", str)
+}
+
+// Bool returns bool type value.
+func (k *Key) Bool() (bool, error) {
+	return parseBool(k.String())
+}
+
+// Float64 returns float64 type value.
+func (k *Key) Float64() (float64, error) {
+	return strconv.ParseFloat(k.String(), 64)
+}
+
+// Int returns int type value.
+func (k *Key) Int() (int, error) {
+	v, err := strconv.ParseInt(k.String(), 0, 64)
+	return int(v), err
+}
+
+// Int64 returns int64 type value.
+func (k *Key) Int64() (int64, error) {
+	return strconv.ParseInt(k.String(), 0, 64)
+}
+
+// Uint returns uint type valued.
+func (k *Key) Uint() (uint, error) {
+	u, e := strconv.ParseUint(k.String(), 0, 64)
+	return uint(u), e
+}
+
+// Uint64 returns uint64 type value.
+func (k *Key) Uint64() (uint64, error) {
+	return strconv.ParseUint(k.String(), 0, 64)
+}
+
+// Duration returns time.Duration type value.
+func (k *Key) Duration() (time.Duration, error) {
+	return time.ParseDuration(k.String())
+}
+
+// TimeFormat parses with given format and returns time.Time type value.
+func (k *Key) TimeFormat(format string) (time.Time, error) {
+	return time.Parse(format, k.String())
+}
+
+// Time parses with RFC3339 format and returns time.Time type value.
+func (k *Key) Time() (time.Time, error) {
+	return k.TimeFormat(time.RFC3339)
+}
+
+// MustString returns default value if key value is empty.
+func (k *Key) MustString(defaultVal string) string {
+	val := k.String()
+	if len(val) == 0 {
+		k.value = defaultVal
+		return defaultVal
+	}
+	return val
+}
+
+// MustBool always returns value without error,
+// it returns false if error occurs.
+func (k *Key) MustBool(defaultVal ...bool) bool {
+	val, err := k.Bool()
+	if len(defaultVal) > 0 && err != nil {
+		k.value = strconv.FormatBool(defaultVal[0])
+		return defaultVal[0]
+	}
+	return val
+}
+
+// MustFloat64 always returns value without error,
+// it returns 0.0 if error occurs.
+func (k *Key) MustFloat64(defaultVal ...float64) float64 {
+	val, err := k.Float64()
+	if len(defaultVal) > 0 && err != nil {
+		k.value = strconv.FormatFloat(defaultVal[0], 'f', -1, 64)
+		return defaultVal[0]
+	}
+	return val
+}
+
+// MustInt always returns value without error,
+// it returns 0 if error occurs.
+func (k *Key) MustInt(defaultVal ...int) int {
+	val, err := k.Int()
+	if len(defaultVal) > 0 && err != nil {
+		k.value = strconv.FormatInt(int64(defaultVal[0]), 10)
+		return defaultVal[0]
+	}
+	return val
+}
+
+// MustInt64 always returns value without error,
+// it returns 0 if error occurs.
+func (k *Key) MustInt64(defaultVal ...int64) int64 {
+	val, err := k.Int64()
+	if len(defaultVal) > 0 && err != nil {
+		k.value = strconv.FormatInt(defaultVal[0], 10)
+		return defaultVal[0]
+	}
+	return val
+}
+
+// MustUint always returns value without error,
+// it returns 0 if error occurs.
+func (k *Key) MustUint(defaultVal ...uint) uint {
+	val, err := k.Uint()
+	if len(defaultVal) > 0 && err != nil {
+		k.value = strconv.FormatUint(uint64(defaultVal[0]), 10)
+		return defaultVal[0]
+	}
+	return val
+}
+
+// MustUint64 always returns value without error,
+// it returns 0 if error occurs.
+func (k *Key) MustUint64(defaultVal ...uint64) uint64 {
+	val, err := k.Uint64()
+	if len(defaultVal) > 0 && err != nil {
+		k.value = strconv.FormatUint(defaultVal[0], 10)
+		return defaultVal[0]
+	}
+	return val
+}
+
+// MustDuration always returns value without error,
+// it returns zero value if error occurs.
+func (k *Key) MustDuration(defaultVal ...time.Duration) time.Duration {
+	val, err := k.Duration()
+	if len(defaultVal) > 0 && err != nil {
+		k.value = defaultVal[0].String()
+		return defaultVal[0]
+	}
+	return val
+}
+
+// MustTimeFormat always parses with given format and returns value without error,
+// it returns zero value if error occurs.
+func (k *Key) MustTimeFormat(format string, defaultVal ...time.Time) time.Time {
+	val, err := k.TimeFormat(format)
+	if len(defaultVal) > 0 && err != nil {
+		k.value = defaultVal[0].Format(format)
+		return defaultVal[0]
+	}
+	return val
+}
+
+// MustTime always parses with RFC3339 format and returns value without error,
+// it returns zero value if error occurs.
+func (k *Key) MustTime(defaultVal ...time.Time) time.Time {
+	return k.MustTimeFormat(time.RFC3339, defaultVal...)
+}
+
+// In always returns value without error,
+// it returns default value if error occurs or doesn't fit into candidates.
+func (k *Key) In(defaultVal string, candidates []string) string {
+	val := k.String()
+	for _, cand := range candidates {
+		if val == cand {
+			return val
+		}
+	}
+	return defaultVal
+}
+
+// InFloat64 always returns value without error,
+// it returns default value if error occurs or doesn't fit into candidates.
+func (k *Key) InFloat64(defaultVal float64, candidates []float64) float64 {
+	val := k.MustFloat64()
+	for _, cand := range candidates {
+		if val == cand {
+			return val
+		}
+	}
+	return defaultVal
+}
+
+// InInt always returns value without error,
+// it returns default value if error occurs or doesn't fit into candidates.
+func (k *Key) InInt(defaultVal int, candidates []int) int {
+	val := k.MustInt()
+	for _, cand := range candidates {
+		if val == cand {
+			return val
+		}
+	}
+	return defaultVal
+}
+
+// InInt64 always returns value without error,
+// it returns default value if error occurs or doesn't fit into candidates.
+func (k *Key) InInt64(defaultVal int64, candidates []int64) int64 {
+	val := k.MustInt64()
+	for _, cand := range candidates {
+		if val == cand {
+			return val
+		}
+	}
+	return defaultVal
+}
+
+// InUint always returns value without error,
+// it returns default value if error occurs or doesn't fit into candidates.
+func (k *Key) InUint(defaultVal uint, candidates []uint) uint {
+	val := k.MustUint()
+	for _, cand := range candidates {
+		if val == cand {
+			return val
+		}
+	}
+	return defaultVal
+}
+
+// InUint64 always returns value without error,
+// it returns default value if error occurs or doesn't fit into candidates.
+func (k *Key) InUint64(defaultVal uint64, candidates []uint64) uint64 {
+	val := k.MustUint64()
+	for _, cand := range candidates {
+		if val == cand {
+			return val
+		}
+	}
+	return defaultVal
+}
+
+// InTimeFormat always parses with given format and returns value without error,
+// it returns default value if error occurs or doesn't fit into candidates.
+func (k *Key) InTimeFormat(format string, defaultVal time.Time, candidates []time.Time) time.Time {
+	val := k.MustTimeFormat(format)
+	for _, cand := range candidates {
+		if val == cand {
+			return val
+		}
+	}
+	return defaultVal
+}
+
+// InTime always parses with RFC3339 format and returns value without error,
+// it returns default value if error occurs or doesn't fit into candidates.
+func (k *Key) InTime(defaultVal time.Time, candidates []time.Time) time.Time {
+	return k.InTimeFormat(time.RFC3339, defaultVal, candidates)
+}
+
+// RangeFloat64 checks if value is in given range inclusively,
+// and returns default value if it's not.
+func (k *Key) RangeFloat64(defaultVal, min, max float64) float64 {
+	val := k.MustFloat64()
+	if val < min || val > max {
+		return defaultVal
+	}
+	return val
+}
+
+// RangeInt checks if value is in given range inclusively,
+// and returns default value if it's not.
+func (k *Key) RangeInt(defaultVal, min, max int) int {
+	val := k.MustInt()
+	if val < min || val > max {
+		return defaultVal
+	}
+	return val
+}
+
+// RangeInt64 checks if value is in given range inclusively,
+// and returns default value if it's not.
+func (k *Key) RangeInt64(defaultVal, min, max int64) int64 {
+	val := k.MustInt64()
+	if val < min || val > max {
+		return defaultVal
+	}
+	return val
+}
+
+// RangeTimeFormat checks if value with given format is in given range inclusively,
+// and returns default value if it's not.
+func (k *Key) RangeTimeFormat(format string, defaultVal, min, max time.Time) time.Time {
+	val := k.MustTimeFormat(format)
+	if val.Unix() < min.Unix() || val.Unix() > max.Unix() {
+		return defaultVal
+	}
+	return val
+}
+
+// RangeTime checks if value with RFC3339 format is in given range inclusively,
+// and returns default value if it's not.
+func (k *Key) RangeTime(defaultVal, min, max time.Time) time.Time {
+	return k.RangeTimeFormat(time.RFC3339, defaultVal, min, max)
+}
+
+// Strings returns list of string divided by given delimiter.
+func (k *Key) Strings(delim string) []string {
+	str := k.String()
+	if len(str) == 0 {
+		return []string{}
+	}
+
+	runes := []rune(str)
+	vals := make([]string, 0, 2)
+	var buf bytes.Buffer
+	escape := false
+	idx := 0
+	for {
+		if escape {
+			escape = false
+			if runes[idx] != '\\' && !strings.HasPrefix(string(runes[idx:]), delim) {
+				buf.WriteRune('\\')
+			}
+			buf.WriteRune(runes[idx])
+		} else {
+			if runes[idx] == '\\' {
+				escape = true
+			} else if strings.HasPrefix(string(runes[idx:]), delim) {
+				idx += len(delim) - 1
+				vals = append(vals, strings.TrimSpace(buf.String()))
+				buf.Reset()
+			} else {
+				buf.WriteRune(runes[idx])
+			}
+		}
+		idx++
+		if idx == len(runes) {
+			break
+		}
+	}
+
+	if buf.Len() > 0 {
+		vals = append(vals, strings.TrimSpace(buf.String()))
+	}
+
+	return vals
+}
+
+// StringsWithShadows returns list of string divided by given delimiter.
+// Shadows will also be appended if any.
+func (k *Key) StringsWithShadows(delim string) []string {
+	vals := k.ValueWithShadows()
+	results := make([]string, 0, len(vals)*2)
+	for i := range vals {
+		if len(vals) == 0 {
+			continue
+		}
+
+		results = append(results, strings.Split(vals[i], delim)...)
+	}
+
+	for i := range results {
+		results[i] = k.transformValue(strings.TrimSpace(results[i]))
+	}
+	return results
+}
+
+// Float64s returns list of float64 divided by given delimiter. Any invalid input will be treated as zero value.
+func (k *Key) Float64s(delim string) []float64 {
+	vals, _ := k.parseFloat64s(k.Strings(delim), true, false)
+	return vals
+}
+
+// Ints returns list of int divided by given delimiter. Any invalid input will be treated as zero value.
+func (k *Key) Ints(delim string) []int {
+	vals, _ := k.parseInts(k.Strings(delim), true, false)
+	return vals
+}
+
+// Int64s returns list of int64 divided by given delimiter. Any invalid input will be treated as zero value.
+func (k *Key) Int64s(delim string) []int64 {
+	vals, _ := k.parseInt64s(k.Strings(delim), true, false)
+	return vals
+}
+
+// Uints returns list of uint divided by given delimiter. Any invalid input will be treated as zero value.
+func (k *Key) Uints(delim string) []uint {
+	vals, _ := k.parseUints(k.Strings(delim), true, false)
+	return vals
+}
+
+// Uint64s returns list of uint64 divided by given delimiter. Any invalid input will be treated as zero value.
+func (k *Key) Uint64s(delim string) []uint64 {
+	vals, _ := k.parseUint64s(k.Strings(delim), true, false)
+	return vals
+}
+
+// TimesFormat parses with given format and returns list of time.Time divided by given delimiter.
+// Any invalid input will be treated as zero value (0001-01-01 00:00:00 +0000 UTC).
+func (k *Key) TimesFormat(format, delim string) []time.Time {
+	vals, _ := k.parseTimesFormat(format, k.Strings(delim), true, false)
+	return vals
+}
+
+// Times parses with RFC3339 format and returns list of time.Time divided by given delimiter.
+// Any invalid input will be treated as zero value (0001-01-01 00:00:00 +0000 UTC).
+func (k *Key) Times(delim string) []time.Time {
+	return k.TimesFormat(time.RFC3339, delim)
+}
+
+// ValidFloat64s returns list of float64 divided by given delimiter. If some value is not float, then
+// it will not be included to result list.
+func (k *Key) ValidFloat64s(delim string) []float64 {
+	vals, _ := k.parseFloat64s(k.Strings(delim), false, false)
+	return vals
+}
+
+// ValidInts returns list of int divided by given delimiter. If some value is not integer, then it will
+// not be included to result list.
+func (k *Key) ValidInts(delim string) []int {
+	vals, _ := k.parseInts(k.Strings(delim), false, false)
+	return vals
+}
+
+// ValidInt64s returns list of int64 divided by given delimiter. If some value is not 64-bit integer,
+// then it will not be included to result list.
+func (k *Key) ValidInt64s(delim string) []int64 {
+	vals, _ := k.parseInt64s(k.Strings(delim), false, false)
+	return vals
+}
+
+// ValidUints returns list of uint divided by given delimiter. If some value is not unsigned integer,
+// then it will not be included to result list.
+func (k *Key) ValidUints(delim string) []uint {
+	vals, _ := k.parseUints(k.Strings(delim), false, false)
+	return vals
+}
+
+// ValidUint64s returns list of uint64 divided by given delimiter. If some value is not 64-bit unsigned
+// integer, then it will not be included to result list.
+func (k *Key) ValidUint64s(delim string) []uint64 {
+	vals, _ := k.parseUint64s(k.Strings(delim), false, false)
+	return vals
+}
+
+// ValidTimesFormat parses with given format and returns list of time.Time divided by given delimiter.
+func (k *Key) ValidTimesFormat(format, delim string) []time.Time {
+	vals, _ := k.parseTimesFormat(format, k.Strings(delim), false, false)
+	return vals
+}
+
+// ValidTimes parses with RFC3339 format and returns list of time.Time divided by given delimiter.
+func (k *Key) ValidTimes(delim string) []time.Time {
+	return k.ValidTimesFormat(time.RFC3339, delim)
+}
+
+// StrictFloat64s returns list of float64 divided by given delimiter or error on first invalid input.
+func (k *Key) StrictFloat64s(delim string) ([]float64, error) {
+	return k.parseFloat64s(k.Strings(delim), false, true)
+}
+
+// StrictInts returns list of int divided by given delimiter or error on first invalid input.
+func (k *Key) StrictInts(delim string) ([]int, error) {
+	return k.parseInts(k.Strings(delim), false, true)
+}
+
+// StrictInt64s returns list of int64 divided by given delimiter or error on first invalid input.
+func (k *Key) StrictInt64s(delim string) ([]int64, error) {
+	return k.parseInt64s(k.Strings(delim), false, true)
+}
+
+// StrictUints returns list of uint divided by given delimiter or error on first invalid input.
+func (k *Key) StrictUints(delim string) ([]uint, error) {
+	return k.parseUints(k.Strings(delim), false, true)
+}
+
+// StrictUint64s returns list of uint64 divided by given delimiter or error on first invalid input.
+func (k *Key) StrictUint64s(delim string) ([]uint64, error) {
+	return k.parseUint64s(k.Strings(delim), false, true)
+}
+
+// StrictTimesFormat parses with given format and returns list of time.Time divided by given delimiter
+// or error on first invalid input.
+func (k *Key) StrictTimesFormat(format, delim string) ([]time.Time, error) {
+	return k.parseTimesFormat(format, k.Strings(delim), false, true)
+}
+
+// StrictTimes parses with RFC3339 format and returns list of time.Time divided by given delimiter
+// or error on first invalid input.
+func (k *Key) StrictTimes(delim string) ([]time.Time, error) {
+	return k.StrictTimesFormat(time.RFC3339, delim)
+}
+
+// parseFloat64s transforms strings to float64s.
+func (k *Key) parseFloat64s(strs []string, addInvalid, returnOnInvalid bool) ([]float64, error) {
+	vals := make([]float64, 0, len(strs))
+	for _, str := range strs {
+		val, err := strconv.ParseFloat(str, 64)
+		if err != nil && returnOnInvalid {
+			return nil, err
+		}
+		if err == nil || addInvalid {
+			vals = append(vals, val)
+		}
+	}
+	return vals, nil
+}
+
+// parseInts transforms strings to ints.
+func (k *Key) parseInts(strs []string, addInvalid, returnOnInvalid bool) ([]int, error) {
+	vals := make([]int, 0, len(strs))
+	for _, str := range strs {
+		valInt64, err := strconv.ParseInt(str, 0, 64)
+		val := int(valInt64)
+		if err != nil && returnOnInvalid {
+			return nil, err
+		}
+		if err == nil || addInvalid {
+			vals = append(vals, val)
+		}
+	}
+	return vals, nil
+}
+
+// parseInt64s transforms strings to int64s.
+func (k *Key) parseInt64s(strs []string, addInvalid, returnOnInvalid bool) ([]int64, error) {
+	vals := make([]int64, 0, len(strs))
+	for _, str := range strs {
+		val, err := strconv.ParseInt(str, 0, 64)
+		if err != nil && returnOnInvalid {
+			return nil, err
+		}
+		if err == nil || addInvalid {
+			vals = append(vals, val)
+		}
+	}
+	return vals, nil
+}
+
+// parseUints transforms strings to uints.
+func (k *Key) parseUints(strs []string, addInvalid, returnOnInvalid bool) ([]uint, error) {
+	vals := make([]uint, 0, len(strs))
+	for _, str := range strs {
+		val, err := strconv.ParseUint(str, 0, 0)
+		if err != nil && returnOnInvalid {
+			return nil, err
+		}
+		if err == nil || addInvalid {
+			vals = append(vals, uint(val))
+		}
+	}
+	return vals, nil
+}
+
+// parseUint64s transforms strings to uint64s.
+func (k *Key) parseUint64s(strs []string, addInvalid, returnOnInvalid bool) ([]uint64, error) {
+	vals := make([]uint64, 0, len(strs))
+	for _, str := range strs {
+		val, err := strconv.ParseUint(str, 0, 64)
+		if err != nil && returnOnInvalid {
+			return nil, err
+		}
+		if err == nil || addInvalid {
+			vals = append(vals, val)
+		}
+	}
+	return vals, nil
+}
+
+// parseTimesFormat transforms strings to times in given format.
+func (k *Key) parseTimesFormat(format string, strs []string, addInvalid, returnOnInvalid bool) ([]time.Time, error) {
+	vals := make([]time.Time, 0, len(strs))
+	for _, str := range strs {
+		val, err := time.Parse(format, str)
+		if err != nil && returnOnInvalid {
+			return nil, err
+		}
+		if err == nil || addInvalid {
+			vals = append(vals, val)
+		}
+	}
+	return vals, nil
+}
+
+// SetValue changes key value.
+func (k *Key) SetValue(v string) {
+	if k.s.f.BlockMode {
+		k.s.f.lock.Lock()
+		defer k.s.f.lock.Unlock()
+	}
+
+	k.value = v
+	k.s.keysHash[k.name] = v
+}

vendor/github.com/go-ini/ini/parser.go

@@ -0,0 +1,487 @@
+// Copyright 2015 Unknwon
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package ini
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+var pythonMultiline = regexp.MustCompile("^(\\s+)([^\n]+)")
+
+type parserOptions struct {
+	IgnoreContinuation          bool
+	IgnoreInlineComment         bool
+	AllowPythonMultilineValues  bool
+	SpaceBeforeInlineComment    bool
+	UnescapeValueDoubleQuotes   bool
+	UnescapeValueCommentSymbols bool
+	PreserveSurroundedQuote     bool
+}
+
+type parser struct {
+	buf     *bufio.Reader
+	options parserOptions
+
+	isEOF   bool
+	count   int
+	comment *bytes.Buffer
+}
+
+func newParser(r io.Reader, opts parserOptions) *parser {
+	return &parser{
+		buf:     bufio.NewReader(r),
+		options: opts,
+		count:   1,
+		comment: &bytes.Buffer{},
+	}
+}
+
+// BOM handles header of UTF-8, UTF-16 LE and UTF-16 BE's BOM format.
+// http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding
+func (p *parser) BOM() error {
+	mask, err := p.buf.Peek(2)
+	if err != nil && err != io.EOF {
+		return err
+	} else if len(mask) < 2 {
+		return nil
+	}
+
+	switch {
+	case mask[0] == 254 && mask[1] == 255:
+		fallthrough
+	case mask[0] == 255 && mask[1] == 254:
+		p.buf.Read(mask)
+	case mask[0] == 239 && mask[1] == 187:
+		mask, err := p.buf.Peek(3)
+		if err != nil && err != io.EOF {
+			return err
+		} else if len(mask) < 3 {
+			return nil
+		}
+		if mask[2] == 191 {
+			p.buf.Read(mask)
+		}
+	}
+	return nil
+}
+
+func (p *parser) readUntil(delim byte) ([]byte, error) {
+	data, err := p.buf.ReadBytes(delim)
+	if err != nil {
+		if err == io.EOF {
+			p.isEOF = true
+		} else {
+			return nil, err
+		}
+	}
+	return data, nil
+}
+
+func cleanComment(in []byte) ([]byte, bool) {
+	i := bytes.IndexAny(in, "#;")
+	if i == -1 {
+		return nil, false
+	}
+	return in[i:], true
+}
+
+func readKeyName(delimiters string, in []byte) (string, int, error) {
+	line := string(in)
+
+	// Check if key name surrounded by quotes.
+	var keyQuote string
+	if line[0] == '"' {
+		if len(line) > 6 && string(line[0:3]) == `"""` {
+			keyQuote = `"""`
+		} else {
+			keyQuote = `"`
+		}
+	} else if line[0] == '`' {
+		keyQuote = "`"
+	}
+
+	// Get out key name
+	endIdx := -1
+	if len(keyQuote) > 0 {
+		startIdx := len(keyQuote)
+		// FIXME: fail case -> """"""name"""=value
+		pos := strings.Index(line[startIdx:], keyQuote)
+		if pos == -1 {
+			return "", -1, fmt.Errorf("missing closing key quote: %s", line)
+		}
+		pos += startIdx
+
+		// Find key-value delimiter
+		i := strings.IndexAny(line[pos+startIdx:], delimiters)
+		if i < 0 {
+			return "", -1, ErrDelimiterNotFound{line}
+		}
+		endIdx = pos + i
+		return strings.TrimSpace(line[startIdx:pos]), endIdx + startIdx + 1, nil
+	}
+
+	endIdx = strings.IndexAny(line, delimiters)
+	if endIdx < 0 {
+		return "", -1, ErrDelimiterNotFound{line}
+	}
+	return strings.TrimSpace(line[0:endIdx]), endIdx + 1, nil
+}
+
+func (p *parser) readMultilines(line, val, valQuote string) (string, error) {
+	for {
+		data, err := p.readUntil('\n')
+		if err != nil {
+			return "", err
+		}
+		next := string(data)
+
+		pos := strings.LastIndex(next, valQuote)
+		if pos > -1 {
+			val += next[:pos]
+
+			comment, has := cleanComment([]byte(next[pos:]))
+			if has {
+				p.comment.Write(bytes.TrimSpace(comment))
+			}
+			break
+		}
+		val += next
+		if p.isEOF {
+			return "", fmt.Errorf("missing closing key quote from '%s' to '%s'", line, next)
+		}
+	}
+	return val, nil
+}
+
+func (p *parser) readContinuationLines(val string) (string, error) {
+	for {
+		data, err := p.readUntil('\n')
+		if err != nil {
+			return "", err
+		}
+		next := strings.TrimSpace(string(data))
+
+		if len(next) == 0 {
+			break
+		}
+		val += next
+		if val[len(val)-1] != '\\' {
+			break
+		}
+		val = val[:len(val)-1]
+	}
+	return val, nil
+}
+
+// hasSurroundedQuote check if and only if the first and last characters
+// are quotes \" or \'.
+// It returns false if any other parts also contain same kind of quotes.
+func hasSurroundedQuote(in string, quote byte) bool {
+	return len(in) >= 2 && in[0] == quote && in[len(in)-1] == quote &&
+		strings.IndexByte(in[1:], quote) == len(in)-2
+}
+
+func (p *parser) readValue(in []byte, bufferSize int) (string, error) {
+
+	line := strings.TrimLeftFunc(string(in), unicode.IsSpace)
+	if len(line) == 0 {
+		if p.options.AllowPythonMultilineValues && len(in) > 0 && in[len(in)-1] == '\n' {
+			return p.readPythonMultilines(line, bufferSize)
+		}
+		return "", nil
+	}
+
+	var valQuote string
+	if len(line) > 3 && string(line[0:3]) == `"""` {
+		valQuote = `"""`
+	} else if line[0] == '`' {
+		valQuote = "`"
+	} else if p.options.UnescapeValueDoubleQuotes && line[0] == '"' {
+		valQuote = `"`
+	}
+
+	if len(valQuote) > 0 {
+		startIdx := len(valQuote)
+		pos := strings.LastIndex(line[startIdx:], valQuote)
+		// Check for multi-line value
+		if pos == -1 {
+			return p.readMultilines(line, line[startIdx:], valQuote)
+		}
+
+		if p.options.UnescapeValueDoubleQuotes && valQuote == `"` {
+			return strings.Replace(line[startIdx:pos+startIdx], `\"`, `"`, -1), nil
+		}
+		return line[startIdx : pos+startIdx], nil
+	}
+
+	lastChar := line[len(line)-1]
+	// Won't be able to reach here if value only contains whitespace
+	line = strings.TrimSpace(line)
+	trimmedLastChar := line[len(line)-1]
+
+	// Check continuation lines when desired
+	if !p.options.IgnoreContinuation && trimmedLastChar == '\\' {
+		return p.readContinuationLines(line[:len(line)-1])
+	}
+
+	// Check if ignore inline comment
+	if !p.options.IgnoreInlineComment {
+		var i int
+		if p.options.SpaceBeforeInlineComment {
+			i = strings.Index(line, " #")
+			if i == -1 {
+				i = strings.Index(line, " ;")
+			}
+
+		} else {
+			i = strings.IndexAny(line, "#;")
+		}
+
+		if i > -1 {
+			p.comment.WriteString(line[i:])
+			line = strings.TrimSpace(line[:i])
+		}
+
+	}
+
+	// Trim single and double quotes
+	if (hasSurroundedQuote(line, '\'') ||
+		hasSurroundedQuote(line, '"')) && !p.options.PreserveSurroundedQuote {
+		line = line[1 : len(line)-1]
+	} else if len(valQuote) == 0 && p.options.UnescapeValueCommentSymbols {
+		if strings.Contains(line, `\;`) {
+			line = strings.Replace(line, `\;`, ";", -1)
+		}
+		if strings.Contains(line, `\#`) {
+			line = strings.Replace(line, `\#`, "#", -1)
+		}
+	} else if p.options.AllowPythonMultilineValues && lastChar == '\n' {
+		return p.readPythonMultilines(line, bufferSize)
+	}
+
+	return line, nil
+}
+
+func (p *parser) readPythonMultilines(line string, bufferSize int) (string, error) {
+	parserBufferPeekResult, _ := p.buf.Peek(bufferSize)
+	peekBuffer := bytes.NewBuffer(parserBufferPeekResult)
+
+	for {
+		peekData, peekErr := peekBuffer.ReadBytes('\n')
+		if peekErr != nil {
+			if peekErr == io.EOF {
+				return line, nil
+			}
+			return "", peekErr
+		}
+
+		peekMatches := pythonMultiline.FindStringSubmatch(string(peekData))
+		if len(peekMatches) != 3 {
+			return line, nil
+		}
+
+		// NOTE: Return if not a python-ini multi-line value.
+		currentIdentSize := len(peekMatches[1])
+		if currentIdentSize <= 0 {
+			return line, nil
+		}
+
+		// NOTE: Just advance the parser reader (buffer) in-sync with the peek buffer.
+		_, err := p.readUntil('\n')
+		if err != nil {
+			return "", err
+		}
+
+		line += fmt.Sprintf("\n%s", peekMatches[2])
+	}
+}
+
+// parse parses data through an io.Reader.
+func (f *File) parse(reader io.Reader) (err error) {
+	p := newParser(reader, parserOptions{
+		IgnoreContinuation:          f.options.IgnoreContinuation,
+		IgnoreInlineComment:         f.options.IgnoreInlineComment,
+		AllowPythonMultilineValues:  f.options.AllowPythonMultilineValues,
+		SpaceBeforeInlineComment:    f.options.SpaceBeforeInlineComment,
+		UnescapeValueDoubleQuotes:   f.options.UnescapeValueDoubleQuotes,
+		UnescapeValueCommentSymbols: f.options.UnescapeValueCommentSymbols,
+		PreserveSurroundedQuote:     f.options.PreserveSurroundedQuote,
+	})
+	if err = p.BOM(); err != nil {
+		return fmt.Errorf("BOM: %v", err)
+	}
+
+	// Ignore error because default section name is never empty string.
+	name := DefaultSection
+	if f.options.Insensitive {
+		name = strings.ToLower(DefaultSection)
+	}
+	section, _ := f.NewSection(name)
+
+	// This "last" is not strictly equivalent to "previous one" if current key is not the first nested key
+	var isLastValueEmpty bool
+	var lastRegularKey *Key
+
+	var line []byte
+	var inUnparseableSection bool
+
+	// NOTE: Iterate and increase `currentPeekSize` until
+	// the size of the parser buffer is found.
+	// TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`.
+	parserBufferSize := 0
+	// NOTE: Peek 1kb at a time.
+	currentPeekSize := 1024
+
+	if f.options.AllowPythonMultilineValues {
+		for {
+			peekBytes, _ := p.buf.Peek(currentPeekSize)
+			peekBytesLength := len(peekBytes)
+
+			if parserBufferSize >= peekBytesLength {
+				break
+			}
+
+			currentPeekSize *= 2
+			parserBufferSize = peekBytesLength
+		}
+	}
+
+	for !p.isEOF {
+		line, err = p.readUntil('\n')
+		if err != nil {
+			return err
+		}
+
+		if f.options.AllowNestedValues &&
+			isLastValueEmpty && len(line) > 0 {
+			if line[0] == ' ' || line[0] == '\t' {
+				lastRegularKey.addNestedValue(string(bytes.TrimSpace(line)))
+				continue
+			}
+		}
+
+		line = bytes.TrimLeftFunc(line, unicode.IsSpace)
+		if len(line) == 0 {
+			continue
+		}
+
+		// Comments
+		if line[0] == '#' || line[0] == ';' {
+			// Note: we do not care ending line break,
+			// it is needed for adding second line,
+			// so just clean it once at the end when set to value.
+			p.comment.Write(line)
+			continue
+		}
+
+		// Section
+		if line[0] == '[' {
+			// Read to the next ']' (TODO: support quoted strings)
+			closeIdx := bytes.LastIndexByte(line, ']')
+			if closeIdx == -1 {
+				return fmt.Errorf("unclosed section: %s", line)
+			}
+
+			name := string(line[1:closeIdx])
+			section, err = f.NewSection(name)
+			if err != nil {
+				return err
+			}
+
+			comment, has := cleanComment(line[closeIdx+1:])
+			if has {
+				p.comment.Write(comment)
+			}
+
+			section.Comment = strings.TrimSpace(p.comment.String())
+
+			// Reset aotu-counter and comments
+			p.comment.Reset()
+			p.count = 1
+
+			inUnparseableSection = false
+			for i := range f.options.UnparseableSections {
+				if f.options.UnparseableSections[i] == name ||
+					(f.options.Insensitive && strings.ToLower(f.options.UnparseableSections[i]) == strings.ToLower(name)) {
+					inUnparseableSection = true
+					continue
+				}
+			}
+			continue
+		}
+
+		if inUnparseableSection {
+			section.isRawSection = true
+			section.rawBody += string(line)
+			continue
+		}
+
+		kname, offset, err := readKeyName(f.options.KeyValueDelimiters, line)
+		if err != nil {
+			// Treat as boolean key when desired, and whole line is key name.
+			if IsErrDelimiterNotFound(err) {
+				switch {
+				case f.options.AllowBooleanKeys:
+					kname, err := p.readValue(line, parserBufferSize)
+					if err != nil {
+						return err
+					}
+					key, err := section.NewBooleanKey(kname)
+					if err != nil {
+						return err
+					}
+					key.Comment = strings.TrimSpace(p.comment.String())
+					p.comment.Reset()
+					continue
+
+				case f.options.SkipUnrecognizableLines:
+					continue
+				}
+			}
+			return err
+		}
+
+		// Auto increment.
+		isAutoIncr := false
+		if kname == "-" {
+			isAutoIncr = true
+			kname = "#" + strconv.Itoa(p.count)
+			p.count++
+		}
+
+		value, err := p.readValue(line[offset:], parserBufferSize)
+		if err != nil {
+			return err
+		}
+		isLastValueEmpty = len(value) == 0
+
+		key, err := section.NewKey(kname, value)
+		if err != nil {
+			return err
+		}
+		key.isAutoIncrement = isAutoIncr
+		key.Comment = strings.TrimSpace(p.comment.String())
+		p.comment.Reset()
+		lastRegularKey = key
+	}
+	return nil
+}

vendor/github.com/go-ini/ini/section.go

@@ -0,0 +1,256 @@
+// Copyright 2014 Unknwon
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package ini
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+// Section represents a config section.
+type Section struct {
+	f        *File
+	Comment  string
+	name     string
+	keys     map[string]*Key
+	keyList  []string
+	keysHash map[string]string
+
+	isRawSection bool
+	rawBody      string
+}
+
+func newSection(f *File, name string) *Section {
+	return &Section{
+		f:        f,
+		name:     name,
+		keys:     make(map[string]*Key),
+		keyList:  make([]string, 0, 10),
+		keysHash: make(map[string]string),
+	}
+}
+
+// Name returns name of Section.
+func (s *Section) Name() string {
+	return s.name
+}
+
+// Body returns rawBody of Section if the section was marked as unparseable.
+// It still follows the other rules of the INI format surrounding leading/trailing whitespace.
+func (s *Section) Body() string {
+	return strings.TrimSpace(s.rawBody)
+}
+
+// SetBody updates body content only if section is raw.
+func (s *Section) SetBody(body string) {
+	if !s.isRawSection {
+		return
+	}
+	s.rawBody = body
+}
+
+// NewKey creates a new key to given section.
+func (s *Section) NewKey(name, val string) (*Key, error) {
+	if len(name) == 0 {
+		return nil, errors.New("error creating new key: empty key name")
+	} else if s.f.options.Insensitive {
+		name = strings.ToLower(name)
+	}
+
+	if s.f.BlockMode {
+		s.f.lock.Lock()
+		defer s.f.lock.Unlock()
+	}
+
+	if inSlice(name, s.keyList) {
+		if s.f.options.AllowShadows {
+			if err := s.keys[name].addShadow(val); err != nil {
+				return nil, err
+			}
+		} else {
+			s.keys[name].value = val
+			s.keysHash[name] = val
+		}
+		return s.keys[name], nil
+	}
+
+	s.keyList = append(s.keyList, name)
+	s.keys[name] = newKey(s, name, val)
+	s.keysHash[name] = val
+	return s.keys[name], nil
+}
+
+// NewBooleanKey creates a new boolean type key to given section.
+func (s *Section) NewBooleanKey(name string) (*Key, error) {
+	key, err := s.NewKey(name, "true")
+	if err != nil {
+		return nil, err
+	}
+
+	key.isBooleanType = true
+	return key, nil
+}
+
+// GetKey returns key in section by given name.
+func (s *Section) GetKey(name string) (*Key, error) {
+	if s.f.BlockMode {
+		s.f.lock.RLock()
+	}
+	if s.f.options.Insensitive {
+		name = strings.ToLower(name)
+	}
+	key := s.keys[name]
+	if s.f.BlockMode {
+		s.f.lock.RUnlock()
+	}
+
+	if key == nil {
+		// Check if it is a child-section.
+		sname := s.name
+		for {
+			if i := strings.LastIndex(sname, "."); i > -1 {
+				sname = sname[:i]
+				sec, err := s.f.GetSection(sname)
+				if err != nil {
+					continue
+				}
+				return sec.GetKey(name)
+			}
+			break
+		}
+		return nil, fmt.Errorf("error when getting key of section '%s': key '%s' not exists", s.name, name)
+	}
+	return key, nil
+}
+
+// HasKey returns true if section contains a key with given name.
+func (s *Section) HasKey(name string) bool {
+	key, _ := s.GetKey(name)
+	return key != nil
+}
+
+// Deprecated: Use "HasKey" instead.
+func (s *Section) Haskey(name string) bool {
+	return s.HasKey(name)
+}
+
+// HasValue returns true if section contains given raw value.
+func (s *Section) HasValue(value string) bool {
+	if s.f.BlockMode {
+		s.f.lock.RLock()
+		defer s.f.lock.RUnlock()
+	}
+
+	for _, k := range s.keys {
+		if value == k.value {
+			return true
+		}
+	}
+	return false
+}
+
+// Key assumes named Key exists in section and returns a zero-value when not.
+func (s *Section) Key(name string) *Key {
+	key, err := s.GetKey(name)
+	if err != nil {
+		// It's OK here because the only possible error is empty key name,
+		// but if it's empty, this piece of code won't be executed.
+		key, _ = s.NewKey(name, "")
+		return key
+	}
+	return key
+}
+
+// Keys returns list of keys of section.
+func (s *Section) Keys() []*Key {
+	keys := make([]*Key, len(s.keyList))
+	for i := range s.keyList {
+		keys[i] = s.Key(s.keyList[i])
+	}
+	return keys
+}
+
+// ParentKeys returns list of keys of parent section.
+func (s *Section) ParentKeys() []*Key {
+	var parentKeys []*Key
+	sname := s.name
+	for {
+		if i := strings.LastIndex(sname, "."); i > -1 {
+			sname = sname[:i]
+			sec, err := s.f.GetSection(sname)
+			if err != nil {
+				continue
+			}
+			parentKeys = append(parentKeys, sec.Keys()...)
+		} else {
+			break
+		}
+
+	}
+	return parentKeys
+}
+
+// KeyStrings returns list of key names of section.
+func (s *Section) KeyStrings() []string {
+	list := make([]string, len(s.keyList))
+	copy(list, s.keyList)
+	return list
+}
+
+// KeysHash returns keys hash consisting of names and values.
+func (s *Section) KeysHash() map[string]string {
+	if s.f.BlockMode {
+		s.f.lock.RLock()
+		defer s.f.lock.RUnlock()
+	}
+
+	hash := map[string]string{}
+	for key, value := range s.keysHash {
+		hash[key] = value
+	}
+	return hash
+}
+
+// DeleteKey deletes a key from section.
+func (s *Section) DeleteKey(name string) {
+	if s.f.BlockMode {
+		s.f.lock.Lock()
+		defer s.f.lock.Unlock()
+	}
+
+	for i, k := range s.keyList {
+		if k == name {
+			s.keyList = append(s.keyList[:i], s.keyList[i+1:]...)
+			delete(s.keys, name)
+			delete(s.keysHash, name)
+			return
+		}
+	}
+}
+
+// ChildSections returns a list of child sections of current section.
+// For example, "[parent.child1]" and "[parent.child12]" are child sections
+// of section "[parent]".
+func (s *Section) ChildSections() []*Section {
+	prefix := s.name + "."
+	children := make([]*Section, 0, 3)
+	for _, name := range s.f.sectionList {
+		if strings.HasPrefix(name, prefix) {
+			children = append(children, s.f.sections[name])
+		}
+	}
+	return children
+}

vendor/github.com/go-ini/ini/struct.go

@@ -0,0 +1,548 @@
+// Copyright 2014 Unknwon
+//
+// Licensed under the Apache License, Version 2.0 (the "License"): you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+package ini
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+	"time"
+	"unicode"
+)
+
+// NameMapper represents a ini tag name mapper.
+type NameMapper func(string) string
+
+// Built-in name getters.
+var (
+	// AllCapsUnderscore converts to format ALL_CAPS_UNDERSCORE.
+	AllCapsUnderscore NameMapper = func(raw string) string {
+		newstr := make([]rune, 0, len(raw))
+		for i, chr := range raw {
+			if isUpper := 'A' <= chr && chr <= 'Z'; isUpper {
+				if i > 0 {
+					newstr = append(newstr, '_')
+				}
+			}
+			newstr = append(newstr, unicode.ToUpper(chr))
+		}
+		return string(newstr)
+	}
+	// TitleUnderscore converts to format title_underscore.
+	TitleUnderscore NameMapper = func(raw string) string {
+		newstr := make([]rune, 0, len(raw))
+		for i, chr := range raw {
+			if isUpper := 'A' <= chr && chr <= 'Z'; isUpper {
+				if i > 0 {
+					newstr = append(newstr, '_')
+				}
+				chr -= ('A' - 'a')
+			}
+			newstr = append(newstr, chr)
+		}
+		return string(newstr)
+	}
+)
+
+func (s *Section) parseFieldName(raw, actual string) string {
+	if len(actual) > 0 {
+		return actual
+	}
+	if s.f.NameMapper != nil {
+		return s.f.NameMapper(raw)
+	}
+	return raw
+}
+
+func parseDelim(actual string) string {
+	if len(actual) > 0 {
+		return actual
+	}
+	return ","
+}
+
+var reflectTime = reflect.TypeOf(time.Now()).Kind()
+
+// setSliceWithProperType sets proper values to slice based on its type.
+func setSliceWithProperType(key *Key, field reflect.Value, delim string, allowShadow, isStrict bool) error {
+	var strs []string
+	if allowShadow {
+		strs = key.StringsWithShadows(delim)
+	} else {
+		strs = key.Strings(delim)
+	}
+
+	numVals := len(strs)
+	if numVals == 0 {
+		return nil
+	}
+
+	var vals interface{}
+	var err error
+
+	sliceOf := field.Type().Elem().Kind()
+	switch sliceOf {
+	case reflect.String:
+		vals = strs
+	case reflect.Int:
+		vals, err = key.parseInts(strs, true, false)
+	case reflect.Int64:
+		vals, err = key.parseInt64s(strs, true, false)
+	case reflect.Uint:
+		vals, err = key.parseUints(strs, true, false)
+	case reflect.Uint64:
+		vals, err = key.parseUint64s(strs, true, false)
+	case reflect.Float64:
+		vals, err = key.parseFloat64s(strs, true, false)
+	case reflectTime:
+		vals, err = key.parseTimesFormat(time.RFC3339, strs, true, false)
+	default:
+		return fmt.Errorf("unsupported type '[]%s'", sliceOf)
+	}
+	if err != nil && isStrict {
+		return err
+	}
+
+	slice := reflect.MakeSlice(field.Type(), numVals, numVals)
+	for i := 0; i < numVals; i++ {
+		switch sliceOf {
+		case reflect.String:
+			slice.Index(i).Set(reflect.ValueOf(vals.([]string)[i]))
+		case reflect.Int:
+			slice.Index(i).Set(reflect.ValueOf(vals.([]int)[i]))
+		case reflect.Int64:
+			slice.Index(i).Set(reflect.ValueOf(vals.([]int64)[i]))
+		case reflect.Uint:
+			slice.Index(i).Set(reflect.ValueOf(vals.([]uint)[i]))
+		case reflect.Uint64:
+			slice.Index(i).Set(reflect.ValueOf(vals.([]uint64)[i]))
+		case reflect.Float64:
+			slice.Index(i).Set(reflect.ValueOf(vals.([]float64)[i]))
+		case reflectTime:
+			slice.Index(i).Set(reflect.ValueOf(vals.([]time.Time)[i]))
+		}
+	}
+	field.Set(slice)
+	return nil
+}
+
+func wrapStrictError(err error, isStrict bool) error {
+	if isStrict {
+		return err
+	}
+	return nil
+}
+
+// setWithProperType sets proper value to field based on its type,
+// but it does not return error for failing parsing,
+// because we want to use default value that is already assigned to struct.
+func setWithProperType(t reflect.Type, key *Key, field reflect.Value, delim string, allowShadow, isStrict bool) error {
+	switch t.Kind() {
+	case reflect.String:
+		if len(key.String()) == 0 {
+			return nil
+		}
+		field.SetString(key.String())
+	case reflect.Bool:
+		boolVal, err := key.Bool()
+		if err != nil {
+			return wrapStrictError(err, isStrict)
+		}
+		field.SetBool(boolVal)
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		durationVal, err := key.Duration()
+		// Skip zero value
+		if err == nil && int64(durationVal) > 0 {
+			field.Set(reflect.ValueOf(durationVal))
+			return nil
+		}
+
+		intVal, err := key.Int64()
+		if err != nil {
+			return wrapStrictError(err, isStrict)
+		}
+		field.SetInt(intVal)
+	//	byte is an alias for uint8, so supporting uint8 breaks support for byte
+	case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		durationVal, err := key.Duration()
+		// Skip zero value
+		if err == nil && uint64(durationVal) > 0 {
+			field.Set(reflect.ValueOf(durationVal))
+			return nil
+		}
+
+		uintVal, err := key.Uint64()
+		if err != nil {
+			return wrapStrictError(err, isStrict)
+		}
+		field.SetUint(uintVal)
+
+	case reflect.Float32, reflect.Float64:
+		floatVal, err := key.Float64()
+		if err != nil {
+			return wrapStrictError(err, isStrict)
+		}
+		field.SetFloat(floatVal)
+	case reflectTime:
+		timeVal, err := key.Time()
+		if err != nil {
+			return wrapStrictError(err, isStrict)
+		}
+		field.Set(reflect.ValueOf(timeVal))
+	case reflect.Slice:
+		return setSliceWithProperType(key, field, delim, allowShadow, isStrict)
+	default:
+		return fmt.Errorf("unsupported type '%s'", t)
+	}
+	return nil
+}
+
+func parseTagOptions(tag string) (rawName string, omitEmpty bool, allowShadow bool) {
+	opts := strings.SplitN(tag, ",", 3)
+	rawName = opts[0]
+	if len(opts) > 1 {
+		omitEmpty = opts[1] == "omitempty"
+	}
+	if len(opts) > 2 {
+		allowShadow = opts[2] == "allowshadow"
+	}
+	return rawName, omitEmpty, allowShadow
+}
+
+func (s *Section) mapTo(val reflect.Value, isStrict bool) error {
+	if val.Kind() == reflect.Ptr {
+		val = val.Elem()
+	}
+	typ := val.Type()
+
+	for i := 0; i < typ.NumField(); i++ {
+		field := val.Field(i)
+		tpField := typ.Field(i)
+
+		tag := tpField.Tag.Get("ini")
+		if tag == "-" {
+			continue
+		}
+
+		rawName, _, allowShadow := parseTagOptions(tag)
+		fieldName := s.parseFieldName(tpField.Name, rawName)
+		if len(fieldName) == 0 || !field.CanSet() {
+			continue
+		}
+
+		isStruct := tpField.Type.Kind() == reflect.Struct
+		isStructPtr := tpField.Type.Kind() == reflect.Ptr && tpField.Type.Elem().Kind() == reflect.Struct
+		isAnonymous := tpField.Type.Kind() == reflect.Ptr && tpField.Anonymous
+		if isAnonymous {
+			field.Set(reflect.New(tpField.Type.Elem()))
+		}
+
+		if isAnonymous || isStruct || isStructPtr {
+			if sec, err := s.f.GetSection(fieldName); err == nil {
+				// Only set the field to non-nil struct value if we have
+				// a section for it. Otherwise, we end up with a non-nil
+				// struct ptr even though there is no data.
+				if isStructPtr && field.IsNil() {
+					field.Set(reflect.New(tpField.Type.Elem()))
+				}
+				if err = sec.mapTo(field, isStrict); err != nil {
+					return fmt.Errorf("error mapping field(%s): %v", fieldName, err)
+				}
+				continue
+			}
+		}
+
+		if key, err := s.GetKey(fieldName); err == nil {
+			delim := parseDelim(tpField.Tag.Get("delim"))
+			if err = setWithProperType(tpField.Type, key, field, delim, allowShadow, isStrict); err != nil {
+				return fmt.Errorf("error mapping field(%s): %v", fieldName, err)
+			}
+		}
+	}
+	return nil
+}
+
+// MapTo maps section to given struct.
+func (s *Section) MapTo(v interface{}) error {
+	typ := reflect.TypeOf(v)
+	val := reflect.ValueOf(v)
+	if typ.Kind() == reflect.Ptr {
+		typ = typ.Elem()
+		val = val.Elem()
+	} else {
+		return errors.New("cannot map to non-pointer struct")
+	}
+
+	return s.mapTo(val, false)
+}
+
+// StrictMapTo maps section to given struct in strict mode,
+// which returns all possible error including value parsing error.
+func (s *Section) StrictMapTo(v interface{}) error {
+	typ := reflect.TypeOf(v)
+	val := reflect.ValueOf(v)
+	if typ.Kind() == reflect.Ptr {
+		typ = typ.Elem()
+		val = val.Elem()
+	} else {
+		return errors.New("cannot map to non-pointer struct")
+	}
+
+	return s.mapTo(val, true)
+}
+
+// MapTo maps file to given struct.
+func (f *File) MapTo(v interface{}) error {
+	return f.Section("").MapTo(v)
+}
+
+// StrictMapTo maps file to given struct in strict mode,
+// which returns all possible error including value parsing error.
+func (f *File) StrictMapTo(v interface{}) error {
+	return f.Section("").StrictMapTo(v)
+}
+
+// MapToWithMapper maps data sources to given struct with name mapper.
+func MapToWithMapper(v interface{}, mapper NameMapper, source interface{}, others ...interface{}) error {
+	cfg, err := Load(source, others...)
+	if err != nil {
+		return err
+	}
+	cfg.NameMapper = mapper
+	return cfg.MapTo(v)
+}
+
+// StrictMapToWithMapper maps data sources to given struct with name mapper in strict mode,
+// which returns all possible error including value parsing error.
+func StrictMapToWithMapper(v interface{}, mapper NameMapper, source interface{}, others ...interface{}) error {
+	cfg, err := Load(source, others...)
+	if err != nil {
+		return err
+	}
+	cfg.NameMapper = mapper
+	return cfg.StrictMapTo(v)
+}
+
+// MapTo maps data sources to given struct.
+func MapTo(v, source interface{}, others ...interface{}) error {
+	return MapToWithMapper(v, nil, source, others...)
+}
+
+// StrictMapTo maps data sources to given struct in strict mode,
+// which returns all possible error including value parsing error.
+func StrictMapTo(v, source interface{}, others ...interface{}) error {
+	return StrictMapToWithMapper(v, nil, source, others...)
+}
+
+// reflectSliceWithProperType does the opposite thing as setSliceWithProperType.
+func reflectSliceWithProperType(key *Key, field reflect.Value, delim string, allowShadow bool) error {
+	slice := field.Slice(0, field.Len())
+	if field.Len() == 0 {
+		return nil
+	}
+	sliceOf := field.Type().Elem().Kind()
+
+	if allowShadow {
+		var keyWithShadows *Key
+		for i := 0; i < field.Len(); i++ {
+			var val string
+			switch sliceOf {
+			case reflect.String:
+				val = slice.Index(i).String()
+			case reflect.Int, reflect.Int64:
+				val = fmt.Sprint(slice.Index(i).Int())
+			case reflect.Uint, reflect.Uint64:
+				val = fmt.Sprint(slice.Index(i).Uint())
+			case reflect.Float64:
+				val = fmt.Sprint(slice.Index(i).Float())
+			case reflectTime:
+				val = slice.Index(i).Interface().(time.Time).Format(time.RFC3339)
+			default:
+				return fmt.Errorf("unsupported type '[]%s'", sliceOf)
+			}
+
+			if i == 0 {
+				keyWithShadows = newKey(key.s, key.name, val)
+			} else {
+				keyWithShadows.AddShadow(val)
+			}
+		}
+		key = keyWithShadows
+		return nil
+	}
+
+	var buf bytes.Buffer
+	for i := 0; i < field.Len(); i++ {
+		switch sliceOf {
+		case reflect.String:
+			buf.WriteString(slice.Index(i).String())
+		case reflect.Int, reflect.Int64:
+			buf.WriteString(fmt.Sprint(slice.Index(i).Int()))
+		case reflect.Uint, reflect.Uint64:
+			buf.WriteString(fmt.Sprint(slice.Index(i).Uint()))
+		case reflect.Float64:
+			buf.WriteString(fmt.Sprint(slice.Index(i).Float()))
+		case reflectTime:
+			buf.WriteString(slice.Index(i).Interface().(time.Time).Format(time.RFC3339))
+		default:
+			return fmt.Errorf("unsupported type '[]%s'", sliceOf)
+		}
+		buf.WriteString(delim)
+	}
+	key.SetValue(buf.String()[:buf.Len()-len(delim)])
+	return nil
+}
+
+// reflectWithProperType does the opposite thing as setWithProperType.
+func reflectWithProperType(t reflect.Type, key *Key, field reflect.Value, delim string, allowShadow bool) error {
+	switch t.Kind() {
+	case reflect.String:
+		key.SetValue(field.String())
+	case reflect.Bool:
+		key.SetValue(fmt.Sprint(field.Bool()))
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		key.SetValue(fmt.Sprint(field.Int()))
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		key.SetValue(fmt.Sprint(field.Uint()))
+	case reflect.Float32, reflect.Float64:
+		key.SetValue(fmt.Sprint(field.Float()))
+	case reflectTime:
+		key.SetValue(fmt.Sprint(field.Interface().(time.Time).Format(time.RFC3339)))
+	case reflect.Slice:
+		return reflectSliceWithProperType(key, field, delim, allowShadow)
+	default:
+		return fmt.Errorf("unsupported type '%s'", t)
+	}
+	return nil
+}
+
+// CR: copied from encoding/json/encode.go with modifications of time.Time support.
+// TODO: add more test coverage.
+func isEmptyValue(v reflect.Value) bool {
+	switch v.Kind() {
+	case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
+		return v.Len() == 0
+	case reflect.Bool:
+		return !v.Bool()
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return v.Int() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return v.Uint() == 0
+	case reflect.Float32, reflect.Float64:
+		return v.Float() == 0
+	case reflect.Interface, reflect.Ptr:
+		return v.IsNil()
+	case reflectTime:
+		t, ok := v.Interface().(time.Time)
+		return ok && t.IsZero()
+	}
+	return false
+}
+
+func (s *Section) reflectFrom(val reflect.Value) error {
+	if val.Kind() == reflect.Ptr {
+		val = val.Elem()
+	}
+	typ := val.Type()
+
+	for i := 0; i < typ.NumField(); i++ {
+		field := val.Field(i)
+		tpField := typ.Field(i)
+
+		tag := tpField.Tag.Get("ini")
+		if tag == "-" {
+			continue
+		}
+
+		rawName, omitEmpty, allowShadow := parseTagOptions(tag)
+		if omitEmpty && isEmptyValue(field) {
+			continue
+		}
+
+		fieldName := s.parseFieldName(tpField.Name, rawName)
+		if len(fieldName) == 0 || !field.CanSet() {
+			continue
+		}
+
+		if (tpField.Type.Kind() == reflect.Ptr && tpField.Anonymous) ||
+			(tpField.Type.Kind() == reflect.Struct && tpField.Type.Name() != "Time") {
+			// Note: The only error here is section doesn't exist.
+			sec, err := s.f.GetSection(fieldName)
+			if err != nil {
+				// Note: fieldName can never be empty here, ignore error.
+				sec, _ = s.f.NewSection(fieldName)
+			}
+
+			// Add comment from comment tag
+			if len(sec.Comment) == 0 {
+				sec.Comment = tpField.Tag.Get("comment")
+			}
+
+			if err = sec.reflectFrom(field); err != nil {
+				return fmt.Errorf("error reflecting field (%s): %v", fieldName, err)
+			}
+			continue
+		}
+
+		// Note: Same reason as secion.
+		key, err := s.GetKey(fieldName)
+		if err != nil {
+			key, _ = s.NewKey(fieldName, "")
+		}
+
+		// Add comment from comment tag
+		if len(key.Comment) == 0 {
+			key.Comment = tpField.Tag.Get("comment")
+		}
+
+		if err = reflectWithProperType(tpField.Type, key, field, parseDelim(tpField.Tag.Get("delim")), allowShadow); err != nil {
+			return fmt.Errorf("error reflecting field (%s): %v", fieldName, err)
+		}
+
+	}
+	return nil
+}
+
+// ReflectFrom reflects secion from given struct.
+func (s *Section) ReflectFrom(v interface{}) error {
+	typ := reflect.TypeOf(v)
+	val := reflect.ValueOf(v)
+	if typ.Kind() == reflect.Ptr {
+		typ = typ.Elem()
+		val = val.Elem()
+	} else {
+		return errors.New("cannot reflect from non-pointer struct")
+	}
+
+	return s.reflectFrom(val)
+}
+
+// ReflectFrom reflects file from given struct.
+func (f *File) ReflectFrom(v interface{}) error {
+	return f.Section("").ReflectFrom(v)
+}
+
+// ReflectFromWithMapper reflects data sources from given struct with name mapper.
+func ReflectFromWithMapper(cfg *File, v interface{}, mapper NameMapper) error {
+	cfg.NameMapper = mapper
+	return cfg.ReflectFrom(v)
+}
+
+// ReflectFrom reflects data sources from given struct.
+func ReflectFrom(cfg *File, v interface{}) error {
+	return ReflectFromWithMapper(cfg, v, nil)
+}

vendor/github.com/minio/minio-go/.gitignore

@@ -0,0 +1,3 @@
+*~
+*.test
+validator

vendor/github.com/minio/minio-go/.travis.yml

@@ -0,0 +1,28 @@
+sudo: false
+language: go
+
+os:
+- linux
+
+env:
+- ARCH=x86_64
+- ARCH=i686
+
+go:
+- 1.11.x
+- tip
+
+matrix:
+  fast_finish: true
+  allow_failures:
+  - go: tip
+
+addons:
+  apt:
+    packages:
+      - devscripts
+
+script:
+- diff -au <(gofmt -d .) <(printf "")
+- diff -au <(licensecheck --check '.go$' --recursive --lines 0 * | grep -v -w 'Apache (v2.0)') <(printf "")
+- make

vendor/github.com/minio/minio-go/CONTRIBUTING.md

@@ -0,0 +1,23 @@
+
+###  Developer Guidelines
+
+``minio-go`` welcomes your contribution. To make the process as seamless as possible, we ask for the following:
+
+* Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes.
+    - Fork it
+    - Create your feature branch (git checkout -b my-new-feature)
+    - Commit your changes (git commit -am 'Add some feature')
+    - Push to the branch (git push origin my-new-feature)
+    - Create new Pull Request
+
+* When you're ready to create a pull request, be sure to:
+    - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
+    - Run `go fmt`
+    - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
+    - Make sure `go test -race ./...` and `go build` completes.
+      NOTE: go test runs functional tests and requires you to have a AWS S3 account. Set them as environment variables
+      ``ACCESS_KEY`` and ``SECRET_KEY``. To run shorter version of the tests please use ``go test -short -race ./...``
+
+* Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project
+    - `minio-go` project is strictly conformant with Golang style
+    - if you happen to observe offending code, please feel free to send a pull request

vendor/github.com/minio/minio-go/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/minio/minio-go/MAINTAINERS.md

@@ -0,0 +1,35 @@
+# For maintainers only
+
+## Responsibilities
+
+Please go through this link [Maintainer Responsibility](https://gist.github.com/abperiasamy/f4d9b31d3186bbd26522)
+
+### Making new releases
+Tag and sign your release commit, additionally this step requires you to have access to Minio's trusted private key.
+```sh
+$ export GNUPGHOME=/media/${USER}/minio/trusted
+$ git tag -s 4.0.0
+$ git push
+$ git push --tags
+```
+
+### Update version
+Once release has been made update `libraryVersion` constant in `api.go` to next to be released version.
+
+```sh
+$ grep libraryVersion api.go
+      libraryVersion = "4.0.1"
+```
+
+Commit your changes
+```
+$ git commit -a -m "Update version for next release" --author "Minio Trusted <trusted@minio.io>"
+```
+
+### Announce
+Announce new release by adding release notes at https://github.com/minio/minio-go/releases from `trusted@minio.io` account. Release notes requires two sections `highlights` and `changelog`. Highlights is a bulleted list of salient features in this release and Changelog contains list of all commits since the last release.
+
+To generate `changelog`
+```sh
+$ git log --no-color --pretty=format:'-%d %s (%cr) <%an>' <last_release_tag>..<latest_release_tag>
+```