From be7bc9335c4e3420b4387f5b6be7db1a344e4363 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Thu, 19 Jun 2025 14:50:39 +0800 Subject: [PATCH] fix: use goid to identify differnt goroutine (#22722) Co-authored-by: Qiu Jian --- go.mod | 5 +- go.sum | 10 +- pkg/appsrv/appsrv.go | 28 +-- pkg/appsrv/appsrv_test.go | 2 +- pkg/cloudcommon/app/app.go | 2 +- pkg/cloudcommon/db/lockman/inmemory.go | 82 +++---- pkg/cloudcommon/db/lockman/inmemory_test.go | 79 +++++++ pkg/cloudcommon/options/options.go | 2 + pkg/cloudcommon/policy/policy.go | 79 +++---- pkg/httperrors/errors.go | 4 + pkg/httperrors/httperrors.go | 4 + pkg/util/samlutils/demo/service.go | 2 +- vendor/github.com/petermattis/goid/.gitignore | 4 + vendor/github.com/petermattis/goid/LICENSE | 202 ++++++++++++++++++ vendor/github.com/petermattis/goid/README.md | 4 + vendor/github.com/petermattis/goid/goid.go | 35 +++ .../github.com/petermattis/goid/goid_gccgo.go | 26 +++ .../github.com/petermattis/goid/goid_go1.3.c | 23 ++ .../github.com/petermattis/goid/goid_go1.3.go | 22 ++ .../github.com/petermattis/goid/goid_go1.4.go | 35 +++ .../github.com/petermattis/goid/goid_go1.4.s | 18 ++ .../github.com/petermattis/goid/goid_go1.5.go | 28 +++ .../github.com/petermattis/goid/goid_go1.5.s | 44 ++++ .../github.com/petermattis/goid/goid_slow.go | 24 +++ .../petermattis/goid/runtime_gccgo_go1.8.go | 17 ++ .../petermattis/goid/runtime_go1.23.go | 38 ++++ .../petermattis/goid/runtime_go1.5.go | 57 +++++ .../petermattis/goid/runtime_go1.6.go | 43 ++++ .../petermattis/goid/runtime_go1.9.go | 37 ++++ vendor/modules.txt | 7 +- .../pkg/apis/compute/storage_const.go | 1 + .../cloudmux/pkg/multicloud/aliyun/aliyun.go | 2 +- .../x/cloudmux/pkg/multicloud/ctyun/ctyun.go | 3 + .../multicloud/ctyun/latitud_and_longitude.go | 53 ----- .../x/cloudmux/pkg/multicloud/ctyun/region.go | 34 +-- .../pkg/multicloud/ctyun/securitygroup.go | 11 +- .../x/cloudmux/pkg/multicloud/ctyun/zone.go | 2 +- .../x/cloudmux/pkg/multicloud/esxi/storage.go | 6 +- .../cloudmux/pkg/multicloud/qcloud/qcloud.go | 2 +- .../x/pkg/util/printutils/printjson.go | 49 +++-- 40 files changed, 914 insertions(+), 212 deletions(-) create mode 100644 vendor/github.com/petermattis/goid/.gitignore create mode 100644 vendor/github.com/petermattis/goid/LICENSE create mode 100644 vendor/github.com/petermattis/goid/README.md create mode 100644 vendor/github.com/petermattis/goid/goid.go create mode 100644 vendor/github.com/petermattis/goid/goid_gccgo.go create mode 100644 vendor/github.com/petermattis/goid/goid_go1.3.c create mode 100644 vendor/github.com/petermattis/goid/goid_go1.3.go create mode 100644 vendor/github.com/petermattis/goid/goid_go1.4.go create mode 100644 vendor/github.com/petermattis/goid/goid_go1.4.s create mode 100644 vendor/github.com/petermattis/goid/goid_go1.5.go create mode 100644 vendor/github.com/petermattis/goid/goid_go1.5.s create mode 100644 vendor/github.com/petermattis/goid/goid_slow.go create mode 100644 vendor/github.com/petermattis/goid/runtime_gccgo_go1.8.go create mode 100644 vendor/github.com/petermattis/goid/runtime_go1.23.go create mode 100644 vendor/github.com/petermattis/goid/runtime_go1.5.go create mode 100644 vendor/github.com/petermattis/goid/runtime_go1.6.go create mode 100644 vendor/github.com/petermattis/goid/runtime_go1.9.go diff --git a/go.mod b/go.mod index 8366ce61d0..49fb05ccdd 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/miekg/dns v1.1.25 github.com/minio/minio-go v6.0.14+incompatible github.com/mitchellh/go-wordwrap v1.0.1 + github.com/petermattis/goid v0.0.0-20250508124226-395b08cebbdb github.com/pierrec/lz4/v4 v4.1.15 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 @@ -95,12 +96,12 @@ require ( k8s.io/cri-api v0.22.17 k8s.io/klog/v2 v2.20.0 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250616091331-d7f1f89a7d90 + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250618092159-24a4b21d2439 yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 - yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0 + yunion.io/x/pkg v1.10.4-0.20250618071935-840c253075b0 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 yunion.io/x/sqlchemy v1.1.3-0.20250531010554-ce98f840b833 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c diff --git a/go.sum b/go.sum index 403bc3a146..2c002c47ac 100644 --- a/go.sum +++ b/go.sum @@ -713,6 +713,8 @@ github.com/openzipkin/zipkin-go-opentracing v0.3.4/go.mod h1:js2AbwmHW0YD9DwIw2J github.com/oracle/oci-go-sdk v24.3.0+incompatible h1:x4mcfb4agelf1O4/1/auGlZ1lr97jXRSSN5MxTgG/zU= github.com/oracle/oci-go-sdk v24.3.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/petermattis/goid v0.0.0-20250508124226-395b08cebbdb h1:3PrKuO92dUTMrQ9dx0YNejC6U/Si6jqKmyQ9vWjwqR4= +github.com/petermattis/goid v0.0.0-20250508124226-395b08cebbdb/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -1410,8 +1412,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250616091331-d7f1f89a7d90 h1:Cbzs++XW+hHXLI5cufagP6hoo+L+7vXU1bZOvyqTJnE= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250616091331-d7f1f89a7d90/go.mod h1:FXxAEbdNfWXX9gjME3K2nJhkydHY5EKEUZb+RLEzVwQ= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250618092159-24a4b21d2439 h1:LpoTE3tGWCh7r9IcEhI82wpXgIGj+lU2skW2GWwUTM4= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250618092159-24a4b21d2439/go.mod h1:FXxAEbdNfWXX9gjME3K2nJhkydHY5EKEUZb+RLEzVwQ= yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 h1:msG4SiDSVU7CrXH06WuHlNEZXIooTcmNbfrIGHuIHBU= yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= @@ -1425,8 +1427,8 @@ yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 h1:Hu/4ERvoWaN6aiFs4h4/yvVB yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc= yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= -yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0 h1:iKWkBMKazSijYNhOaSh4qBuIu+PmXYhEAMNwrxaXL4Q= -yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM= +yunion.io/x/pkg v1.10.4-0.20250618071935-840c253075b0 h1:diTsDq4t6jntGdAKtHZYx9eRWkLg0KyMTp0WFdueT+0= +yunion.io/x/pkg v1.10.4-0.20250618071935-840c253075b0/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= yunion.io/x/sqlchemy v1.1.3-0.20250531010554-ce98f840b833 h1:XTFC1naKYkciCQDLm9izpzHXfTenmmtYsTpVKrsN5hE= diff --git a/pkg/appsrv/appsrv.go b/pkg/appsrv/appsrv.go index f07e823d4e..fed1bb9406 100644 --- a/pkg/appsrv/appsrv.go +++ b/pkg/appsrv/appsrv.go @@ -93,13 +93,13 @@ const ( var quitHandlerRegisted bool -func NewApplication(name string, connMax int, db bool) *Application { +func NewApplication(name string, connMax int, queueSize int, db bool) *Application { app := Application{name: name, context: ctx.CtxWithTime(), connMax: connMax, - session: NewWorkerManager("HttpRequestWorkerManager", connMax, DEFAULT_BACKLOG, db), - readSession: NewWorkerManager("HttpGetRequestWorkerManager", connMax, DEFAULT_BACKLOG, db), - systemSession: NewWorkerManager("InternalHttpRequestWorkerManager", 1, DEFAULT_BACKLOG, false), + session: NewWorkerManager("HttpRequestWorkerManager", connMax, connMax*queueSize, db), + readSession: NewWorkerManager("HttpGetRequestWorkerManager", connMax, connMax*queueSize, db), + systemSession: NewWorkerManager("InternalHttpRequestWorkerManager", 1, queueSize, false), roots: make(map[string]*RadixNode), rootLock: &sync.RWMutex{}, idleTimeout: DEFAULT_IDLE_TIMEOUT, @@ -424,7 +424,7 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri task.appParams.Body, _ = jsonutils.Parse(data) r.Body = io.NopCloser(bytes.NewBuffer(data)) } - session.Run( + inqueue := session.Run( task, currentWorker, func(err error) { @@ -432,13 +432,17 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri task.fw.closeChannels() }, ) - runErr := task.fw.wait(task.ctx, currentWorker) - if runErr != nil { - switch je := runErr.(type) { - case *httputils.JSONClientError: - httperrors.GeneralServerError(task.ctx, w, je) - default: - httperrors.InternalServerError(task.ctx, w, "Internal server error") + if !inqueue { + httperrors.TooManyRequestsError(task.ctx, w, "Request queue is full") + } else { + runErr := task.fw.wait(task.ctx, currentWorker) + if runErr != nil { + switch je := runErr.(type) { + case *httputils.JSONClientError: + httperrors.GeneralServerError(task.ctx, w, je) + default: + httperrors.InternalServerError(task.ctx, w, "Internal server error") + } } } task.fw.closeChannels() diff --git a/pkg/appsrv/appsrv_test.go b/pkg/appsrv/appsrv_test.go index cbb35a98e6..8f2283fd7e 100644 --- a/pkg/appsrv/appsrv_test.go +++ b/pkg/appsrv/appsrv_test.go @@ -49,7 +49,7 @@ type ApplicationTestSuit struct { } func (suite *ApplicationTestSuit) SetupTest() { - suite.app = NewApplication("test", 4, false) + suite.app = NewApplication("test", 4, 10, false) suite.app.AddHandler("GET", "/delay", func(ctx context.Context, w http.ResponseWriter, r *http.Request) { time.Sleep(time.Second * 1) Send(w, "delay pong") diff --git a/pkg/cloudcommon/app/app.go b/pkg/cloudcommon/app/app.go index 4c68d42536..a4a0030ce6 100644 --- a/pkg/cloudcommon/app/app.go +++ b/pkg/cloudcommon/app/app.go @@ -30,7 +30,7 @@ import ( func InitApp(options *common_options.BaseOptions, dbAccess bool) *appsrv.Application { // cache := appsrv.NewCache(options.AuthTokenCacheSize) log.Infof("RequestWorkerCount: %d", options.RequestWorkerCount) - app := appsrv.NewApplication(options.ApplicationID, options.RequestWorkerCount, dbAccess) + app := appsrv.NewApplication(options.ApplicationID, options.RequestWorkerCount, options.RequestWorkerQueueSize, dbAccess) app.CORSAllowHosts(options.CorsHosts) app.SetDefaultTimeout(time.Duration(options.DefaultProcessTimeoutSeconds) * time.Second) // app.SetContext(appsrv.APP_CONTEXT_KEY_CACHE, cache) diff --git a/pkg/cloudcommon/db/lockman/inmemory.go b/pkg/cloudcommon/db/lockman/inmemory.go index 67d0c688be..8a5b80f26f 100644 --- a/pkg/cloudcommon/db/lockman/inmemory.go +++ b/pkg/cloudcommon/db/lockman/inmemory.go @@ -19,10 +19,12 @@ import ( "runtime/debug" "sync" + "github.com/petermattis/goid" + "yunion.io/x/log" ) -const ( +var ( debug_log = false ) @@ -34,15 +36,15 @@ type SInMemoryLockRecord struct { key string lock *sync.Mutex cond *sync.Cond - holder context.Context + holder int64 depth int waiter *FIFO } -func newInMemoryLockRecord(ctx context.Context) *SInMemoryLockRecord { +func newInMemoryLockRecord(ctxDummy context.Context) *SInMemoryLockRecord { lock := &sync.Mutex{} cond := sync.NewCond(lock) - rec := SInMemoryLockRecord{lock: lock, cond: cond, holder: ctx, depth: 0, waiter: NewFIFO()} + rec := SInMemoryLockRecord{lock: lock, cond: cond, holder: -1, depth: 0, waiter: NewFIFO()} return &rec } @@ -51,24 +53,29 @@ func (rec *SInMemoryLockRecord) fatalf(fmtStr string, args ...interface{}) { log.Fatalf(fmtStr, args...) } -func (rec *SInMemoryLockRecord) lockContext(ctx context.Context) { +func (rec *SInMemoryLockRecord) lockContext(ctxDummy context.Context) { rec.lock.Lock() defer rec.lock.Unlock() - if rec.holder == nil { - rec.holder = ctx + curGoid := goid.Get() + + if rec.holder < 0 { + if debug_log { + log.Debugf("lockContext: curGoid=[%d] key=[%s] create new record", curGoid, rec.key) + } + rec.holder = curGoid rec.depth = 1 return } if debug_log { - log.Debugf("rec.hold=[%p] ctx=[%p] %v key=[%s]", rec.holder, ctx, rec.holder == ctx, rec.key) + log.Debugf("rec.hold=[%d] ctx=[%d] %v key=[%s]", rec.holder, curGoid, rec.holder == curGoid, rec.key) } - if rec.holder == ctx { + if rec.holder == curGoid { rec.depth += 1 if debug_log { - log.Infof("lockContext: same ctx, depth: %d [%p] key=[%s]", rec.depth, rec.holder, rec.key) + log.Infof("lockContext: same ctx, depth: %d holder=[%d] ctx=[%d] key=[%s]", rec.depth, rec.holder, curGoid, rec.key) } if rec.depth > 32 { // XXX MUST BE BUG ??? @@ -79,57 +86,59 @@ func (rec *SInMemoryLockRecord) lockContext(ctx context.Context) { // check rec.waiter.Enum(func(ele interface{}) { - electx := ele.(context.Context) - if electx == ctx { - rec.fatalf("try to lock from a waiter context???? key=[%s]", rec.key) + electx := ele.(int64) + if electx == curGoid { + rec.fatalf("try to lock from a waiter context???? curGoid=[%d] waiterGoid=[%d] key=[%s]", curGoid, electx, rec.key) } }) - rec.waiter.Push(ctx) + rec.waiter.Push(curGoid) if debug_log { - log.Debugf("waiter size %d after push", rec.waiter.Len()) - log.Debugf("Start to wait ... [%p] key=[%s]", ctx, rec.key) + log.Debugf("waiter size %d after push curGoid=[%d]", rec.waiter.Len(), curGoid) + log.Debugf("Start to wait ... holder=[%d] curGoid [%d] key=[%s]", rec.holder, curGoid, rec.key) } - for rec.holder != nil { + for rec.holder >= 0 { rec.cond.Wait() } if debug_log { - log.Debugf("End of wait ... [%p] key=[%s]", ctx, rec.key) + log.Debugf("End of wait ... holder=[%d] curGoid [%d] key=[%s]", rec.holder, curGoid, rec.key) } - rec.waiter.Pop(ctx) + rec.waiter.Pop(curGoid) if debug_log { - log.Debugf("waiter size %d after pop key=[%s]", rec.waiter.Len(), rec.key) + log.Debugf("waiter size %d after pop curGoid=[%d] key=[%s]", rec.waiter.Len(), curGoid, rec.key) } - rec.holder = ctx + rec.holder = curGoid rec.depth = 1 } -func (rec *SInMemoryLockRecord) unlockContext(ctx context.Context) (needClean bool) { +func (rec *SInMemoryLockRecord) unlockContext(ctxDummy context.Context) (needClean bool) { rec.lock.Lock() defer rec.lock.Unlock() - if rec.holder != ctx { - rec.fatalf("try to unlock a wait context??? key=[%s]", rec.key) + curGoid := goid.Get() + + if rec.holder != curGoid { + rec.fatalf("try to unlock a wait context??? key=[%s] holder=[%d] curGoid=[%d]", rec.key, rec.holder, curGoid) } if debug_log { - log.Debugf("unlockContext depth %d [%p] key=[%s]", rec.depth, ctx, rec.key) + log.Debugf("unlockContext depth %d curGoid=[%d] key=[%s]", rec.depth, curGoid, rec.key) } rec.depth -= 1 if rec.depth <= 0 { if debug_log { - log.Debugf("depth 0, to release lock for context [%p] key=[%s]", ctx, rec.key) + log.Debugf("depth 0, to release lock for context curGoid=[%d] key=[%s]", curGoid, rec.key) } - rec.holder = nil + rec.holder = -1 if rec.waiter.Len() == 0 { return true } @@ -154,11 +163,11 @@ func NewInMemoryLockManager() ILockManager { return &lockMan } -func (lockman *SInMemoryLockManager) getRecordWithLock(ctx context.Context, key string) *SInMemoryLockRecord { +func (lockman *SInMemoryLockManager) getRecordWithLock(ctx context.Context, key string, new bool) *SInMemoryLockRecord { lockman.tableLock.Lock() defer lockman.tableLock.Unlock() - return lockman.getRecord(ctx, key, true) + return lockman.getRecord(ctx, key, new) } func (lockman *SInMemoryLockManager) getRecord(ctx context.Context, key string, new bool) *SInMemoryLockRecord { @@ -175,23 +184,16 @@ func (lockman *SInMemoryLockManager) getRecord(ctx context.Context, key string, } func (lockman *SInMemoryLockManager) LockKey(ctx context.Context, key string) { - record := lockman.getRecordWithLock(ctx, key) - + record := lockman.getRecordWithLock(ctx, key, true) record.lockContext(ctx) } func (lockman *SInMemoryLockManager) UnlockKey(ctx context.Context, key string) { - lockman.tableLock.Lock() - defer lockman.tableLock.Unlock() - - record := lockman.getRecord(ctx, key, false) + record := lockman.getRecordWithLock(ctx, key, false) if record == nil { - log.Errorf("BUG: unlock an non-existent lock\n%s", debug.Stack()) + log.Errorf("BUG: unlock an non-existent lock ctx: %p key: %s\n%s", ctx, key, debug.Stack()) return } - needClean := record.unlockContext(ctx) - if needClean { - delete(lockman.lockTable, key) - } + record.unlockContext(ctx) } diff --git a/pkg/cloudcommon/db/lockman/inmemory_test.go b/pkg/cloudcommon/db/lockman/inmemory_test.go index 0efc249bfd..e56f17efeb 100644 --- a/pkg/cloudcommon/db/lockman/inmemory_test.go +++ b/pkg/cloudcommon/db/lockman/inmemory_test.go @@ -16,9 +16,17 @@ package lockman import ( "context" + "fmt" "math/rand" + "os" + "sync" "testing" "time" + + "github.com/petermattis/goid" + + "yunion.io/x/log" + "yunion.io/x/pkg/utils" ) func TestInMemoryLockManager(t *testing.T) { @@ -56,3 +64,74 @@ func TestRunManu(t *testing.T) { } t.Logf("complete") } + +type app struct { + ctx context.Context + key string + lockman ILockManager +} + +func (app *app) run() { + app.lockman.LockKey(app.ctx, app.key) + defer app.lockman.UnlockKey(app.ctx, app.key) + + fmt.Printf("run for goid: %d key %s\n", goid.Get(), app.key) +} + +type emptyKey struct{} + +func TestRunManu3(t *testing.T) { + for i := 0; i < 100; i++ { + TestRunManu2(t) + } +} + +func TestRunManu2(t *testing.T) { + rand.Seed(100) + lockman := NewInMemoryLockManager() + + debug_log = true + + // 使用 WaitGroup 来等待所有 goroutine 完成 + var wg sync.WaitGroup + MAX_GOROUTINE := 4096 + MAX_KEY := 4 + + complete := make(chan struct{}) + + go func() { + for { + select { + case <-time.After(time.Second): + t.Logf("timeout") + utils.DumpAllGoroutineStack(os.Stdout) + case <-complete: + return + } + } + }() + + bgCtx := context.Background() + + // 为每个 goroutine 创建独立的 context + for i := 0; i < MAX_GOROUTINE; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + // 为每个 goroutine 创建新的 context + ctx := context.WithValue(bgCtx, emptyKey{}, id) + log.Infof("ctx for id %d: %p", id, ctx) + app := &app{ + ctx: ctx, + key: fmt.Sprintf("test-%d", id%MAX_KEY), + lockman: lockman, + } + app.run() + }(i) + } + + // 等待所有 goroutine 完成 + wg.Wait() + close(complete) + t.Logf("complete") +} diff --git a/pkg/cloudcommon/options/options.go b/pkg/cloudcommon/options/options.go index aec76fe6cc..20ed0d7061 100644 --- a/pkg/cloudcommon/options/options.go +++ b/pkg/cloudcommon/options/options.go @@ -71,6 +71,8 @@ type BaseOptions struct { ApplicationID string `help:"Application ID"` RequestWorkerCount int `default:"8" help:"Request worker thread count, default is 8"` + RequestWorkerQueueSize int `default:"10" help:"Request worker queue size, default is 10"` + TaskWorkerCount int `default:"4" help:"Task manager worker thread count, default is 4"` LocalTaskWorkerCount int `default:"4" help:"Worker thread count that runs local tasks, default is 4"` diff --git a/pkg/cloudcommon/policy/policy.go b/pkg/cloudcommon/policy/policy.go index e37d11fd7b..e23b058b1e 100644 --- a/pkg/cloudcommon/policy/policy.go +++ b/pkg/cloudcommon/policy/policy.go @@ -31,12 +31,12 @@ import ( "yunion.io/x/onecloud/pkg/apis" identity_api "yunion.io/x/onecloud/pkg/apis/identity" - "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules/identity" + "yunion.io/x/onecloud/pkg/util/ctx" "yunion.io/x/onecloud/pkg/util/hashcache" "yunion.io/x/onecloud/pkg/util/rbacutils" "yunion.io/x/onecloud/pkg/util/tagutils" @@ -76,7 +76,7 @@ type SPolicyManager struct { policyCache *hashcache.Cache // policy cache permissionCache *hashcache.Cache // permission cache - fetchWorker *appsrv.SWorkerManager + // fetchWorker *appsrv.SWorkerManager lock *sync.Mutex } @@ -146,19 +146,19 @@ func (manager *SPolicyManager) init(refreshInterval time.Duration, workerCount i defaultFetcherFuncAddr := reflect.ValueOf(DefaultPolicyFetcher).Pointer() remoteFetcherFuncAddr := reflect.ValueOf(auth.FetchMatchPolicies).Pointer() log.Debugf("DefaultPolicyFetcher: %x RemotePolicyFetcher: %x", defaultFetcherFuncAddr, remoteFetcherFuncAddr) - var isDB bool - if defaultFetcherFuncAddr == remoteFetcherFuncAddr { - // remote fetcher, so start watcher - isDB = false - } else { - isDB = true - } + // var isDB bool + // if defaultFetcherFuncAddr == remoteFetcherFuncAddr { + // remote fetcher, so start watcher + // isDB = false + // } else { + // isDB = true + // } - if workerCount <= 0 { - workerCount = 1 - } - log.Infof("policy fetch worker count %d", workerCount) - manager.fetchWorker = appsrv.NewWorkerManager("policyFetchWorker", workerCount, 2048, isDB) + // if workerCount <= 0 { + // workerCount = 1 + // } + // log.Infof("policy fetch worker count %d", workerCount) + // manager.fetchWorker = appsrv.NewWorkerManager("policyFetchWorker", workerCount, 2048, isDB) } func getMaskedLoginIp(userCred mcclient.TokenCredential) string { @@ -255,53 +255,22 @@ func (manager *SPolicyManager) Allow(targetScope rbacscope.TRbacScope, userCred return rbacutils.PolicyDeny } -type fetchResult struct { - output *mcclient.SFetchMatchPoliciesOutput - err error -} - -type policyTask struct { - manager *SPolicyManager - key string - userCred mcclient.TokenCredential - resChan chan fetchResult -} - -func (t *policyTask) Run() { - val := t.manager.policyCache.AtomicGet(t.key) - result := fetchResult{} - if gotypes.IsNil(val) { - pg, err := DefaultPolicyFetcher(context.Background(), t.userCred) - if err != nil { - result.err = errors.Wrap(err, "DefaultPolicyFetcher") - } else { - t.manager.policyCache.AtomicSet(t.key, pg) - result.output = pg - } - } else { - result.output = val.(*mcclient.SFetchMatchPoliciesOutput) - } - t.resChan <- result -} - -func (t *policyTask) Dump() string { - return "" -} - func (manager *SPolicyManager) fetchMatchedPolicies(userCred mcclient.TokenCredential) (*mcclient.SFetchMatchPoliciesOutput, error) { key := policyKey(userCred) - task := policyTask{ - manager: manager, - key: key, - userCred: userCred, + val := manager.policyCache.AtomicGet(key) + if !gotypes.IsNil(val) { + // cache hit + return val.(*mcclient.SFetchMatchPoliciesOutput), nil } - task.resChan = make(chan fetchResult) - manager.fetchWorker.Run(&task, nil, nil) + pg, err := DefaultPolicyFetcher(ctx.CtxWithTime(), userCred) + if err != nil { + return nil, errors.Wrap(err, "DefaultPolicyFetcher") + } - res := <-task.resChan - return res.output, res.err + manager.policyCache.AtomicSet(key, pg) + return pg, nil } func (manager *SPolicyManager) allow(scope rbacscope.TRbacScope, userCred mcclient.TokenCredential, service string, resource string, action string, extra ...string) rbacutils.SPolicyResult { diff --git a/pkg/httperrors/errors.go b/pkg/httperrors/errors.go index 561918f029..52f40ac116 100644 --- a/pkg/httperrors/errors.go +++ b/pkg/httperrors/errors.go @@ -207,6 +207,10 @@ func NewTooLargeEntityError(msg string, params ...interface{}) *httputils.JSONCl return httputils.NewJsonClientError(httpErrorCode[ErrTooLarge], string(ErrTooLarge), msg, params...) } +func NewTooManyRequestsError(msg string, params ...interface{}) *httputils.JSONClientError { + return httputils.NewJsonClientError(httpErrorCode[ErrTooManyRequests], string(ErrTooManyRequests), msg, params...) +} + func NewServiceAbnormalError(msg string, params ...interface{}) *httputils.JSONClientError { return httputils.NewJsonClientError(httpErrorCode[ErrServiceAbnormal], string(ErrServiceAbnormal), msg, params...) } diff --git a/pkg/httperrors/httperrors.go b/pkg/httperrors/httperrors.go index b983077ff0..0a5c5cbe35 100644 --- a/pkg/httperrors/httperrors.go +++ b/pkg/httperrors/httperrors.go @@ -205,3 +205,7 @@ func ProtectedResourceError(ctx context.Context, w http.ResponseWriter, msg stri func NoProjectError(ctx context.Context, w http.ResponseWriter, msg string, params ...interface{}) { JsonClientError(ctx, w, NewNoProjectError(msg, params...)) } + +func TooManyRequestsError(ctx context.Context, w http.ResponseWriter, msg string, params ...interface{}) { + JsonClientError(ctx, w, NewTooManyRequestsError(msg, params...)) +} diff --git a/pkg/util/samlutils/demo/service.go b/pkg/util/samlutils/demo/service.go index ece88e8132..d4cc36b09b 100644 --- a/pkg/util/samlutils/demo/service.go +++ b/pkg/util/samlutils/demo/service.go @@ -101,7 +101,7 @@ func prepareServer() error { return errors.Wrapf(httperrors.ErrInputParameter, "cert %s not found", options.Cert) } - app := appsrv.NewApplication("samldemo", 4, false) + app := appsrv.NewApplication("samldemo", 4, 10, false) saml, err := samlutils.NewSAMLInstance(options.Entity, options.Cert, options.Key) if err != nil { diff --git a/vendor/github.com/petermattis/goid/.gitignore b/vendor/github.com/petermattis/goid/.gitignore new file mode 100644 index 0000000000..2b9d6b5528 --- /dev/null +++ b/vendor/github.com/petermattis/goid/.gitignore @@ -0,0 +1,4 @@ +*~ +*.test +.*.swp +.DS_Store diff --git a/vendor/github.com/petermattis/goid/LICENSE b/vendor/github.com/petermattis/goid/LICENSE new file mode 100644 index 0000000000..e06d208186 --- /dev/null +++ b/vendor/github.com/petermattis/goid/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/github.com/petermattis/goid/README.md b/vendor/github.com/petermattis/goid/README.md new file mode 100644 index 0000000000..3fd144c2c1 --- /dev/null +++ b/vendor/github.com/petermattis/goid/README.md @@ -0,0 +1,4 @@ +# goid ![Build Status](https://github.com/petermattis/goid/actions/workflows/go.yml/badge.svg) + +Programatically retrieve the current goroutine's ID. See [the CI +configuration](.github/workflows/go.yml) for supported Go versions. diff --git a/vendor/github.com/petermattis/goid/goid.go b/vendor/github.com/petermattis/goid/goid.go new file mode 100644 index 0000000000..408e619929 --- /dev/null +++ b/vendor/github.com/petermattis/goid/goid.go @@ -0,0 +1,35 @@ +// Copyright 2016 Peter Mattis. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. + +package goid + +import ( + "bytes" + "runtime" + "strconv" +) + +func ExtractGID(s []byte) int64 { + s = s[len("goroutine "):] + s = s[:bytes.IndexByte(s, ' ')] + gid, _ := strconv.ParseInt(string(s), 10, 64) + return gid +} + +// Parse the goid from runtime.Stack() output. Slow, but it works. +func getSlow() int64 { + var buf [64]byte + return ExtractGID(buf[:runtime.Stack(buf[:], false)]) +} diff --git a/vendor/github.com/petermattis/goid/goid_gccgo.go b/vendor/github.com/petermattis/goid/goid_gccgo.go new file mode 100644 index 0000000000..31c14d99a9 --- /dev/null +++ b/vendor/github.com/petermattis/goid/goid_gccgo.go @@ -0,0 +1,26 @@ +// Copyright 2018 Peter Mattis. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. + +//go:build gccgo +// +build gccgo + +package goid + +//extern runtime.getg +func getg() *g + +func Get() int64 { + return getg().goid +} diff --git a/vendor/github.com/petermattis/goid/goid_go1.3.c b/vendor/github.com/petermattis/goid/goid_go1.3.c new file mode 100644 index 0000000000..2e3f7ab79d --- /dev/null +++ b/vendor/github.com/petermattis/goid/goid_go1.3.c @@ -0,0 +1,23 @@ +// Copyright 2015 Peter Mattis. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. + +// +build !go1.4 + +#include + +void ·Get(int64 ret) { + ret = g->goid; + USED(&ret); +} diff --git a/vendor/github.com/petermattis/goid/goid_go1.3.go b/vendor/github.com/petermattis/goid/goid_go1.3.go new file mode 100644 index 0000000000..d73b699201 --- /dev/null +++ b/vendor/github.com/petermattis/goid/goid_go1.3.go @@ -0,0 +1,22 @@ +// Copyright 2015 Peter Mattis. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. + +//go:build !go1.4 +// +build !go1.4 + +package goid + +// Get returns the id of the current goroutine. +func Get() int64 diff --git a/vendor/github.com/petermattis/goid/goid_go1.4.go b/vendor/github.com/petermattis/goid/goid_go1.4.go new file mode 100644 index 0000000000..4798980b3f --- /dev/null +++ b/vendor/github.com/petermattis/goid/goid_go1.4.go @@ -0,0 +1,35 @@ +// Copyright 2015 Peter Mattis. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. + +//go:build go1.4 && !go1.5 +// +build go1.4,!go1.5 + +package goid + +import "unsafe" + +var pointerSize = unsafe.Sizeof(uintptr(0)) + +// Backdoor access to runtime·getg(). +func getg() uintptr // in goid_go1.4.s + +// Get returns the id of the current goroutine. +func Get() int64 { + // The goid is the 16th field in the G struct where each field is a + // pointer, uintptr or padded to that size. See runtime.h from the + // Go sources. I'm not aware of a cleaner way to determine the + // offset. + return *(*int64)(unsafe.Pointer(getg() + 16*pointerSize)) +} diff --git a/vendor/github.com/petermattis/goid/goid_go1.4.s b/vendor/github.com/petermattis/goid/goid_go1.4.s new file mode 100644 index 0000000000..21a07d6624 --- /dev/null +++ b/vendor/github.com/petermattis/goid/goid_go1.4.s @@ -0,0 +1,18 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Assembly to get into package runtime without using exported symbols. +// See https://github.com/golang/go/blob/release-branch.go1.4/misc/cgo/test/backdoor/thunk.s + +// +build amd64 amd64p32 arm 386 +// +build go1.4,!go1.5 + +#include "textflag.h" + +#ifdef GOARCH_arm +#define JMP B +#endif + +TEXT ·getg(SB),NOSPLIT,$0-0 + JMP runtime·getg(SB) diff --git a/vendor/github.com/petermattis/goid/goid_go1.5.go b/vendor/github.com/petermattis/goid/goid_go1.5.go new file mode 100644 index 0000000000..4521f79201 --- /dev/null +++ b/vendor/github.com/petermattis/goid/goid_go1.5.go @@ -0,0 +1,28 @@ +// Copyright 2016 Peter Mattis. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. + +//go:build (386 || amd64 || amd64p32 || arm || arm64 || s390x) && gc && go1.5 +// +build 386 amd64 amd64p32 arm arm64 s390x +// +build gc +// +build go1.5 + +package goid + +// Defined in goid_go1.5.s. +func getg() *g + +func Get() int64 { + return getg().goid +} diff --git a/vendor/github.com/petermattis/goid/goid_go1.5.s b/vendor/github.com/petermattis/goid/goid_go1.5.s new file mode 100644 index 0000000000..c49333f14a --- /dev/null +++ b/vendor/github.com/petermattis/goid/goid_go1.5.s @@ -0,0 +1,44 @@ +// Copyright 2021 Peter Mattis. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. + +// Assembly to mimic runtime.getg. + +//go:build (386 || amd64 || amd64p32 || arm || arm64 || s390x) && gc && go1.5 +// +build 386 amd64 amd64p32 arm arm64 s390x +// +build gc +// +build go1.5 + +#include "textflag.h" + +// func getg() *g +TEXT ·getg(SB),NOSPLIT,$0-8 +#ifdef GOARCH_386 + MOVL (TLS), AX + MOVL AX, ret+0(FP) +#endif +#ifdef GOARCH_amd64 + MOVQ (TLS), AX + MOVQ AX, ret+0(FP) +#endif +#ifdef GOARCH_arm + MOVW g, ret+0(FP) +#endif +#ifdef GOARCH_arm64 + MOVD g, ret+0(FP) +#endif +#ifdef GOARCH_s390x + MOVD g, ret+0(FP) +#endif + RET diff --git a/vendor/github.com/petermattis/goid/goid_slow.go b/vendor/github.com/petermattis/goid/goid_slow.go new file mode 100644 index 0000000000..8bdb4357e0 --- /dev/null +++ b/vendor/github.com/petermattis/goid/goid_slow.go @@ -0,0 +1,24 @@ +// Copyright 2016 Peter Mattis. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. + +//go:build (go1.4 && !go1.5 && !amd64 && !amd64p32 && !arm && !386) || (go1.5 && !386 && !amd64 && !amd64p32 && !arm && !arm64 && !s390x) +// +build go1.4,!go1.5,!amd64,!amd64p32,!arm,!386 go1.5,!386,!amd64,!amd64p32,!arm,!arm64,!s390x + +package goid + +// Get returns the id of the current goroutine. +func Get() int64 { + return getSlow() +} diff --git a/vendor/github.com/petermattis/goid/runtime_gccgo_go1.8.go b/vendor/github.com/petermattis/goid/runtime_gccgo_go1.8.go new file mode 100644 index 0000000000..dfcb74e0c4 --- /dev/null +++ b/vendor/github.com/petermattis/goid/runtime_gccgo_go1.8.go @@ -0,0 +1,17 @@ +//go:build gccgo && go1.8 +// +build gccgo,go1.8 + +package goid + +// https://github.com/gcc-mirror/gcc/blob/releases/gcc-7/libgo/go/runtime/runtime2.go#L329-L354 + +type g struct { + _panic uintptr + _defer uintptr + m uintptr + syscallsp uintptr + syscallpc uintptr + param uintptr + atomicstatus uint32 + goid int64 // Here it is! +} diff --git a/vendor/github.com/petermattis/goid/runtime_go1.23.go b/vendor/github.com/petermattis/goid/runtime_go1.23.go new file mode 100644 index 0000000000..146d81734a --- /dev/null +++ b/vendor/github.com/petermattis/goid/runtime_go1.23.go @@ -0,0 +1,38 @@ +//go:build gc && go1.23 +// +build gc,go1.23 + +package goid + +type stack struct { + lo uintptr + hi uintptr +} + +type gobuf struct { + sp uintptr + pc uintptr + g uintptr + ctxt uintptr + ret uintptr + lr uintptr + bp uintptr +} + +type g struct { + stack stack + stackguard0 uintptr + stackguard1 uintptr + + _panic uintptr + _defer uintptr + m uintptr + sched gobuf + syscallsp uintptr + syscallpc uintptr + syscallbp uintptr + stktopsp uintptr + param uintptr + atomicstatus uint32 + stackLock uint32 + goid int64 // Here it is! +} diff --git a/vendor/github.com/petermattis/goid/runtime_go1.5.go b/vendor/github.com/petermattis/goid/runtime_go1.5.go new file mode 100644 index 0000000000..6ce2ab8eee --- /dev/null +++ b/vendor/github.com/petermattis/goid/runtime_go1.5.go @@ -0,0 +1,57 @@ +// Copyright 2016 Peter Mattis. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. See the AUTHORS file +// for names of contributors. + +//go:build go1.5 && !go1.6 +// +build go1.5,!go1.6 + +package goid + +// Just enough of the structs from runtime/runtime2.go to get the offset to goid. +// See https://github.com/golang/go/blob/release-branch.go1.5/src/runtime/runtime2.go + +type stack struct { + lo uintptr + hi uintptr +} + +type gobuf struct { + sp uintptr + pc uintptr + g uintptr + ctxt uintptr + ret uintptr + lr uintptr + bp uintptr +} + +type g struct { + stack stack + stackguard0 uintptr + stackguard1 uintptr + + _panic uintptr + _defer uintptr + m uintptr + stackAlloc uintptr + sched gobuf + syscallsp uintptr + syscallpc uintptr + stkbar []uintptr + stkbarPos uintptr + param uintptr + atomicstatus uint32 + stackLock uint32 + goid int64 // Here it is! +} diff --git a/vendor/github.com/petermattis/goid/runtime_go1.6.go b/vendor/github.com/petermattis/goid/runtime_go1.6.go new file mode 100644 index 0000000000..983d55bc47 --- /dev/null +++ b/vendor/github.com/petermattis/goid/runtime_go1.6.go @@ -0,0 +1,43 @@ +//go:build gc && go1.6 && !go1.9 +// +build gc,go1.6,!go1.9 + +package goid + +// Just enough of the structs from runtime/runtime2.go to get the offset to goid. +// See https://github.com/golang/go/blob/release-branch.go1.6/src/runtime/runtime2.go + +type stack struct { + lo uintptr + hi uintptr +} + +type gobuf struct { + sp uintptr + pc uintptr + g uintptr + ctxt uintptr + ret uintptr + lr uintptr + bp uintptr +} + +type g struct { + stack stack + stackguard0 uintptr + stackguard1 uintptr + + _panic uintptr + _defer uintptr + m uintptr + stackAlloc uintptr + sched gobuf + syscallsp uintptr + syscallpc uintptr + stkbar []uintptr + stkbarPos uintptr + stktopsp uintptr + param uintptr + atomicstatus uint32 + stackLock uint32 + goid int64 // Here it is! +} diff --git a/vendor/github.com/petermattis/goid/runtime_go1.9.go b/vendor/github.com/petermattis/goid/runtime_go1.9.go new file mode 100644 index 0000000000..f9ef8f5ffe --- /dev/null +++ b/vendor/github.com/petermattis/goid/runtime_go1.9.go @@ -0,0 +1,37 @@ +//go:build gc && go1.9 && !go1.23 +// +build gc,go1.9,!go1.23 + +package goid + +type stack struct { + lo uintptr + hi uintptr +} + +type gobuf struct { + sp uintptr + pc uintptr + g uintptr + ctxt uintptr + ret uintptr + lr uintptr + bp uintptr +} + +type g struct { + stack stack + stackguard0 uintptr + stackguard1 uintptr + + _panic uintptr + _defer uintptr + m uintptr + sched gobuf + syscallsp uintptr + syscallpc uintptr + stktopsp uintptr + param uintptr + atomicstatus uint32 + stackLock uint32 + goid int64 // Here it is! +} diff --git a/vendor/modules.txt b/vendor/modules.txt index eaff5f6404..b5bbb3c7dc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1044,6 +1044,9 @@ github.com/openzipkin/zipkin-go-opentracing/wire # github.com/oracle/oci-go-sdk v24.3.0+incompatible ## explicit github.com/oracle/oci-go-sdk/common +# github.com/petermattis/goid v0.0.0-20250508124226-395b08cebbdb +## explicit; go 1.17 +github.com/petermattis/goid # github.com/philhofer/fwd v1.0.0 ## explicit github.com/philhofer/fwd @@ -1849,7 +1852,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250616091331-d7f1f89a7d90 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250618092159-24a4b21d2439 ## explicit; go 1.21 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing @@ -1946,7 +1949,7 @@ yunion.io/x/log/hooks yunion.io/x/ovsdb/cli_util yunion.io/x/ovsdb/schema/ovn_nb yunion.io/x/ovsdb/types -# yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0 +# yunion.io/x/pkg v1.10.4-0.20250618071935-840c253075b0 ## explicit; go 1.18 yunion.io/x/pkg/appctx yunion.io/x/pkg/errors diff --git a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go index a31e46bd3a..19d9c13f40 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go +++ b/vendor/yunion.io/x/cloudmux/pkg/apis/compute/storage_const.go @@ -21,6 +21,7 @@ const ( STORAGE_VSAN = "vsan" STORAGE_NFS = "nfs" STORAGE_CIFS = "cifs" + STORAGE_PMEM = "pmem" STORAGE_PUBLIC_CLOUD = "cloud" STORAGE_CLOUD_EFFICIENCY = "cloud_efficiency" diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/aliyun.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/aliyun.go index c76981f78e..5894606d0d 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/aliyun.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/aliyun.go @@ -808,7 +808,7 @@ func (self *SAliyunClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) accounts, err := self.ListAccounts() if err != nil { - if e, ok := errors.Cause(err).(*alierr.ServerError); ok && e.ErrorCode() == "EntityNotExists.ResourceDirectory" { + if e, ok := errors.Cause(err).(*alierr.ServerError); ok && (e.ErrorCode() == "EntityNotExists.ResourceDirectory" || e.ErrorCode() == "NoPermission") { return ret, nil } return nil, errors.Wrapf(err, "ListAccounts") diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/ctyun.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/ctyun.go index ef4eaa7faa..c5118060ac 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/ctyun.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/ctyun.go @@ -323,6 +323,9 @@ func (self *SCtyunClient) request(method httputils.THttpMethod, service, resourc func (self *SCtyunClient) GetIRegions() ([]cloudprovider.ICloudRegion, error) { ret := []cloudprovider.ICloudRegion{} for i := range self.regions { + if !self.regions[i].OpenapiAvailable { + continue + } self.regions[i].client = self ret = append(ret, &self.regions[i]) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/latitud_and_longitude.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/latitud_and_longitude.go index 686fb442a8..f36401b555 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/latitud_and_longitude.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/latitud_and_longitude.go @@ -19,59 +19,6 @@ import ( "yunion.io/x/cloudmux/pkg/cloudprovider" ) -var CtyunRegionIdMap = map[string]string{ - "07323cf87fa811ea977e0242ac110002": "cn-lasa-4", - "200000001627": "cn-fuzhou-5", - "200000001681": "cn-ln-liaoyang-1", - "200000001703": "cn-sd-qd20-public-ctcloud", - "200000001704": "cn-gx-nn23-public-ctcloud", - "200000001781": "cn-hb-wh41-public-ctcloud", - "200000001788": "cn-wulumuqi-27", - "200000001790": "cn-sh36-public-ctcloud", - "200000001852": "cn-huabei2-public-ctcloud", - "200000001858": "cfbr-fsao1-public-ctcloud", - "200000001859": "cfae-fdxb1-public-ctcloud", - "200000001860": "cfde-ffra1-public-ctcloud", - "200000001861": "cfsg-fsin3-public-ctcloud", - "200000002368": "cn-xinan1-public-ctcloud", - "200000002401": "cn-hn-cs42-public-ctcloud", - "200000002527": "cn-jx-khn5-public-ctcloud", - "21c52b2a876e11ea9f6a0242ac110002": "cn-hangzhou-2", - "22c0f506ef1d11ea80620242ac110002": "cn-nanjing-4", - "276826f4313311eaaae30242ac110002": "cn-wuhan-3", - "2cdd393e876f11ea98880242ac110002": "cn-yaan-2", - "4009c41a876e11eabdc50242ac110002": "cn-wuhu-1", - "415089caaea711eab0790242ac110002": "cn-kunming-2", - "45d9efdad66f11ec9aab0242ac110002": "cn-hefei2", - "461f819e6e3e11ea9ad30242ac110002": "cn-fujian-3", - "49829300a71211ea95240242ac110002": "cn-jinzhong-2", - "52c69bbc042411ec8dac0242ac110002": "cn-nanjing-5", - "6019b5007a0b11eab5db0242ac110002": "cn-huhehaote-6", - "60a39fca876e11ea91cf0242ac110002": "cn-nanjing-3", - "705213b6876e11eaa5740242ac110002": "cn-haikou-2", - "7dcbf0ba919c11ea83d60242ac110002": "cn-tianjin-2", - "8062c840876e11ea9d060242ac110002": "cn-wuhan-5", - "8d11979c4d5d11eab0520242ac110002": "cn-foshan-3", - "8ef3dba6876e11ea8c2a0242ac110002": "cn-nanning-2", - "9833d24065a211eaa6070242ac110002": "cn-chenzhou-4", - "9859b8964d5d11eaba270242ac110002": "cn-jiangxi-2", - "990ba31c22ec11eaaebd0242ac110002": "cn-hunan-3", - "995b39bae63811ec8c4b0242ac110002": "cn-guangzhou-5", - "a10d954c70f411eab3650242ac110002": "cn-chongqing-2", - "a17034a4794111eaaa590242ac110002": "cn-shanghai-7", - "a2ed23940b3911ea98040242ac110002": "cn-shanxi-2", - "aaf589124d5d11eaa04d0242ac110002": "cn-guiyang-1", - "ad51908ca3db11ea96c20242ac110002": "cn-haerbin-2", - "aefabf04a3df11eaa3650242ac110002": "cn-zhengzhou-2", - "b6bb383e876c11ea8a5e0242ac110002": "cn-beijing-5", - "b7e069bc876e11eaa4c00242ac110002": "cn-xian-4", - "bb9fdb42056f11eda1610242ac110002": "cn-huadong1-public-ctcloud", - "d7d93102848711ea9ff10242ac110002": "cn-fuzhou-4", - "dc3aceb4412211ecb8e70242ac110002": "cn-xian-5", - "dff35c48876e11eaadc90242ac110002": "cn-lanzhou-3", - "eeed8c16e13111e9a5b40242ac110002": "cn-nanjing-2", -} - var LatitudeAndLongitude = map[string]cloudprovider.SGeographicInfo{ "cn-beijing-5": api.RegionBeijing, "cn-hangzhou-2": api.RegionHangzhou, diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/region.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/region.go index 04d4424579..396bc29bc2 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/region.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/region.go @@ -37,12 +37,14 @@ type SRegion struct { product *SProduct - IsMultiZones bool - RegionParent string - RegionId string - RegionType string - ZoneList []string - RegionName string + IsMultiZones bool + OpenapiAvailable bool + RegionParent string + RegionId string + RegionCode string + RegionType string + ZoneList []string + RegionName string } func (self *SRegion) list(service, res string, params map[string]interface{}) (jsonutils.JSONObject, error) { @@ -118,9 +120,8 @@ func (self *SRegion) CreateISecurityGroup(opts *cloudprovider.SecurityGroupCreat } func (self *SRegion) GetId() string { - id, ok := CtyunRegionIdMap[self.RegionId] - if ok { - return id + if len(self.RegionCode) > 0 { + return self.RegionCode } return self.RegionId } @@ -136,25 +137,12 @@ func (self *SRegion) GetI18n() cloudprovider.SModelI18nTable { return table } -func (self *SRegion) getProduct() (*SProduct, error) { - if !gotypes.IsNil(self.product) { - return self.product, nil - } - var err error - self.product, err = self.GetProduct() - return self.product, err -} - func (self *SRegion) GetGlobalId() string { return fmt.Sprintf("%s/%s", self.client.GetAccessEnv(), self.GetId()) } func (self *SRegion) GetStatus() string { - product, err := self.getProduct() - if err != nil { - return api.CLOUD_REGION_STATUS_OUTOFSERVICE - } - if len(product.Other.Region) == 0 { + if !self.OpenapiAvailable { return api.CLOUD_REGION_STATUS_OUTOFSERVICE } return api.CLOUD_REGION_STATUS_INSERVER diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/securitygroup.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/securitygroup.go index 02a4e16fb0..ef0b65e185 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/securitygroup.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/securitygroup.go @@ -154,8 +154,15 @@ func (self *SSecurityGroup) CreateRule(opts *cloudprovider.SecurityGroupRuleCrea } for i := range self.SecurityGroupRuleList { if !utils.IsInStringArray(self.SecurityGroupRuleList[i].Id, ruleIds) { - self.SecurityGroupRuleList[i].secgroup = self - return &self.SecurityGroupRuleList[i], nil + if self.SecurityGroupRuleList[i].GetDirection() == opts.Direction && + self.SecurityGroupRuleList[i].GetProtocol() == opts.Protocol && + self.SecurityGroupRuleList[i].GetPriority() == opts.Priority && + self.SecurityGroupRuleList[i].GetPorts() == opts.Ports && + self.SecurityGroupRuleList[i].DestCidrIP == opts.CIDR && + self.SecurityGroupRuleList[i].GetAction() == opts.Action { + self.SecurityGroupRuleList[i].secgroup = self + return &self.SecurityGroupRuleList[i], nil + } } } time.Sleep(time.Second * 3) diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/zone.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/zone.go index 282cc3f4e2..b3ea5ddf99 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/zone.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/ctyun/zone.go @@ -72,7 +72,7 @@ func (self *SZone) GetIHostById(id string) (cloudprovider.ICloudHost, error) { } func (self *SZone) GetStorages() ([]SStorage, error) { - product, err := self.region.getProduct() + product, err := self.region.GetProduct() if err != nil { return nil, errors.Wrapf(err, "getProduct") } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/storage.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/storage.go index 2c6b722af1..bfeeded259 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/storage.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/storage.go @@ -147,6 +147,8 @@ func (self *SDatastore) getVolumeId() (string, error) { return fsInfo.Vmfs.Uuid, nil case *types.NasDatastoreInfo: return fmt.Sprintf("%s:%s", fsInfo.Nas.RemoteHost, fsInfo.Nas.RemotePath), nil + case *types.PMemDatastoreInfo: + return strings.TrimPrefix(fsInfo.Url, "ds:///"), nil } if moStore.Summary.Type == "vsan" { vsanId := moStore.Summary.Url @@ -414,10 +416,12 @@ func (self *SDatastore) GetStorageType() string { } case "nfs", "nfs41": return api.STORAGE_NFS - case "vsan": + case "vsan", "vsand": return api.STORAGE_VSAN case "cifs": return api.STORAGE_CIFS + case "pmem": + return api.STORAGE_PMEM default: log.Fatalf("unsupported datastore type %s", moStore.Summary.Type) return "" diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go index af3ef291f4..603d21e1f2 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/qcloud/qcloud.go @@ -924,7 +924,7 @@ func (client *SQcloudClient) fetchBuckets() error { func (client *SQcloudClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { nodes, err := client.DescribeOrganizationMembers() - if err != nil && errors.Cause(err) != cloudprovider.ErrNotFound { + if err != nil && errors.Cause(err) != cloudprovider.ErrNotFound && errors.Cause(err) != cloudprovider.ErrNoPermission { return nil, err } subAccount := cloudprovider.SSubAccount{} diff --git a/vendor/yunion.io/x/pkg/util/printutils/printjson.go b/vendor/yunion.io/x/pkg/util/printutils/printjson.go index ef16d432b1..d1119d7935 100644 --- a/vendor/yunion.io/x/pkg/util/printutils/printjson.go +++ b/vendor/yunion.io/x/pkg/util/printutils/printjson.go @@ -26,9 +26,33 @@ import ( "yunion.io/x/pkg/util/sets" ) +type StatusInfo struct { + Status string `json:"status"` + TotalCount int64 `json:"total_count"` + PendingDeletedCount int64 `json:"pending_deleted_count"` +} + +type StatusInfoList []StatusInfo + +func (a StatusInfoList) Len() int { return len(a) } +func (a StatusInfoList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a StatusInfoList) Less(i, j int) bool { + if a[i].TotalCount != a[j].TotalCount { + return a[i].TotalCount > a[j].TotalCount + } + if a[i].PendingDeletedCount != a[j].PendingDeletedCount { + return a[i].PendingDeletedCount > a[j].PendingDeletedCount + } + return a[i].Status < a[j].Status +} + +type TotalCountWithStatusInfo struct { + StatusInfo StatusInfoList `json:"status_info"` +} + func PrintJSONList(list *ListResult, columns []string) { colsWithData := make([]string, 0) - if columns == nil || len(columns) == 0 { + if len(columns) == 0 { colsWithDataMap := make(map[string]bool) for _, obj := range list.Data { objdict, _ := obj.(*jsonutils.JSONDict) @@ -140,21 +164,22 @@ func PrintJSONList(list *ListResult, columns []string) { } fmt.Println("*** ", title, " ***") if list.Totals != nil { - if totalDict, ok := list.Totals.(*jsonutils.JSONDict); ok { - totalMap, err := totalDict.GetMap() - if err != nil { - fmt.Println("error to convert totals to JSONDict") - } else { - segs := make([]string, 0) - for k, v := range totalMap { - segs = append(segs, fmt.Sprintf("%s: %s", k, v.String())) - } - fmt.Printf("***%s***\n", strings.Join(segs, ", ")) + totalWithStatusInfo := TotalCountWithStatusInfo{} + err := list.Totals.Unmarshal(&totalWithStatusInfo) + if err != nil { + fmt.Println("error to unmarshal totals to TotalCountWithStatusInfo", err) + } else if len(totalWithStatusInfo.StatusInfo) > 0 { + sort.Sort(totalWithStatusInfo.StatusInfo) + pt := prettytable.NewPrettyTableWithTryTermWidth([]string{"#", "Status", "Count", "PendingDeletedCount"}, tryTermWidth) + rows := make([][]string, 0) + for i, statusInfo := range totalWithStatusInfo.StatusInfo { + rows = append(rows, []string{fmt.Sprintf("#%d", i+1), statusInfo.Status, strconv.FormatInt(statusInfo.TotalCount, 10), strconv.FormatInt(statusInfo.PendingDeletedCount, 10)}) } + fmt.Print(pt.GetString(rows)) } } if colTruncated { - fmt.Println(fmt.Sprintf("!!!Some text truncated, set env %s=-1 to show full text!!!", OS_MAX_COLUMN_TEXT_LENGTH)) + fmt.Println("!!!Some text truncated, set env", OS_MAX_COLUMN_TEXT_LENGTH, "=-1 to show full text!!!") } }