feature: service config update support

cmd/climc/shell/policies.go

@@ -17,8 +17,6 @@ package shell
 import (
 	"fmt"
 	"io/ioutil"
-	"os"
-	"os/exec"
 	"strings"
 	"time"
 
@@ -32,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
 
 func init() {
@@ -212,33 +211,13 @@ func init() {
 			return err
 		}
 
-		tmpfile, err := ioutil.TempFile("", "policy-blob")
-		if err != nil {
-			return err
-		}
-		defer os.Remove(tmpfile.Name()) // clean up
-
-		if _, err := tmpfile.Write([]byte(yaml)); err != nil {
-			return err
-		}
-		if err := tmpfile.Close(); err != nil {
-			return err
-		}
-
-		cmd := exec.Command("vim", tmpfile.Name())
-		cmd.Stdin = os.Stdin
-		cmd.Stdout = os.Stdout
-		err = cmd.Run()
+		yaml, err = shellutils.Edit(yaml)
 		if err != nil {
 			return err
 		}
 
 		params := jsonutils.NewDict()
-		policyBytes, err := ioutil.ReadFile(tmpfile.Name())
-		if err != nil {
-			return err
-		}
-		params.Add(jsonutils.NewString(string(policyBytes)), "policy")
+		params.Add(jsonutils.NewString(yaml), "policy")
 
 		result, err = modules.Policies.Patch(s, policyId, params)
 		if err != nil {

cmd/climc/shell/services.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
 )
 
 func init() {
@@ -239,4 +240,34 @@ func init() {
 		return nil
 	})
 
+	type ServiceConfigEditOptions struct {
+		SERVICE string `help:"service name or id"`
+	}
+	R(&ServiceConfigEditOptions{}, "service-config-edit", "Edit config yaml of a service", func(s *mcclient.ClientSession, args *ServiceConfigEditOptions) error {
+		conf, err := modules.ServicesV3.GetSpecific(s, args.SERVICE, "config", nil)
+		if err != nil {
+			return err
+		}
+		confJson, err := conf.Get("config")
+		if err != nil {
+			return err
+		}
+		content, err := shellutils.Edit(confJson.YAMLString())
+		if err != nil {
+			return err
+		}
+		yamlJson, err := jsonutils.ParseYAML(content)
+		if err != nil {
+			return err
+		}
+		config := jsonutils.NewDict()
+		config.Add(yamlJson, "config")
+		nconf, err := modules.ServicesV3.PerformAction(s, args.SERVICE, "config", config)
+		if err != nil {
+			return err
+		}
+		fmt.Println(nconf.PrettyString())
+		return nil
+	})
+
 }

pkg/apigateway/options/options.go

@@ -35,3 +35,14 @@ type GatewayOptions struct {
 var (
 	Options GatewayOptions
 )
+
+func OnOptionsChange(oldO, newO interface{}) bool {
+	oldOpts := oldO.(*GatewayOptions)
+	newOpts := newO.(*GatewayOptions)
+
+	if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) {
+		return true
+	}
+
+	return false
+}

pkg/apigateway/service/service.go

@@ -41,10 +41,7 @@ func StartService() {
 		log.Infof("Auth complete.")
 	})
 
-	err := app_common.MergeServiceConfig(opts, api.SERVICE_TYPE, api.SERVICE_VERSION)
-	if err != nil {
-		log.Fatalf("[MERGE CONFIG] Fail to merge service config %s", err)
-	}
+	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
 
 	if opts.DisableModuleApiVersion {
 		mcclient.DisableApiVersionByModule()

pkg/apis/identity/consts.go

@@ -89,17 +89,30 @@ var (
 		},
 	}
 
-	BlacklistOptionMap = map[string][]string{
+	CommonWhitelistOptionMap = map[string][]string{
 		"default": []string{
-			"region",
-			"sql_connection",
+			"default_quota_value",
+			"enable_rbac",
+			"non_default_domain_projects",
+			"time_zone",
+		},
+	}
+
+	ServiceBlacklistOptionMap = map[string][]string{
+		"default": []string{
+			"help",
+			"version",
 			"config",
+			"pid_file",
+
+			"region",
 			"application_id",
 			"log_level",
+			"log_verbose_level",
 			"temp_path",
-			"auto_sync_table",
 			"address",
 			"port",
+			"port_v2",
 			"admin_port",
 			"notify_admin_users",
 			"session_endpoint_type",
@@ -107,12 +120,30 @@ var (
 			"admin_project",
 			"admin_user",
 			"auth_url",
-			"default_aws_instance_type_file",
-			"port_v2",
 			"enable_ssl",
 			"ssl_certfile",
 			"ssl_keyfile",
 			"ssl_ca_certs",
+
+			"is_slave_node",
+			"config_sync_period_seconds",
+
+			"sql_connection",
+			"auto_sync_table",
+			"exit_after_db_init",
+			"global_virtual_resource_namespace",
+			"debug_sqlchemy",
+			"lockman_method",
+			"etcd_lock_prefix",
+			"etcd_lock_ttl",
+			"etcd_endpoints",
+			"etcd_username",
+			"etcd_password",
+			"etcd_use_tls",
+			"etcd_skip_tls_verify",
+			"etcd_cacert",
+			"etcd_cert",
+			"etcd_key",
 		},
 	}
 )

pkg/appsrv/workers.go

@@ -37,10 +37,16 @@ func enableDebug() {
 	isDebug = true
 }
 
-var workerManagers []*SWorkerManager
+var (
+	workerManagers []*SWorkerManager
+
+	workerManagerLock *sync.Mutex
+)
 
 func init() {
 	workerManagers = make([]*SWorkerManager, 0)
+
+	workerManagerLock = &sync.Mutex{}
 }
 
 type SWorker struct {
@@ -166,6 +172,9 @@ func NewWorkerManagerIgnoreOverflow(name string, workerCount int, backlog int, d
 		ignoreOverflow: ignoreOverflow,
 	}
 
+	workerManagerLock.Lock()
+	defer workerManagerLock.Unlock()
+
 	workerManagers = append(workerManagers, &manager)
 	return &manager
 }
@@ -252,6 +261,13 @@ type SWorkerManagerStates struct {
 	DetachWorkerCnt int
 }
 
+func (s SWorkerManagerStates) IsBusy() bool {
+	if s.QueueCnt == 0 && s.ActiveWorkerCnt == 0 && s.DetachWorkerCnt == 0 {
+		return false
+	}
+	return true
+}
+
 func (wm *SWorkerManager) getState() SWorkerManagerStates {
 	state := SWorkerManagerStates{}
 

pkg/appsrv/workers_watchdog.go

@@ -0,0 +1,78 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package appsrv
+
+import (
+	"time"
+
+	"yunion.io/x/log"
+)
+
+const (
+	WATCHDOG_SLEEP_SECONDS = 30
+)
+
+var (
+	busyWorkers map[*SWorkerManager]int
+
+	exitFlag bool
+)
+
+func init() {
+	busyWorkers = make(map[*SWorkerManager]int)
+
+	watchdog()
+}
+
+func watchdog() {
+	do_worker_watchdog()
+
+	time.AfterFunc(time.Second*WATCHDOG_SLEEP_SECONDS, watchdog)
+}
+
+func do_worker_watchdog() {
+	log.Debugf("worker manager watchdog runing")
+
+	for _, w := range workerManagers {
+		stats := w.getState()
+		busy := stats.IsBusy()
+		if busy {
+			if _, ok := busyWorkers[w]; ok {
+				busyWorkers[w] += 1
+			} else {
+				busyWorkers[w] = 1
+			}
+		} else {
+			if _, ok := busyWorkers[w]; ok {
+				delete(busyWorkers, w)
+			}
+		}
+	}
+	if len(busyWorkers) > 0 {
+		for w, k := range busyWorkers {
+			if k > 1 {
+				log.Warningf("WorkerManager %s has been busy for %d cycles...", w.name, k)
+			}
+		}
+	} else {
+		if exitFlag {
+			log.Fatalln("System is idle, no worker is busy, exitFlag is set, to exit ...")
+		}
+	}
+}
+
+func SetExitFlag() {
+	exitFlag = true
+}

pkg/cloudcommon/options/changes.go

@@ -0,0 +1,38 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+func OnBaseOptionsChange(oOpts, nOpts interface{}) bool {
+	oldOpts := oOpts.(*BaseOptions)
+	newOpts := nOpts.(*BaseOptions)
+
+	if oldOpts.RequestWorkerCount != newOpts.RequestWorkerCount {
+		return true
+	}
+	if oldOpts.TimeZone != newOpts.TimeZone {
+		return true
+	}
+	return false
+}
+
+func OnCommonOptionsChange(oOpts, nOpts interface{}) bool {
+	oldOpts := oOpts.(*CommonOptions)
+	newOpts := nOpts.(*CommonOptions)
+
+	if OnBaseOptionsChange(&oldOpts.BaseOptions, &newOpts.BaseOptions) {
+		return true
+	}
+	return false
+}

pkg/cloudcommon/options/manager.go

@@ -0,0 +1,105 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+import (
+	"reflect"
+	"time"
+
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/appsrv"
+)
+
+const (
+	MIN_REFRESH_INTERVAL_SECONDS = 30
+)
+
+type TOptionsChangeFunc func(oldOpts, newOpts interface{}) bool
+
+type SOptionManager struct {
+	serviceType    string
+	serviceVersion string
+
+	options interface{}
+
+	session IServiceConfigSession
+
+	refreshInterval time.Duration
+
+	onOptionsChange TOptionsChangeFunc
+}
+
+var (
+	OptionManager *SOptionManager
+)
+
+func StartOptionManager(option interface{}, refreshSeconds int, serviceType, serviceVersion string, onChange TOptionsChangeFunc) {
+	StartOptionManagerWithSessionDriver(option, refreshSeconds, serviceType, serviceVersion, onChange, newServiceConfigSession())
+}
+
+func StartOptionManagerWithSessionDriver(options interface{}, refreshSeconds int, serviceType, serviceVersion string, onChange TOptionsChangeFunc, session IServiceConfigSession) {
+	log.Infof("OptionManager start to fetch service configs ...")
+	if refreshSeconds <= MIN_REFRESH_INTERVAL_SECONDS {
+		// a minimal 30 seconds refresh interval
+		refreshSeconds = MIN_REFRESH_INTERVAL_SECONDS
+	}
+	refreshInterval := time.Duration(refreshSeconds) * time.Second
+	OptionManager = &SOptionManager{
+		serviceType:     serviceType,
+		serviceVersion:  serviceVersion,
+		options:         options,
+		session:         session,
+		refreshInterval: refreshInterval,
+		onOptionsChange: onChange,
+	}
+	OptionManager.firstSync()
+}
+
+func (manager *SOptionManager) newOptions() interface{} {
+	optType := reflect.ValueOf(manager.options).Elem().Type()
+	return reflect.New(optType).Interface()
+}
+
+func copyOptions(dst, src interface{}) {
+	dstValue := reflect.ValueOf(dst).Elem()
+	dstValue.Set(reflect.ValueOf(src).Elem())
+}
+
+func (manager *SOptionManager) doSync(first bool) {
+	newOpts := manager.newOptions()
+	copyOptions(newOpts, manager.options)
+	merged := manager.session.Merge(newOpts, manager.serviceType, manager.serviceVersion)
+
+	if merged && !reflect.DeepEqual(newOpts, manager.options) {
+		log.Infof("Service config changed ...")
+		if !first && manager.onOptionsChange != nil && manager.onOptionsChange(manager.options, newOpts) {
+			log.Infof("Option changes detected and going to restart the program...")
+			appsrv.SetExitFlag()
+		}
+		copyOptions(manager.options, newOpts)
+		manager.session.Upload()
+	}
+}
+
+func (manager *SOptionManager) firstSync() {
+	manager.doSync(true)
+	time.AfterFunc(manager.refreshInterval, manager.sync)
+}
+
+func (manager *SOptionManager) sync() {
+	manager.doSync(false)
+	time.AfterFunc(manager.refreshInterval, manager.sync)
+}

pkg/cloudcommon/options/mergeconf.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package app
+package options
 
 import (
 	"context"
@@ -56,46 +56,66 @@ func getServiceConfig(s *mcclient.ClientSession, serviceId string) (jsonutils.JS
 	return defConf, nil
 }
 
-func MergeServiceConfig(opts interface{}, serviceType string, serviceVersion string) error {
+type IServiceConfigSession interface {
+	Merge(opts interface{}, serviceType string, serviceVersion string) bool
+	Upload()
+}
+
+type mcclientServiceConfigSession struct {
+	session   *mcclient.ClientSession
+	serviceId string
+	config    *jsonutils.JSONDict
+}
+
+func newServiceConfigSession() IServiceConfigSession {
+	return &mcclientServiceConfigSession{}
+}
+
+func (s *mcclientServiceConfigSession) Merge(opts interface{}, serviceType string, serviceVersion string) bool {
 	merged := false
-	conf := jsonutils.Marshal(opts).(*jsonutils.JSONDict)
-	region, _ := conf.GetString("region")
-	epType, _ := conf.GetString("session_endpoint_type")
-	s := auth.AdminSession(context.Background(), region, "", epType, "")
-	serviceId, _ := getServiceIdByType(s, serviceType, serviceVersion)
-	if len(serviceId) > 0 {
-		serviceConf, err := getServiceConfig(s, serviceId)
+	s.config = jsonutils.Marshal(opts).(*jsonutils.JSONDict)
+	region, _ := s.config.GetString("region")
+	epType, _ := s.config.GetString("session_endpoint_type")
+	s.session = auth.AdminSession(context.Background(), region, "", epType, "")
+	s.serviceId, _ = getServiceIdByType(s.session, serviceType, serviceVersion)
+	if len(s.serviceId) > 0 {
+		serviceConf, err := getServiceConfig(s.session, s.serviceId)
 		if err != nil {
 			log.Errorf("getServiceConfig for %s failed: %s", serviceType, err)
 		} else {
-			conf.Update(serviceConf)
+			s.config.Update(serviceConf)
 			merged = true
 		}
 	}
-	commonServiceId, _ := getServiceIdByType(s, consts.COMMON_SERVICE, "")
+	commonServiceId, _ := getServiceIdByType(s.session, consts.COMMON_SERVICE, "")
 	if len(commonServiceId) > 0 {
-		commonConf, err := getServiceConfig(s, commonServiceId)
+		commonConf, err := getServiceConfig(s.session, commonServiceId)
 		if err != nil {
 			log.Errorf("getServiceConfig for %s failed: %s", consts.COMMON_SERVICE, err)
 		} else {
-			conf.Update(commonConf)
+			s.config.Update(commonConf)
 			merged = true
 		}
 	}
 	if merged {
-		err := conf.Unmarshal(opts)
-		if err != nil {
-			return errors.Wrap(err, "conf.Unmarshal")
+		err := s.config.Unmarshal(opts)
+		if err == nil {
+			return true
 		}
-		if len(serviceId) > 0 {
-			nconf := jsonutils.NewDict()
-			nconf.Add(conf, "config", "default")
-			_, err := modules.ServicesV3.PerformAction(s, serviceId, "config", nconf)
-			if err != nil {
-				// ignore the error
-				log.Errorf("fail to save config: %s", err)
-			}
+		log.Errorf("s.config.Unmarshal fail %s", err)
+	}
+	return false
+}
+
+func (s *mcclientServiceConfigSession) Upload() {
+	// upload service config
+	if len(s.serviceId) > 0 {
+		nconf := jsonutils.NewDict()
+		nconf.Add(s.config, "config", "default")
+		_, err := modules.ServicesV3.PerformAction(s.session, s.serviceId, "config", nconf)
+		if err != nil {
+			// ignore the error
+			log.Errorf("fail to save config: %s", err)
 		}
 	}
-	return nil
 }

pkg/cloudcommon/options/options.go

@@ -73,6 +73,8 @@ type BaseOptions struct {
 	RbacPolicySyncPeriodSeconds      int  `help:"policy sync interval in seconds, default 5 minutes" default:"300"`
 	RbacPolicySyncFailedRetrySeconds int  `help:"seconds to wait after a failed sync, default 30 seconds" default:"30"`
 
+	ConfigSyncPeriodSeconds int `help:"service config sync interval in seconds, default 300 seconds/5 minutes" default:"300"`
+
 	IsSlaveNode        bool `help:"Region service slave node"`
 	CronJobWorkerCount int  `help:"Cron job worker count" default:"4"`
 
@@ -80,7 +82,7 @@ type BaseOptions struct {
 
 	CalculateQuotaUsageIntervalSeconds int `help:"interval to calculate quota usages, default 30 minutes" default:"900"`
 
-	NonDefaultDomainProjects bool `help:"allow projects in non-default domains" default:"false"`
+	NonDefaultDomainProjects bool `help:"allow projects in non-default domains" default:"false" json:",allowfalse"`
 
 	TimeZone string `help:"time zone" default:"Asia/Shanghai"`
 

pkg/compute/options/options.go

@@ -133,3 +133,13 @@ type SCapabilityOptions struct {
 var (
 	Options ComputeOptions
 )
+
+func OnOptionsChange(oldO, newO interface{}) bool {
+	oldOpts := oldO.(*ComputeOptions)
+	newOpts := newO.(*ComputeOptions)
+
+	if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) {
+		return true
+	}
+	return false
+}

pkg/compute/service/service.go

@@ -61,14 +61,11 @@ func StartService() {
 	db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
 	defer cloudcommon.CloseDB()
 
-	err := app_common.MergeServiceConfig(opts, api.SERVICE_TYPE, api.SERVICE_VERSION)
-	if err != nil {
-		log.Fatalf("[MERGE CONFIG] Fail to merge service config %s", err)
-	}
+	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
 
 	options.InitNameSyncResources()
 
-	err = setInfluxdbRetentionPolicy()
+	err := setInfluxdbRetentionPolicy()
 	if err != nil {
 		log.Errorf("setInfluxdbRetentionPolicy fail: %s", err)
 	}

pkg/image/options/options.go

@@ -46,3 +46,14 @@ type SImageOptions struct {
 var (
 	Options SImageOptions
 )
+
+func OnOptionsChange(oldO, newO interface{}) bool {
+	oldOpts := oldO.(*SImageOptions)
+	newOpts := newO.(*SImageOptions)
+
+	if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) {
+		return true
+	}
+
+	return false
+}

pkg/image/service/service.go

@@ -93,10 +93,7 @@ func StartService() {
 
 	db.EnsureAppInitSyncDB(app, dbOpts, models.InitDB)
 
-	err := app_common.MergeServiceConfig(opts, api.SERVICE_TYPE, api.SERVICE_VERSION)
-	if err != nil {
-		log.Fatalf("[MERGE CONFIG] Fail to merge service config %s", err)
-	}
+	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
 
 	go models.CheckImages()
 

pkg/keystone/models/configs.go

@@ -19,13 +19,14 @@ import (
 	"sort"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/onecloud/pkg/keystone/options"
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 )
 
 type SConfigOptionManager struct {
@@ -183,7 +184,7 @@ func (manager *SConfigOptionManager) syncConfigs(model db.IModel, newOpts TConfi
 	return nil
 }
 
-func getConfigOptions(conf api.TConfigs, model db.IModel, blackList map[string][]string, sensitiveList map[string][]string) (TConfigOptions, TConfigOptions) {
+func getConfigOptions(conf api.TConfigs, model db.IModel, whiteList map[string][]string, blackList map[string][]string, sensitiveList map[string][]string) (TConfigOptions, TConfigOptions) {
 	options := make(TConfigOptions, 0)
 	sensitive := make(TConfigOptions, 0)
 	for group, groupConf := range conf {
@@ -194,12 +195,21 @@ func getConfigOptions(conf api.TConfigs, model db.IModel, blackList map[string][
 			opt.Group = group
 			opt.Option = optKey
 			opt.Value = optVal
-			if v, ok := blackList[group]; ok && utils.IsInStringArray(optKey, v) {
-				// skip
-			} else if v, ok := sensitiveList[group]; ok && utils.IsInStringArray(optKey, v) {
+			if v, ok := sensitiveList[group]; ok && utils.IsInStringArray(optKey, v) {
 				sensitive = append(sensitive, opt)
 			} else {
-				options = append(options, opt)
+				if whiteList != nil {
+					if v, ok := whiteList[group]; ok && utils.IsInStringArray(optKey, v) {
+						options = append(options, opt)
+					}
+				} else if blackList != nil {
+					if v, ok := blackList[group]; ok && utils.IsInStringArray(optKey, v) {
+					} else {
+						options = append(options, opt)
+					}
+				} else {
+					options = append(options, opt)
+				}
 			}
 		}
 	}
@@ -297,8 +307,8 @@ func GetConfigs(model db.IModel, all bool) (api.TConfigs, error) {
 	return config2map(opts), nil
 }
 
-func saveConfigs(action string, model db.IModel, opts api.TConfigs, blackList map[string][]string, sensitiveConfs map[string][]string) error {
-	whiteListedOpts, sensitiveOpts := getConfigOptions(opts, model, blackList, sensitiveConfs)
+func saveConfigs(action string, model db.IModel, opts api.TConfigs, whiteList map[string][]string, blackList map[string][]string, sensitiveConfs map[string][]string) error {
+	whiteListedOpts, sensitiveOpts := getConfigOptions(opts, model, whiteList, blackList, sensitiveConfs)
 	if action == "update" {
 		err := WhitelistedConfigManager.updateConfigs(whiteListedOpts)
 		if err != nil {
@@ -330,47 +340,65 @@ func saveConfigs(action string, model db.IModel, opts api.TConfigs, blackList ma
 	return nil
 }
 
-func MergeServiceConfig(opts *options.SKeystoneOptions) error {
+type dbServiceConfigSession struct {
+	config  *jsonutils.JSONDict
+	service *SService
+}
+
+func NewServiceConfigSession() common_options.IServiceConfigSession {
+	return &dbServiceConfigSession{}
+}
+
+func (s *dbServiceConfigSession) Merge(opts interface{}, serviceType string, serviceVersion string) bool {
 	merged := false
-	conf := jsonutils.Marshal(opts).(*jsonutils.JSONDict)
-	service, _ := ServiceManager.fetchServiceByType(api.SERVICE_TYPE)
-	if service != nil {
-		serviceConf, err := GetConfigs(service, false)
+	s.config = jsonutils.Marshal(opts).(*jsonutils.JSONDict)
+	s.service, _ = ServiceManager.fetchServiceByType(serviceType)
+	if s.service != nil {
+		serviceConf, err := GetConfigs(s.service, false)
 		if err != nil {
-			return errors.Wrap(err, "GetConfigs service")
+			log.Errorf("GetConfigs for %s fail: %s", serviceType, err)
+		} else {
+			serviceConfJson := jsonutils.Marshal(serviceConf["default"])
+			s.config.Update(serviceConfJson)
+			merged = true
 		}
-		serviceConfJson := jsonutils.Marshal(serviceConf["default"])
-		conf.Update(serviceConfJson)
-		merged = true
 	}
 	commonService, _ := ServiceManager.fetchServiceByType(consts.COMMON_SERVICE)
 	if commonService != nil {
 		commonConf, err := GetConfigs(commonService, false)
 		if err != nil {
-			return errors.Wrap(err, "GetConfigs commonService")
+			log.Errorf("GetConfigs for %s fail: %s", consts.COMMON_SERVICE, err)
+		} else {
+			commonConfJson := jsonutils.Marshal(commonConf["default"])
+			s.config.Update(commonConfJson)
+			merged = true
 		}
-		commonConfJson := jsonutils.Marshal(commonConf["default"])
-		conf.Update(commonConfJson)
-		merged = true
 	}
 	if merged {
-		err := conf.Unmarshal(opts)
-		if err != nil {
-			return errors.Wrap(err, "conf.Unmarshal")
-		}
-		if service != nil {
-			nconf := jsonutils.NewDict()
-			nconf.Add(conf, "default")
-			tconf := api.TConfigs{}
-			err = nconf.Unmarshal(tconf)
-			if err != nil {
-				return errors.Wrap(err, "conf.Unmarshal(tconf)")
-			}
-			err = saveConfigs("", service, tconf, api.BlacklistOptionMap, nil)
-			if err != nil {
-				return errors.Wrap(err, "saveConfigs")
-			}
+		err := s.config.Unmarshal(opts)
+		if err == nil {
+			return true
 		}
+		log.Errorf("s.config.Unmarshal fail %s", err)
+	}
+	return false
+}
+
+func (s *dbServiceConfigSession) Upload() {
+	if s.service == nil {
+		return
+	}
+	nconf := jsonutils.NewDict()
+	nconf.Add(s.config, "default")
+	tconf := api.TConfigs{}
+	err := nconf.Unmarshal(tconf)
+	if err != nil {
+		log.Errorf("nconf.Unmarshal fail %s", err)
+		return
+	}
+	err = saveConfigs("", s.service, tconf, nil, api.ServiceBlacklistOptionMap, nil)
+	if err != nil {
+		log.Errorf("saveConfigs fail %s", err)
+		return
 	}
-	return nil
 }

pkg/keystone/models/identity_provider.go

@@ -228,7 +228,7 @@ func (ident *SIdentityProvider) PerformConfig(ctx context.Context, userCred mccl
 		return nil, httperrors.NewInputParameterError("invalid input data")
 	}
 	action, _ := data.GetString("action")
-	err = saveConfigs(action, ident, opts, nil, api.SensitiveDomainConfigMap)
+	err = saveConfigs(action, ident, opts, nil, nil, api.SensitiveDomainConfigMap)
 	if err != nil {
 		return nil, httperrors.NewInternalServerError("saveConfig fail %s", err)
 	}
@@ -327,7 +327,7 @@ func (ident *SIdentityProvider) PostCreate(ctx context.Context, userCred mcclien
 		log.Errorf("parse config error %s", err)
 		return
 	}
-	err = saveConfigs("", ident, opts, nil, api.SensitiveDomainConfigMap)
+	err = saveConfigs("", ident, opts, nil, nil, api.SensitiveDomainConfigMap)
 	if err != nil {
 		log.Errorf("saveConfig fail %s", err)
 		return

pkg/keystone/models/services.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/identity"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -159,6 +160,14 @@ func (service *SService) AllowPerformConfig(ctx context.Context, userCred mcclie
 	return db.IsAdminAllowUpdateSpec(userCred, service, "config")
 }
 
+func (service *SService) isCommonService() bool {
+	if service.Type == consts.COMMON_SERVICE {
+		return true
+	} else {
+		return false
+	}
+}
+
 func (service *SService) PerformConfig(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
 	action, _ := data.GetString("action")
 	opts := api.TConfigs{}
@@ -166,7 +175,11 @@ func (service *SService) PerformConfig(ctx context.Context, userCred mcclient.To
 	if err != nil {
 		return nil, httperrors.NewInputParameterError("invalid input data")
 	}
-	err = saveConfigs(action, service, opts, api.BlacklistOptionMap, nil)
+	if service.isCommonService() {
+		err = saveConfigs(action, service, opts, api.CommonWhitelistOptionMap, nil, nil)
+	} else {
+		err = saveConfigs(action, service, opts, nil, api.ServiceBlacklistOptionMap, nil)
+	}
 	if err != nil {
 		return nil, httperrors.NewInternalServerError("saveConfig fail %s", err)
 	}

pkg/keystone/options/options.go

@@ -27,7 +27,7 @@ type SKeystoneOptions struct {
 
 	TokenExpirationSeconds int    `default:"86400" help:"token expiration seconds" token:"expiration"`
 	FernetKeyRepository    string `help:"fernet key repo directory" token:"key_repository" default:"/etc/yunion/keystone/fernet-keys"`
-	SetupCredentialKeys    bool   `help:"setup standalone fernet keys for credentials" token:"setup_credential_key" default:"false"`
+	SetupCredentialKeys    bool   `help:"setup standalone fernet keys for credentials" token:"setup_credential_key" default:"false" json:",allowfalse"`
 
 	BootstrapAdminUserPassword string `help:"bootstreap sysadmin user password" default:"sysadmin"`
 
@@ -47,3 +47,14 @@ type SKeystoneOptions struct {
 var (
 	Options SKeystoneOptions
 )
+
+func OnOptionsChange(oldOptions, newOptions interface{}) bool {
+	oldOpts := oldOptions.(*SKeystoneOptions)
+	newOpts := newOptions.(*SKeystoneOptions)
+
+	if options.OnBaseOptionsChange(&oldOpts.BaseOptions, &newOpts.BaseOptions) {
+		return true
+	}
+
+	return false
+}

pkg/keystone/service/service.go

@@ -21,15 +21,13 @@ import (
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/golang-plus/uuid"
 
-	"yunion.io/x/log"
-
 	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
-	"yunion.io/x/onecloud/pkg/cloudcommon/policy" // "yunion.io/x/onecloud/pkg/keystone/keys"
+	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/keystone/cronjobs"
 	_ "yunion.io/x/onecloud/pkg/keystone/driver/cas"
 	_ "yunion.io/x/onecloud/pkg/keystone/driver/ldap"
@@ -76,10 +74,7 @@ func StartService() {
 
 	app_common.InitBaseAuth(&opts.BaseOptions)
 
-	err := models.MergeServiceConfig(opts)
-	if err != nil {
-		log.Fatalf("[MERGE CONFIG] Fail to merge service config: %s", err)
-	}
+	common_options.StartOptionManagerWithSessionDriver(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, "", options.OnOptionsChange, models.NewServiceConfigSession())
 
 	if !opts.IsSlaveNode {
 		cron := cronman.InitCronJobManager(true, opts.CronJobWorkerCount)

pkg/util/shellutils/edit.go

@@ -0,0 +1,53 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shellutils
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+
+	"yunion.io/x/pkg/errors"
+)
+
+func Edit(yaml string) (string, error) {
+	tmpfile, err := ioutil.TempFile("", "policy-blob")
+	if err != nil {
+		return "", errors.Wrap(err, "ioutil.TempFile")
+	}
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	if _, err := tmpfile.Write([]byte(yaml)); err != nil {
+		return "", errors.Wrap(err, "tmpfile.Write")
+	}
+	if err := tmpfile.Close(); err != nil {
+		return "", errors.Wrap(err, "tmpfile.Close")
+	}
+
+	cmd := exec.Command("vim", tmpfile.Name())
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	err = cmd.Run()
+	if err != nil {
+		return "", errors.Wrap(err, "cmd.Run")
+	}
+
+	policyBytes, err := ioutil.ReadFile(tmpfile.Name())
+	if err != nil {
+		return "", errors.Wrap(err, "ioutil.ReadFile")
+	}
+
+	return string(policyBytes), nil
+}

scripts/rbac/domainadmin.yaml

@@ -3,6 +3,4 @@ roles:
   - domain_admin
 scope: domain
 policy:
-  *:
-    *:
-      *: allow
+  '*': allow

scripts/rbac/member.yaml

@@ -1,8 +1,9 @@
 # rbac for normal user, not allow for delete
 scope: project
 policy:
-  *:
-    *:
-      *: allow
+  '*':
+    '*':
+      '*': allow
       create: deny
+      update: deny
       delete: deny

scripts/rbac/projectowner.yaml

@@ -3,6 +3,4 @@ roles:
   - project_owner
 scope: project
 policy:
-  *:
-    *:
-      *: allow
+  '*': allow

scripts/rbac/sysadmin.yaml

@@ -5,4 +5,4 @@ roles:
   - admin
 scope: system
 policy:
-  *: allow
+  '*': allow