Merge pull request #9113 from yousong/automated-cherry-pick-of-#9109-upstream-release-3.3

Automated cherry pick of #9109: fix(host): make sure it won't match vpc guests by accident

go.mod

@@ -131,7 +131,7 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6
+	yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769
 	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
 	yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6

go.sum

@@ -1128,6 +1128,8 @@ yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNp
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
 yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6 h1:RPpnTSCZwa9VQuWAf7S2dRjnZlYTIM0NH/j5vvUpk9g=
 yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4=
+yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960RuOa9I55imIGrigSdjs=

pkg/cloudcommon/db/jointbase.go

@@ -34,7 +34,7 @@ import (
 type SJointResourceBase struct {
 	SResourceBase
 
-	RowId int64 `primary:"true" auto_increment:"true"`
+	RowId int64 `primary:"true" auto_increment:"true" list:"user"`
 }
 
 type SJointResourceBaseManager struct {

pkg/cloudprovider/securitygroup.go

@@ -125,9 +125,6 @@ func AddDefaultRule(rules []SecurityRule, defaultRule SecurityRule, localRuleStr
 		defaultRule.Priority = max
 	}
 	defaultRule.Priority -= int(order)
-	if onlyAllowRules {
-		defaultRule.Priority = -1
-	}
 	return append(rules, defaultRule)
 }
 

pkg/compute/models/guest_actions.go

@@ -2023,6 +2023,12 @@ func (self *SGuest) PerformChangeIpaddr(ctx context.Context, userCred mcclient.T
 			}
 			return nil, httperrors.NewBadRequestError(err.Error())
 		}
+		if _, err := db.Update(&ngn[0], func() error {
+			ngn[0].EipId = gn.EipId
+			return nil
+		}); err != nil {
+			return nil, err
+		}
 
 		return ngn, nil
 	}()

pkg/compute/models/regiondrivers.go

@@ -16,7 +16,6 @@ package models
 
 import (
 	"context"
-	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -155,7 +154,6 @@ type IDBInstanceDriver interface {
 
 	RequestCreateDBInstance(ctx context.Context, userCred mcclient.TokenCredential, dbinstance *SDBInstance, task taskman.ITask) error
 	RequestCreateDBInstanceBackup(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, backup *SDBInstanceBackup, task taskman.ITask) error
-	RequestRenewDBInstance(instance *SDBInstance, bc billing.SBillingCycle) (time.Time, error)
 	RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, instance *SDBInstance, task taskman.ITask) error
 
 	IsSupportedDBInstance() bool

pkg/compute/regiondrivers/aws.go

@@ -60,7 +60,7 @@ func (self *SAwsRegionDriver) GetDefaultSecurityGroupInRule() cloudprovider.Secu
 }
 
 func (self *SAwsRegionDriver) GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule {
-	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:allow any")}
+	return cloudprovider.SecurityRule{SecurityRule: *secrules.MustParseSecurityRule("out:deny any")}
 }
 
 func (self *SAwsRegionDriver) GetSecurityGroupRuleMaxPriority() int {

pkg/compute/regiondrivers/base.go

@@ -17,7 +17,6 @@ package regiondrivers
 import (
 	"context"
 	"fmt"
-	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
@@ -302,10 +301,6 @@ func (self *SBaseRegionDriver) InitDBInstanceUser(ctx context.Context, dbinstanc
 	return nil
 }
 
-func (self *SBaseRegionDriver) RequestRenewDBInstance(instance *models.SDBInstance, bc billing.SBillingCycle) (time.Time, error) {
-	return time.Time{}, fmt.Errorf("Not Implement RequestRenewDBInstance")
-}
-
 func (self *SBaseRegionDriver) RequestChangeDBInstanceConfig(ctx context.Context, userCred mcclient.TokenCredential, instance *models.SDBInstance, task taskman.ITask) error {
 	return fmt.Errorf("Not Implement RequestChangeDBInstanceConfig")
 }

pkg/compute/regiondrivers/secgroup_aws_test.go

@@ -34,7 +34,7 @@ func TestAwsRuleSync(t *testing.T) {
 
 	data := []TestData{
 		{
-			Name: "Test out deny rules",
+			Name: "Test remove out allow rules",
 			LocalRules: secrules.SecurityRuleSet{
 				localRuleWithPriority("out:deny any", 1),
 			},
@@ -49,6 +49,30 @@ func TestAwsRuleSync(t *testing.T) {
 				remoteRuleWithName("", "out:allow any", 1),
 			},
 		},
+		{
+			Name: "Test out deny rules",
+			LocalRules: secrules.SecurityRuleSet{
+				localRuleWithPriority("out:deny any", 1),
+			},
+			RemoteRules: []cloudprovider.SecurityRule{},
+			Common:      []cloudprovider.SecurityRule{},
+			InAdds:      []cloudprovider.SecurityRule{},
+			OutAdds:     []cloudprovider.SecurityRule{},
+			InDels:      []cloudprovider.SecurityRule{},
+			OutDels:     []cloudprovider.SecurityRule{},
+		},
+		{
+			Name:        "Test out allow rules",
+			LocalRules:  secrules.SecurityRuleSet{},
+			RemoteRules: []cloudprovider.SecurityRule{},
+			Common:      []cloudprovider.SecurityRule{},
+			InAdds:      []cloudprovider.SecurityRule{},
+			OutAdds: []cloudprovider.SecurityRule{
+				remoteRuleWithName("", "out:allow any", 0),
+			},
+			InDels:  []cloudprovider.SecurityRule{},
+			OutDels: []cloudprovider.SecurityRule{},
+		},
 	}
 
 	for _, d := range data {

pkg/compute/tasks/dbinstance_renew_task.go

@@ -16,14 +16,16 @@ package tasks
 
 import (
 	"context"
-	"fmt"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/util/billing"
 	"yunion.io/x/onecloud/pkg/util/logclient"
@@ -37,33 +39,62 @@ func init() {
 	taskman.RegisterTask(DBInstanceRenewTask{})
 }
 
+func (self *DBInstanceRenewTask) taskFailed(ctx context.Context, rds *models.SDBInstance, err error) {
+	db.OpsLog.LogEvent(rds, db.ACT_REW_FAIL, err, self.UserCred)
+	logclient.AddActionLogWithStartable(self, rds, logclient.ACT_RENEW, err, self.UserCred, false)
+	rds.SetStatus(self.GetUserCred(), api.DBINSTANCE_RENEW_FAILED, err.Error())
+	self.SetStageFailed(ctx, err.Error())
+}
+
 func (self *DBInstanceRenewTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
-	instance := obj.(*models.SDBInstance)
+	rds := obj.(*models.SDBInstance)
 
-	durationStr, _ := self.GetParams().GetString("duration")
-	bc, _ := billing.ParseBillingCycle(durationStr)
-
-	exp, err := instance.GetRegion().GetDriver().RequestRenewDBInstance(instance, bc)
+	duration, _ := self.GetParams().GetString("duration")
+	bc, err := billing.ParseBillingCycle(duration)
 	if err != nil {
-		msg := fmt.Sprintf("RequestRenewDBInstance failed %s", err)
-		log.Errorf(msg)
-		db.OpsLog.LogEvent(instance, db.ACT_REW_FAIL, msg, self.UserCred)
-		logclient.AddActionLogWithStartable(self, instance, logclient.ACT_RENEW, msg, self.UserCred, false)
-		instance.SetStatus(self.GetUserCred(), api.DBINSTANCE_RENEW_FAILED, msg)
-		self.SetStageFailed(ctx, msg)
+		self.taskFailed(ctx, rds, errors.Wrapf(err, "ParseBillingCycle(%s)", duration))
 		return
 	}
 
-	err = instance.SaveRenewInfo(ctx, self.UserCred, &bc, &exp, "")
+	iRds, err := rds.GetIDBInstance()
 	if err != nil {
-		msg := fmt.Sprintf("SaveRenewInfo fail %s", err)
-		log.Errorf(msg)
-		self.SetStageFailed(ctx, msg)
+		self.taskFailed(ctx, rds, errors.Wrapf(err, "GetIDBInstance"))
+		return
+	}
+	oldExpired := iRds.GetExpiredAt()
+
+	err = iRds.Renew(bc)
+	if err != nil {
+		self.taskFailed(ctx, rds, errors.Wrapf(err, "iRds.Renew"))
 		return
 	}
 
-	logclient.AddActionLogWithStartable(self, instance, logclient.ACT_RENEW, nil, self.UserCred, true)
+	err = cloudprovider.WaitCreated(15*time.Second, 5*time.Minute, func() bool {
+		err := iRds.Refresh()
+		if err != nil {
+			log.Errorf("failed refresh rds %s error: %v", rds.Name, err)
+		}
+		newExipred := iRds.GetExpiredAt()
+		if newExipred.After(oldExpired) {
+			return true
+		}
+		return false
+	})
+	if err != nil {
+		self.taskFailed(ctx, rds, errors.Wrapf(err, "wait expired time refresh"))
+		return
+	}
 
-	models.StartResourceSyncStatusTask(ctx, self.UserCred, instance, "DBInstanceSyncStatusTask", self.GetTaskId())
+	logclient.AddActionLogWithStartable(self, rds, logclient.ACT_RENEW, map[string]string{"duration": duration}, self.UserCred, true)
+
+	self.SetStage("OnSyncstatusComplete", nil)
+	models.StartResourceSyncStatusTask(ctx, self.UserCred, rds, "DBInstanceSyncStatusTask", self.GetTaskId())
+}
+
+func (self *DBInstanceRenewTask) OnSyncstatusComplete(ctx context.Context, rds *models.SDBInstance, data jsonutils.JSONObject) {
 	self.SetStageComplete(ctx, nil)
 }
+
+func (self *DBInstanceRenewTask) OnSyncstatusCompleteFailed(ctx context.Context, rds *models.SDBInstance, reason jsonutils.JSONObject) {
+	self.SetStageFailed(ctx, reason.String())
+}

pkg/hostman/guestman/qemu-kvm.go

@@ -994,10 +994,13 @@ func (s *SKVMGuestInstance) ExecSuspendTask(ctx context.Context) {
 func (s *SKVMGuestInstance) GetNicDescMatch(mac, ip, port, bridge string) jsonutils.JSONObject {
 	nics, _ := s.Desc.GetArray("nics")
 	for _, nic := range nics {
+		nicBridge, _ := nic.GetString("bridge")
+		if bridge == "" && nicBridge != "" && nicBridge == options.HostOptions.OvnIntegrationBridge {
+			continue
+		}
 		nicMac, _ := nic.GetString("mac")
 		nicIp, _ := nic.GetString("ip")
 		nicPort, _ := nic.GetString("ifname")
-		nicBridge, _ := nic.GetString("bridge")
 		if (len(mac) == 0 || netutils2.MacEqual(nicMac, mac)) &&
 			(len(ip) == 0 || nicIp == ip) &&
 			(len(port) == 0 || nicPort == port) &&

pkg/monitor/service/service.go

@@ -43,7 +43,7 @@ import (
 
 func StartService() {
 	opts := &options.Options
-	common_options.ParseOptions(opts, os.Args, "alerter.conf", "alerter")
+	common_options.ParseOptions(opts, os.Args, "monitor.conf", "monitor")
 
 	commonOpts := &opts.CommonOptions
 	common_app.InitAuth(commonOpts, func() {

pkg/multicloud/huawei/object.go

@@ -46,6 +46,10 @@ func (o *SObject) GetAcl() cloudprovider.TBucketACLType {
 	input.Bucket = o.bucket.Name
 	input.Key = o.Key
 	output, err := obscli.GetObjectAcl(input)
+	if err != nil {
+		log.Errorf("GetObjectAcl error: %v", err)
+		return acl
+	}
 	acl = obsAcl2CannedAcl(output.Grants)
 	return acl
 }

pkg/scheduler/cache/candidate/hosts.go

@@ -194,7 +194,7 @@ func NewGuestReservedResourceUsedByBuilder(b *HostBuilder, host *computemodels.S
 	for _, g := range gst {
 		dSize := guestDiskSize(&g, true)
 		disk += int64(dSize)
-		if o.GetOptions().IgnoreNonrunningGuests && !utils.IsInStringArray(g.Status, computeapi.VM_RUNNING_STATUS) {
+		if o.GetOptions().IgnoreNonrunningGuests && (g.Status == computeapi.VM_READY) {
 			continue
 		}
 		cpu += int64(g.VcpuCount)

pkg/vpcagent/models/modelset.go

@@ -15,6 +15,8 @@
 package models
 
 import (
+	"fmt"
+
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -33,7 +35,7 @@ type (
 	SecurityGroupRules map[string]*SecurityGroupRule
 	Elasticips         map[string]*Elasticip
 
-	Guestnetworks  map[string]*Guestnetwork  // key: guestId/ifname
+	Guestnetworks  map[string]*Guestnetwork  // key: rowId
 	Guestsecgroups map[string]*Guestsecgroup // key: guestId/secgroupId
 
 	DnsRecords map[string]*DnsRecord
@@ -332,7 +334,8 @@ func (set Guestnetworks) NewModel() db.IModel {
 
 func (set Guestnetworks) AddModel(i db.IModel) {
 	m := i.(*Guestnetwork)
-	set[m.GuestId+"/"+m.Ifname] = m
+	k := fmt.Sprintf("%d", m.RowId)
+	set[k] = m
 }
 
 func (set Guestnetworks) Copy() apihelper.IModelSet {

vendor/modules.txt

@@ -1007,7 +1007,7 @@ sigs.k8s.io/yaml
 yunion.io/x/executor/apis
 yunion.io/x/executor/client
 yunion.io/x/executor/server
-# yunion.io/x/jsonutils v0.0.0-20201014025429-7ce189228db6
+# yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769
 yunion.io/x/jsonutils
 # yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 yunion.io/x/log

vendor/yunion.io/x/jsonutils/jsonutils.go

@@ -201,6 +201,15 @@ ret:
 				case 't':
 					buffer = append(buffer, '\t')
 					i++
+				case 'b':
+					buffer = append(buffer, '\b')
+					i++
+				case 'f':
+					buffer = append(buffer, '\f')
+					i++
+				case '\\':
+					buffer = append(buffer, '\\')
+					i++
 				default:
 					buffer = append(buffer, str[i])
 					i++
@@ -268,25 +277,69 @@ func parseJSONValue(str []byte, offset int) (JSONObject, int, error) {
 	}
 }
 
+// https://www.ietf.org/rfc/rfc4627.txt
+//
+//         string = quotation-mark *char quotation-mark
+//
+//         char = unescaped /
+//                escape (
+//                    %x22 /          ; "    quotation mark  U+0022
+//                    %x5C /          ; \    reverse solidus U+005C
+//                    %x2F /          ; /    solidus         U+002F
+//                    %x62 /          ; b    backspace       U+0008
+//                    %x66 /          ; f    form feed       U+000C
+//                    %x6E /          ; n    line feed       U+000A
+//                    %x72 /          ; r    carriage return U+000D
+//                    %x74 /          ; t    tab             U+0009
+//                    %x75 4HEXDIG )  ; uXXXX                U+XXXX
+//
+//         escape = %x5C              ; \
+//
+//         quotation-mark = %x22      ; "
+//
+//         unescaped = %x20-21 / %x23-5B / %x5D-10FFFF
+//
+func escapeJsonChar(sb *strings.Builder, ch byte) {
+	switch ch {
+	case '"':
+		sb.Write([]byte{'\\', '"'})
+	case '\\':
+		sb.Write([]byte{'\\', '\\'})
+	case '\b':
+		sb.Write([]byte{'\\', 'b'})
+	case '\f':
+		sb.Write([]byte{'\\', 'f'})
+	case '\n':
+		sb.Write([]byte{'\\', 'n'})
+	case '\r':
+		sb.Write([]byte{'\\', 'r'})
+	case '\t':
+		sb.Write([]byte{'\\', 't'})
+	default:
+		sb.WriteByte(ch)
+		/*if ((ch >= 0x20 && ch <= 0x21) || (ch >= 0x23 || ch <= 0x5B) || (ch >= 0x5D && ch <= 0x10FFFF)) && ch != 0x81 && ch != 0x8d && ch != 0x8f && ch != 0x90 && ch != 0x9d {
+			sb.WriteRune(ch)
+		} else if ch <= 0xff {
+			sb.Write([]byte{'\\', 'x'})
+			sb.WriteString(fmt.Sprintf("%02x", ch))
+		} else if ch <= 0xffff {
+			sb.Write([]byte{'\\', 'u'})
+			sb.WriteString(fmt.Sprintf("%04x", ch))
+		} else {
+			sb.Write([]byte{'\\', 'u'})
+			sb.WriteString(fmt.Sprintf("%04x", ch>>16))
+			sb.Write([]byte{'\\', 'u'})
+			sb.WriteString(fmt.Sprintf("%04x", (ch & 0xffff)))
+		}*/
+	}
+}
+
 func quoteString(str string) string {
 	sb := &strings.Builder{}
 	sb.Grow(len(str) + 2)
 	sb.WriteByte('"')
-	for i := 0; i < len(str); i++ {
-		switch c := str[i]; c {
-		case '"':
-			sb.Write([]byte{'\\', '"'})
-		case '\r':
-			sb.Write([]byte{'\\', 'r'})
-		case '\n':
-			sb.Write([]byte{'\\', 'n'})
-		case '\t':
-			sb.Write([]byte{'\\', 't'})
-		case '\\':
-			sb.Write([]byte{'\\', '\\'})
-		default:
-			sb.WriteByte(c)
-		}
+	for i := 0; i < len(str); i += 1 {
+		escapeJsonChar(sb, str[i])
 	}
 	sb.WriteByte('"')
 	return sb.String()

vendor/yunion.io/x/jsonutils/marshal.go

@@ -217,7 +217,11 @@ func Marshal(obj interface{}) JSONObject {
 	if obj == nil {
 		return JSONNull
 	}
-	objValue := reflect.Indirect(reflect.ValueOf(obj))
+	val := reflect.ValueOf(obj)
+	if kind := val.Kind(); val.IsZero() && kind == reflect.Ptr {
+		return JSONNull
+	}
+	objValue := reflect.Indirect(val)
 	return marshalValue(objValue, nil)
 }
 

vendor/yunion.io/x/jsonutils/unmarshal.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 	"time"
 
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/sortedmap"
@@ -293,7 +294,7 @@ func (this *JSONString) unmarshalValue(val reflect.Value) error {
 		if len(this.data) > 0 {
 			tm, err = timeutils.ParseTimeStr(this.data)
 			if err != nil {
-				return errors.Wrap(err, "timeutils.ParseTimeStr")
+				log.Warningf("timeutils.ParseTimeStr %s %s", this.data, err)
 			}
 		} else {
 			tm = time.Time{}