Merge pull request #10225 from ioito/automated-cherry-pick-of-#10224-upstream-release-3.7

Automated cherry pick of #10224: fix(region): optimized aws cloudtrail sync

pkg/multicloud/aws/aws.go

@@ -285,15 +285,6 @@ func (client *SAwsClient) getAwsRoute53Session() (*session.Session, error) {
 	return session, nil
 }
 
-func (client *SAwsClient) getAwsCloudtrailSession() (*session.Session, error) {
-	session, err := client.getDefaultSession()
-	if err != nil {
-		return nil, errors.Wrap(err, "client.getDefaultSession()")
-	}
-	session.ClientConfig(CLOUD_TRAIL_SERVICE_NAME)
-	return session, nil
-}
-
 func (self *SAwsClient) invalidateIBuckets() {
 	self.iBuckets = nil
 }

pkg/multicloud/aws/cloudtrail.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudtrail"
 
 	"yunion.io/x/jsonutils"
@@ -112,28 +113,54 @@ func (self *SEvent) GetCreatedAt() time.Time {
 	return self.EventTime
 }
 
-func (self *SAwsClient) LookupEvents(start, end time.Time) ([]SEvent, error) {
+func (self *SRegion) getAwsCloudtrailSession() (*session.Session, error) {
+	session, err := self.getAwsSession()
+	if err != nil {
+		return nil, errors.Wrap(err, "client.getDefaultSession()")
+	}
+	session.ClientConfig(CLOUD_TRAIL_SERVICE_NAME)
+	return session, nil
+}
+
+func (self *SRegion) LookupEvents(start, end time.Time, withReadEvent bool) ([]SEvent, error) {
 	s, err := self.getAwsCloudtrailSession()
 	if err != nil {
 		return nil, errors.Wrapf(err, "getAwsCloudtrailSession")
 	}
 	client := cloudtrail.New(s)
-	input := &cloudtrail.LookupEventsInput{}
+	input := &cloudtrail.LookupEventsInput{LookupAttributes: []*cloudtrail.LookupAttribute{}}
 	if !start.IsZero() {
 		input.SetStartTime(start)
 	}
 	if !end.IsZero() {
 		input.SetEndTime(end)
 	}
+	if !withReadEvent {
+		readonly := "ReadOnly"
+		val := "false"
+		input.LookupAttributes = append(input.LookupAttributes, &cloudtrail.LookupAttribute{
+			AttributeKey:   &readonly,
+			AttributeValue: &val,
+		})
+	}
 	events := []SEvent{}
 	nextToken := ""
 	for {
 		if len(nextToken) > 0 {
 			input.SetNextToken(nextToken)
 		}
-		output, err := client.LookupEvents(input)
-		if err != nil {
-			return nil, errors.Wrapf(err, "LookupEvents")
+		var output *cloudtrail.LookupEventsOutput = nil
+		for {
+			output, err = client.LookupEvents(input)
+			if err != nil {
+				if strings.Contains(err.Error(), "ThrottlingException") {
+					log.Warningf("LookupEvents ThrottlingException, try after 3 seconds")
+					time.Sleep(time.Second * 3)
+					continue
+				}
+				return nil, errors.Wrapf(err, "LookupEvents(%s, %s)", start, end)
+			}
+			break
 		}
 		for i := range output.Events {
 			err := FillZero(output.Events[i])
@@ -145,6 +172,9 @@ func (self *SAwsClient) LookupEvents(start, end time.Time) ([]SEvent, error) {
 			if err != nil {
 				return nil, errors.Wrapf(err, "jsonutils.Update")
 			}
+			if strings.Contains(event.CloudTrailEvent, "awsRegion") && !strings.Contains(event.CloudTrailEvent, fmt.Sprintf(`"awsRegion":"%s"`, self.RegionId)) {
+				continue
+			}
 			events = append(events, event)
 		}
 		nextToken = ""
@@ -159,25 +189,12 @@ func (self *SAwsClient) LookupEvents(start, end time.Time) ([]SEvent, error) {
 }
 
 func (self *SRegion) GetICloudEvents(start time.Time, end time.Time, withReadEvent bool) ([]cloudprovider.ICloudEvent, error) {
-	var events []SEvent
-	var err error
-	for {
-		events, err = self.client.LookupEvents(start, end)
-		if err != nil {
-			if strings.Contains(err.Error(), "ThrottlingException") {
-				log.Warningf("LookupEvents ThrottlingException, try after 3 seconds")
-				time.Sleep(time.Second * 3)
-				continue
-			}
-			return nil, errors.Wrapf(err, "LookupEvents(%s, %s)", start, end)
-		}
-		break
+	events, err := self.LookupEvents(start, end, withReadEvent)
+	if err != nil {
+		return nil, errors.Wrapf(err, "LookupEvents(%s, %s)", start, end)
 	}
 	ret := []cloudprovider.ICloudEvent{}
 	for i := range events {
-		if !withReadEvent && events[i].ReadOnly == "true" {
-			continue
-		}
 		ret = append(ret, &events[i])
 	}
 	return ret, nil

pkg/multicloud/aws/shell/cloudtrail.go

@@ -23,11 +23,12 @@ import (
 
 func init() {
 	type EventListOptions struct {
-		Start time.Time
-		End   time.Time
+		Start        time.Time
+		End          time.Time
+		WithReadOnly bool
 	}
 	shellutils.R(&EventListOptions{}, "event-list", "List events", func(cli *aws.SRegion, args *EventListOptions) error {
-		events, err := cli.GetClient().LookupEvents(args.Start, args.End)
+		events, err := cli.LookupEvents(args.Start, args.End, args.WithReadOnly)
 		if err != nil {
 			return err
 		}