fix: add latest changes from CWAgent

cfg/aws/credentials.go

@@ -5,10 +5,25 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/sts"
+)
+
+const (
+	bjsPartition          = "aws-cn"
+	pdtPartition          = "aws-us-gov"
+	lckPartition          = "aws-iso-b"
+	dcaPartition          = "aws-iso"
+	classicFallbackRegion = "us-east-1"
+	bjsFallbackRegion     = "cn-north-1"
+	pdtFallbackRegion     = "us-gov-west-1"
+	lckFallbackRegion     = "us-isob-east-1"
+	dcaFallbackRegion     = "us-iso-east-1"
 )
 
 type CredentialConfig struct {
@@ -21,6 +36,17 @@ type CredentialConfig struct {
 	Token     string
 }
 
+type stsCredentialProvider struct {
+	regional, partitional, fallbackProvider *stscreds.AssumeRoleProvider
+}
+
+func (s *stsCredentialProvider) IsExpired() bool {
+	if s.fallbackProvider != nil {
+		return s.fallbackProvider.IsExpired()
+	}
+	return s.regional.IsExpired()
+}
+
 type RootCredentialsProvider struct {
 	Name        func() string
 	Credentials func(*CredentialConfig) *credentials.Credentials
@@ -75,7 +101,7 @@ func (c *CredentialConfig) assumeCredentials() client.ConfigProvider {
 	config := &aws.Config{
 		Region: aws.String(c.Region),
 	}
-	config.Credentials = stscreds.NewCredentials(rootCredentials, c.RoleARN)
+	config.Credentials = newStsCredentials(rootCredentials, c.RoleARN, c.Region)
 	return getSession(config)
 }
 
@@ -87,6 +113,83 @@ func (c *CredentialConfig) Credentials() client.ConfigProvider {
 	}
 }
 
+func (s *stsCredentialProvider) Retrieve() (credentials.Value, error) {
+	if s.fallbackProvider != nil {
+		return s.fallbackProvider.Retrieve()
+	}
+
+	v, err := s.regional.Retrieve()
+
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == sts.ErrCodeRegionDisabledException {
+			log.Printf("D! The regional STS endpoint is deactivated and going to fall back to partitional STS endpoint\n")
+			s.fallbackProvider = s.partitional
+			return s.partitional.Retrieve()
+		}
+	}
+
+	return v, err
+}
+
+func newStsCredentials(c client.ConfigProvider, roleARN string, region string) *credentials.Credentials {
+	regional := &stscreds.AssumeRoleProvider{
+		Client: sts.New(c, &aws.Config{
+			Region:              aws.String(region),
+			STSRegionalEndpoint: endpoints.RegionalSTSEndpoint,
+		}),
+		RoleARN:  roleARN,
+		Duration: stscreds.DefaultDuration,
+	}
+
+	fallbackRegion := getFallbackRegion(region)
+
+	partitional := &stscreds.AssumeRoleProvider{
+		Client: sts.New(c, &aws.Config{
+			Region:              aws.String(fallbackRegion),
+			Endpoint:            aws.String(getFallbackEndpoint(fallbackRegion)),
+			STSRegionalEndpoint: endpoints.RegionalSTSEndpoint,
+		}),
+		RoleARN:  roleARN,
+		Duration: stscreds.DefaultDuration,
+	}
+
+	return credentials.NewCredentials(&stsCredentialProvider{regional: regional, partitional: partitional})
+}
+
+// The partitional STS endpoint used to fallback when regional STS endpoint is not activated.
+func getFallbackEndpoint(region string) string {
+	partition := getPartition(region)
+	endpoint, _ := partition.EndpointFor("sts", region)
+	log.Printf("D! STS partitional endpoint retrieved: %s", endpoint.URL)
+	return endpoint.URL
+}
+
+// Get the region in the partition where STS endpoint cannot be deactivated by customers which is used to fallback.
+// NOTE: Some Regions are not enabled by default, such as the Asia Pacific Hong Kong Region. In that case, when you
+// manually enable the Region, the regional STS endpoints will always be activated and cannot be deactivated.
+// Refer to: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html
+func getFallbackRegion(region string) string {
+	partition := getPartition(region)
+	switch partition.ID() {
+	case bjsPartition:
+		return bjsFallbackRegion
+	case pdtPartition:
+		return pdtFallbackRegion
+	case dcaPartition:
+		return dcaFallbackRegion
+	case lckPartition:
+		return lckFallbackRegion
+	default:
+		return classicFallbackRegion
+	}
+}
+
+// Get the partition information based on the region name
+func getPartition(region string) endpoints.Partition {
+	partition, _ := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), region)
+	return partition
+}
+
 func init() {
 	//Initialize the default root credentials chain
 	staticCredentialsProvider := RootCredentialsProvider{

cmd/config-downloader/downloader.go

@@ -210,7 +210,7 @@ func main() {
 
 	if multiConfig != "remove" {
 		outputFilePath = filepath.Join(outputDir, outputFilePath+context.TmpFileSuffix)
-		err = ioutil.WriteFile(outputFilePath, []byte(config), os.ModePerm)
+		err = ioutil.WriteFile(outputFilePath, []byte(config), 0644)
 		if err != nil {
 			panic(fmt.Sprintf("Failed to write the json file %v: %v\n", outputFilePath, err))
 		} else {

plugins/inputs/logfile/logfile.go

@@ -275,6 +275,11 @@ func (t *LogFile) getTargetFiles(fileconfig *FileConfig) ([]string, error) {
 	var targetFileName string
 	var targetModTime time.Time
 	for matchedFileName, matchedFileInfo := range g.Match() {
+		// we do not allow customer to monitor the file in t.FileStateFolder, it will monitor all of the state files
+		if t.FileStateFolder != "" && strings.HasPrefix(matchedFileName, t.FileStateFolder) {
+			continue
+		}
+
 		if isCompressedFile(matchedFileName) {
 			continue
 		}

translator/translate/util/placeholderUtil.go

@@ -1,23 +1,26 @@
 package util
 
 import (
-	"github.com/aws/amazon-cloudwatch-agent/translator/util/ec2util"
 	"log"
 	"net"
 	"os"
-
 	"strings"
+
+	"github.com/aws/amazon-cloudwatch-agent/translator/translate/agent"
+	"github.com/aws/amazon-cloudwatch-agent/translator/util/ec2util"
 )
 
 const (
 	instanceIdPlaceholder    = "{instance_id}"
 	hostnamePlaceholder      = "{hostname}"
 	localHostnamePlaceholder = "{local_hostname}" //regardless of ec2 metadata
 	ipAddressPlaceholder     = "{ip_address}"
+	awsRegionPlaceholder     = "{aws_region}"
 
 	unknownInstanceId = "i-UNKNOWN"
 	unknownHostname   = "UNKNOWN-HOST"
 	unknownIpAddress  = "UNKNOWN-IP"
+	unknownAwsRegion  = "UNKNOWN-REGION"
 )
 
 //resolve place holder for log group and log stream.
@@ -49,8 +52,14 @@ func GetMetadataInfo() map[string]string {
 	if ipAddress == "" {
 		ipAddress = getIpAddress()
 	}
+
+	awsRegion := agent.Global_Config.Region
+	if awsRegion == "" {
+		awsRegion = unknownAwsRegion
+	}
+
 	return map[string]string{instanceIdPlaceholder: instanceID, hostnamePlaceholder: hostname,
-		localHostnamePlaceholder: localHostname, ipAddressPlaceholder: ipAddress}
+		localHostnamePlaceholder: localHostname, ipAddressPlaceholder: ipAddress, awsRegionPlaceholder: awsRegion}
 }
 
 func getHostName() string {

translator/util/ecsutil/ecsutil.go

@@ -2,12 +2,13 @@ package ecsutil
 
 import (
 	"encoding/json"
-	"github.com/aws/amazon-cloudwatch-agent/translator/config"
-	"github.com/aws/amazon-cloudwatch-agent/translator/util/httpclient"
 	"log"
 	"os"
 	"strings"
 	"sync"
+
+	"github.com/aws/amazon-cloudwatch-agent/translator/config"
+	"github.com/aws/amazon-cloudwatch-agent/translator/util/httpclient"
 )
 
 const (
@@ -42,9 +43,10 @@ func initECSUtilSingleton() (newInstance *ecsUtil) {
 	if os.Getenv(config.RUN_IN_CONTAINER) != config.RUN_IN_CONTAINER_TRUE {
 		return
 	}
+	log.Println("I! attempt to access ECS task metadata to determine whether I'm running in ECS.")
 	ecsMetadataResponse, err := newInstance.getECSMetadata()
 	if err != nil {
-		log.Println("E! getting information from ECS task metadata fail: ", err)
+		log.Printf("I! access ECS task metadata fail with response %v, assuming I'm not running in ECS.\n", err)
 		return
 	}