diff --git a/cfg/aws/credentials.go b/cfg/aws/credentials.go index ff5555a8cc4..0e3db1e436e 100644 --- a/cfg/aws/credentials.go +++ b/cfg/aws/credentials.go @@ -5,10 +5,25 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" +) + +const ( + bjsPartition = "aws-cn" + pdtPartition = "aws-us-gov" + lckPartition = "aws-iso-b" + dcaPartition = "aws-iso" + classicFallbackRegion = "us-east-1" + bjsFallbackRegion = "cn-north-1" + pdtFallbackRegion = "us-gov-west-1" + lckFallbackRegion = "us-isob-east-1" + dcaFallbackRegion = "us-iso-east-1" ) type CredentialConfig struct { @@ -21,6 +36,17 @@ type CredentialConfig struct { Token string } +type stsCredentialProvider struct { + regional, partitional, fallbackProvider *stscreds.AssumeRoleProvider +} + +func (s *stsCredentialProvider) IsExpired() bool { + if s.fallbackProvider != nil { + return s.fallbackProvider.IsExpired() + } + return s.regional.IsExpired() +} + type RootCredentialsProvider struct { Name func() string Credentials func(*CredentialConfig) *credentials.Credentials @@ -75,7 +101,7 @@ func (c *CredentialConfig) assumeCredentials() client.ConfigProvider { config := &aws.Config{ Region: aws.String(c.Region), } - config.Credentials = stscreds.NewCredentials(rootCredentials, c.RoleARN) + config.Credentials = newStsCredentials(rootCredentials, c.RoleARN, c.Region) return getSession(config) } @@ -87,6 +113,83 @@ func (c *CredentialConfig) Credentials() client.ConfigProvider { } } +func (s *stsCredentialProvider) Retrieve() (credentials.Value, error) { + if s.fallbackProvider != nil { + return s.fallbackProvider.Retrieve() + } + + v, err := s.regional.Retrieve() + + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == sts.ErrCodeRegionDisabledException { + log.Printf("D! The regional STS endpoint is deactivated and going to fall back to partitional STS endpoint\n") + s.fallbackProvider = s.partitional + return s.partitional.Retrieve() + } + } + + return v, err +} + +func newStsCredentials(c client.ConfigProvider, roleARN string, region string) *credentials.Credentials { + regional := &stscreds.AssumeRoleProvider{ + Client: sts.New(c, &aws.Config{ + Region: aws.String(region), + STSRegionalEndpoint: endpoints.RegionalSTSEndpoint, + }), + RoleARN: roleARN, + Duration: stscreds.DefaultDuration, + } + + fallbackRegion := getFallbackRegion(region) + + partitional := &stscreds.AssumeRoleProvider{ + Client: sts.New(c, &aws.Config{ + Region: aws.String(fallbackRegion), + Endpoint: aws.String(getFallbackEndpoint(fallbackRegion)), + STSRegionalEndpoint: endpoints.RegionalSTSEndpoint, + }), + RoleARN: roleARN, + Duration: stscreds.DefaultDuration, + } + + return credentials.NewCredentials(&stsCredentialProvider{regional: regional, partitional: partitional}) +} + +// The partitional STS endpoint used to fallback when regional STS endpoint is not activated. +func getFallbackEndpoint(region string) string { + partition := getPartition(region) + endpoint, _ := partition.EndpointFor("sts", region) + log.Printf("D! STS partitional endpoint retrieved: %s", endpoint.URL) + return endpoint.URL +} + +// Get the region in the partition where STS endpoint cannot be deactivated by customers which is used to fallback. +// NOTE: Some Regions are not enabled by default, such as the Asia Pacific Hong Kong Region. In that case, when you +// manually enable the Region, the regional STS endpoints will always be activated and cannot be deactivated. +// Refer to: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html +func getFallbackRegion(region string) string { + partition := getPartition(region) + switch partition.ID() { + case bjsPartition: + return bjsFallbackRegion + case pdtPartition: + return pdtFallbackRegion + case dcaPartition: + return dcaFallbackRegion + case lckPartition: + return lckFallbackRegion + default: + return classicFallbackRegion + } +} + +// Get the partition information based on the region name +func getPartition(region string) endpoints.Partition { + partition, _ := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), region) + return partition +} + func init() { //Initialize the default root credentials chain staticCredentialsProvider := RootCredentialsProvider{ diff --git a/cmd/config-downloader/downloader.go b/cmd/config-downloader/downloader.go index 96ac9cfb98f..da4ce12f0d1 100644 --- a/cmd/config-downloader/downloader.go +++ b/cmd/config-downloader/downloader.go @@ -210,7 +210,7 @@ func main() { if multiConfig != "remove" { outputFilePath = filepath.Join(outputDir, outputFilePath+context.TmpFileSuffix) - err = ioutil.WriteFile(outputFilePath, []byte(config), os.ModePerm) + err = ioutil.WriteFile(outputFilePath, []byte(config), 0644) if err != nil { panic(fmt.Sprintf("Failed to write the json file %v: %v\n", outputFilePath, err)) } else { diff --git a/plugins/inputs/logfile/logfile.go b/plugins/inputs/logfile/logfile.go index cc4645fc004..b0001cb687a 100644 --- a/plugins/inputs/logfile/logfile.go +++ b/plugins/inputs/logfile/logfile.go @@ -275,6 +275,11 @@ func (t *LogFile) getTargetFiles(fileconfig *FileConfig) ([]string, error) { var targetFileName string var targetModTime time.Time for matchedFileName, matchedFileInfo := range g.Match() { + // we do not allow customer to monitor the file in t.FileStateFolder, it will monitor all of the state files + if t.FileStateFolder != "" && strings.HasPrefix(matchedFileName, t.FileStateFolder) { + continue + } + if isCompressedFile(matchedFileName) { continue } diff --git a/translator/totomlconfig/sampleConfig/log_and_scroll_linux.conf b/translator/totomlconfig/sampleConfig/log_and_scroll_linux.conf deleted file mode 100755 index eedb60501d6..00000000000 --- a/translator/totomlconfig/sampleConfig/log_and_scroll_linux.conf +++ /dev/null @@ -1,91 +0,0 @@ -[agent] - collection_jitter = "0s" - debug = false - flush_interval = "1s" - flush_jitter = "0s" - hostname = "" - interval = "60s" - logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" - logtarget = "lumberjack" - metric_batch_size = 1000 - metric_buffer_limit = 10000 - omit_hostname = false - precision = "" - quiet = false - round_interval = false - -[inputs] - - [[inputs.logfile]] - destination = "cloudwatchlogs" - file_state_folder = "/opt/aws/amazon-cloudwatch-agent/logs/state" - - [[inputs.logfile.file_config]] - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" - from_beginning = true - log_group_name = "amazon-cloudwatch-agent.log" - log_stream_name = "amazon-cloudwatch-agent.log" - multi_line_start_pattern = "{timestamp_regex}" - pipe = false - timestamp_layout = "02 Jan 2006 15:04:05" - timestamp_regex = "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})" - timezone = "UTC" - - [[inputs.logfile.file_config]] - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/test.log" - from_beginning = true - log_group_name = "test.log" - log_stream_name = "test.log" - pipe = false - timezone = "UTC" - [[inputs.tail.file_config]] - blacklist = "agent.log*|env.log|profiler.log|\\.\\d$" - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/*" - from_beginning = true - log_group_name = "EC2-System-Logs" - log_stream_name = "multi_log_stream" - pipe = false - publish_multi_logs = true - timezone = "UTC" - [inputs.logfile.tags] - metricPath = "logs" - - [[inputs.logfile]] - destination = "scroll" - file_state_folder = "/opt/aws/amazon-cloudwatch-agent/logs/state/scroll" - name_override = "raw_log_line" - - [[inputs.logfile.file_config]] - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" - from_beginning = true - log_group_name = "amazon-cloudwatch-agent.log" - log_type = "SERVICE" - multi_line_start_pattern = "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2})" - pipe = false - - [[inputs.logfile.file_config]] - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/test.log" - from_beginning = true - log_group_name = "test.log" - multi_line_start_pattern = "{timestamp_regex}" - pipe = false - timestamp_layout = "02 Jan 2006 15:04:05" - timestamp_regex = "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})" - timezone = "UTC" - -[outputs] - - [[outputs.cloudwatchlogs]] - force_flush_interval = "60s" - log_stream_name = "LOG_STREAM_NAME" - region = "us-west-2" - tagexclude = ["metricPath"] - [outputs.cloudwatchlogs.tagpass] - metricPath = ["logs"] - - [[outputs.cloudwatchscroll]] - endpoint_override = "https://ingress.cell0001.prod.us-east-1.scroll.aws.a2z.com" - file_state_folder = "/opt/aws/amazon-cloudwatch-agent/logs/state/scroll" - force_flush_interval = "60s" - region = "us-west-2" - role_arn = "log_role_arn_value_test" diff --git a/translator/totomlconfig/sampleConfig/log_and_scroll_linux.json b/translator/totomlconfig/sampleConfig/log_and_scroll_linux.json deleted file mode 100644 index bb5abc2051a..00000000000 --- a/translator/totomlconfig/sampleConfig/log_and_scroll_linux.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "logs": { - "logs_collected": { - "files": { - "collect_list": [ - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log", - "log_group_name": "amazon-cloudwatch-agent.log", - "log_stream_name": "amazon-cloudwatch-agent.log", - "multi_line_start_pattern": "{timestamp_format}", - "timestamp_format": "%d %b %Y %H:%M:%S", - "timezone": "UTC" - }, - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/test.log", - "log_group_name": "test.log", - "log_stream_name": "test.log", - "timezone": "UTC" - }, - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/*", - "blacklist": "agent.log*|env.log|profiler.log|\\.\\d$", - "publish_multi_logs": true, - "log_group_name": "EC2-System-Logs", - "log_stream_name": "multi_log_stream", - "timezone": "UTC" - } - ] - } - }, - "log_stream_name": "LOG_STREAM_NAME", - "force_flush_interval": 60 - }, - "scroll": { - "endpoint_override":"https://ingress.cell0001.prod.us-east-1.scroll.aws.a2z.com", - "logs_collected": { - "files": { - "collect_list": [ - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log", - "log_group_name": "amazon-cloudwatch-agent.log", - "log_type": "SERVICE", - "multi_line_start_pattern": "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2})" - }, - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/test.log", - "log_group_name": "test.log", - "log_stream_name": "test.log", - "multi_line_start_pattern": "{timestamp_format}", - "timestamp_format": "%d %b %Y %H:%M:%S", - "timezone": "UTC" - } - ] - } - }, - "force_flush_interval": 60, - "credentials": { - "role_arn": "log_role_arn_value_test" - } - } -} \ No newline at end of file diff --git a/translator/translate/util/placeholderUtil.go b/translator/translate/util/placeholderUtil.go index c32ac42b159..d240211447c 100644 --- a/translator/translate/util/placeholderUtil.go +++ b/translator/translate/util/placeholderUtil.go @@ -1,12 +1,13 @@ package util import ( - "github.com/aws/amazon-cloudwatch-agent/translator/util/ec2util" "log" "net" "os" - "strings" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" + "github.com/aws/amazon-cloudwatch-agent/translator/util/ec2util" ) const ( @@ -14,10 +15,12 @@ const ( hostnamePlaceholder = "{hostname}" localHostnamePlaceholder = "{local_hostname}" //regardless of ec2 metadata ipAddressPlaceholder = "{ip_address}" + awsRegionPlaceholder = "{aws_region}" unknownInstanceId = "i-UNKNOWN" unknownHostname = "UNKNOWN-HOST" unknownIpAddress = "UNKNOWN-IP" + unknownAwsRegion = "UNKNOWN-REGION" ) //resolve place holder for log group and log stream. @@ -49,8 +52,14 @@ func GetMetadataInfo() map[string]string { if ipAddress == "" { ipAddress = getIpAddress() } + + awsRegion := agent.Global_Config.Region + if awsRegion == "" { + awsRegion = unknownAwsRegion + } + return map[string]string{instanceIdPlaceholder: instanceID, hostnamePlaceholder: hostname, - localHostnamePlaceholder: localHostname, ipAddressPlaceholder: ipAddress} + localHostnamePlaceholder: localHostname, ipAddressPlaceholder: ipAddress, awsRegionPlaceholder: awsRegion} } func getHostName() string { diff --git a/translator/util/ecsutil/ecsutil.go b/translator/util/ecsutil/ecsutil.go index 15319df9cb8..bc386a68337 100644 --- a/translator/util/ecsutil/ecsutil.go +++ b/translator/util/ecsutil/ecsutil.go @@ -2,12 +2,13 @@ package ecsutil import ( "encoding/json" - "github.com/aws/amazon-cloudwatch-agent/translator/config" - "github.com/aws/amazon-cloudwatch-agent/translator/util/httpclient" "log" "os" "strings" "sync" + + "github.com/aws/amazon-cloudwatch-agent/translator/config" + "github.com/aws/amazon-cloudwatch-agent/translator/util/httpclient" ) const ( @@ -42,9 +43,10 @@ func initECSUtilSingleton() (newInstance *ecsUtil) { if os.Getenv(config.RUN_IN_CONTAINER) != config.RUN_IN_CONTAINER_TRUE { return } + log.Println("I! attempt to access ECS task metadata to determine whether I'm running in ECS.") ecsMetadataResponse, err := newInstance.getECSMetadata() if err != nil { - log.Println("E! getting information from ECS task metadata fail: ", err) + log.Printf("I! access ECS task metadata fail with response %v, assuming I'm not running in ECS.\n", err) return }