From abf880ab09069394733e67562683c39496a94347 Mon Sep 17 00:00:00 2001 From: Ping Xiang Date: Wed, 24 Jun 2020 15:41:16 -0700 Subject: [PATCH 1/6] fix: add STS regional Endpoint support with fallback path As global endpoint is under the deprecation path, we will use regional STS endpoint by default. To make sure our current customers will not be affected by this change, whenever regional STS endpoint is not available, we will always fallback to the STS endpoint of a particular partition which is always active. --- cfg/aws/credentials.go | 105 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 104 insertions(+), 1 deletion(-) diff --git a/cfg/aws/credentials.go b/cfg/aws/credentials.go index ff5555a8cc4..0e3db1e436e 100644 --- a/cfg/aws/credentials.go +++ b/cfg/aws/credentials.go @@ -5,10 +5,25 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" +) + +const ( + bjsPartition = "aws-cn" + pdtPartition = "aws-us-gov" + lckPartition = "aws-iso-b" + dcaPartition = "aws-iso" + classicFallbackRegion = "us-east-1" + bjsFallbackRegion = "cn-north-1" + pdtFallbackRegion = "us-gov-west-1" + lckFallbackRegion = "us-isob-east-1" + dcaFallbackRegion = "us-iso-east-1" ) type CredentialConfig struct { @@ -21,6 +36,17 @@ type CredentialConfig struct { Token string } +type stsCredentialProvider struct { + regional, partitional, fallbackProvider *stscreds.AssumeRoleProvider +} + +func (s *stsCredentialProvider) IsExpired() bool { + if s.fallbackProvider != nil { + return s.fallbackProvider.IsExpired() + } + return s.regional.IsExpired() +} + type RootCredentialsProvider struct { Name func() string Credentials func(*CredentialConfig) *credentials.Credentials @@ -75,7 +101,7 @@ func (c *CredentialConfig) assumeCredentials() client.ConfigProvider { config := &aws.Config{ Region: aws.String(c.Region), } - config.Credentials = stscreds.NewCredentials(rootCredentials, c.RoleARN) + config.Credentials = newStsCredentials(rootCredentials, c.RoleARN, c.Region) return getSession(config) } @@ -87,6 +113,83 @@ func (c *CredentialConfig) Credentials() client.ConfigProvider { } } +func (s *stsCredentialProvider) Retrieve() (credentials.Value, error) { + if s.fallbackProvider != nil { + return s.fallbackProvider.Retrieve() + } + + v, err := s.regional.Retrieve() + + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == sts.ErrCodeRegionDisabledException { + log.Printf("D! The regional STS endpoint is deactivated and going to fall back to partitional STS endpoint\n") + s.fallbackProvider = s.partitional + return s.partitional.Retrieve() + } + } + + return v, err +} + +func newStsCredentials(c client.ConfigProvider, roleARN string, region string) *credentials.Credentials { + regional := &stscreds.AssumeRoleProvider{ + Client: sts.New(c, &aws.Config{ + Region: aws.String(region), + STSRegionalEndpoint: endpoints.RegionalSTSEndpoint, + }), + RoleARN: roleARN, + Duration: stscreds.DefaultDuration, + } + + fallbackRegion := getFallbackRegion(region) + + partitional := &stscreds.AssumeRoleProvider{ + Client: sts.New(c, &aws.Config{ + Region: aws.String(fallbackRegion), + Endpoint: aws.String(getFallbackEndpoint(fallbackRegion)), + STSRegionalEndpoint: endpoints.RegionalSTSEndpoint, + }), + RoleARN: roleARN, + Duration: stscreds.DefaultDuration, + } + + return credentials.NewCredentials(&stsCredentialProvider{regional: regional, partitional: partitional}) +} + +// The partitional STS endpoint used to fallback when regional STS endpoint is not activated. +func getFallbackEndpoint(region string) string { + partition := getPartition(region) + endpoint, _ := partition.EndpointFor("sts", region) + log.Printf("D! STS partitional endpoint retrieved: %s", endpoint.URL) + return endpoint.URL +} + +// Get the region in the partition where STS endpoint cannot be deactivated by customers which is used to fallback. +// NOTE: Some Regions are not enabled by default, such as the Asia Pacific Hong Kong Region. In that case, when you +// manually enable the Region, the regional STS endpoints will always be activated and cannot be deactivated. +// Refer to: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_enable-regions.html +func getFallbackRegion(region string) string { + partition := getPartition(region) + switch partition.ID() { + case bjsPartition: + return bjsFallbackRegion + case pdtPartition: + return pdtFallbackRegion + case dcaPartition: + return dcaFallbackRegion + case lckPartition: + return lckFallbackRegion + default: + return classicFallbackRegion + } +} + +// Get the partition information based on the region name +func getPartition(region string) endpoints.Partition { + partition, _ := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), region) + return partition +} + func init() { //Initialize the default root credentials chain staticCredentialsProvider := RootCredentialsProvider{ From 525744f13b3b8523a8c3efc40bb85ff64c557e7a Mon Sep 17 00:00:00 2001 From: Ping Xiang Date: Wed, 24 Jun 2020 17:48:57 -0700 Subject: [PATCH 2/6] fix: add region as a placeholder in log group/stream name With this change, customer can specify a aws region placeholder in the log group/stream name like this: "logs": { "logs_collected": { "files": { "collect_list": [ { "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log", "log_group_name": "{hostname}-{aws_region}", "log_stream_name": "{aws_region}" } ] } } } config-translator will replace the placeholder with proper values --- translator/translate/util/placeholderUtil.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/translator/translate/util/placeholderUtil.go b/translator/translate/util/placeholderUtil.go index c32ac42b159..d240211447c 100644 --- a/translator/translate/util/placeholderUtil.go +++ b/translator/translate/util/placeholderUtil.go @@ -1,12 +1,13 @@ package util import ( - "github.com/aws/amazon-cloudwatch-agent/translator/util/ec2util" "log" "net" "os" - "strings" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" + "github.com/aws/amazon-cloudwatch-agent/translator/util/ec2util" ) const ( @@ -14,10 +15,12 @@ const ( hostnamePlaceholder = "{hostname}" localHostnamePlaceholder = "{local_hostname}" //regardless of ec2 metadata ipAddressPlaceholder = "{ip_address}" + awsRegionPlaceholder = "{aws_region}" unknownInstanceId = "i-UNKNOWN" unknownHostname = "UNKNOWN-HOST" unknownIpAddress = "UNKNOWN-IP" + unknownAwsRegion = "UNKNOWN-REGION" ) //resolve place holder for log group and log stream. @@ -49,8 +52,14 @@ func GetMetadataInfo() map[string]string { if ipAddress == "" { ipAddress = getIpAddress() } + + awsRegion := agent.Global_Config.Region + if awsRegion == "" { + awsRegion = unknownAwsRegion + } + return map[string]string{instanceIdPlaceholder: instanceID, hostnamePlaceholder: hostname, - localHostnamePlaceholder: localHostname, ipAddressPlaceholder: ipAddress} + localHostnamePlaceholder: localHostname, ipAddressPlaceholder: ipAddress, awsRegionPlaceholder: awsRegion} } func getHostName() string { From 4df01d51a48591d7622e5113a430e907fdfd2644 Mon Sep 17 00:00:00 2001 From: Ping Xiang Date: Wed, 24 Jun 2020 17:56:21 -0700 Subject: [PATCH 3/6] fix: change downloaded json file mode to 0644 for security reason --- cmd/config-downloader/downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/config-downloader/downloader.go b/cmd/config-downloader/downloader.go index 96ac9cfb98f..da4ce12f0d1 100644 --- a/cmd/config-downloader/downloader.go +++ b/cmd/config-downloader/downloader.go @@ -210,7 +210,7 @@ func main() { if multiConfig != "remove" { outputFilePath = filepath.Join(outputDir, outputFilePath+context.TmpFileSuffix) - err = ioutil.WriteFile(outputFilePath, []byte(config), os.ModePerm) + err = ioutil.WriteFile(outputFilePath, []byte(config), 0644) if err != nil { panic(fmt.Sprintf("Failed to write the json file %v: %v\n", outputFilePath, err)) } else { From 8125f9f061e1b92cdba7de6295db760ef12cf380 Mon Sep 17 00:00:00 2001 From: Ping Xiang Date: Wed, 24 Jun 2020 20:20:46 -0700 Subject: [PATCH 4/6] fix: update log to explain the attempt of accessing ECS metadata --- translator/util/ecsutil/ecsutil.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/translator/util/ecsutil/ecsutil.go b/translator/util/ecsutil/ecsutil.go index 15319df9cb8..bc386a68337 100644 --- a/translator/util/ecsutil/ecsutil.go +++ b/translator/util/ecsutil/ecsutil.go @@ -2,12 +2,13 @@ package ecsutil import ( "encoding/json" - "github.com/aws/amazon-cloudwatch-agent/translator/config" - "github.com/aws/amazon-cloudwatch-agent/translator/util/httpclient" "log" "os" "strings" "sync" + + "github.com/aws/amazon-cloudwatch-agent/translator/config" + "github.com/aws/amazon-cloudwatch-agent/translator/util/httpclient" ) const ( @@ -42,9 +43,10 @@ func initECSUtilSingleton() (newInstance *ecsUtil) { if os.Getenv(config.RUN_IN_CONTAINER) != config.RUN_IN_CONTAINER_TRUE { return } + log.Println("I! attempt to access ECS task metadata to determine whether I'm running in ECS.") ecsMetadataResponse, err := newInstance.getECSMetadata() if err != nil { - log.Println("E! getting information from ECS task metadata fail: ", err) + log.Printf("I! access ECS task metadata fail with response %v, assuming I'm not running in ECS.\n", err) return } From 8c0dbc930fef489da8aa09932386090db5ca753a Mon Sep 17 00:00:00 2001 From: Ping Xiang Date: Wed, 24 Jun 2020 21:39:28 -0700 Subject: [PATCH 5/6] fix: remove internal config files --- .../sampleConfig/log_and_scroll_linux.conf | 91 ------------------- .../sampleConfig/log_and_scroll_linux.json | 61 ------------- 2 files changed, 152 deletions(-) delete mode 100755 translator/totomlconfig/sampleConfig/log_and_scroll_linux.conf delete mode 100644 translator/totomlconfig/sampleConfig/log_and_scroll_linux.json diff --git a/translator/totomlconfig/sampleConfig/log_and_scroll_linux.conf b/translator/totomlconfig/sampleConfig/log_and_scroll_linux.conf deleted file mode 100755 index eedb60501d6..00000000000 --- a/translator/totomlconfig/sampleConfig/log_and_scroll_linux.conf +++ /dev/null @@ -1,91 +0,0 @@ -[agent] - collection_jitter = "0s" - debug = false - flush_interval = "1s" - flush_jitter = "0s" - hostname = "" - interval = "60s" - logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" - logtarget = "lumberjack" - metric_batch_size = 1000 - metric_buffer_limit = 10000 - omit_hostname = false - precision = "" - quiet = false - round_interval = false - -[inputs] - - [[inputs.logfile]] - destination = "cloudwatchlogs" - file_state_folder = "/opt/aws/amazon-cloudwatch-agent/logs/state" - - [[inputs.logfile.file_config]] - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" - from_beginning = true - log_group_name = "amazon-cloudwatch-agent.log" - log_stream_name = "amazon-cloudwatch-agent.log" - multi_line_start_pattern = "{timestamp_regex}" - pipe = false - timestamp_layout = "02 Jan 2006 15:04:05" - timestamp_regex = "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})" - timezone = "UTC" - - [[inputs.logfile.file_config]] - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/test.log" - from_beginning = true - log_group_name = "test.log" - log_stream_name = "test.log" - pipe = false - timezone = "UTC" - [[inputs.tail.file_config]] - blacklist = "agent.log*|env.log|profiler.log|\\.\\d$" - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/*" - from_beginning = true - log_group_name = "EC2-System-Logs" - log_stream_name = "multi_log_stream" - pipe = false - publish_multi_logs = true - timezone = "UTC" - [inputs.logfile.tags] - metricPath = "logs" - - [[inputs.logfile]] - destination = "scroll" - file_state_folder = "/opt/aws/amazon-cloudwatch-agent/logs/state/scroll" - name_override = "raw_log_line" - - [[inputs.logfile.file_config]] - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" - from_beginning = true - log_group_name = "amazon-cloudwatch-agent.log" - log_type = "SERVICE" - multi_line_start_pattern = "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2})" - pipe = false - - [[inputs.logfile.file_config]] - file_path = "/opt/aws/amazon-cloudwatch-agent/logs/test.log" - from_beginning = true - log_group_name = "test.log" - multi_line_start_pattern = "{timestamp_regex}" - pipe = false - timestamp_layout = "02 Jan 2006 15:04:05" - timestamp_regex = "(\\d{2} \\w{3} \\d{4} \\d{2}:\\d{2}:\\d{2})" - timezone = "UTC" - -[outputs] - - [[outputs.cloudwatchlogs]] - force_flush_interval = "60s" - log_stream_name = "LOG_STREAM_NAME" - region = "us-west-2" - tagexclude = ["metricPath"] - [outputs.cloudwatchlogs.tagpass] - metricPath = ["logs"] - - [[outputs.cloudwatchscroll]] - endpoint_override = "https://ingress.cell0001.prod.us-east-1.scroll.aws.a2z.com" - file_state_folder = "/opt/aws/amazon-cloudwatch-agent/logs/state/scroll" - force_flush_interval = "60s" - region = "us-west-2" - role_arn = "log_role_arn_value_test" diff --git a/translator/totomlconfig/sampleConfig/log_and_scroll_linux.json b/translator/totomlconfig/sampleConfig/log_and_scroll_linux.json deleted file mode 100644 index bb5abc2051a..00000000000 --- a/translator/totomlconfig/sampleConfig/log_and_scroll_linux.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "logs": { - "logs_collected": { - "files": { - "collect_list": [ - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log", - "log_group_name": "amazon-cloudwatch-agent.log", - "log_stream_name": "amazon-cloudwatch-agent.log", - "multi_line_start_pattern": "{timestamp_format}", - "timestamp_format": "%d %b %Y %H:%M:%S", - "timezone": "UTC" - }, - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/test.log", - "log_group_name": "test.log", - "log_stream_name": "test.log", - "timezone": "UTC" - }, - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/*", - "blacklist": "agent.log*|env.log|profiler.log|\\.\\d$", - "publish_multi_logs": true, - "log_group_name": "EC2-System-Logs", - "log_stream_name": "multi_log_stream", - "timezone": "UTC" - } - ] - } - }, - "log_stream_name": "LOG_STREAM_NAME", - "force_flush_interval": 60 - }, - "scroll": { - "endpoint_override":"https://ingress.cell0001.prod.us-east-1.scroll.aws.a2z.com", - "logs_collected": { - "files": { - "collect_list": [ - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log", - "log_group_name": "amazon-cloudwatch-agent.log", - "log_type": "SERVICE", - "multi_line_start_pattern": "(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2})" - }, - { - "file_path": "/opt/aws/amazon-cloudwatch-agent/logs/test.log", - "log_group_name": "test.log", - "log_stream_name": "test.log", - "multi_line_start_pattern": "{timestamp_format}", - "timestamp_format": "%d %b %Y %H:%M:%S", - "timezone": "UTC" - } - ] - } - }, - "force_flush_interval": 60, - "credentials": { - "role_arn": "log_role_arn_value_test" - } - } -} \ No newline at end of file From 1a50a87d6cf63ba1446f318c8fbea7b12ebd8f6e Mon Sep 17 00:00:00 2001 From: Ping Xiang Date: Wed, 24 Jun 2020 22:07:31 -0700 Subject: [PATCH 6/6] fix: don't monitor files in file state folder File state folder contains the state files generated by agent. These files keep track of the reading offset for the log files tracked by agent. Users are not supposed to monitor the files in the file state folder. With some misconfiguration, if the state files are also monitored by agent, new state files tracking existing state files will be created. This is not desired behavior and should be not allowed. --- plugins/inputs/logfile/logfile.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/inputs/logfile/logfile.go b/plugins/inputs/logfile/logfile.go index cc4645fc004..b0001cb687a 100644 --- a/plugins/inputs/logfile/logfile.go +++ b/plugins/inputs/logfile/logfile.go @@ -275,6 +275,11 @@ func (t *LogFile) getTargetFiles(fileconfig *FileConfig) ([]string, error) { var targetFileName string var targetModTime time.Time for matchedFileName, matchedFileInfo := range g.Match() { + // we do not allow customer to monitor the file in t.FileStateFolder, it will monitor all of the state files + if t.FileStateFolder != "" && strings.HasPrefix(matchedFileName, t.FileStateFolder) { + continue + } + if isCompressedFile(matchedFileName) { continue }