Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions changelog/27518.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
```release-note:bug
agent: Fixed an issue causing excessive CPU usage during normal operation
```

```release-note:bug
proxy: Fixed an issue causing excessive CPU usage during normal operation
```
39 changes: 16 additions & 23 deletions command/agent/template/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -246,31 +246,24 @@ func (ts *Server) Run(ctx context.Context, incoming chan string, templates []*ct
ts.runner.Stop()
return nil
}
default:
// We are using default instead of a new case block to prioritize the case where <-incoming has a new value over
// receiving an error message from the consul-template server
select {
case err := <-ts.runner.ServerErrCh:
var responseError *api.ResponseError
ok := errors.As(err, &responseError)
if !ok {
ts.logger.Error("template server: could not extract error response")
continue
}
if responseError.StatusCode == 403 && strings.Contains(responseError.Error(), logical.ErrInvalidToken.Error()) && !tokenRenewalInProgress.Load() {
ts.logger.Info("template server: received invalid token error")

// Drain the error channel before sending a new error
select {
case <-invalidTokenCh:
default:
}
invalidTokenCh <- err
}
default:
case err := <-ts.runner.ServerErrCh:
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if this is a possible scenario -

  • IncomingToken receives a new token at the same time template sends an error back to ServerErrCh
  • We reauthenticate first by honoring the ServerErrCh select first. Now IncomingCh has two values in the channel
  • We try using the first token in IncomingCh but there is an error. Another error is sent to ServerErrCh
  • Again the SeverErrCh is honored first and we reauthenticate.

We are now stuck in a loop where we always honor the token one behind valid token.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, that's a good point! I do think it's likely in this scenario that both tokens will be valid, but it's still not a great state to be in. I'll rework this to drain the incoming channel in the same place we drain the invalid token channel. I think that should prevent any looping

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I definitely understand why we had it the way we had it before though, but I do think this might be the best fix, and the only situation it would struggle is if we have the two channels filled exactly simultaneously

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like this!! Thanks for adding Violet!

var responseError *api.ResponseError
ok := errors.As(err, &responseError)
if !ok {
ts.logger.Error("template server: could not extract error response")
continue
}

if responseError.StatusCode == 403 && strings.Contains(responseError.Error(), logical.ErrInvalidToken.Error()) && !tokenRenewalInProgress.Load() {
ts.logger.Info("template server: received invalid token error")

// Drain the error channel and incoming channel before sending a new error
select {
case <-invalidTokenCh:
case <-incoming:
default:
}
invalidTokenCh <- err
}
}
}
}
Expand Down
12 changes: 3 additions & 9 deletions command/agentproxyshared/auth/auth.go
Original file line number Diff line number Diff line change
Expand Up @@ -563,18 +563,12 @@ func (ah *AuthHandler) Run(ctx context.Context, am AuthMethod) error {
// Set authenticated when authentication succeeds
metrics.SetGauge([]string{ah.metricsSignifier, "authenticated"}, 1)
ah.logger.Info("renewed auth token")

case <-credCh:
ah.logger.Info("auth method found new credentials, re-authenticating")
break LifetimeWatcherLoop
default:
select {
case <-ah.InvalidToken:
ah.logger.Info("invalid token found, re-authenticating")
break LifetimeWatcherLoop
default:
continue
}
case <-ah.InvalidToken:
ah.logger.Info("invalid token found, re-authenticating")
break LifetimeWatcherLoop
}
}
}
Expand Down