Skip to content

Commit 7ad0806

Browse files
martintmkmartintmk
authored
[Docs] Apply chaos selectively (#1923)
1 parent 1dcfc01 commit 7ad0806

File tree

3 files changed

+226
-3
lines changed

3 files changed

+226
-3
lines changed

‎.github/wordlist.txt‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
alloc
22
apis
3+
ASP.NET
34
async
45
azurefunctions
56
bcl
@@ -13,6 +14,7 @@ deserialization
1314
dotnet
1415
dotnetrocks
1516
durations
17+
enricher
1618
eshoponcontainers
1719
extensibility
1820
flurl
@@ -23,16 +25,17 @@ jittered
2325
json
2426
loggingpolicy
2527
markdownsnippets
28+
middleware
2629
minver
2730
moq
2831
namespace
2932
natively
3033
ndc
3134
nuget
3235
oss
33-
pcl
34-
parallelize
3536
paas
37+
parallelize
38+
pcl
3639
pluralsight
3740
pollydocs
3841
pre
@@ -67,4 +70,3 @@ uwp
6770
waitandretry
6871
wpf
6972
xunit
70-
enricher

‎docs/chaos/index.md‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,113 @@ All the strategies' options implement the [`ChaosStrategyOptions`](xref:Polly.Si
9090
> If both `Enabled` and `EnabledGenerator` are specified then `Enabled` will be ignored.
9191
9292
[simmy]: https://github.com/Polly-Contrib/Simmy
93+
94+
## Patterns
95+
96+
### Inject chaos selectively
97+
98+
You aim to dynamically adjust the frequency and timing of chaos injection. For instance, in pre-production and test environments, it's sensible to consistently inject chaos. This proactive approach helps in preparing for potential failures. In production environments, however, you may prefer to limit chaos to certain users and tenants, ensuring that regular users remain unaffected. The chaos API offers the flexibility needed to manage these varying scenarios.
99+
100+
Additionally, you have the option to dynamically alter the injection rate and simulate extreme scenarios by setting the injection rate to *1.0 (100%)*. Exercise caution when applying this high rate, restricting it to a subset of tenants and users to avoid rendering the system unusable for regular users.
101+
102+
The following example illustrates how to configure chaos strategies accordingly:
103+
104+
<!-- snippet: chaos-selective -->
105+
```cs
106+
services.AddResiliencePipeline("chaos-pipeline", (builder, context) =>
107+
{
108+
var environment = context.ServiceProvider.GetRequiredService<IHostEnvironment>();
109+
110+
builder.AddChaosFault(new ChaosFaultStrategyOptions
111+
{
112+
EnabledGenerator = args =>
113+
{
114+
// Enable chaos in development and staging environments.
115+
if (environment.IsDevelopment() || environment.IsStaging())
116+
{
117+
return ValueTask.FromResult(true);
118+
}
119+
120+
// Enable chaos for specific users or tenants, even in production environments.
121+
if (ShouldEnableChaos(args.Context))
122+
{
123+
return ValueTask.FromResult(true);
124+
}
125+
126+
return ValueTask.FromResult(false);
127+
},
128+
InjectionRateGenerator = args =>
129+
{
130+
if (environment.IsStaging())
131+
{
132+
// 1% chance of failure on staging environments.
133+
return ValueTask.FromResult(0.01);
134+
}
135+
136+
if (environment.IsDevelopment())
137+
{
138+
// 5% chance of failure on development environments.
139+
return ValueTask.FromResult(0.05);
140+
}
141+
142+
// The context can carry information to help determine the injection rate.
143+
// For instance, in production environments, you might have certain test users or tenants
144+
// for whom you wish to inject chaos.
145+
if (ResolveInjectionRate(args.Context, out double injectionRate))
146+
{
147+
return ValueTask.FromResult(injectionRate);
148+
}
149+
150+
// No chaos on production environments.
151+
return ValueTask.FromResult(0.0);
152+
},
153+
FaultGenerator = new FaultGenerator()
154+
.AddException<TimeoutException>()
155+
.AddException<HttpRequestException>()
156+
});
157+
});
158+
```
159+
<!-- endSnippet -->
160+
161+
We suggest encapsulating the chaos decisions and injection rate in a shared class, such as `IChaosManager`:
162+
163+
<!-- snippet: chaos-manager -->
164+
```cs
165+
public interface IChaosManager
166+
{
167+
bool IsChaosEnabled(ResilienceContext context);
168+
169+
double GetInjectionRate(ResilienceContext context);
170+
}
171+
```
172+
<!-- endSnippet -->
173+
174+
This approach allows you to consistently apply and manage chaos-related settings across various chaos strategies by reusing `IChaosManager`. By centralizing the logic for enabling chaos and determining injection rates, you can ensure uniformity and ease of maintenance across your application and reuse it across multiple chaos strategies:
175+
176+
<!-- snippet: chaos-selective-manager -->
177+
```cs
178+
services.AddResiliencePipeline("chaos-pipeline", (builder, context) =>
179+
{
180+
var chaosManager = context.ServiceProvider.GetRequiredService<IChaosManager>();
181+
182+
builder
183+
.AddChaosFault(new ChaosFaultStrategyOptions
184+
{
185+
EnabledGenerator = args => ValueTask.FromResult(chaosManager.IsChaosEnabled(args.Context)),
186+
InjectionRateGenerator = args => ValueTask.FromResult(chaosManager.GetInjectionRate(args.Context)),
187+
FaultGenerator = new FaultGenerator()
188+
.AddException<TimeoutException>()
189+
.AddException<HttpRequestException>()
190+
})
191+
.AddChaosLatency(new ChaosLatencyStrategyOptions
192+
{
193+
EnabledGenerator = args => ValueTask.FromResult(chaosManager.IsChaosEnabled(args.Context)),
194+
InjectionRateGenerator = args => ValueTask.FromResult(chaosManager.GetInjectionRate(args.Context)),
195+
Latency = TimeSpan.FromSeconds(60)
196+
});
197+
});
198+
```
199+
<!-- endSnippet -->
200+
201+
> [!NOTE]
202+
> An alternative method involves using [`Microsoft.Extensions.AsyncState`](https://www.nuget.org/packages/Microsoft.Extensions.AsyncState) for storing information relevant to chaos injection decisions. This can be particularly useful in frameworks like ASP.NET Core. For instance, you could implement a middleware that retrieves user information from `HttpContext`, assesses the user type, and then stores this data in `IAsyncContext<ChaosUser>`. Subsequently, `IChaosManager` can access `IAsyncContext<ChaosUser>` to retrieve this information. This approach eliminates the need to manually insert such data into `ResilienceContext` for each call within the resilience pipeline, thereby streamlining the process.

‎src/Snippets/Docs/Chaos.Index.cs‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
using System.Net.Http;
2+
using Microsoft.Extensions.DependencyInjection;
3+
using Microsoft.Extensions.Hosting;
24
using Polly.CircuitBreaker;
35
using Polly.Retry;
46
using Polly.Simmy;
7+
using Polly.Simmy.Fault;
8+
using Polly.Simmy.Latency;
59

610
namespace Snippets.Docs;
711

@@ -35,5 +39,112 @@ public static void Usage()
3539
#endregion
3640
}
3741

42+
public static void ApplyChaosSelectively(IServiceCollection services)
43+
{
44+
#region chaos-selective
45+
46+
services.AddResiliencePipeline("chaos-pipeline", (builder, context) =>
47+
{
48+
var environment = context.ServiceProvider.GetRequiredService<IHostEnvironment>();
49+
50+
builder.AddChaosFault(new ChaosFaultStrategyOptions
51+
{
52+
EnabledGenerator = args =>
53+
{
54+
// Enable chaos in development and staging environments.
55+
if (environment.IsDevelopment() || environment.IsStaging())
56+
{
57+
return ValueTask.FromResult(true);
58+
}
59+
60+
// Enable chaos for specific users or tenants, even in production environments.
61+
if (ShouldEnableChaos(args.Context))
62+
{
63+
return ValueTask.FromResult(true);
64+
}
65+
66+
return ValueTask.FromResult(false);
67+
},
68+
InjectionRateGenerator = args =>
69+
{
70+
if (environment.IsStaging())
71+
{
72+
// 1% chance of failure on staging environments.
73+
return ValueTask.FromResult(0.01);
74+
}
75+
76+
if (environment.IsDevelopment())
77+
{
78+
// 5% chance of failure on development environments.
79+
return ValueTask.FromResult(0.05);
80+
}
81+
82+
// The context can carry information to help determine the injection rate.
83+
// For instance, in production environments, you might have certain test users or tenants
84+
// for whom you wish to inject chaos.
85+
if (ResolveInjectionRate(args.Context, out double injectionRate))
86+
{
87+
return ValueTask.FromResult(injectionRate);
88+
}
89+
90+
// No chaos on production environments.
91+
return ValueTask.FromResult(0.0);
92+
},
93+
FaultGenerator = new FaultGenerator()
94+
.AddException<TimeoutException>()
95+
.AddException<HttpRequestException>()
96+
});
97+
});
98+
99+
#endregion
100+
}
101+
102+
public static void ApplyChaosSelectivelyWithChaosManager(IServiceCollection services)
103+
{
104+
#region chaos-selective-manager
105+
106+
services.AddResiliencePipeline("chaos-pipeline", (builder, context) =>
107+
{
108+
var chaosManager = context.ServiceProvider.GetRequiredService<IChaosManager>();
109+
110+
builder
111+
.AddChaosFault(new ChaosFaultStrategyOptions
112+
{
113+
EnabledGenerator = args => ValueTask.FromResult(chaosManager.IsChaosEnabled(args.Context)),
114+
InjectionRateGenerator = args => ValueTask.FromResult(chaosManager.GetInjectionRate(args.Context)),
115+
FaultGenerator = new FaultGenerator()
116+
.AddException<TimeoutException>()
117+
.AddException<HttpRequestException>()
118+
})
119+
.AddChaosLatency(new ChaosLatencyStrategyOptions
120+
{
121+
EnabledGenerator = args => ValueTask.FromResult(chaosManager.IsChaosEnabled(args.Context)),
122+
InjectionRateGenerator = args => ValueTask.FromResult(chaosManager.GetInjectionRate(args.Context)),
123+
Latency = TimeSpan.FromSeconds(60)
124+
});
125+
});
126+
127+
#endregion
128+
}
129+
130+
private static bool ResolveInjectionRate(ResilienceContext context, out double injectionRate)
131+
{
132+
injectionRate = 0.0;
133+
return false;
134+
}
135+
136+
private static bool ShouldEnableChaos(ResilienceContext context) => true;
137+
38138
private static ValueTask RestartRedisAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask;
139+
140+
#region chaos-manager
141+
142+
public interface IChaosManager
143+
{
144+
bool IsChaosEnabled(ResilienceContext context);
145+
146+
double GetInjectionRate(ResilienceContext context);
147+
}
148+
149+
#endregion
39150
}

0 commit comments

Comments
 (0)