diff --git a/src/Polly.Contrib.WaitAndRetry.Specs/DecorrelatedJitterBackoffSpecs.cs b/src/Polly.Contrib.WaitAndRetry.Specs/AwsDecorrelatedJitterBackoffSpecs.cs similarity index 80% rename from src/Polly.Contrib.WaitAndRetry.Specs/DecorrelatedJitterBackoffSpecs.cs rename to src/Polly.Contrib.WaitAndRetry.Specs/AwsDecorrelatedJitterBackoffSpecs.cs index c83a9ab..53f0dac 100644 --- a/src/Polly.Contrib.WaitAndRetry.Specs/DecorrelatedJitterBackoffSpecs.cs +++ b/src/Polly.Contrib.WaitAndRetry.Specs/AwsDecorrelatedJitterBackoffSpecs.cs @@ -1,11 +1,12 @@ using FluentAssertions; using System; using System.Collections.Generic; +using System.Linq; using Xunit; namespace Polly.Contrib.WaitAndRetry.Specs { - public sealed class DecorrelatedJitterBackoffSpecs + public sealed class AwsDecorrelatedJitterBackoffSpecs { [Fact] public void Backoff_WithMinDelayLessThanZero_ThrowsException() @@ -18,7 +19,7 @@ public void Backoff_WithMinDelayLessThanZero_ThrowsException() const int seed = 1; // Act - Action act = () => Backoff.DecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); + Action act = () => Backoff.AwsDecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); // Assert act.Should().Throw() @@ -36,7 +37,7 @@ public void Backoff_WithMaxDelayLessThanMinDelay_ThrowsException() const int seed = 1; // Act - Action act = () => Backoff.DecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); + Action act = () => Backoff.AwsDecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); // Assert act.Should().Throw() @@ -54,7 +55,7 @@ public void Backoff_WithRetryCountLessThanZero_ThrowsException() const int seed = 1; // Act - Action act = () => Backoff.DecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); + Action act = () => Backoff.AwsDecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); // Assert act.Should().Throw() @@ -72,7 +73,7 @@ public void Backoff_WithRetryEqualToZero_ResultIsEmpty() const int seed = 1; // Act - IEnumerable result = Backoff.DecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); + IEnumerable result = Backoff.AwsDecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); // Assert result.Should().NotBeNull(); @@ -90,10 +91,11 @@ public void Backoff_WithFastFirstEqualToTrue_ResultIsZero() const int seed = 1; // Act - IEnumerable result = Backoff.DecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); + IEnumerable result = Backoff.AwsDecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); // Assert result.Should().NotBeNull(); + result = result.ToList(); result.Should().HaveCount(retryCount); bool first = true; @@ -123,10 +125,11 @@ public void Backoff_ResultIsInRange() const int seed = 100; // Act - IEnumerable result = Backoff.DecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); + IEnumerable result = Backoff.AwsDecorrelatedJitterBackoff(minDelay, maxDelay, retryCount, seed, fastFirst); // Assert result.Should().NotBeNull(); + result = result.ToList(); result.Should().HaveCount(retryCount); foreach (TimeSpan timeSpan in result) diff --git a/src/Polly.Contrib.WaitAndRetry.Specs/DecorrelatedJitterBackoffV2Specs.cs b/src/Polly.Contrib.WaitAndRetry.Specs/DecorrelatedJitterBackoffV2Specs.cs new file mode 100644 index 0000000..ce14d6f --- /dev/null +++ b/src/Polly.Contrib.WaitAndRetry.Specs/DecorrelatedJitterBackoffV2Specs.cs @@ -0,0 +1,167 @@ +using FluentAssertions; +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; +using Xunit.Abstractions; + +namespace Polly.Contrib.WaitAndRetry.Specs +{ + public sealed class DecorrelatedJitterBackoffV2Specs + { + private readonly ITestOutputHelper testOutputHelper; + + public DecorrelatedJitterBackoffV2Specs(ITestOutputHelper testOutputHelper) + { + this.testOutputHelper = testOutputHelper; + } + + [Fact] + public void Backoff_WithMeanFirstDelayLessThanZero_ThrowsException() + { + // Arrange + var medianFirstDelay = new TimeSpan(-1); + const int retryCount = 3; + const bool fastFirst = false; + const int seed = 1; + + // Act + Action act = () => Backoff.DecorrelatedJitterBackoffV2(medianFirstDelay, retryCount, seed, fastFirst); + + // Assert + act.Should().Throw() + .And.ParamName.Should().Be("medianFirstRetryDelay"); + } + + [Fact] + public void Backoff_WithRetryCountLessThanZero_ThrowsException() + { + // Arrange + var medianFirstDelay = TimeSpan.FromSeconds(1); + const int retryCount = -1; + const bool fastFirst = false; + const int seed = 1; + + // Act + Action act = () => Backoff.DecorrelatedJitterBackoffV2(medianFirstDelay, retryCount, seed, fastFirst); + + // Assert + act.Should().Throw() + .And.ParamName.Should().Be("retryCount"); + } + + [Fact] + public void Backoff_WithRetryEqualToZero_ResultIsEmpty() + { + // Arrange + var medianFirstDelay = TimeSpan.FromSeconds(2); + const int retryCount = 0; + const bool fastFirst = false; + const int seed = 1; + + // Act + IEnumerable result = Backoff.DecorrelatedJitterBackoffV2(medianFirstDelay, retryCount, seed, fastFirst); + + // Assert + result.Should().NotBeNull(); + result.Should().BeEmpty(); + } + + [Fact] + public void Backoff_WithFastFirstEqualToTrue_ResultIsZero() + { + // Arrange + var medianFirstDelay = TimeSpan.FromSeconds(2); + const int retryCount = 10; + const bool fastFirst = true; + const int seed = 1; + + // Act + IEnumerable result = Backoff.DecorrelatedJitterBackoffV2(medianFirstDelay, retryCount, seed, fastFirst); + + // Assert + result.Should().NotBeNull(); + result = result.ToList(); + result.Should().HaveCount(retryCount); + + bool first = true; + int t = 0; + foreach (TimeSpan timeSpan in result) + { + if (first) + { + timeSpan.Should().Be(TimeSpan.FromMilliseconds(0)); + first = false; + } + else + { + t++; + AssertOnRetryDelayForTry(t, timeSpan, medianFirstDelay); + } + } + } + + [Fact] + public void Backoff_ResultIsInRange() + { + // Arrange + var medianFirstDelay = TimeSpan.FromSeconds(1); + const int retryCount = 6; + const bool fastFirst = false; + const int seed = 23456; + + // Act + IEnumerable result = Backoff.DecorrelatedJitterBackoffV2(medianFirstDelay, retryCount, seed, fastFirst); + + // Assert + result.Should().NotBeNull(); + result = result.ToList(); + result.Should().HaveCount(retryCount); + + int t = 0; + foreach (TimeSpan timeSpan in result) + { + t++; + AssertOnRetryDelayForTry(t, timeSpan, medianFirstDelay); + } + } + + public static IEnumerable SeedRange => Enumerable.Range(0, 1000).Select(o => new object[] {o}).ToArray(); + + [Theory] + [MemberData(nameof(SeedRange))] + public void Backoff_ResultIsInRange_WideTest(int seed) + { + // Arrange + var medianFirstDelay = TimeSpan.FromSeconds(3); + const int retryCount = 6; + const bool fastFirst = false; + + // Act + IEnumerable result = Backoff.DecorrelatedJitterBackoffV2(medianFirstDelay, retryCount, seed, fastFirst); + + // Assert + result.Should().NotBeNull(); + result = result.ToList(); + result.Should().HaveCount(retryCount); + + int t = 0; + foreach (TimeSpan timeSpan in result) + { + t++; + AssertOnRetryDelayForTry(t, timeSpan, medianFirstDelay); + } + } + + private void AssertOnRetryDelayForTry(int t, TimeSpan calculatedDelay, TimeSpan medianFirstDelay) + { + /*testOutputHelper.WriteLine($"Try {t}, delay: {calculatedDelay.TotalSeconds} seconds; given median first delay {medianFirstDelay.TotalSeconds} seconds.");*/ + + calculatedDelay.Should().BeGreaterOrEqualTo(TimeSpan.Zero); + + int upperLimitFactor = t < 2 ? (int)Math.Pow(2, t + 1) : (int)(Math.Pow(2, t + 1) - Math.Pow(2, t - 1)); + + calculatedDelay.Should().BeLessOrEqualTo(TimeSpan.FromTicks(medianFirstDelay.Ticks * upperLimitFactor)); + } + } +} \ No newline at end of file diff --git a/src/Polly.Contrib.WaitAndRetry/Backoff.DecorrelatedJitter.cs b/src/Polly.Contrib.WaitAndRetry/Backoff.AwsDecorrelatedJitter.cs similarity index 88% rename from src/Polly.Contrib.WaitAndRetry/Backoff.DecorrelatedJitter.cs rename to src/Polly.Contrib.WaitAndRetry/Backoff.AwsDecorrelatedJitter.cs index 4964119..d8f3b13 100644 --- a/src/Polly.Contrib.WaitAndRetry/Backoff.DecorrelatedJitter.cs +++ b/src/Polly.Contrib.WaitAndRetry/Backoff.AwsDecorrelatedJitter.cs @@ -3,12 +3,12 @@ namespace Polly.Contrib.WaitAndRetry { - partial class Backoff // .DecorrelatedJitter + partial class Backoff // .AwsDecorrelatedJitter { /// /// Generates sleep durations in an jittered manner, making sure to mitigate any correlations. /// For example: 117ms, 236ms, 141ms, 424ms, ... - /// For background, see https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/. + /// Per the formula from https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/. /// /// The minimum duration value to use for the wait before each retry. /// The maximum duration value to use for the wait before each retry. @@ -16,7 +16,7 @@ partial class Backoff // .DecorrelatedJitter /// An optional seed to use. /// If not specified, will use a shared instance with a random seed, per Microsoft recommendation for maximum randomness. /// Whether the first retry will be immediate or not. - public static IEnumerable DecorrelatedJitterBackoff(TimeSpan minDelay, TimeSpan maxDelay, int retryCount, int? seed = null, bool fastFirst = false) + public static IEnumerable AwsDecorrelatedJitterBackoff(TimeSpan minDelay, TimeSpan maxDelay, int retryCount, int? seed = null, bool fastFirst = false) { if (minDelay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(minDelay), minDelay, "should be >= 0ms"); if (maxDelay < minDelay) throw new ArgumentOutOfRangeException(nameof(maxDelay), maxDelay, $"should be >= {minDelay}"); diff --git a/src/Polly.Contrib.WaitAndRetry/Backoff.DecorrelatedJitterV2.cs b/src/Polly.Contrib.WaitAndRetry/Backoff.DecorrelatedJitterV2.cs new file mode 100644 index 0000000..d83d606 --- /dev/null +++ b/src/Polly.Contrib.WaitAndRetry/Backoff.DecorrelatedJitterV2.cs @@ -0,0 +1,68 @@ +using System; +using System.Collections.Generic; + +namespace Polly.Contrib.WaitAndRetry +{ + partial class Backoff // .DecorrelatedJitterV2 + { + /// + /// Generates sleep durations in an exponentially backing-off, jittered manner, making sure to mitigate any correlations. + /// For example: 850ms, 1455ms, 3060ms. + /// Per discussion in Polly issue 530, the jitter of this implementation exhibits fewer spikes and a smoother distribution than the AWS jitter formula. + /// + /// The median delay to target before the first retry, call it f (= f * 2^0). + /// Choose this value both to approximate the first delay, and to scale the remainder of the series. + /// Subsequent retries will (over a large sample size) have a median approximating retries at time f * 2^1, f * 2^2 ... f * 2^t etc for try t. + /// The actual amount of delay-before-retry for try t may be distributed between 0 and f * (2^(t+1) - 2^(t-1)) for t >= 2; + /// or between 0 and f * 2^(t+1), for t is 0 or 1. + /// The maximum number of retries to use, in addition to the original call. + /// An optional seed to use. + /// If not specified, will use a shared instance with a random seed, per Microsoft recommendation for maximum randomness. + /// Whether the first retry will be immediate or not. + public static IEnumerable DecorrelatedJitterBackoffV2(TimeSpan medianFirstRetryDelay, int retryCount, int? seed = null, bool fastFirst = false) + { + if (medianFirstRetryDelay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(medianFirstRetryDelay), medianFirstRetryDelay, "should be >= 0ms"); + if (retryCount < 0) throw new ArgumentOutOfRangeException(nameof(retryCount), retryCount, "should be >= 0"); + + if (retryCount == 0) + return Empty(); + + return Enumerate(medianFirstRetryDelay, retryCount, fastFirst, new ConcurrentRandom(seed)); + + // The original author/credit for this jitter formula is @george-polevoy . Jitter formula used with permission as described at https://github.com/App-vNext/Polly/issues/530#issuecomment-526555979 + // Minor adaptations (pFactor = 4.0 and rpScalingFactor = 1 / 1.4d) by @reisenberger, to scale the formula output for easier parameterisation to users. + + IEnumerable Enumerate(TimeSpan scaleFirstTry, int maxRetries, bool fast, ConcurrentRandom random) + { + // A factor used within the formula to help smooth the first calculated delay. + const double pFactor = 4.0; + + // A factor used to scale the median values of the retry times generated by the formula to be _near_ whole seconds, to aid Polly user comprehension. + // This factor allows the median values to fall approximately at 1, 2, 4 etc seconds, instead of 1.4, 2.8, 5.6, 11.2. + const double rpScalingFactor = 1 / 1.4d; + + int i = 0; + if (fast) + { + i++; + yield return TimeSpan.Zero; + } + + long targetTicksFirstDelay = scaleFirstTry.Ticks; + + double prev = 0.0; + for (; i < maxRetries; i++) + { + double t = (double)i + random.NextDouble(); + double next = Math.Pow(2, t) * Math.Tanh(Math.Sqrt(pFactor * t)); + + double formulaIntrinsicValue = next - prev; + yield return TimeSpan.FromTicks((long)(formulaIntrinsicValue * rpScalingFactor * targetTicksFirstDelay)); + + prev = next; + } + + } + } + } +} \ No newline at end of file diff --git a/src/Polly.Contrib.WaitAndRetry/ConcurrentRandom.cs b/src/Polly.Contrib.WaitAndRetry/ConcurrentRandom.cs index 0825970..21ca2ed 100644 --- a/src/Polly.Contrib.WaitAndRetry/ConcurrentRandom.cs +++ b/src/Polly.Contrib.WaitAndRetry/ConcurrentRandom.cs @@ -15,6 +15,9 @@ internal sealed class ConcurrentRandom // Also note that in concurrency testing, using a 'new Random()' for every thread ended up // being highly correlated. On NetFx this is maybe due to the same seed somehow being used // in each instance, but either way the singleton approach mitigated the problem. + + // For more discussion of different approaches to randomization in concurrent scenarios: https://github.com/App-vNext/Polly/issues/530#issuecomment-439680613 + private static readonly Random s_random = new Random(); private readonly Random _random;