For some part of my system I need to add retry logic for reading from the database. I have a number of repositories with async and sync read methods that I can't change. I found a simple solution - interception of all read methods with AsyncInterceptor and add retry read policy with Polly when database exception caught. Polly retries reading with some intervals.
Interceptor code:
public class RetriableReadAsyncInterceptor : IAsyncInterceptor
{
public void InterceptSynchronous(IInvocation invocation)
{
invocation.ReturnValue = InternalInterceptSync(invocation);
}
public void InterceptAsynchronous(IInvocation invocation)
{
throw new NotImplementedException();
}
public void InterceptAsynchronous<TResult>(IInvocation invocation)
{
invocation.ReturnValue = InternalInterceptAsync<TResult>(invocation);
}
private IEnumerable<TimeSpan> RetryIntervals =>
new[]
{
TimeSpan.FromSeconds(1),
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(10),
TimeSpan.FromSeconds(15)
};
private object InternalInterceptSync(IInvocation invocation)
{
return Policy
.Handle<DatabaseException>()
.WaitAndRetry(RetryIntervals, (exception, timeSpan) =>
{
Console.WriteLine($"Exception {timeSpan}");
})
.Execute(() =>
{
invocation.Proceed();
return invocation.ReturnValue;
});
}
private async Task<TResult> InternalInterceptAsync<TResult>(IInvocation invocation)
{
return await Policy
.Handle<DatabaseException>()
.WaitAndRetryAsync(RetryIntervals, (exception, timeSpan) =>
{
Console.WriteLine($"Exception {timeSpan}");
})
.ExecuteAsync(async () =>
{
invocation.Proceed();
var task = (Task<TResult>)invocation.ReturnValue;
return await task;
});
}
}
Repository code:
public class Repository : IRepository
{
private int _exceptionsCoutner;
public Entity GetById(int id)
{
if (_exceptionsCoutner <= 2)
{
_exceptionsCoutner++;
throw new DatabaseException();
}
//read from db
return new Entity {Id = id};
}
public async Task<Entity> GetByIdAsync(int id)
{
if (_exceptionsCoutner <= 2)
{
_exceptionsCoutner++;
throw new DatabaseException();
}
//read from db
return await Task.FromResult(new Entity { Id = id });
}
}
Sync version of GetById works as expected (retries with intervals):
Exception 00:00:01
Exception 00:00:05
Exception 00:00:10
Async version of GetById retries but not waits for time interval elapsed:
Exception 00:00:01
Exception 00:00:01
Exception 00:00:01
I can't understand where is the problem. If you have any thoughts - please share.
Full example can be found here.
It was a kind of 'chicken and egg' problem which can be solved now with newer version of Castle.Core (I tried version 4.4.0) leveraging the invocation.CaptureProceedInfo method:
private Task<TResult> InternalInterceptAsync<TResult>(IInvocation invocation)
{
var capture = invocation.CaptureProceedInfo();
return Policy
.Handle<DatabaseException>()
.WaitAndRetryAsync(RetryIntervals, (exception, timeSpan) =>
{
Console.WriteLine($"Exception {timeSpan}");
})
.ExecuteAsync(async () =>
{
capture.Invoke();
var task = (Task<TResult>)invocation.ReturnValue;
return await task;
});
}
OK, here is my naive implementation of retry:
public class Retry
{
public static async Task<TResult> DoAsync<TResult, TException>(
Func<Task<TResult>> action,
TimeSpan retryInterval,
int maxAttemptCount = 3)
where TException : Exception
{
TException exception = null;
var startDateTime = DateTime.UtcNow;
for (var attempted = 0; attempted < maxAttemptCount; attempted++)
{
try
{
return await action().ConfigureAwait(false);
}
catch (TException ex)
{
exception = ex;
Console.WriteLine($"Exception {DateTime.UtcNow - startDateTime}");
await Task.Delay(retryInterval); //doesnt work
//Thread.Sleep(retryInterval); works!
}
}
throw exception;
}
}
And interceptor:
private async Task<TResult> InternalInterceptAsync<TResult>(IInvocation invocation)
{
return await Retry.DoAsync<TResult, DatabaseException>(async () =>
{
invocation.Proceed();
var task = (Task<TResult>) invocation.ReturnValue;
return await task.ConfigureAwait(false);
},
TimeSpan.FromSeconds(3),
4);
}
Implementation with blocking Tread.Sleep works well, but with Task.Delay dont.
Related
I've got two Transient Fault Handling/Retry pattern implementations.
The issue is that theTask.Run swallows the exception and it doesn't rethrow it out of the Task.Run scope.
If I await the Task.Run it would work, but I cannot do that in my real use case.
public static class PollyRetry
{
public static T Do<T>(Func<T> action, TimeSpan retryWait, int retryCount = 0)
{
var policyResult = Policy
.Handle<Exception>()
.WaitAndRetry(retryCount, retryAttempt => retryWait)
.ExecuteAndCapture(action);
if (policyResult.Outcome == OutcomeType.Failure)
{
throw policyResult.FinalException;
}
return policyResult.Result;
}
public static async Task<T> DoAsync<T>(Func<Task<T>> action, TimeSpan retryWait, int retryCount = 0)
{
var policyResult = await Policy
.Handle<Exception>()
.WaitAndRetryAsync(retryCount, retryAttempt => retryWait)
.ExecuteAndCaptureAsync(action);
if (policyResult.Outcome == OutcomeType.Failure)
{
throw policyResult.FinalException;
}
return policyResult.Result;
}
}
public static class Retry
{
public static void Do(Action action, TimeSpan retryInterval, int retryCount = 3)
{
Do<object?>(() =>
{
action();
return null;
}, retryInterval, retryCount);
}
public static async Task DoAsync(Func<Task> action, TimeSpan retryInterval, int retryCount = 3)
{
await DoAsync<object?>(async () =>
{
await action();
return null;
}, retryInterval, retryCount);
}
public static T Do<T>(Func<T> action, TimeSpan retryInterval, int retryCount = 3)
{
var exceptions = new List<Exception>();
for (var count = 1; count <= retryCount; count++)
{
try
{
return action();
}
catch (Exception ex)
{
exceptions.Add(ex);
if (count < retryCount)
{
Thread.Sleep(retryInterval);
}
}
}
throw new AggregateException(exceptions);
}
public static async Task<T> DoAsync<T>(Func<Task<T>> func, TimeSpan retryInterval, int retryCount = 3)
{
var exceptions = new List<Exception>();
for (var count = 1; count <= retryCount; count++)
{
try
{
return await func();
}
catch (Exception ex)
{
exceptions.Add(ex);
if (count < retryCount)
{
await Task.Delay(retryInterval);
}
}
}
throw new AggregateException(exceptions);
}
}
public sealed class WebSocketClient
{
private readonly Channel<string> _receiveChannel;
private readonly Channel<string> _sendChannel;
public WebSocketClient()
{
_receiveChannel = Channel.CreateBounded<string>(new BoundedChannelOptions(10)
{
SingleWriter = true,
SingleReader = false,
FullMode = BoundedChannelFullMode.DropOldest
});
_sendChannel = Channel.CreateBounded<string>(new BoundedChannelOptions(10)
{
SingleReader = true,
SingleWriter = false,
FullMode = BoundedChannelFullMode.Wait
});
}
public async Task StartWithRetry(Uri uri)
{
await Retry.DoAsync(() => Task.FromResult(StartAsync(uri)), TimeSpan.FromSeconds(5), 5);
}
public async Task StartAsync(Uri uri)
{
using var ws = new ClientWebSocket();
await ws.ConnectAsync(uri, default);
if (ws.State == WebSocketState.Open)
{
const string message = "{\"op\": \"subscribe\", \"args\": [\"orderBookL2_25:XBTUSD\"]}";
var buffer = new ArraySegment<byte>(Encoding.UTF8.GetBytes(message));
await ws.SendAsync(buffer, WebSocketMessageType.Text, true, default);
}
_ = Task.Run(async () =>
{
while (await _receiveChannel.Reader.WaitToReadAsync())
{
while (_receiveChannel.Reader.TryRead(out var message))
{
Console.WriteLine($"Message: {message}");
}
}
});
_ = Task.Run(async () =>
{
// This throws WebSocketException with ex.WebSocketErrorCode == WebSocketError.ConnectionClosedPrematurely
while (true)
{
ValueWebSocketReceiveResult receiveResult;
using var buffer = MemoryPool<byte>.Shared.Rent(4096);
await using var ms = new MemoryStream(buffer.Memory.Length);
do
{
receiveResult = await ws.ReceiveAsync(buffer.Memory, default);
if (receiveResult.MessageType == WebSocketMessageType.Close)
{
break;
}
await ms.WriteAsync(buffer.Memory[..receiveResult.Count]);
} while (!receiveResult.EndOfMessage);
ms.Seek(0, SeekOrigin.Begin);
if (receiveResult.MessageType == WebSocketMessageType.Text)
{
using var reader = new StreamReader(ms, Encoding.UTF8);
var message = await reader.ReadToEndAsync();
await _receiveChannel.Writer.WriteAsync(message);
}
else if (receiveResult.MessageType == WebSocketMessageType.Close)
{
break;
}
}
});
}
}
Minimal Reproducible Example
var code = new MinimalReproducibleCode();
await code.StartWithRetry();
public sealed class MinimalReproducibleCode
{
public async Task StartWithRetry()
{
await Retry.DoAsync(() => Task.FromResult(StartAsync()), TimeSpan.FromSeconds(5), 5);
}
public Task StartAsync()
{
Console.WriteLine("This has just started");
_ = Task.Run(() =>
{
while (true)
{
Console.WriteLine("Code is working");
throw new DivideByZeroException();
}
});
return Task.CompletedTask;
}
}
public static class Retry
{
public static void Do(Action action, TimeSpan retryInterval, int retryCount = 3)
{
_ = Do<object?>(() =>
{
action();
return null;
}, retryInterval, retryCount);
}
public static async Task DoAsync(Func<Task> action, TimeSpan retryInterval, int retryCount = 3)
{
_ = await DoAsync<object?>(async () =>
{
await action();
return null;
}, retryInterval, retryCount);
}
public static async Task DoAsync<TException>(
Func<Task> action,
Func<TException, bool> exceptionFilter,
TimeSpan retryInterval,
int retryCount = 3) where TException : Exception
{
_ = await DoAsync<object?>(async () =>
{
await action();
return null;
}, retryInterval, retryCount);
}
public static T Do<T>(Func<T> action, TimeSpan retryWait, int retryCount = 3)
{
var policyResult = Policy
.Handle<Exception>()
.WaitAndRetry(retryCount, retryAttempt => retryWait)
.ExecuteAndCapture(action);
if (policyResult.Outcome == OutcomeType.Failure)
{
throw policyResult.FinalException;
}
return policyResult.Result;
}
public static async Task<T> DoAsync<T>(Func<Task<T>> action, TimeSpan retryWait, int retryCount = 3)
{
var policyResult = await Policy
.Handle<Exception>()
.WaitAndRetryAsync(retryCount, retryAttempt => retryWait)
.ExecuteAndCaptureAsync(action);
if (policyResult.Outcome == OutcomeType.Failure)
{
throw policyResult.FinalException;
}
return policyResult.Result;
}
public static async Task<T> DoAsync<T, TException>(
Func<Task<T>> action,
Func<TException, bool> exceptionFilter,
TimeSpan retryWait,
int retryCount = 0) where TException : Exception
{
var policyResult = await Policy
.Handle(exceptionFilter)
.WaitAndRetryAsync(retryCount, retryAttempt => retryWait)
.ExecuteAndCaptureAsync(action);
if (policyResult.Outcome == OutcomeType.Failure)
{
throw policyResult.FinalException;
}
return policyResult.Result;
}
}
OK, based on your code, here's how to make it work:
public async Task StartWithRetry()
{
await Retry.DoAsync(() => StartAsync(), TimeSpan.FromSeconds(5), 5);
}
public async Task StartAsync()
{
Console.WriteLine("This has just started");
await Task.Run(() =>
{
while (true)
{
Console.WriteLine("Code is working");
throw new DivideByZeroException();
}
});
}
You need to await the Task.Run and not fire-and-forget it.
When I run the above code I get:
This has just started
Code is working
This has just started
Code is working
This has just started
Code is working
This has just started
Code is working
This has just started
Code is working
This has just started
Code is working
DivideByZeroException
Attempted to divide by zero.
I am exploring Azure Function running on .net 5 and I found out about the new middleware capabilities.
I have built a dummy middleware like this one:
public sealed class ExceptionLoggingMiddleware : IFunctionsWorkerMiddleware
{
private readonly ILogger<ExceptionLoggingMiddleware> m_logger;
public ExceptionLoggingMiddleware(ILogger<ExceptionLoggingMiddleware> logger)
{
m_logger = logger;
}
public async Task Invoke(FunctionContext context, FunctionExecutionDelegate next)
{
try
{
await next(context);
}
catch (Exception unhandledException)
{
m_logger.LogCritical(unhandledException, "Unhandled exception caught: {UnhandledException}", unhandledException.Message);
}
}
}
In my use case, the Azure Function is an HTTP triggered function:
public sealed class StorageAccountsFunction
{
private readonly ILogger<StorageAccountsFunction> m_logger;
public StorageAccountsFunction
(
ILogger<StorageAccountsFunction> logger
)
{
m_logger = logger;
}
[Function("v1-post-storage-account")]
public async Task<HttpResponseData> CreateAsync
(
[HttpTrigger(AuthorizationLevel.Anonymous, "POST", Route = "v1/storage-accounts")]
HttpRequestData httpRequestData,
FunctionContext context
)
{
m_logger.LogInformation("Processing a request to create a new storage account");
throw new Exception("Oh no! Oh well..");
}
}
In my Function App running in-process on .net core 3.1, each Function had the responsibility of catching the unhandled exception (via a base class) and returned the appropriate HTTP status code.
I would like to have that logic sit in a middleware instead to have it centralized and avoid any future mistakes.
Question
The exception is caught by the middleware properly. However, I do not see how I can alter the response and return something more appropriate, instead of a 500 Internal Server Error that I get right now?
According to this issue, there is currently no official implementation regarding this, but they also mention a "hacky workaround" until the proper functionality is implemented directly into Azure functions
We created an extension method for FunctionContext:
internal static class FunctionUtilities
{
internal static HttpRequestData GetHttpRequestData(this FunctionContext context)
{
var keyValuePair = context.Features.SingleOrDefault(f => f.Key.Name == "IFunctionBindingsFeature");
var functionBindingsFeature = keyValuePair.Value;
var type = functionBindingsFeature.GetType();
var inputData = type.GetProperties().Single(p => p.Name == "InputData").GetValue(functionBindingsFeature) as IReadOnlyDictionary<string, object>;
return inputData?.Values.SingleOrDefault(o => o is HttpRequestData) as HttpRequestData;
}
internal static void InvokeResult(this FunctionContext context, HttpResponseData response)
{
var keyValuePair = context.Features.SingleOrDefault(f => f.Key.Name == "IFunctionBindingsFeature");
var functionBindingsFeature = keyValuePair.Value;
var type = functionBindingsFeature.GetType();
var result = type.GetProperties().Single(p => p.Name == "InvocationResult");
result.SetValue(functionBindingsFeature, response);
}
}
The usage in the middleware looks like this:
public async Task Invoke(FunctionContext context, FunctionExecutionDelegate next)
{
try
{
await next(context);
}
catch (Exception ex)
{
if (ex.InnerException is *NameOfExceptionYouNeed* e)
{
var req = context.GetHttpRequestData();
var res = await req.ErrorResponseAsync(e.Message);
context.InvokeResult(res);
return;
}
throw;
}
}
This is natively supported now as of version 1.8.0 of Microsoft.Azure.Functions.Worker.
The FunctionContextHttpRequestExtensions class was introduced so now you can just
using Microsoft.Azure.Functions.Worker;
public class MyMiddleware : IFunctionsWorkerMiddleware
{
public async Task Invoke(FunctionContext context, FunctionExecutionDelegate next)
{
// To access the RequestData
var req = await context.GetHttpRequestDataAsync();
// To set the ResponseData
var res = req!.CreateResponse();
await res.WriteStringAsync("Please login first", HttpStatusCode.Unauthorized);
context.GetInvocationResult().Value = res;
}
}
This code works for me. It is based on the example here: https://github.com/Azure/azure-functions-dotnet-worker/blob/main/samples/CustomMiddleware/ExceptionHandlingMiddleware.cs
public async Task Invoke(FunctionContext context, FunctionExecutionDelegate next)
{
// Simple example which always fails. Use the following in an error condition
var httpReqData = await context.GetHttpRequestDataAsync();
if (httpReqData != null)
{
var newHttpResponse = httpReqData.CreateResponse(HttpStatusCode.InternalServerError);
await newHttpResponse.WriteAsJsonAsync(new { ResponseStatus = "Invocation failed!" }, newHttpResponse.StatusCode);
context.GetInvocationResult().Value = newHttpResponse;
}
}
I have a try/catch block in both my application and unit test. I'm trying to create a test that catches an exception being thrown in my actor. When debugging through the code I get the exception being thrown but in my test I never receive the exception.
Application:
public class FooActor : ReceiveActor {
private readonly IFooService fooService;
private readonly IChildActorFactory childCreatorFactory;
private IActorRef barActor;
public FooActor(IFooService fooService, IChildActorFactory childCreatorFactory) {
this.fooService = fooService;
this.childCreatorFactory = childCreatorFactory;
ReceiveAsync<bool>(async (x) => await StartAsync(x).ConfigureAwait(false));
}
protected override void PreStart() {
barActor = childCreatorFactory.Create<BarActor>(Context, "BarActor");
}
public async Task StartAsync(bool start) {
try {
if (start) {
var fooInformation = await fooService.GetInformationAsync().ConfigureAwait(false);
if (fooInformation != null) {
barActor.Tell(fooInformation);
}
}
} catch (Exception exception) {
throw new Exception($"Unhandled exception. Actor {Self.Path.Name};", exception);
}
}
}
Test:
[Fact]
public void StartAsync_ThrowsException_ExceptionThrown() {
using (var mock = AutoMock.GetLoose()) {
//Arrange
Sys.UseAutofac(mock.Container);
var mockChildActorFactory = mock.Mock<IChildActorFactory>();
var mockBarService = mock.Mock<IBarService>();
mockBarService.Setup(x => x.GetInformationAsync()).Throws(new Exception());
var props = Props.Create(() => new FooActor(mockBarService.Object, mockChildActorFactory.Object));
var fooActorName = "FooActor";
var fooActor = new TestActorRef<FooActor>(Sys, props, null, fooActorName);
try {
// Act
fooActor.Receive(true);
} catch (Exception exception) {
// Assert
Assert.Equal($"Unhandled Exception. Actor { fooActorName }.", exception.Message);
}
}
}
The problem is in the async/await operators in ReceiveAsync method:
ReceiveAsync<bool>(async (x) => await StartAsync(x).ConfigureAwait(false));
when execution context reached the await operation it just start Task at thread pool(simplified) and returns to the caller. I.e. when
fooActor.Receive(true);
completed, the actual task with StartAsync(x) may not be started yet. So when actual exception is thrown your test is already finished without any exception.
I'm currently trying to test the following code in an application that makes use of the Microsoft Bot Framework.
public async Task ResumeAfterCalculation_v2FormDialog(IDialogContext context, IAwaitable<Calculation_v2Form> result)
{
try
{
var extractedCalculationForm = await result;
//Removed additional code
}
catch (FormCanceledException ex)
{
var reply = "You have canceled the operation.";
await _chat.PostAsync(context, reply);
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
finally
{
context.Done<object>(null);
}
}
When a user types 'quit' to the bot the 'await result' code throws a FormCanceledException and the code quits the form.
When creating a test I implemented a class to mock the IAwaitable:
public class TaskAwaiterHelper<T> : IAwaiter<T>, IAwaitable<T>
{
public Task<T> Task { get; }
public TaskAwaiterHelper(T obj)
{
this.Task = System.Threading.Tasks.Task.FromResult(obj);
}
public TaskAwaiterHelper(Task<T> task)
{
this.Task = task;
}
public bool IsCompleted { get { return Task.IsCompleted; } }
public void OnCompleted(Action action)
{
SynchronizationContext context = SynchronizationContext.Current;
TaskScheduler scheduler = context == null ? TaskScheduler.Current
: TaskScheduler.FromCurrentSynchronizationContext();
Task.ContinueWith(ignored => action(), scheduler);
}
public T GetResult()
{
return Task.Result;
}
public IAwaiter<T> GetAwaiter()
{
return this;
}
}
I then created the following test:
[Fact]
public async Task ResumeAfterCalculation_v2FormDialog_WasCancelled_ThenCallsDone()
{
//Arrange
var chat = new Mock<IChatHelper>();
var calculationApi = new Mock<ICalculationApi>();
var dialogContextMock = new Mock<IDialogContext>();
var rootLuisDialog = new RootLuisDialog(chat.Object, calculationApi.Object);
var taskAwaiter = new TaskAwaiterHelper<Calculation_v2Form>(new Task<Calculation_v2Form>(() =>
{
throw new FormCanceledException("Error created for test test", null);
}));
taskAwaiter.Task.Start();
//Act
await rootLuisDialog.ResumeAfterCalculation_v2FormDialog(dialogContextMock.Object, taskAwaiter);
//Assert
chat.Verify(c => c.PostAsync(dialogContextMock.Object, "You have canceled the operation."), Times.Once());
dialogContextMock.Verify(t => t.Done<object>(null), Times.Once());
}
Now whatever I try to do I the exception that's being thrown in the IAwaitable is being wrapped in an AggregateException, so we always end up in the catch (Exception ex) instead of the desired catch (FormCanceledException ex)
Is there a way to make a Task throw a specific Exception instead of an AggregateException (I mean there should be as the bot framework itself seems to be able to do it).
I just found the answer, I basically created a new class:
public class ExceptionThrower : IAwaitable<Calculation_v2Form>
{
public IAwaiter<Calculation_v2Form> GetAwaiter()
{
throw new FormCanceledException("Error created for test test", null);
}
}
And just provided this to the method:
var exceptionThrower = new ExceptionThrower();
await rootLuisDialog.ResumeAfterCalculation_v2FormDialog(dialogContextMock.Object, exceptionThrower);
I have some logic for Task and Task<T>.
Is there any way to avoid duplicating code ?
My current code is following:
public async Task<SocialNetworkUserInfo> GetMe()
{
return await WrapException(() => new SocialNetworkUserInfo());
}
public async Task AuthenticateAsync()
{
await WrapException(() => _facebook.Authenticate());
}
public async Task<T> WrapException<T>(Func<Task<T>> task)
{
try
{
return await task();
}
catch (FacebookNoInternetException ex)
{
throw new NoResponseException(ex.Message, ex, true);
}
catch (FacebookException ex)
{
throw new SocialNetworkException("Social network call failed", ex);
}
}
public async Task WrapException(Func<Task> task)
{
try
{
await task();
}
catch (FacebookNoInternetException ex)
{
throw new NoResponseException(ex.Message, ex, true);
}
catch (FacebookException ex)
{
throw new SocialNetworkException("Social network call failed", ex);
}
}
You can make the Task overload call the other one, and return a dummy value.
public async Task WrapException(Func<Task> task)
{
await WrapException<object>(async () => {
await task();
return null;
});
}
Or, since the async keyword is unnecessary here:
public Task WrapException(Func<Task> task)
{
return WrapException<object>(async () => {
await task();
return null;
});
}
Assuming that Func does not itself throw, the following would work.
public async Task<T> WrapException<T>(Func<Task<T>> task)
{
var actualTask = task();
await WrapException((Task)actualTask);
return actualTask.Result;
}
We know that Result won't block or throw since WrapException ensured it ran to completion.