Develop Reference

Possible HttpClient deadlock

I have a decorator class that adds the ability to rate limit http requests for hitting an API:
public class RateLimitedHttpClient : IHttpClient
{
public RateLimitedHttpClient(System.Net.Http.HttpClient client)
{
_client = client;
_client.Timeout = TimeSpan.FromMinutes(30);
//ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls;
}
public async Task<string> ReadAsync(string url)
{
if (!_sw.IsRunning)
_sw.Start();
await Delay();
using var response = await _client.GetAsync(url);
return await response.Content.ReadAsStringAsync();
}
private async Task Delay()
{
var totalElapsed = GetTimeElapsedSinceLastRequest();
while (totalElapsed < MinTimeBetweenRequests)
{
await Task.Delay(MinTimeBetweenRequests - totalElapsed);
totalElapsed = GetTimeElapsedSinceLastRequest();
};
_timeElapsedOfLastHttpRequest = (int)_sw.Elapsed.TotalMilliseconds;
}
private int GetTimeElapsedSinceLastRequest()
{
return (int)_sw.Elapsed.TotalMilliseconds - _timeElapsedOfLastHttpRequest;
}
private readonly System.Net.Http.HttpClient _client;
private readonly Stopwatch _sw = new Stopwatch();
private int _timeElapsedOfLastHttpRequest;
private const int MinTimeBetweenRequests = 100;
}
However, I'm noticing that at the line indicated below I get a message in the debugger that says that the next statement will execute when the current thread returns.
var epsDataPoints = await _downloader.GetEPSData(cik);
foreach (var eps in epsDataPoints)
{
// getting VS2019 debugger message here!
// assuming that the line above is deadlocking....
Console.WriteLine($"{cik} :: {eps.DateInterval} :: {eps.EPS}");
}
When I open up the Task Manager, the network bandwidth goes to 0 and everything stops with the application other than it sitting at the Console.WriteLine above.
The EPSDownloader class that uses the IHttpClient is below:
public class EPSDownloader
{
public EPSDownloader(IHttpClient client)
{
_client = client;
}
public async Task<IEnumerable<EPSDataPoint>> GetEPSData(int cik)
{
var epsDataPoints = new Dictionary<LocalDate, EPSDataPoint>();
var reportLinks = await GetReportLinks(cik);
foreach (var reportLink in reportLinks)
{
var xbrlLink = await GetXBRLLink(reportLink);
var epsData = await GetEPSData(xbrlLink);
foreach (var eps in epsData)
{
if (!epsDataPoints.ContainsKey(eps.DateInterval.End))
epsDataPoints.Add(eps.DateInterval.End, eps);
}
}
var list = epsDataPoints.OrderBy(d => d.Key).Select(e => e.Value).ToList();
return list;
}
private async Task<IList<string>> GetReportLinks(int cik)
{
// move this url elsewhere
var url = "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=" + cik +
"&type=10&dateb=&owner=include&count=100";
var srBody = await _client.ReadAsync(url); // consider moving this to srPage
var srPage = new SearchResultsPage(srBody);
return srPage.GetAllReportLinks();
}
private async Task<string> GetXBRLLink(string link)
{
var url = SEC_HOSTNAME + link;
var fdBody = await _client.ReadAsync(url);
var fdPage = new FilingDetailsPage(fdBody);
return fdPage.GetInstanceDocumentLink();
}
private async Task<IList<EPSDataPoint>> GetEPSData(string xbrlLink)
{
var xbrlBody = await _client.ReadAsync(SEC_HOSTNAME + xbrlLink);
var xbrlDoc = new XBRLDocument(xbrlBody);
return xbrlDoc.GetAllQuarterlyEPSData();
}
private readonly IHttpClient _client;
private const string SEC_HOSTNAME = "https://www.sec.gov";
}
It seems to be that there is an issue with HttpClient, but I don't know why. No exceptions are being thrown, but I do occasionally see that threads have exited with code 0.
Update: I actually restarted my computer while the application was running and it began running fine again for about 20 minutes before the Task Manager showed 0 for the network speed and the application just sat there.

Request is not async

public class RollingRequests
{
private const int DefaultNumSimultaneousRequests = 10;
private readonly HttpClient _client; // Don't worry about disposing see https://stackoverflow.com/questions/15705092/do-httpclient-and-httpclienthandler-have-to-be-disposed
private readonly HttpCompletionOption _httpCompletionOption;
private readonly int _numSimultaneousRequests;
public RollingRequests() : this(DefaultNumSimultaneousRequests)
{
}
public RollingRequests(int windowSize) : this(new HttpClient(), windowSize)
{
}
public RollingRequests(HttpClient client, int numSimultaneousRequests, HttpCompletionOption httpCompletionOption = HttpCompletionOption.ResponseContentRead)
{
_client = client;
_numSimultaneousRequests = numSimultaneousRequests;
_httpCompletionOption = httpCompletionOption;
}
public async Task ExecuteAsync(List<string> urls, CancellationToken cancellationToken, Action<HttpResponseHeaders, string> requestCallback = null)
{
var nextIndex = 0;
var activeTasks = new List<Task<Tuple<string, HttpResponseMessage>>>();
var startingIndex = Math.Min(_numSimultaneousRequests, urls.Count);
for (nextIndex = 0; nextIndex < startingIndex; nextIndex++)
{
activeTasks.Add(RequestUrlAsync(urls[nextIndex], cancellationToken));
}
while (activeTasks.Count > 0)
{
var finishedTask = await Task.WhenAny(activeTasks).ConfigureAwait(false);
activeTasks.Remove(finishedTask);
var retryUrl = await ProcessTask(await finishedTask, requestCallback).ConfigureAwait(false);
// If retrying, add the URL to the end of the queue
if (retryUrl != null)
{
urls.Add(retryUrl);
}
if (nextIndex < urls.Count)
{
activeTasks.Add(RequestUrlAsync(urls[nextIndex], cancellationToken));
nextIndex++;
}
}
}
private async Task<string> ProcessTask(Tuple<string, HttpResponseMessage> result, Action<HttpResponseHeaders, string> requestCallback = null)
{
var url = result.Item1;
using (var response = result.Item2)
{
if (!response.IsSuccessStatusCode)
{
return url;
}
if (requestCallback != null)
{
string content = null;
if (_httpCompletionOption == HttpCompletionOption.ResponseContentRead)
{
content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}
requestCallback(response.Headers, content);
}
return null;
}
}
private async Task<Tuple<string, HttpResponseMessage>> RequestUrlAsync(string url, CancellationToken ct)
{
var response = await _client.GetAsync(url, _httpCompletionOption, ct).ConfigureAwait(false);
return new Tuple<string, HttpResponseMessage>(url, response);
}
}
This is a class that allows for X simultaneous requests to be on-going at once. When I am unit-testing this class and I moq the HttpClient giving each request a 1 second delay the initial activeTasks.Add is taking 5 seconds if I have 5 requests, suggesting to me that RequestUrlAsync isn't truly async.
Can anyone spot the issue?
Edit:
This is how I am sleeping the mocked client
_messageHandlerMock
.Protected()
.Setup<Task<HttpResponseMessage>>(MethodToMoq, ItExpr.IsAny<HttpRequestMessage>(), ItExpr.IsAny<CancellationToken>())
.Callback(() => Thread.Sleep(1000))
.ReturnsAsync(callback)
.Verifiable();

I tested your class RollingRequests with actual urls and works as expected. Then I replaced await _client.GetAsync(... with await Task.Delay(1000) and continued working as expected. Then replaced the same line with Thread.Sleep(1000) and replicated your problem.
Moral lesson: avoid blocking the current thread when running asynchronous code!

(It would be easier to answer if you had provided a Minimal, Reproducible Example)
Mixing Thread.Sleep with asynchronous code isn't a good idea because it is a blocking call.
Mocking internals should also be avoided.
Here's a simple example of a test that takes about 1 second to execute 10 requests:
async Task Test()
{
var httpClient = new HttpClient(new TestHttpMessageHandler());
var ticks = Environment.TickCount;
await Task.WhenAll(Enumerable.Range(0, 10).Select(_ => httpClient.GetAsync("https://stackoverflow.com/")));
Console.WriteLine($"{Environment.TickCount - ticks}ms");
}
class TestHttpMessageHandler : HttpMessageHandler
{
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
await Task.Delay(1000);
return new HttpResponseMessage();
}
}

forceclient async call never received

I'm unit testing one of my async methods but it turns out to be a bit tricky.
In my code I execute two actions witch I try to verify with NSubstitute like so
[TestMethod]
public async Task GivenDebatchingHandler_WhenCommandReceived_EventIsPublishedAndStatusUpdated()
{
// arrange
var forceClient = Substitute.For<IForceClient>();
forceClient.UpdateAsync(EntityNames.EventStore, Arg.Any<string>(), Arg.Any<ExpandoObject>()).Returns(info => Task.FromResult(new SuccessResponse { Success = true }));
var messageHandlerContext = Substitute.For<IMessageHandlerContext>();
var handler = new DebatchingHandler(forceClient);
var #event = new KlantManagementEnvelopeCreatedEvent { Header = new Header { MessageId = "UnitTest" } };
var cmd = new PublishMultipleKlantManagementEnvelopeCreatedEventsCommand { EventsLargePayload = new List<KlantManagementEnvelopeCreatedEvent>(new[] { #event }) };
// act
await handler.Handle(cmd, messageHandlerContext).ConfigureAwait(false);
// assert
await messageHandlerContext.Received().Publish(#event, Arg.Any<PublishOptions>()).ConfigureAwait(false);
await forceClient.Received().UpdateAsync(EntityNames.EventStore, "UnitTest", Arg.Any<ExpandoObject>()).ConfigureAwait(false);
}
The publish is received, but the UpdateAsync is not. This is the code under test:
public async Task Handle(PublishMultipleKlantManagementEnvelopeCreatedEventsCommand message, IMessageHandlerContext context)
{
await Task.WhenAll(message.EventsLargePayload.Select(#event => this.ProcessEvent(#event, context))).ConfigureAwait(false);
}
public async Task ProcessEvent(KlantManagementEnvelopeCreatedEvent envelopeCreatedEvent, IMessageHandlerContext context)
{
await context.Publish(envelopeCreatedEvent).ConfigureAwait(false);
var eventStoreRecord = new EventStore__c { Status__c = EventStoreStatus.Published.ToName() };
await this.forceClient.UpdateAsync(EntityNames.EventStore, envelopeCreatedEvent.Header.MessageId, eventStoreRecord).ConfigureAwait(false);
}
Why is the UpdateAsync call never received?

The test has Arg.Any<ExpandoObject>() in the UpdateAsync assert.
Unless EventStore__c in the method under test derives from ExpandoObject then I don't think that assertion would match.
Try using
await forceClient.Received()
.UpdateAsync(EntityNames.EventStore, "UnitTest", Arg.Any<EventStore__c>())
.ConfigureAwait(false);

Converting async Task Response to String

First of all, I would like to say, I'm quite new to C#.
I'm trying to create a POST request which sends some data to a PHP file somewhere on a different server.
Now, after the request is send I would like to see the response, as I'm sending back a JSON string from the server as a success message.
When I use the following code:
public MainPage()
{
this.InitializeComponent();
Windows.UI.ViewManagement.ApplicationView.GetForCurrentView().SetDesiredBoundsMode(Windows.UI.ViewManagement.ApplicationViewBoundsMode.UseCoreWindow);
responseBlockTxt.Text = start();
}
public string start()
{
var response = sendRequest();
System.Diagnostics.Debug.WriteLine(response);
return "";
}
public async Task<string> sendRequest()
{
using (var client = new HttpClient())
{
var values = new Dictionary<string, string>
{
{ "vote", "true" },
{ "slug", "the-slug" }
};
var content = new FormUrlEncodedContent(values);
var response = await client.PostAsync("URL/api.php", content);
var responseString = await response.Content.ReadAsStringAsync();
return responseString;
}
}
The output is:
System.Threading.Tasks.Task`1[System.String]
So, how would I see all the results from this?

Go Async all the way. Avoid blocking calls when calling async methods. async void is allowed in event handlers so update page to perform the call on load event
Read up on Async/Await - Best Practices in Asynchronous Programming
And then update your code accordingly
public MainPage() {
this.InitializeComponent();
Windows.UI.ViewManagement.ApplicationView.GetForCurrentView().SetDesiredBoundsMode(Windows.UI.ViewManagement.ApplicationViewBoundsMode.UseCoreWindow);
this.Loaded += OnLoaded;
}
public async void OnLoaded(object sender, RoutedEventArgs e) {
responseBlockTxt.Text = await start();
}
public async Task<string> start() {
var response = await sendRequest();
System.Diagnostics.Debug.WriteLine(response);
return response;
}
private static HttpClient client = new HttpClient();
public async Task<string> sendRequest() {
var values = new Dictionary<string, string> {
{ "vote", "true" },
{ "slug", "the-slug" }
};
var content = new FormUrlEncodedContent(values);
using(var response = await client.PostAsync("URL/api.php", content)) {
var responseString = await response.Content.ReadAsStringAsync();
return responseString;
}
}

I Guess
public string start()
{
var response = sendRequest();
Task<String> t = sendRequest();
System.Diagnostics.Debug.WriteLine(t.Result);
return "";
}
public async Task<string> sendRequest()
{
using (var client = new HttpClient())
{
var values = new Dictionary<string, string>
{
{ "vote", "true" },
{ "slug", "the-slug" }
};
var content = new FormUrlEncodedContent(values);
var response = await client.PostAsync("URL/api.php", content);
var responseString = await response.Content.ReadAsStringAsync();
return responseString;
}
}

The problem is in the start method, the SendRequest method returns a Task<string> and that's what you get on your response variable. Since you are attempting to run an async method synchronously you have to do some extra stuff, try this:
public string start()
{
var response = sendRequest().ConfigureAwait(true)
.GetAwaiter()
.GetResult();
System.Diagnostics.Debug.WriteLine(response);
return "";
}
That get the actual result inside your awaitable Task<string>. If you want to find some more info on this take a look at this question

public string start()
{
var response = sendRequest().ConfigureAwait(true)
.GetAwaiter()
.GetResult();
System.Diagnostics.Debug.WriteLine(response);
return "";
}
I have Tried this. It is working perfectly.

Task.WhenAll doesn't resume

Below code is simplified version what i am trying to do. The code supposed to reach Console.ReadKey() line very quickly but it never happens. Task.WhenAll never resume next line. What is wrong on the code ?
class Program
{
static void Main(string[] args)
{
DoWorkAsync().Wait();
Console.ReadKey();
}
static async Task DoWorkAsync()
{
var block = new ActionBlock<Job>(async (task) =>
{
await task.DoAsync();
});
var jobs = Enumerable.Range(0, 2).Select(i => new Job());
foreach (var job in jobs)
{
await block.SendAsync(job);
}
await Task.WhenAll(jobs.Select(c => c.Completion));
}
public class Job
{
TaskCompletionSource<bool> completionSource = new TaskCompletionSource<bool>();
public Task<bool> Completion { get { return completionSource.Task; } }
public async Task DoAsync()
{
await Task.Delay(100);
completionSource.SetResult(true);
}
}

It will help.
var jobs = Enumerable.Range(0, 2).Select(i => new Job()).ToList();
It happens because of multiple enumeration of jobs
In this line await Task.WhenAll(jobs.Select(c => c.Completion)); you are waiting for new jobs, which are not send to block.