Develop Reference

C# .NET Core 2.2 Web Api ~ upload file as stream does not work

I want to upload a file es stream using REST.
I implement the code like I find at dotnetcoretutorials.com, which looks quit good.
And than I add a index.htm inside my project folder:
<form>
<input type="file" id="myfile"/>
<input type="button" onclick="uploadFile();" value="Upload" />
</form>
<script type="text/javascript">
function uploadFile() {
var xhr = new XMLHttpRequest();
var file = document.getElementById('myfile').files[0];
xhr.open("POST", "api/upload");
xhr.setRequestHeader("filename", file.name);
xhr.send(file);
}
I start the project with IIS and call the index.html in my browser. I choose a file and click the "Update" -button. The chosen file is not copied anywhere.
Why the upload does not work?
The file myfile.temp is not written. I get this exception:
index.html:11 Access to XMLHttpRequest at 'file:///C:/Users/frank.mehlhop/source/repos/FileTransfer/FileTransfer/wwwroot/api/upload' from origin 'null' has been blocked by CORS policy: Cross origin requests are only supported for protocol schemes: http, data, chrome, chrome-extension, https.
I tried at the cmd
.\chrome.exe --allow-file-access-from-files file:///C:\Users\frankmehlhop\source\repos\FileTransfer\FileTransfer\wwwroot\index.html
, but no changes. At my breakpoints the code does not stop, when I start my App, the breakpoints are marked yellow with
The breakpoint will not currently be hit. No symbols have been loaded for this document.
UploadController.cs:
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.ModelBinding;
using System.Threading.Tasks;
namespace FileTransfer.Controllers
{
[Produces("application/json")]
[Route("api/upload")]
public class UploadController : Controller
{
// https://learn.microsoft.com/de-de/aspnet/core/mvc/models/file-uploads?view=aspnetcore-2.2#uploading-large-files-with-streaming
// https://dotnetcoretutorials.com/2017/03/12/uploading-files-asp-net-core/
[HttpPost]
[DisableFormValueModelBinding]
public async Task<IActionResult> Index()
{
FormValueProvider formModel;
using (var stream = System.IO.File.Create("c:\\temp\\myfile.temp"))
{
formModel = await Request.StreamFile(stream);
}
var viewModel = new MyViewModel();
var bindingSuccessful = await TryUpdateModelAsync(viewModel, prefix: "",
valueProvider: formModel);
if (!bindingSuccessful)
{
if (!ModelState.IsValid)
{
return BadRequest(ModelState);
}
}
return Ok(viewModel);
}
public class MyViewModel
{
public string Username { get; set; }
}
}
FileStreamHelper.cs
public static class FileStreamingHelper
{
private static readonly FormOptions _defaultFormOptions = new FormOptions();
public static async Task<FormValueProvider> StreamFile(this HttpRequest request, Stream targetStream)
{
if (!MultipartRequestHelper.IsMultipartContentType(request.ContentType))
{
throw new Exception($"Expected a multipart request, but got {request.ContentType}");
}
// Used to accumulate all the form url encoded key value pairs in the
// request.
var formAccumulator = new KeyValueAccumulator();
string targetFilePath = null;
var boundary = MultipartRequestHelper.GetBoundary(
MediaTypeHeaderValue.Parse(request.ContentType),
_defaultFormOptions.MultipartBoundaryLengthLimit);
var reader = new MultipartReader(boundary, request.Body);
var section = await reader.ReadNextSectionAsync();
while (section != null)
{
ContentDispositionHeaderValue contentDisposition;
var hasContentDispositionHeader = ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out contentDisposition);
if (hasContentDispositionHeader)
{
if (MultipartRequestHelper.HasFileContentDisposition(contentDisposition))
{
await section.Body.CopyToAsync(targetStream);
}
else if (MultipartRequestHelper.HasFormDataContentDisposition(contentDisposition))
{
// Content-Disposition: form-data; name="key"
//
// value
// Do not limit the key name length here because the
// multipart headers length limit is already in effect.
var key = HeaderUtilities.RemoveQuotes(contentDisposition.Name);
var encoding = GetEncoding(section);
using (var streamReader = new StreamReader(
section.Body,
encoding,
detectEncodingFromByteOrderMarks: true,
bufferSize: 1024,
leaveOpen: true))
{
// The value length limit is enforced by MultipartBodyLengthLimit
var value = await streamReader.ReadToEndAsync();
if (String.Equals(value, "undefined", StringComparison.OrdinalIgnoreCase))
{
value = String.Empty;
}
formAccumulator.Append(key.Value, value); // For .NET Core <2.0 remove ".Value" from key
if (formAccumulator.ValueCount > _defaultFormOptions.ValueCountLimit)
{
throw new InvalidDataException($"Form key count limit {_defaultFormOptions.ValueCountLimit} exceeded.");
}
}
}
}
// Drains any remaining section body that has not been consumed and
// reads the headers for the next section.
section = await reader.ReadNextSectionAsync();
}
// Bind form data to a model
var formValueProvider = new FormValueProvider(
BindingSource.Form,
new FormCollection(formAccumulator.GetResults()),
CultureInfo.CurrentCulture);
return formValueProvider;
}
private static Encoding GetEncoding(MultipartSection section)
{
MediaTypeHeaderValue mediaType;
var hasMediaTypeHeader = MediaTypeHeaderValue.TryParse(section.ContentType, out mediaType);
// UTF-7 is insecure and should not be honored. UTF-8 will succeed in
// most cases.
if (!hasMediaTypeHeader || Encoding.UTF7.Equals(mediaType.Encoding))
{
return Encoding.UTF8;
}
return mediaType.Encoding;
}
}
DisableFormValueModelBindingAttribute.cs
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)]
public class DisableFormValueModelBindingAttribute : Attribute, IResourceFilter
{
public void OnResourceExecuting(ResourceExecutingContext context)
{
var formValueProviderFactory = context.ValueProviderFactories
.OfType<FormValueProviderFactory>()
.FirstOrDefault();
if (formValueProviderFactory != null)
{
context.ValueProviderFactories.Remove(formValueProviderFactory);
}
var jqueryFormValueProviderFactory = context.ValueProviderFactories
.OfType<JQueryFormValueProviderFactory>()
.FirstOrDefault();
if (jqueryFormValueProviderFactory != null)
{
context.ValueProviderFactories.Remove(jqueryFormValueProviderFactory);
}
}
public void OnResourceExecuted(ResourceExecutedContext context)
{
}
}
MultipartRequestHelper.cs
public static class MultipartRequestHelper
{
// Content-Type: multipart/form-data; boundary="----WebKitFormBoundarymx2fSWqWSd0OxQqq"
// The spec says 70 characters is a reasonable limit.
public static string GetBoundary(MediaTypeHeaderValue contentType, int lengthLimit)
{
var boundary = HeaderUtilities.RemoveQuotes(contentType.Boundary).ToString();
if (string.IsNullOrWhiteSpace(boundary))
{
throw new InvalidDataException("Missing content-type boundary.");
}
if (boundary.Length > lengthLimit)
{
throw new InvalidDataException(
$"Multipart boundary length limit {lengthLimit} exceeded.");
}
return boundary;
}
public static bool IsMultipartContentType(string contentType)
{
return !string.IsNullOrEmpty(contentType)
&& contentType.IndexOf("multipart/", StringComparison.OrdinalIgnoreCase) >= 0;
}
public static bool HasFormDataContentDisposition(ContentDispositionHeaderValue contentDisposition)
{
// Content-Disposition: form-data; name="key";
return contentDisposition != null
&& contentDisposition.DispositionType.Equals("form-data")
&& string.IsNullOrEmpty(contentDisposition.FileName.ToString())
&& string.IsNullOrEmpty(contentDisposition.FileNameStar.ToString());
}
public static bool HasFileContentDisposition(ContentDispositionHeaderValue contentDisposition)
{
// Content-Disposition: form-data; name="myfile1"; filename="Misc 002.jpg"
return contentDisposition != null
&& contentDisposition.DispositionType.Equals("form-data")
&& (!string.IsNullOrEmpty(contentDisposition.FileName.ToString())
|| !string.IsNullOrEmpty(contentDisposition.FileNameStar.ToString()));
}
}

Is it better to use HttpClient or WebRequest in ASP.NET Core to read the content of a file line by line asynchronously remotely?

I plan to read a remote file line by line asynchronously using https://github.com/Dasync/AsyncEnumerable (since there is not yet Async Streams [C# 8 maybe]: https://github.com/dotnet/csharplang/blob/master/proposals/async-streams.md):
public static class StringExtensions
{
public static AsyncEnumerable<string> ReadLinesAsyncViaHttpClient(this string uri)
{
return new AsyncEnumerable<string>(async yield =>
{
using (var httpClient = new HttpClient())
{
using (var responseStream = await httpClient.GetStreamAsync(uri))
{
using (var streamReader = new StreamReader(responseStream))
{
while(true)
{
var line = await streamReader.ReadLineAsync();
if (line != null)
{
await yield.ReturnAsync(line);
}
else
{
return;
}
}
}
}
}
});
}
public static AsyncEnumerable<string> ReadLinesAsyncViaWebRequest(this string uri)
{
return new AsyncEnumerable<string>(async yield =>
{
var request = WebRequest.Create(uri);
using (var response = request.GetResponse())
{
using (var responseStream = response.GetResponseStream())
{
using (var streamReader = new StreamReader(responseStream))
{
while(true)
{
var line = await streamReader.ReadLineAsync();
if (line != null)
{
await yield.ReturnAsync(line);
}
else
{
return;
}
}
}
}
}
});
}
}
It seems that they both run just fine in a simple Console application like below:
public class Program
{
public static async Task Main(string[] args)
{
// Or any other remote file
const string url = #"https://gist.githubusercontent.com/dgrtwo/a30d99baa9b7bfc9f2440b355ddd1f75/raw/700ab5bb0b5f8f5a14377f5103dbe921d4238216/by_tag_year.csv";
await url.ReadLinesAsyncViaWebRequest().ForEachAsync(line =>
{
Console.WriteLine(line, Color.GreenYellow);
});
await url.ReadLinesAsyncViaHttpClient().ForEachAsync(line =>
{
Console.WriteLine(line, Color.Purple);
});
}
}
... but I have some concerns if it is used as part of an ASP.NET Core WebAPI to process the lines and then push them using PushStreamContent:
https://learn.microsoft.com/en-us/previous-versions/aspnet/hh995285(v=vs.108)
https://blog.stephencleary.com/2016/10/async-pushstreamcontent.html
The idea would be to have a pipeline of data which leverages async / await so that the number of threads in use is as low as possible and also to avoid an increase in memory (which leverage the enumerable-like feature of AsyncEnumerable).
I read several articles but it seems it's all non .NET Core versions and I don't really know if there would be some potential performance issues / caveats in regard to what I would like to achieve?
Difference between HttpRequest, HttpWebRequest and WebRequest
http://www.diogonunes.com/blog/webclient-vs-httpclient-vs-httpwebrequest/
An example of "business" case would be:
using System;
using System.Collections.Async;
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
namespace WebApplicationTest.Controllers
{
[Route("api/[controller]")]
[ApiController]
public class DumbValuesController : ControllerBase
{
private static readonly Random Random = new Random();
// GET api/values
[HttpGet]
public async Task<IActionResult> DumbGetAsync([FromQuery] string fileUri)
{
using (var streamWriter = new StreamWriter(HttpContext.Response.Body))
{
await fileUri.ReadLinesAsyncViaHttpClient().ForEachAsync(async line =>
{
// Some dumb process on each (maybe big line)
line += Random.Next(0, 100 + 1);
await streamWriter.WriteLineAsync(line);
});
}
return Ok();
}
}
}

We have access to the source code for .NET Core. So you can look.
The underlying implementation of both end up using HttpClientHandler (the implementation of that class is split up into 4 files).
You can see this from the source code of both HttpClient and HttpWebRequest (which WebRequest uses).
So I suspect you won't notice any difference in the performance of either.
HttpClient is the latest one to be written, so that's why its use is encouraged. And for the reasons mentioned in the article you linked to: http://www.diogonunes.com/blog/webclient-vs-httpclient-vs-httpwebrequest/

With the latest release of .Net Core 6.0, WebRequest will be declared as deprecated. Microsoft recommended to use HttpClient instead
https://learn.microsoft.com/en-us/dotnet/core/compatibility/networking/6.0/webrequest-deprecated

API File Upload using HTTP content not exposed in swagger

I am implementing a swagger interface into an existing web API. The current API controller exposes an async upload function which uses the Request.Content to transport an image asynchronously. The code that has been used is explained in this article.
My api controller:
[HttpPost]
[Route("foo/bar/upload")]
public async Task<HttpResponseMessage> Upload()
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var provider = await Request.Content.ReadAsMultipartAsync(new InMemoryMultipartFormDataStreamProvider());
NameValueCollection formData = provider.FormData;
HttpResponseMessage response;
//access files
IList<HttpContent> files = provider.Files;
if (files.Count > 0)
{
HttpContent file1 = files[0];
using (Stream input = await file1.ReadAsStreamAsync())
{
object responseObj = ExternalProcessInputStream(input)
response = Request.CreateResponse(HttpStatusCode.OK, responseObj);
}
}
else
{
response = Request.CreateResponse(HttpStatusCode.BadRequest);
}
return response;
}
This works dandy, but when i expose this through swagger i have a parameterless function, which returns an error when used.
My question is how can supply a proper value to test this method with?

You'll need to add a custom IOperationFilter to handle this.
Given you have a controller like so:
[ValidateMimeMultipartContentFilter]
[HttpPost, Route("softwarepackage")]
public Task<SoftwarePackageModel> UploadSingleFile()
{
var streamProvider = new MultipartFormDataStreamProvider(ServerUploadFolder);
var task = Request.Content.ReadAsMultipartAsync(streamProvider).ContinueWith<SoftwarePackageModel>(t =>
{
var firstFile = streamProvider.FileData.FirstOrDefault();
if (firstFile != null)
{
// Do something with firstFile.LocalFileName
}
return new SoftwarePackageModel
{
};
});
return task;
}
You then need to create an Swashbuckle.Swagger.IOperationFilter to add a file upload parameter to your function like:
public class FileOperationFilter : IOperationFilter
{
public void Apply(Operation operation, SchemaRegistry schemaRegistry, ApiDescription apiDescription)
{
if (operation.operationId.ToLower() == "softwarepackage_uploadsinglefile")
{
if (operation.parameters == null)
operation.parameters = new List<Parameter>(1);
else
operation.parameters.Clear();
operation.parameters.Add(new Parameter
{
name = "File",
#in = "formData",
description = "Upload software package",
required = true,
type = "file"
});
operation.consumes.Add("application/form-data");
}
}
}
And in your Swagger config you'll need to register the filter:
config.EnableSwagger(c => {... c.OperationFilter<FileOperationFilter>(); ... });
To top this up, I also added a FilterAttribute to filter out Multipart content:
public class ValidateMimeMultipartContentFilter : ActionFilterAttribute
{
public override void OnActionExecuting(HttpActionContext actionContext)
{
if (!actionContext.Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
}
public override void OnActionExecuted(HttpActionExecutedContext actionExecutedContext)
{
}
}

UWP Windows.Web.HttpClient fake for unit test

I try to do unit-test REST communication logic for UWP client. With reference to the answer for System.Web.HttpClient, I found that Windows.Net.HttpClient also accepts an arguement called IHttpFilter.
So, I try to make custom response with IHttpFilter but I don't know correct way to make a response.
public class TestFilter : IHttpFilter
{
public IAsyncOperationWithProgress<HttpResponseMessage, HttpProgress> SendRequestAsync(HttpRequestMessage request)
{
if (request.Method == HttpMethod.Get)
{
// response fake response for GET...
}
}
public void Dispose()
{
// do nothing
}
}
And the target method for unit-test is as below.
public async Task<string> PostResult(HttpClient httpClient, string username)
{
var json = new JsonObject
{
{"Username",
JsonValue.CreateStringValue(string.IsNullOrEmpty(username) ? CommonKey.UnAuthorizedUserPartitionKey : username)
},
};
var content = new HttpStringContent(json.Stringify());
content.Headers.ContentType = new HttpMediaTypeHeaderValue("application/json");
// I want to make below line testable...
var response = await httpClient.PostAsync(new Uri(Common.ProcessUrl), content);
try
{
response.EnsureSuccessStatusCode();
return null;
}
catch (Exception exception)
{
return exception.Message ?? "EMPTY ERROR MESSAGE";
}
}
Note that It's NOT duplicate question related to System.Web.HttpClient mocking/faking. What I ask is Windows.Web.HttpClient specifically. I failed to implement with it.
Note that, Windows.Web.Http.IHttpClient is internal accessible and HttpClient is sealed. So hard to do Mock or inherit-and-override it.

While I agree with some that there are better ways to test HttpClient calls, I'll answer your question of how to create a "fake" response with an IHttpFilter implementation (System.Runtime.InteropServices.WindowsRuntime is your friend)
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices.WindowsRuntime;
using System.Threading;
using System.Threading.Tasks;
using Windows.Foundation;
using Windows.Web.Http;
using Windows.Web.Http.Filters;
namespace Project.UnitTesting
{
public class FakeResponseFilter : IHttpFilter
{
private readonly Dictionary<Uri, HttpResponseMessage> _fakeResponses = new Dictionary<Uri, HttpResponseMessage>();
public void AddFakeResponse(Uri uri, HttpResponseMessage responseMessage)
{
_fakeResponses.Add(uri, responseMessage);
}
public void Dispose()
{
// Nothing to dispose
}
public IAsyncOperationWithProgress<HttpResponseMessage, HttpProgress> SendRequestAsync(HttpRequestMessage request)
{
if (_fakeResponses.ContainsKey(request.RequestUri))
{
var fakeResponse = _fakeResponses[request.RequestUri];
return DownloadStringAsync(fakeResponse);
}
// Alternatively, you might want to throw here if a request comes
// in that is not in the _fakeResponses dictionary.
return DownloadStringAsync(new HttpResponseMessage(HttpStatusCode.NotFound) { RequestMessage = request });
}
private IAsyncOperationWithProgress<HttpResponseMessage, HttpProgress> DownloadStringAsync(HttpResponseMessage message)
{
return AsyncInfo.Run(delegate (CancellationToken cancellationToken, IProgress<HttpProgress> progress)
{
progress.Report(new HttpProgress());
try
{
return Task.FromResult(message);
}
finally
{
progress.Report(new HttpProgress());
}
});
}
}
}

Compress HTTP GET Response

I am currently working on migrating few of my MVC3 Controllers to MVC4 Api Controllers.
I have implemented Compression mechanism for MVC3 controller Get Method Responses by inherting ActionFilterAttribute and overriding OnActionExecutiong method. After some Research I found that I need to use ActionFilterMethod from System.Web.HttpFilters. It would be great if somebody can share piece of sample code to get me started for this compressing HTTP response using GZip

The easiest is to enable compression directly at IIS level.
If you want to do it at the application level you could write a custom delegating message handler as shown in the following post:
public class CompressHandler : DelegatingHandler
{
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
return base.SendAsync(request, cancellationToken).ContinueWith<HttpResponseMessage>((responseToCompleteTask) =>
{
HttpResponseMessage response = responseToCompleteTask.Result;
if (response.RequestMessage.Headers.AcceptEncoding != null)
{
string encodingType = response.RequestMessage.Headers.AcceptEncoding.First().Value;
response.Content = new CompressedContent(response.Content, encodingType);
}
return response;
},
TaskContinuationOptions.OnlyOnRanToCompletion);
}
}
public class CompressedContent : HttpContent
{
private HttpContent originalContent;
private string encodingType;
public CompressedContent(HttpContent content, string encodingType)
{
if (content == null)
{
throw new ArgumentNullException("content");
}
if (encodingType == null)
{
throw new ArgumentNullException("encodingType");
}
originalContent = content;
this.encodingType = encodingType.ToLowerInvariant();
if (this.encodingType != "gzip" && this.encodingType != "deflate")
{
throw new InvalidOperationException(string.Format("Encoding '{0}' is not supported. Only supports gzip or deflate encoding.", this.encodingType));
}
// copy the headers from the original content
foreach (KeyValuePair<string, IEnumerable<string>> header in originalContent.Headers)
{
this.Headers.AddWithoutValidation(header.Key, header.Value);
}
this.Headers.ContentEncoding.Add(encodingType);
}
protected override bool TryComputeLength(out long length)
{
length = -1;
return false;
}
protected override Task SerializeToStreamAsync(Stream stream, TransportContext context)
{
Stream compressedStream = null;
if (encodingType == "gzip")
{
compressedStream = new GZipStream(stream, CompressionMode.Compress, leaveOpen: true);
}
else if (encodingType == "deflate")
{
compressedStream = new DeflateStream(stream, CompressionMode.Compress, leaveOpen: true);
}
return originalContent.CopyToAsync(compressedStream).ContinueWith(tsk =>
{
if (compressedStream != null)
{
compressedStream.Dispose();
}
});
}
}
All that's left now is to register the handler in Application_Start:
GlobalConfiguration.Configuration.MessageHandlers.Add(new CompressHandler());

If you are using IIS 7+, I would say leave the compression to IIS as it supports GZIP compression. Just turn it on.
On the other hand, compression is too close to the metal for the controller. Ideally controller should work in much higher level than bytes and streams.

Use a class and write the following code
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method)]
public class CompressFilter : ActionFilterAttribute
{
public override void OnActionExecuted(HttpActionExecutedContext context)
{
var acceptedEncoding = context.Response.RequestMessage.Headers.AcceptEncoding.First().Value;
if (!acceptedEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase)
&& !acceptedEncoding.Equals("deflate", StringComparison.InvariantCultureIgnoreCase))
{
return;
}
context.Response.Content = new CompressedContent(context.Response.Content, acceptedEncoding);
}
}
Now create another class and write the following code.
public class CompressedContent : HttpContent
{
private readonly string _encodingType;
private readonly HttpContent _originalContent;
public CompressedContent(HttpContent content, string encodingType = "gzip")
{
if (content == null)
{
throw new ArgumentNullException("content");
}
_originalContent = content;
_encodingType = encodingType.ToLowerInvariant();
foreach (var header in _originalContent.Headers)
{
Headers.TryAddWithoutValidation(header.Key, header.Value);
}
Headers.ContentEncoding.Add(encodingType);
}
protected override bool TryComputeLength(out long length)
{
length = -1;
return false;
}
protected override Task SerializeToStreamAsync(Stream stream, TransportContext context)
{
Stream compressedStream = null;
switch (_encodingType)
{
case "gzip":
compressedStream = new GZipStream(stream, CompressionMode.Compress, true);
break;
case "deflate":
compressedStream = new DeflateStream(stream, CompressionMode.Compress, true);
break;
default:
compressedStream = stream;
break;
}
return _originalContent.CopyToAsync(compressedStream).ContinueWith(tsk =>
{
if (compressedStream != null)
{
compressedStream.Dispose();
}
});
}
}
Now use the following attribute in Controller or in any api action method like this
[Route("GetData")]
[CompressFilter]
public HttpResponseMessage GetData()
{
}