Develop Reference

Working cefsharp POST example

Having a difficult time getting an HTTP POST request/response using cefsharp / ChromiumWebBrowser. I'm unable to find a working example on Stackoverflow, nor in the documentation. Looking to see if anyone has a full example? I'm stuck on if it can be done with a Navigate function (as show in one example), or needs to be a done with a handler / schema.
I'm trying a basic POST to a PHP script. If the data1/data2 match the input, it's return json status:success, otherwise failure. I see in the devtools that the html body comes back with json success, but this code returns or nothing at all. I've tried too different ways to get the response data. I want to grab the JSON response for the C# code to review. Surely there should be an easy way to accomplish this? I want to send an HTTP request and then get the body (json) to parse. If this needs the schema/handler, I cannot find a full example of using this.
namespace BrowserTest
{
public partial class MainForm : Form
{
ChromiumWebBrowser browser = null;
public Loader()
{
browser = new ChromiumWebBrowser("http://localhost/test/"); // Initialize to this page
pBrowserLogin.Controls.Add(browser);
}
private void btnTest_Click(object sender, EventArgs e)
{
byte[] request = Encoding.ASCII.GetBytes("data1=" + txtData1.Text + "&data2=" + txtData2.Text);
PostTest.Navigate(browser, "http://localhost/test/posttest.php", request, "application/x-www-form-urlencoded");
}
}
public static class PostTest
{
public static void Navigate(this IWebBrowser browser, string url, byte[] postDataBytes, string contentType)
{
IFrame frame = browser.GetMainFrame();
IRequest request = frame.CreateRequest();
request.Url = url;
request.Method = "POST";
request.InitializePostData();
var element = request.PostData.CreatePostDataElement();
element.Bytes = postDataBytes;
request.PostData.AddElement(element);
NameValueCollection headers = new NameValueCollection();
headers.Add("Content-Type", contentType);
request.Headers = headers;
frame.LoadRequest(request);
frame.GetTextAsync().ContinueWith(taskHtml =>
{
var html = taskHtml.Result;
System.Windows.Forms.MessageBox.Show(html);
});
string script = string.Format("document.documentElement.outerHTML;");
frame.EvaluateScriptAsync(script).ContinueWith(x =>
{
var response = x.Result;
if (response.Success && response.Result != null)
{
var fullhtml = response.Result;
System.Windows.Forms.MessageBox.Show(fullhtml.ToString());
}
});
}
}
}

Downloading Large Google Drive files with WebClient in C#

I know there are tones of questions on this subject already. After reading all the threads, I decided to get a redirected URL in a confirmation HTML page and then use it as a direct link to download.
As you know, the original URL format of the direct download link is like this.
https://drive.google.com/uc?export=download&id=XXXXX..
But if the size of the target file is big, then it is like this.
https://drive.google.com/uc?export=download&confirm=RRRR&id=XXXXX..
I can get RRRR from the first downloaded data, so I need to try twice in order to download the real file. The concept is very simple enough but I can't get this to work.
class Test
{
class MyWebClient: WebClient
{
CookieContainer c = new CookieContainer();
protected override WebRequest GetWebRequest(Uri u)
{
var r = (HttpWebRequest) base.GetWebRequest(u);
r.CookieContainer = c;
return r;
}
}
static string GetRealURL(string filename)
{
// Some Jobs to Parse....
return directLink;
}
static void Main()
{
MyWebClient wc = new MyWebClient();
string targetLink = "https://drive.google.com/uc?export=download&id=XXXXXXX";
wc.DownloadFile(targetLink, "tempFile.tmp");
targetLink = GetRealURL("tempFile.tmp");
wc.DownloadFile(targetLink, "realFile.dat");
}
}
What did I wrong?
I can get the right download link from the first file, but I get another confirmation page file with another confirm code on the second try. I thought this was because of cookies, so I created my own WebClient class as you can see above.
Also I originally used DownloadFileAsync(), and changed it to DownloadFile() just in case, but the same result..
I'm still thinking it has something to do with cookie things.
What am I missing here?

I had this same problem but had solved it in an HttpClient. I tried via your approach with WebClient and was able to get it to work. You don't show your GetRealUrl() source, but i'm willing to bet in there lies the issue. Here's how I did it:
You need to parse the html response to get the url in the href attribute of the "download anyway" button. It will only have the relative url, (the /uc?export=download... part)
You need to replace the xml escape character & with &
Then you can build the url using thte domain https://drive.google.com
At which point you can download the file. Here's the source (used in a test WPF application):
class MyWebClient : WebClient
{
CookieContainer c = new CookieContainer();
protected override WebRequest GetWebRequest(Uri u)
{
var r = (HttpWebRequest)base.GetWebRequest(u);
r.CookieContainer = c;
return r;
}
}
private async void WebClientTestButtonGdrive_Click(object sender, RoutedEventArgs e)
{
using (MyWebClient client = new MyWebClient())
{
//get the warning page
string htmlPage = await client.DownloadStringTaskAsync("https://drive.google.com/uc?id=XXXXXXX&export=download");
//use HtmlAgilityPack to get the url with the confirm parameter in the url
HtmlDocument document = new HtmlDocument();
document.LoadHtml(htmlPage);
HtmlNode node = document.DocumentNode;
HtmlNode urlNode = node.SelectSingleNode(#"//a[contains(#href, 'XXXXXXX') and contains(#id, 'uc-download-link')]//#href");
string downloadUrl = urlNode.Attributes["href"].Value;
downloadUrl = downloadUrl.Replace("&", "&");
downloadUrl = "https://drive.google.com" + downloadUrl;
//download the file
if (File.Exists("FileToDownload.zip"))
File.Delete("FileToDownload.zip");
await client.DownloadFileTaskAsync(downloadUrl, "FileToDownload.zip");
}
}

C# Expect100Continue header request

I am facing the problem with posting username and password with different domains - one submits the form successfully while the other doesn't(the form data is empty)! The html code on both domains is the same. Here is the sample code- the commented domain doesn't post: Any Help is greatly appreciated!
Note: the domain that runs on nginx posts data successfully while the other on apache doesn't if at all it has got something to do with servers
public class CookieAwareWebClient : System.Net.WebClient
{
private System.Net.CookieContainer Cookies = new System.Net.CookieContainer();
protected override System.Net.WebRequest GetWebRequest(Uri address)
{
System.Net.WebRequest request = base.GetWebRequest(address);
if (request is System.Net.HttpWebRequest)
{
var hwr = request as System.Net.HttpWebRequest;
hwr.CookieContainer = Cookies;
}
return request;
}
}
# Main function
NameValueCollection postData = new NameValueCollection();
postData.Add("username", "abcd");
postData.Add("password", "efgh");
var wc = new CookieAwareWebClient();
//string url = "https://abcd.example.com/service/login/";
string url = "https://efgh.example.com/service/login/";
wc.DownloadString(url);
//writer.WriteLine(wc.ResponseHeaders);
Console.WriteLine(wc.ResponseHeaders);
byte[] results = wc.UploadValues(url, postData);
string text = System.Text.Encoding.ASCII.GetString(results);
Console.WriteLine(text);

The problem was with Expect100Continue header being added automatically each time when a request was made through the program which wasn't handled well on Apache. You have to set the Expect100Continue to false each time when a request is made in the following way. Thanks for the Fiddler lead although I could see it through the dumpcap tool on Amazon EC2 instance! Here is the solution!
# Main function
NameValueCollection postData = new NameValueCollection();
postData.Add("username", "abcd");
postData.Add("password", "efgh");
var wc = new CookieAwareWebClient();
var uri = new Uri("https://abcd.example.com/service/login/");
var servicePoint = ServicePointManager.FindServicePoint(uri);
servicePoint.Expect100Continue = false;
wc.DownloadString(uri);
Console.WriteLine(wc.ResponseHeaders);
byte[] results = wc.UploadValues(uri, postData);
string text = System.Text.Encoding.ASCII.GetString(results);
Console.WriteLine(text);

Howto: download a file keeping the original name in C#?

I have files on a server that can be accessed from a URL formatted like this:
http:// address/Attachments.aspx?id=GUID
I have access to the GUID and need to be able to download multiple files to the same folder.
if you take that URL and throw it in a browser, you will download the file and it will have the original file name.
I want to replicate that behavior in C#. I have tried using the WebClient class's DownloadFile method, but with that you have to specify a new file name. And even worse, DownloadFile will overwrite an existing file. I know I could generate a unique name for every file, but i'd really like the original.
Is it possible to download a file preserving the original file name?
Update:
Using the fantastic answer below to use the WebReqest class I came up with the following which works perfectly:
public override void OnAttachmentSaved(string filePath)
{
var webClient = new WebClient();
//get file name
var request = WebRequest.Create(filePath);
var response = request.GetResponse();
var contentDisposition = response.Headers["Content-Disposition"];
const string contentFileNamePortion = "filename=";
var fileNameStartIndex = contentDisposition.IndexOf(contentFileNamePortion, StringComparison.InvariantCulture) + contentFileNamePortion.Length;
var originalFileNameLength = contentDisposition.Length - fileNameStartIndex;
var originalFileName = contentDisposition.Substring(fileNameStartIndex, originalFileNameLength);
//download file
webClient.UseDefaultCredentials = true;
webClient.DownloadFile(filePath, String.Format(#"C:\inetpub\Attachments Test\{0}", originalFileName));
}
Just had to do a little string manipulation to get the actual filename. I'm so excited. Thanks everyone!

As hinted in comments, the filename will be available in Content-Disposition header. Not sure about how to get its value when using WebClient, but it's fairly simple with WebRequest:
WebRequest request = WebRequest.Create("http://address/Attachments.aspx?id=GUID");
WebResponse response = request.GetResponse();
string originalFileName = response.Headers["Content-Disposition"];
Stream streamWithFileBody = response.GetResponseStream();

C# Console/Server access to web site

I am working on a C# project where I need to get data from a secured web site that does not have an API or web services. My plan is to login, get to the page I need, and parse out the HTML to get to the data bits I need to log to a database. Right now I'm testing with a console app, but eventually this will be converted to an Azure Service bus application.
In order to get to anything, you have to login at their login.cfm page, which means I need to load the username and password input controls on the page and click the submit button. Then navigate to the page I need to parse.
Since I don't have a 'browser' to parse for controls, I am trying to use various C# .NET classes to get to the page, set the username and password, and click submit, but nothing seems to work.
Any examples I can look at, or .NET classes I should be reviewing that were designed for this sort of project?
Thanks!

Use the WebClient class in System.Net
For persistence of session cookie you'll have to make a custom WebClient class.
#region webclient with cookies
public class WebClientX : WebClient
{
public CookieContainer cookies = new CookieContainer();
protected override WebRequest GetWebRequest(Uri location)
{
WebRequest req = base.GetWebRequest(location);
if (req is HttpWebRequest)
(req as HttpWebRequest).CookieContainer = cookies;
return req;
}
protected override WebResponse GetWebResponse(WebRequest request)
{
WebResponse res = base.GetWebResponse(request);
if (res is HttpWebResponse)
cookies.Add((res as HttpWebResponse).Cookies);
return res;
}
}
#endregion
Use a browser add-on like FireBug or the development tools built into Chrome to get the HTTP POST data being sent when you submit a form. Send those POSTs using the WebClientX class and parse the response HTML.
The fastest way to parse HTML when you already know the format is using a simple Regex.Match. So you'd go through the actions in your browser using the development tools to record your POSTs, URLs and HTML content then you'll perform the same tasks using the WebClientX.

Ok, so here is the complete Code to login to one page, then read from a 2nd page after the login.
class Program
{
static void Main(string[] args)
{
string uriString = "http://www.remotesite.com/login.cfm";
// Create a new WebClient instance.
WebClientX myWebClient = new WebClientX();
// Create a new NameValueCollection instance to hold some custom parameters to be posted to the URL.
NameValueCollection myNameValueCollection = new NameValueCollection();
// Add necessary parameter/value pairs to the name/value container.
myNameValueCollection.Add("userid", "myname");
myNameValueCollection.Add("mypassword", "mypassword");
Console.WriteLine("\nUploading to {0} ...", uriString);
// 'The Upload(String,NameValueCollection)' implicitly method sets HTTP POST as the request method.
byte[] responseArray = myWebClient.UploadValues(uriString, myNameValueCollection);
// Decode and display the response.
Console.WriteLine("\nResponse received was :\n{0}", Encoding.ASCII.GetString(responseArray));
Console.WriteLine("\n\n\n pausing...");
Console.ReadKey();
// Go to 2nd page on the site to get additional data
Stream myStream = myWebClient.OpenRead("https://www.remotesite.com/status_results.cfm?t=8&prog=d");
Console.WriteLine("\nDisplaying Data :\n");
StreamReader sr = new StreamReader(myStream);
StringBuilder sb = new StringBuilder();
using (StreamReader reader = new StreamReader(myStream, System.Text.Encoding.UTF8))
{
string line;
while ((line = reader.ReadLine()) != null)
{
sb.Append(line + "\r\n");
}
}
using (StreamWriter outfile = new StreamWriter(#"Logfile1.txt"))
{
outfile.Write(sb.ToString());
}
Console.WriteLine(sb.ToString());
Console.WriteLine("\n\n\n pausing...");
Console.ReadKey();
}
}
public class WebClientX : WebClient
{
public CookieContainer cookies = new CookieContainer();
protected override WebRequest GetWebRequest(Uri location)
// public override WebRequest GetWebRequest(Uri location)
{
WebRequest req = base.GetWebRequest(location);
if (req is HttpWebRequest)
(req as HttpWebRequest).CookieContainer = cookies;
return req;
}
protected override WebResponse GetWebResponse(WebRequest request)
{
WebResponse res = base.GetWebResponse(request);
if (res is HttpWebResponse)
cookies.Add((res as HttpWebResponse).Cookies);
return res;
}
}