Develop Reference

C# HttpWebRequest Use cookie

I have a little problem with cookie handling in C#
So on my web site, I have a login page, once logged in, I am redirected to the home page. I get with HttpWebRequest to connect and follow the redirection, I created a class, here it is :
class webReq
{
private string urlConnection;
private string login;
private string password;
private CookieCollection cookieContainer;
private long executionTime = 0;
public webReq(string urlCo, string login, string pass)
{
this.urlConnection = urlCo;
this.login = login;
this.password = pass;
this.cookieContainer = null;
}
public void StartConnection()
{
string WriteHTML = "D:/REM/Connection.html";
List<string> datas = new List<string>();
datas.Add("Username=" + this.login);
datas.Add("Password=" + this.password);
datas.Add("func=ll.login");
datas.Add("NextURL=/admin/livelink.exe");
datas.Add("loginbutton=Sign in");
string postData = "";
postData = string.Join("&", datas);
var buffer = Encoding.ASCII.GetBytes(postData);
try
{
var watch = System.Diagnostics.Stopwatch.StartNew();
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(this.urlConnection);
request.AllowAutoRedirect = true;
request.Method = "POST";
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1003.1 Safari/535.19";
request.Accept = "text/html, application/xhtml+xml, */*";
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = buffer.Length;
request.CookieContainer = new CookieContainer();
Stream stream = request.GetRequestStream();
stream.Write(buffer, 0, buffer.Length);
stream.Close();
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
stream = response.GetResponseStream();
watch.Stop();
this.executionTime = watch.ElapsedMilliseconds;
StreamReader reader = new StreamReader(stream);
System.IO.File.WriteAllText(WriteHTML, reader.ReadToEnd());
this.cookieContainer = new CookieCollection();
foreach (Cookie cookie in response.Cookies)
{
this.cookieContainer.Add(cookie);
}
}
catch (WebException ex)
{
Console.WriteLine(ex.GetBaseException().ToString());
}
}
}
I load the home page well, and I manage to get a cookie.
So I developed a function to use my cookie to browse the website :
public void connectUrl(string url, int numeroTest)
{
string WriteHTML = "D:/REM/Page"+numeroTest+".html";
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
//Add cookie to request.CookieContainer
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(this.cookieContainer);
var watch = System.Diagnostics.Stopwatch.StartNew();
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream stream = response.GetResponseStream();
watch.Stop();
this.executionTime = watch.ElapsedMilliseconds;
StreamReader reader = new StreamReader(stream);
System.IO.File.WriteAllText(WriteHTML, reader.ReadToEnd());
}
Normally, I have to retrieve three cookies, like on the website :
Only, I can't navigate on the website, I end up on the login page, the cookies are not good, and that I'm in debug, I only loaded one cookie(BrowseSettings) out of the three(LLCookie & LLTZCookie) :
I don't understand why I can't retrieve all the cookies on the website.... If anyone has a solution!

I found the reason why I can't get all the cookies, even if I can't find exactly why it works by disabling redirection, in my StartConnection() method :
request.AllowAutoRedirect = true;

c# webrequest json file missing element value

I'm trying to download this json file to read out some data:
http://ddragon.leagueoflegends.com/cdn/5.7.2/data/en_US/item.json
My problem is now that somehow always the values in the tags attribute is missing.
I already used different methods to retrieve them used different encodings but nothing helped. Also the content length is less than the one which I receive in the browser.
Below are some codes I already tried. As you can see I used the WebClient to download this as a string. Also tried to download it as a file, same result.
Used HttpWebRequest with all header attributes.
Used the HEAD method to receive the content length which is less than in the browser.
I also tried using the MemoryStream and the StreamReader to get the correct data. Nothing worked.
String url = "http://ddragon.leagueoflegends.com/cdn/" + version + "/data/en_US/item.json";
//String json = new WebClient().DownloadString(url);
String json = "";
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36";
request.ContentType = "text/plain; charset=utf-8 ";
request.Accept = "*/*";
request.Method = "GET";
//Get the headers associated with the request.
WebHeaderCollection myWebHeaderCollection = request.Headers;
myWebHeaderCollection.Add("DNT", "1");
myWebHeaderCollection.Add("Accept-Encoding", "gzip, deflate, sdch");
myWebHeaderCollection.Add("Accept-Language", "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4");
HttpWebRequest requestHeader = ((HttpWebRequest) WebRequest.Create(url));
requestHeader.Method = "HEAD";
using (WebResponse resp = requestHeader.GetResponse())
{
Console.WriteLine(resp.ContentLength);
}
Stream responseStream;
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
using (responseStream = response.GetResponseStream())
{
if (responseStream != null)
{
Console.WriteLine(response.ContentLength);
Console.WriteLine(response.ContentType);
StreamReader readStream = null;
if (string.IsNullOrEmpty(response.CharacterSet))
{
readStream = new StreamReader(responseStream);
}
else
{
readStream = new StreamReader(responseStream, Encoding.GetEncoding(response.CharacterSet));
}
json = readStream.ReadToEnd();
readStream.Close();
//using (var memoryStream = new MemoryStream())
//{
// responseStream.CopyTo(memoryStream);
// json = Encoding.UTF8.GetString(memoryStream.ToArray());
//}
}
}
Edit:
Sample Outputs:
Browser: http://pastebin.com/nQBj64L0
C# Client: http://pastebin.com/7a0Gxvcg

Post form data using HttpWebRequest

I want to post some form data to a specified URL that isn't inside my own web application. It has the same domain, such like "domain.client.nl". The web application has a url "web.domain.client.nl" en the url where I want to post to is "idp.domain.client.nl".
But my code does nothing..... does someone knows what I'm doing wrong?
Wouter
StringBuilder postData = new StringBuilder();
postData.Append(HttpUtility.UrlEncode(String.Format("username={0}&", uname)));
postData.Append(HttpUtility.UrlEncode(String.Format("password={0}&", pword)));
postData.Append(HttpUtility.UrlEncode(String.Format("url_success={0}&", urlSuccess)));
postData.Append(HttpUtility.UrlEncode(String.Format("url_failed={0}", urlFailed)));
ASCIIEncoding ascii = new ASCIIEncoding();
byte[] postBytes = ascii.GetBytes(postData.ToString());
// set up request object
HttpWebRequest request;
try
{
request = (HttpWebRequest)HttpWebRequest.Create(WebSiteConstants.UrlIdp);
}
catch (UriFormatException)
{
request = null;
}
if (request == null)
throw new ApplicationException("Invalid URL: " + WebSiteConstants.UrlIdp);
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = postBytes.Length;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
// add post data to request
Stream postStream = request.GetRequestStream();
postStream.Write(postBytes, 0, postBytes.Length);
postStream.Flush();
postStream.Close();

Both the field name and the value should be url encoded.
format of the post data and query string are the same
The .net way of doing is something like this
NameValueCollection outgoingQueryString = HttpUtility.ParseQueryString(String.Empty);
outgoingQueryString.Add("field1","value1");
outgoingQueryString.Add("field2", "value2");
string postdata = outgoingQueryString.ToString();
This will take care of encoding the fields and the value names

Try this:
var request = (HttpWebRequest)WebRequest.Create("http://www.example.com/recepticle.aspx");
var postData = "thing1=hello";
postData += "&thing2=world";
var data = Encoding.ASCII.GetBytes(postData);
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = data.Length;
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
var response = (HttpWebResponse)request.GetResponse();
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();

You are encoding the form incorrectly. You should only encode the values:
StringBuilder postData = new StringBuilder();
postData.Append("username=" + HttpUtility.UrlEncode(uname) + "&");
postData.Append("password=" + HttpUtility.UrlEncode(pword) + "&");
postData.Append("url_success=" + HttpUtility.UrlEncode(urlSuccess) + "&");
postData.Append("url_failed=" + HttpUtility.UrlEncode(urlFailed));
edit
I was incorrect. According to RFC1866 section 8.2.1 both names and values should be encoded.
But for the given example, the names do not have any characters that needs to be encoded, so in this case my code example is correct ;)
The code in the question is still incorrect as it would encode the equal sign which is the reason to why the web server cannot decode it.
A more proper way would have been:
StringBuilder postData = new StringBuilder();
postData.AppendUrlEncoded("username", uname);
postData.AppendUrlEncoded("password", pword);
postData.AppendUrlEncoded("url_success", urlSuccess);
postData.AppendUrlEncoded("url_failed", urlFailed);
//in an extension class
public static void AppendUrlEncoded(this StringBuilder sb, string name, string value)
{
if (sb.Length != 0)
sb.Append("&");
sb.Append(HttpUtility.UrlEncode(name));
sb.Append("=");
sb.Append(HttpUtility.UrlEncode(value));
}

Use this code:
internal void SomeFunction() {
Dictionary<string, string> formField = new Dictionary<string, string>();
formField.Add("Name", "Henry");
formField.Add("Age", "21");
string body = GetBodyStringFromDictionary(formField);
// output : Name=Henry&Age=21
}
internal string GetBodyStringFromDictionary(Dictionary<string, string> formField)
{
string body = string.Empty;
foreach (var pair in formField)
{
body += $"{pair.Key}={pair.Value}&";
}
// delete last "&"
body = body.Substring(0, body.Length - 1);
return body;
}

List<KeyValuePair<string, string>> formField= new List<KeyValuePair<string,string>>();
formField.Add(new KeyValuePair<string, string>("Name", "Henry"));
formField.Add(new KeyValuePair<string, string>("Age", "21"));
var body = string.Join("&", formField.Select(kvp => $"{kvp.Key}={kvp.Value}"));

missing post data by using HttpWebRequest

I got a problem on posting data by using HttpWebRequest.
There is a string(ie. key1=value1&key2=value2&key3=value3) and I have post it to a site (ie. www.*.com/edit), but ,I don't know why that sometimes it's nothing wrong , but sometimes ,the first key=value1 will be missing, only key2=value&key3=value3 that can find in HttpAnalyzer.
public static string SubmitData(string Url, string FormData, CookieContainer _Cc, string ContentType)
{
Stream RequestStream = null, ResponseStream = null; StreamReader Sr = null;
HttpWebRequest HRequest = (HttpWebRequest)WebRequest.Create(Url);
try
{
HRequest.CookieContainer = _Cc;
HRequest.Method = "POST";
HRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)";
HRequest.ContentType = ContentType;
HRequest.ContentLength = FormData.Length;
//byte[] BFromData = new ASCIIEncoding().GetBytes(FormData);
byte[] BFromData = Encoding.ASCII.GetBytes(FormData);
BFromData = Encoding.Convert(Encoding.ASCII, Encoding.UTF8, BFromData);//ascii → utf8
RequestStream = HRequest.GetRequestStream();
RequestStream.Write(BFromData, 0, BFromData.Length);
//RequestStream.Write(utf8Bytes,0,utf8Bytes.Length );
HttpWebResponse HResponse = (HttpWebResponse)HRequest.GetResponse();
ResponseStream = HResponse.GetResponseStream();
Sr = new StreamReader(ResponseStream, Encoding.UTF8);
return Sr.ReadToEnd();
}
catch
{
return "";
}
finally
{
if (null != RequestStream) RequestStream.Close();
if (null != ResponseStream) ResponseStream.Close();
if (null != Sr) Sr.Close();
}
}

Use Fiddler to see how the request looks like when you click on the form then try using this approach and modify what you need for your request.
public static void PostDataAndDoSomething()
{
string URI = "http://www.something.com";
//make your request payload
string requestBody = String.Format("{{'param1': {0}, 'param2': {1}}}",value1, value2); //json format
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(URI); //make request
// set request headers as you need
request.ContentType = "application/json; charset=UTF-8";
request.Accept = "application/json, text/javascript;
request.Method = "POST";
request.UserAgent = "";
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write(requestBody); //write your request payload
}
WebResponse response = request.GetResponse();
string jsonData = String.Empty;
using (var reader = new StreamReader(response.GetResponseStream()))
{
jsonData = reader.ReadToEnd();
}
response.Close();
//do something with your data, deserialize, Regex etc....
}

Cannot login website with httprequest POST in C#?

I'm trying to write a bit of code to login to a website. But it's not working. Please can you give me some advice. This is my a bit of code:
static void Main(string[] args)
{
CookieContainer container = new CookieContainer();
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://pagehype.com/login.php");
request.Method = "POST";
request.Timeout = 10000;
request.ReadWriteTimeout = 30000;
request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3 (.NET CLR 3.5.30729) (Prevx 3.0.5)";
request.CookieContainer = container;
ASCIIEncoding encoding = new ASCIIEncoding();
string postData = "username=user&password=password&processlogin=1&return=";
byte[] data = encoding.GetBytes(postData);
request.ContentType = "application/x-www-form-urlencoded";
request.ContentLength = data.Length;
Stream newStream = request.GetRequestStream();
newStream.Write(data, 0, data.Length);
newStream.Close();
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
StreamReader reader = new StreamReader(response.GetResponseStream());
string htmldoc = reader.ReadToEnd();
response.Close();
Console.Write(htmldoc);
}
Many thanks,

Use http://www.fiddler2.com/fiddler2/ to view the http request sent went you login in using a browser and ensure that the request you build in code is the same.

PHP logins use PHPSESSID cookie. You'll need to capture this and pass it back in the CookieContainer. This is how the server will recognise you as an authenticated user.
The cookie is set in the Set-Cookie header in the initial response. You'll need to parse it to recreate the cookie in your container (don't forget the path (and domain?)
var setCookie = response.GetResponseHeader("Set-Cookie");
response.Close();
container = new CookieContainer();
foreach (var cookie in setCookie.Split(','))
{
var split = cookie.Split(';');
string name = split[0].Split('=')[0];
string value = split[0].Split('=')[1];
var c = new Cookie(name, value);
if (cookie.Contains(" Domain="))
c.Domain = split.Where(x => x.StartsWith(" Domain")).First().Split('=')[1];
else
{
c.Domain = ".pagehype.com";
}
if (cookie.Contains(" Path="))
c.Path = split.Where(x => x.StartsWith(" Path")).First().Split('=')[1];
container.Add(c);
}
Then add this container to your request.