Develop Reference

How to do DIGEST with POST, PUT and PATCH

I am at a total loss here. Im currently having success doing a GET request against a web service that implements DIGEST MD5-sess authentication. This works fine. I get the expected result so I figure my 'BUILD DIGEST AUTH HEADER' method works as intended.
I then get the requirement to also support POST, PUT and PATCH but now I run into a whole heap of problems. First request obviously returns a 401 and I read the info in the WWW-Authenticate header and make a MD5-sess auth header taking into account that this is now a POST, PUT or PATCH request.
var methodString = urlResolverStrategy.ResolverDataModel.HttpMethod.ToString();
var ha2 = CalculateMd5Hash(string.Format("{0}:{1}", methodString, dir));
I keep everything the same but do also obviously add the content to the request on POST, PUT and PATCH. But for some reason I cannot figure out Im locked out of the service. Second request return another 401 even.
Is there something special that needs to be done for DIGEST Auth when method is POST, PUT and PATCH that Im missing?
I have all my requests setup in Postman as well put Postman does not have the same problem. Second call in Postman gets the expected results. A 200 and the expected data on POST, PUT and PATCH.
Im using a slightly modified version DigestAuthFixer.cs that's also available on some other posts here in stackoverflow.
Code below is currently hardcoded to use the MD5-sess method.
public class DigestAuthFixer
{
private static readonly Random random = new Random(DateTime.Now.Millisecond);
private readonly AuthData authData;
readonly UrlResolverStrategyBase urlResolverStrategy;
public HttpStatusCode StatusCode { get; private set; }
public DigestAuthFixer(BasicAuth basicAuth, UrlResolverStrategyBase urlResolverStrategy)
{
// TODO: Complete member initialization
authData = new AuthData
{
Host = urlResolverStrategy.GetHostName(),
User = basicAuth.GetUsername(),
Password = basicAuth.GetPassword(),
};
this.urlResolverStrategy = urlResolverStrategy;
}
private string CalculateMd5Hash(string input)
{
var inputBytes = Encoding.ASCII.GetBytes(input);
var hash = MD5.Create().ComputeHash(inputBytes);
var sb = new StringBuilder();
foreach (var b in hash)
{
sb.Append(b.ToString("x2"));
}
return sb.ToString();
}
private string GrabHeaderVar(string varName, string header)
{
var regHeader = new Regex(string.Format(#"{0}=""([^""]*)""", varName));
var matchHeader = regHeader.Match(header);
if (matchHeader.Success)
{
return matchHeader.Groups[1].Value;
}
throw new ApplicationException(string.Format("Header {0} not found", varName));
}
private string GetDigestHeader(string dir)
{
authData.NC++;
string ha1;
if (authData.Algorithm == "MD5-sess")
{
var ha0 = CalculateMd5Hash(string.Format("{0}:{1}:{2}", authData.User, authData.Realm, authData.Password));
ha1 = CalculateMd5Hash(string.Format("{0}:{1}:{2}", ha0, authData.Nonce, authData.Cnonce));
}
else
{
ha1 = CalculateMd5Hash(string.Format("{0}:{1}:{2}", authData.User, authData.Realm, authData.Password));
}
var methodString = urlResolverStrategy.ResolverDataModel.HttpMethod.ToString();
var ha2 = CalculateMd5Hash(string.Format("{0}:{1}", methodString, dir));
var digestResponse = CalculateMd5Hash(string.Format("{0}:{1}:{2:00000000}:{3}:{4}:{5}", ha1, authData.Nonce, authData.NC, authData.Cnonce, authData.Qop, ha2));
var authString = string.Format("Digest username=\"{0}\", realm=\"{1}\", nonce=\"{2}\", uri=\"{3}\", algorithm=\"{4}\", qop={5}, nc={6:00000000}, cnonce=\"{7}\", response=\"{8}\"", authData.User, authData.Realm, authData.Nonce, dir, authData.Algorithm, authData.Qop, authData.NC, authData.Cnonce, digestResponse);
return authString;
}
public string GrabResponse(string nUrl, string content)
{
var uri = new Uri(authData.Host + nUrl);
var request = (HttpWebRequest)WebRequest.Create(uri);
request.Method = urlResolverStrategy.ResolverDataModel.HttpMethod.ToString();
// If we've got a recent Auth header, re-use it!
if (!string.IsNullOrEmpty(authData.Cnonce) && DateTime.Now.Subtract(authData.CnonceDate).TotalHours < 1.0)
{
request.Headers.Add("Authorization", GetDigestHeader(nUrl));
}
if (!string.IsNullOrEmpty(urlResolverStrategy.ResolverDataModel.IfMatchHeaderValue))
{
request.Headers.Add(HttpHelper.IfMatchHeaderName, urlResolverStrategy.ResolverDataModel.IfMatchHeaderValue);
}
AddContentToBody(request, content);
HttpWebResponse response = null;
try
{
response = (HttpWebResponse)request.GetResponse();
StatusCode = response.StatusCode;
}
catch (WebException ex)
{
// Try to fix a 401 exception by adding a Authorization header
if (ex.Response == null || ((HttpWebResponse)ex.Response).StatusCode != HttpStatusCode.Unauthorized)
throw;
var wwwAuthenticateHeader = ex.Response.Headers["WWW-Authenticate"];
authData.Realm = GrabHeaderVar("realm", wwwAuthenticateHeader);
authData.Nonce = GrabHeaderVar("nonce", wwwAuthenticateHeader);
authData.Qop = GrabHeaderVar("qop", wwwAuthenticateHeader);
authData.Algorithm = "MD5-sess"; // GrabHeaderVar("algorithm", wwwAuthenticateHeader);
authData.NC = 0;
authData.Cnonce = RandomString(8);
authData.CnonceDate = DateTime.Now;
string debug = wwwAuthenticateHeader + Environment.NewLine + nUrl + Environment.NewLine + uri.ToString() + Environment.NewLine + authData.ToString();
var digestRequest = (HttpWebRequest)WebRequest.Create(uri);
digestRequest.Method = urlResolverStrategy.ResolverDataModel.HttpMethod.ToString();
AddContentToBody(digestRequest, content);
var authHeader = GetDigestHeader(nUrl);
debug += uri.ToString() + Environment.NewLine;
debug += nUrl + Environment.NewLine;
debug += authHeader + Environment.NewLine;
digestRequest.Headers.Add("Authorization", authHeader);
if (!string.IsNullOrEmpty(urlResolverStrategy.ResolverDataModel.IfMatchHeaderValue))
{
request.Headers.Add(HttpHelper.IfMatchHeaderName, urlResolverStrategy.ResolverDataModel.IfMatchHeaderValue);
}
HttpWebResponse digestResponse = null;
try
{
//return authHeader;
digestResponse = (HttpWebResponse)digestRequest.GetResponse();
StatusCode = digestResponse.StatusCode;
response = digestResponse;
}
catch (Exception digestRequestEx)
{
if (digestResponse != null)
{
StatusCode = response.StatusCode;
}
else
{
StatusCode = HttpStatusCode.InternalServerError;
}
//return "It broke" + Environment.NewLine + debug;
return "There was a problem with url, username or password (" + digestRequestEx.Message + ")";
}
}
var reader = new StreamReader(response.GetResponseStream());
return reader.ReadToEnd();
}
public string RandomString(int length)
{
var chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
var stringChars = new char[length];
for (int i = 0; i < stringChars.Length; i++)
{
stringChars[i] = chars[random.Next(chars.Length)];
}
return new string(stringChars);
}
private void AddContentToBody(HttpWebRequest request, string content)
{
if (string.IsNullOrEmpty(content))
return;
var data = Encoding.Default.GetBytes(content); // note: choose appropriate encoding
request.ContentLength = data.Length;
request.ContentType = HttpHelper.MediaTypes.Json;
request.Accept = "*/*";
request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.NoCacheNoStore);
//request.Headers.Add("Accept-Encoding", "gzip, deflate, br");
using (var streamWriter = new StreamWriter(request.GetRequestStream()))
{
streamWriter.Write(content);
}
}
}
internal class AuthData
{
public string Host;
public string User;
public string Password;
public string Realm;
public string Nonce;
public string Qop;
public string Cnonce;
public DateTime CnonceDate;
public int NC;
public string Algorithm;
public override string ToString()
{
string newLine = Environment.NewLine;
string result = Host + newLine;
result += User + newLine;
result += Realm + newLine;
result += Nonce + newLine;
result += Qop + newLine;
result += Cnonce + newLine;
result += CnonceDate + newLine;
result += NC + newLine;
result += Algorithm + newLine;
return result;
}
}

So apparently it matters in which order you add all the headers to the request. With hookbin I was able to detect that even though I have put an Authorization header on my digestRequest object it did not follow through to hookbin.
My placing the Authorization header addition to just below the setting the method line it all works...
I had no idea that that could pose a problem. The reason my GET method work is because is because 'content' is empty so no headers are added.
var digestRequest = (HttpWebRequest)WebRequest.Create(uri);
digestRequest.Method = urlResolverStrategy.ResolverDataModel.HttpMethod.ToString();
digestRequest.Headers.Add("Authorization", GetDigestHeader(nUrl));

strava login via httpwebrequest failed

Hi I'm trying to login via https://www.strava.com/session with HttpWebrequest but it doesn't log me in. It gives me an response of 302 which is good but it never redirect me to https://www.strava.com/dashboard.
this is the code that I'm using
Httpclient:
public class HttpClient
{
private const string UserAgent = "Mozilla/5.0";
public CookieCollection CookieCollection;
public HttpWebRequest WebRequest;
public HttpWebResponse WebResponse;
public int code { get; set; }
public string location { get; set; }
public string PostData(string url, string postData, string refer = "")
{
WebRequest = (HttpWebRequest)System.Net.WebRequest.Create(url);
WebRequest.UserAgent = UserAgent;
WebRequest.Referer = refer;
WebRequest.AllowAutoRedirect =false;
WebRequest.Timeout = 10000;
WebRequest.KeepAlive = true;
WebRequest.CookieContainer = new CookieContainer();
if (CookieCollection != null && CookieCollection.Count > 0)
{
WebRequest.CookieContainer.Add(CookieCollection);
}
WebRequest.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
WebRequest.Method = "POST";
try
{
var postBytes = Encoding.UTF8.GetBytes(postData);
WebRequest.ContentLength = postBytes.Length;
var postDataStream = WebRequest.GetRequestStream();
postDataStream.Write(postBytes, 0, postBytes.Length);
postDataStream.Close();
try
{
WebResponse = (HttpWebResponse)WebRequest.GetResponse();
this.code = (int)WebResponse.StatusCode;
this.location = WebResponse.Headers["Location"];
if (WebResponse.StatusCode == HttpStatusCode.OK ||WebResponse.StatusCode == HttpStatusCode.Redirect)
{
WebResponse.Cookies = WebRequest.CookieContainer.GetCookies(WebRequest.RequestUri);
if (WebResponse.Cookies.Count > 0)
{
if (CookieCollection == null)
{
CookieCollection = WebResponse.Cookies;
}
else
{
foreach (Cookie oRespCookie in WebResponse.Cookies)
{
var bMatch = false;
foreach (
var oReqCookie in
CookieCollection.Cast<Cookie>()
.Where(oReqCookie => oReqCookie.Name == oRespCookie.Name))
{
oReqCookie.Value = oRespCookie.Value;
bMatch = true;
break;
}
if (!bMatch)
CookieCollection.Add(oRespCookie);
}
}
}
var reader = new StreamReader(WebResponse.GetResponseStream());
var responseString = reader.ReadToEnd();
reader.Close();
return responseString;
}
}
catch (WebException wex)
{
if (wex.Response != null)
{
using (var errorResponse = (HttpWebResponse)wex.Response)
{
using (var reader = new StreamReader(errorResponse.GetResponseStream()))
{
var error = reader.ReadToEnd();
return error;
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
return "Error in posting data" ;
}
public string GetData(string url, string post = "")
{
var responseStr = string.Empty;
WebRequest = (HttpWebRequest)System.Net.WebRequest.Create(url);
WebRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
WebRequest.Method = "GET";
WebRequest.KeepAlive = true;
WebRequest.Credentials = CredentialCache.DefaultCredentials;
WebRequest.UserAgent = UserAgent;
WebRequest.CookieContainer = new CookieContainer();
if (CookieCollection != null && CookieCollection.Count > 0)
{
WebRequest.CookieContainer.Add(CookieCollection);
}
if (!string.IsNullOrEmpty(post))
{
var postBytes = Encoding.UTF8.GetBytes(post);
WebRequest.ContentLength = postBytes.Length;
var postDataStream = WebRequest.GetRequestStream();
postDataStream.Write(postBytes, 0, postBytes.Length);
postDataStream.Close();
}
WebResponse wresp = null;
try
{
wresp = WebRequest.GetResponse();
var downStream = wresp.GetResponseStream();
if (downStream != null)
{
using (var downReader = new StreamReader(downStream))
{
responseStr = downReader.ReadToEnd();
}
}
return responseStr;
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
if (wresp != null)
{
wresp.Close();
wresp = null;
}
}
finally
{
WebRequest = null;
}
return responseStr;
}
}
Getting crfs-token:
private string GetToken()
{
string token = "";
String sourcestring = hc.GetData(loginURL);
Regex metaTag = new Regex(#"<meta[\s]+[^>]*?name[\s]?=[\s""']+(.*?)[\s""']+content[\s]?=[\s""']+(.*?)[""']+.*?>");
foreach (Match m in metaTag.Matches(sourcestring))
{
if (m.Groups[2].Value.Contains("token"))
{
continue;
}
token = m.Groups[2].Value;
}
return token;
}
Custom keyvaluepair
private string PostParam(Dictionary<string, string> data)
{
var sb = new StringBuilder();
var p = new List<string>();
foreach (KeyValuePair<string, string> pair in data)
{
sb.Clear();
sb.Append(pair.Key).Append("=").Append(pair.Value);
p.Add(sb.ToString());
}
var pp = string.Join("&", p);
return pp;
}
Login:
private HttpClient hc = new HttpClient();
Dictionary data = new Dictionary();
data.Add("utf8", "✓");
data.Add("authenticity_token", GetToken());
data.Add("plan", "");
data.Add("email", "email");
data.Add("password", "password");
hc.PostData(sessionURL,WebUtility.UrlEncode(PostParam(data)), loginURL);
Can someone tell me what I'm doing wrong? if I look the request header when trying to login to strava in browser its the same but still it doesn't log me.

I found the problem.
You need to encode only the token (and the UTF8 character), not the full post data.
This works for me (for some reason I need to run the code two times)
// First time says "logged out"
Dictionary<string, string> data = new Dictionary<string, string>();
data.Add("utf8", WebUtility.UrlEncode("✓"));
string token = GetToken();
string tokenEncoded = WebUtility.UrlEncode(token);
data.Add("authenticity_token", tokenEncoded);
data.Add("plan", "");
data.Add("email", "youremail");
data.Add("password", "yourpwd");
data.Add("remember_me", "on");
string parameters = PostParam(data);
hc.PostData(sessionURL, parameters, loginURL);
// Second time logs in
Dictionary<string, string> data = new Dictionary<string, string>();
data.Add("utf8", WebUtility.UrlEncode("✓"));
string token = GetToken();
string tokenEncoded = WebUtility.UrlEncode(token);
data.Add("authenticity_token", tokenEncoded);
data.Add("plan", "");
data.Add("email", "youremail");
data.Add("password", "yourpwd");
data.Add("remember_me", "on");
string parameters = PostParam(data);
hc.PostData(sessionURL, parameters, loginURL);
Note:
// Keep this on default value "true"
//WebRequest.AllowAutoRedirect = false;

remark: you can use this code (see my previous post) to change activity's status (privacy) after logging in:
Dictionary<string, string> data2 = new Dictionary<string, string>();
data2.Add("utf8", WebUtility.UrlEncode("✓"));
string token2 = GetToken();
string tokenEncoded2 = WebUtility.UrlEncode(token2);
data2.Add("_method", "patch");
data2.Add("authenticity_token", tokenEncoded2);
data2.Add("activity%5Bvisibility%5D", "only_me"); // or "followers_only"
string parameters2 = PostParam(data2);
hc.PostData("https://www.strava.com/activities/youractivityID", parameters2, loginURL);

How can I get html from page with cloudflare ddos portection?

I use htmlagility to get webpage data but I tried everything with page using www.cloudflare.com protection for ddos. The redirect page is not possible to handle in htmlagility because they don't redirect with meta nor js I guess, they check if you have already being checked with a cookie that I failed to simulate with c#. When I get the page, the html code is from the landing cloadflare page.

I also encountered this problem some time ago. The real solution would be solve the challenge the cloudflare websites gives you (you need to compute a correct answer using javascript, send it back, and then you receive a cookie / your token with which you can continue to view the website). So all you would get normally is a page like
In the end, I just called a python-script with a shell-execute. I used the modules provided within this github fork. This could serve as a starting point to implement the circumvention of the cloudflare anti-dDoS page in C# aswell.
FYI, the python script I wrote for my personal usage just wrote the cookie in a file. I read that later again using C# and store it in a CookieJar to continue browsing the page within C#.
#!/usr/bin/env python
import cfscrape
import sys
scraper = cfscrape.create_scraper() # returns a requests.Session object
fd = open("cookie.txt", "w")
c = cfscrape.get_cookie_string(sys.argv[1])
fd.write(str(c))
fd.close()
print(c)
EDIT: To repeat this, this has only LITTLE to do with cookies! Cloudflare forces you to solve a REAL challenge using javascript commands. It's not as easy as accepting a cookie and using it later on. Look at https://github.com/Anorov/cloudflare-scrape/blob/master/cfscrape/init.py and the ~40 lines of javascript emulation to solve the challenge.
Edit2: Instead of writing something to circumvent the protection, I've also seen people using a fully-fledged browser-object (this is not a headless browser) to go to the website and subscribe to certain events when the page is loaded. Use the WebBrowser class to create an infinetly small browser window and subscribe to the appropiate events.
Edit3:
Alright, I actually implemented the C# way to do this. This uses the JavaScript Engine Jint for .NET, available via https://www.nuget.org/packages/Jint
The cookie-handling code is ugly because sometimes the HttpResponse class won't pick up the cookies, although the header contains a Set-Cookie section.
using System;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Web;
using System.Collections;
using System.Threading;
namespace Cloudflare_Evader
{
public class CloudflareEvader
{
/// <summary>
/// Tries to return a webclient with the neccessary cookies installed to do requests for a cloudflare protected website.
/// </summary>
/// <param name="url">The page which is behind cloudflare's anti-dDoS protection</param>
/// <returns>A WebClient object or null on failure</returns>
public static WebClient CreateBypassedWebClient(string url)
{
var JSEngine = new Jint.Engine(); //Use this JavaScript engine to compute the result.
//Download the original page
var uri = new Uri(url);
HttpWebRequest req =(HttpWebRequest) WebRequest.Create(url);
req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0";
//Try to make the usual request first. If this fails with a 503, the page is behind cloudflare.
try
{
var res = req.GetResponse();
string html = "";
using (var reader = new StreamReader(res.GetResponseStream()))
html = reader.ReadToEnd();
return new WebClient();
}
catch (WebException ex) //We usually get this because of a 503 service not available.
{
string html = "";
using (var reader = new StreamReader(ex.Response.GetResponseStream()))
html = reader.ReadToEnd();
//If we get on the landing page, Cloudflare gives us a User-ID token with the cookie. We need to save that and use it in the next request.
var cookie_container = new CookieContainer();
//using a custom function because ex.Response.Cookies returns an empty set ALTHOUGH cookies were sent back.
var initial_cookies = GetAllCookiesFromHeader(ex.Response.Headers["Set-Cookie"], uri.Host);
foreach (Cookie init_cookie in initial_cookies)
cookie_container.Add(init_cookie);
/* solve the actual challenge with a bunch of RegEx's. Copy-Pasted from the python scrapper version.*/
var challenge = Regex.Match(html, "name=\"jschl_vc\" value=\"(\\w+)\"").Groups[1].Value;
var challenge_pass = Regex.Match(html, "name=\"pass\" value=\"(.+?)\"").Groups[1].Value;
var builder = Regex.Match(html, #"setTimeout\(function\(\){\s+(var t,r,a,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n").Groups[1].Value;
builder = Regex.Replace(builder, #"a\.value =(.+?) \+ .+?;", "$1");
builder = Regex.Replace(builder, #"\s{3,}[a-z](?: = |\.).+", "");
//Format the javascript..
builder = Regex.Replace(builder, #"[\n\\']", "");
//Execute it.
long solved = long.Parse(JSEngine.Execute(builder).GetCompletionValue().ToObject().ToString());
solved += uri.Host.Length; //add the length of the domain to it.
Console.WriteLine("***** SOLVED CHALLENGE ******: " + solved);
Thread.Sleep(3000); //This sleeping IS requiered or cloudflare will not give you the token!!
//Retreive the cookies. Prepare the URL for cookie exfiltration.
string cookie_url = string.Format("{0}://{1}/cdn-cgi/l/chk_jschl", uri.Scheme, uri.Host);
var uri_builder = new UriBuilder(cookie_url);
var query = HttpUtility.ParseQueryString(uri_builder.Query);
//Add our answers to the GET query
query["jschl_vc"] = challenge;
query["jschl_answer"] = solved.ToString();
query["pass"] = challenge_pass;
uri_builder.Query = query.ToString();
//Create the actual request to get the security clearance cookie
HttpWebRequest cookie_req = (HttpWebRequest) WebRequest.Create(uri_builder.Uri);
cookie_req.AllowAutoRedirect = false;
cookie_req.CookieContainer = cookie_container;
cookie_req.Referer = url;
cookie_req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0";
//We assume that this request goes through well, so no try-catch
var cookie_resp = (HttpWebResponse)cookie_req.GetResponse();
//The response *should* contain the security clearance cookie!
if (cookie_resp.Cookies.Count != 0) //first check if the HttpWebResponse has picked up the cookie.
foreach (Cookie cookie in cookie_resp.Cookies)
cookie_container.Add(cookie);
else //otherwise, use the custom function again
{
//the cookie we *hopefully* received here is the cloudflare security clearance token.
if (cookie_resp.Headers["Set-Cookie"] != null)
{
var cookies_parsed = GetAllCookiesFromHeader(cookie_resp.Headers["Set-Cookie"], uri.Host);
foreach (Cookie cookie in cookies_parsed)
cookie_container.Add(cookie);
}
else
{
//No security clearence? something went wrong.. return null.
//Console.WriteLine("MASSIVE ERROR: COULDN'T GET CLOUDFLARE CLEARANCE!");
return null;
}
}
//Create a custom webclient with the two cookies we already acquired.
WebClient modedWebClient = new WebClientEx(cookie_container);
modedWebClient.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0");
modedWebClient.Headers.Add("Referer", url);
return modedWebClient;
}
}
/* Credit goes to https://stackoverflow.com/questions/15103513/httpwebresponse-cookies-empty-despite-set-cookie-header-no-redirect
(user https://stackoverflow.com/users/541404/cameron-tinker) for these functions
*/
public static CookieCollection GetAllCookiesFromHeader(string strHeader, string strHost)
{
ArrayList al = new ArrayList();
CookieCollection cc = new CookieCollection();
if (strHeader != string.Empty)
{
al = ConvertCookieHeaderToArrayList(strHeader);
cc = ConvertCookieArraysToCookieCollection(al, strHost);
}
return cc;
}
private static ArrayList ConvertCookieHeaderToArrayList(string strCookHeader)
{
strCookHeader = strCookHeader.Replace("\r", "");
strCookHeader = strCookHeader.Replace("\n", "");
string[] strCookTemp = strCookHeader.Split(',');
ArrayList al = new ArrayList();
int i = 0;
int n = strCookTemp.Length;
while (i < n)
{
if (strCookTemp[i].IndexOf("expires=", StringComparison.OrdinalIgnoreCase) > 0)
{
al.Add(strCookTemp[i] + "," + strCookTemp[i + 1]);
i = i + 1;
}
else
al.Add(strCookTemp[i]);
i = i + 1;
}
return al;
}
private static CookieCollection ConvertCookieArraysToCookieCollection(ArrayList al, string strHost)
{
CookieCollection cc = new CookieCollection();
int alcount = al.Count;
string strEachCook;
string[] strEachCookParts;
for (int i = 0; i < alcount; i++)
{
strEachCook = al[i].ToString();
strEachCookParts = strEachCook.Split(';');
int intEachCookPartsCount = strEachCookParts.Length;
string strCNameAndCValue = string.Empty;
string strPNameAndPValue = string.Empty;
string strDNameAndDValue = string.Empty;
string[] NameValuePairTemp;
Cookie cookTemp = new Cookie();
for (int j = 0; j < intEachCookPartsCount; j++)
{
if (j == 0)
{
strCNameAndCValue = strEachCookParts[j];
if (strCNameAndCValue != string.Empty)
{
int firstEqual = strCNameAndCValue.IndexOf("=");
string firstName = strCNameAndCValue.Substring(0, firstEqual);
string allValue = strCNameAndCValue.Substring(firstEqual + 1, strCNameAndCValue.Length - (firstEqual + 1));
cookTemp.Name = firstName;
cookTemp.Value = allValue;
}
continue;
}
if (strEachCookParts[j].IndexOf("path", StringComparison.OrdinalIgnoreCase) >= 0)
{
strPNameAndPValue = strEachCookParts[j];
if (strPNameAndPValue != string.Empty)
{
NameValuePairTemp = strPNameAndPValue.Split('=');
if (NameValuePairTemp[1] != string.Empty)
cookTemp.Path = NameValuePairTemp[1];
else
cookTemp.Path = "/";
}
continue;
}
if (strEachCookParts[j].IndexOf("domain", StringComparison.OrdinalIgnoreCase) >= 0)
{
strPNameAndPValue = strEachCookParts[j];
if (strPNameAndPValue != string.Empty)
{
NameValuePairTemp = strPNameAndPValue.Split('=');
if (NameValuePairTemp[1] != string.Empty)
cookTemp.Domain = NameValuePairTemp[1];
else
cookTemp.Domain = strHost;
}
continue;
}
}
if (cookTemp.Path == string.Empty)
cookTemp.Path = "/";
if (cookTemp.Domain == string.Empty)
cookTemp.Domain = strHost;
cc.Add(cookTemp);
}
return cc;
}
}
/*Credit goes to https://stackoverflow.com/questions/1777221/using-cookiecontainer-with-webclient-class
(user https://stackoverflow.com/users/129124/pavel-savara) */
public class WebClientEx : WebClient
{
public WebClientEx(CookieContainer container)
{
this.container = container;
}
public CookieContainer CookieContainer
{
get { return container; }
set { container = value; }
}
private CookieContainer container = new CookieContainer();
protected override WebRequest GetWebRequest(Uri address)
{
WebRequest r = base.GetWebRequest(address);
var request = r as HttpWebRequest;
if (request != null)
{
request.CookieContainer = container;
}
return r;
}
protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
{
WebResponse response = base.GetWebResponse(request, result);
ReadCookies(response);
return response;
}
protected override WebResponse GetWebResponse(WebRequest request)
{
WebResponse response = base.GetWebResponse(request);
ReadCookies(response);
return response;
}
private void ReadCookies(WebResponse r)
{
var response = r as HttpWebResponse;
if (response != null)
{
CookieCollection cookies = response.Cookies;
container.Add(cookies);
}
}
}
}
The function will return a webclient with the solved challenges and cookies inside. You can use it as follows:
static void Main(string[] args)
{
WebClient client = null;
while (client == null)
{
Console.WriteLine("Trying..");
client = CloudflareEvader.CreateBypassedWebClient("http://anilinkz.tv");
}
Console.WriteLine("Solved! We're clear to go");
Console.WriteLine(client.DownloadString("http://anilinkz.tv/anime-list"));
Console.ReadLine();
}

A "simple" working method to bypass Cloudflare if you don't use libraries (that sometimes does not work).
Open a "hidden" WebBrowser (size 1,1 or so).
Open the root of your target Cloudflare site.
Get the cookies from WebBrowser.
Use these cookies in WebClient.
Make sure the UserAgent for both WebBrowser and WebClient are identical. Cloudflare will give you a 503 if a mismatch there on the WebClient aftwerwards.
You will need to search here on stack on how to get cookies from WebBrowser and how to modify WebClient so you can set its cookiecontainer + modify the UserAgent on 1 or both so they are identical.
Since the cookies from Cloudflare seems to never expire, you can then serialize the cookies to somewhere temporary and load it each time you run your app, maybe a verification and refetch if failing.
Been doing this for a while and it works quite well. Could not get the C# libs to work for a specific Cloudflare site while they worked on others. No clue to why yet.
This also works behind the scenes on an IIS server, but you will have to set up "frowned upon" settings. That is, run the app pool as SYSTEM or ADMIN and set it to Classic mode.

Nowadays' answer should include Flaresolverr project.
It is meant to be deployed as a container using Docker, so you only have to pass it a port and it's running.
It doesn't impact your project as you don't import a library. It is currently supported. The only bad point I see, is that you need to install Docker to make it work.

Use WebClient to get html of the page,
I wrote following class which handles cookies too,
Just pass CookieContainer instance in constructor.
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Net;
using System.Text;
namespace NitinJS
{
public class SmsWebClient : WebClient
{
public SmsWebClient(CookieContainer container, Dictionary<string, string> Headers)
: this(container)
{
foreach (var keyVal in Headers)
{
this.Headers[keyVal.Key] = keyVal.Value;
}
}
public SmsWebClient(bool flgAddContentType = true)
: this(new CookieContainer(), flgAddContentType)
{
}
public SmsWebClient(CookieContainer container, bool flgAddContentType = true)
{
this.Encoding = Encoding.UTF8;
System.Net.ServicePointManager.Expect100Continue = false;
ServicePointManager.MaxServicePointIdleTime = 2000;
this.container = container;
if (flgAddContentType)
this.Headers["Content-Type"] = "application/json";//"application/x-www-form-urlencoded";
this.Headers["Accept"] = "application/json, text/javascript, */*; q=0.01";// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
//this.Headers["Accept-Encoding"] = "gzip, deflate";
this.Headers["Accept-Language"] = "en-US,en;q=0.5";
this.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; rv:23.0) Gecko/20100101 Firefox/23.0";
this.Headers["X-Requested-With"] = "XMLHttpRequest";
//this.Headers["Connection"] = "keep-alive";
}
private readonly CookieContainer container = new CookieContainer();
protected override WebRequest GetWebRequest(Uri address)
{
WebRequest r = base.GetWebRequest(address);
var request = r as HttpWebRequest;
if (request != null)
{
request.CookieContainer = container;
request.Timeout = 3600000; //20 * 60 * 1000
}
return r;
}
protected override WebResponse GetWebResponse(WebRequest request, IAsyncResult result)
{
WebResponse response = base.GetWebResponse(request, result);
ReadCookies(response);
return response;
}
protected override WebResponse GetWebResponse(WebRequest request)
{
WebResponse response = base.GetWebResponse(request);
ReadCookies(response);
return response;
}
private void ReadCookies(WebResponse r)
{
var response = r as HttpWebResponse;
if (response != null)
{
CookieCollection cookies = response.Cookies;
container.Add(cookies);
}
}
}
}
USAGE:
CookieContainer cookies = new CookieContainer();
SmsWebClient client = new SmsWebClient(cookies);
string html = client.DownloadString("http://www.google.com");

Why the variable doc return null all the time?

Im downloading images from a website.
This is the code:
class MyClient : WebClient
{
public bool HeadOnly { get; set; }
protected override WebRequest GetWebRequest(Uri address)
{
WebRequest req = base.GetWebRequest(address);
if (HeadOnly && req.Method == "GET")
{
req.Method = "HEAD";
}
return req;
}
}
Then the method:
public static HtmlAgilityPack.HtmlDocument getHtmlDocumentWebClient(string url, bool useProxy, string proxyIp, int proxyPort, string usename, string password)
{
HtmlAgilityPack.HtmlDocument doc = null;
try
{
doc = null;
using (MyClient clients = new MyClient())
{
clients.HeadOnly = true;
byte[] body = clients.DownloadData(url);
// note should be 0-length
string type = clients.ResponseHeaders["content-type"];
clients.HeadOnly = false;
// check 'tis not binary... we'll use text/, but could
// check for text/html
if (type == null)
{
return null;
}
else
{
if (type.StartsWith(#"text/html"))
{
string text = clients.DownloadString(url);
try
{
doc = new HtmlAgilityPack.HtmlDocument();
WebClient client = new WebClient();
//client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
client.Credentials = CredentialCache.DefaultCredentials;
client.Proxy = WebRequest.DefaultWebProxy;
if (useProxy)
{
//Proxy
if (!string.IsNullOrEmpty(proxyIp))
{
WebProxy p = new WebProxy(proxyIp, proxyPort);
if (!string.IsNullOrEmpty(usename))
{
if (password == null)
password = string.Empty;
NetworkCredential nc = new NetworkCredential(usename, password);
p.Credentials = nc;
}
}
}
doc.Load(client.OpenRead(url));
}
catch
{
}
}
}
}
if (doc == null)
{
MessageBox.Show("Doc is null " + doc + " The link that did it was " + url);
}
}
catch
{
}
return doc;
}
I tried to add general try catch to the method but it still getting to the MessageBox and not to the catch.
Anyway the link of the image im trying to download is:
http://members.tripod.com/~DannyWest/bundy.jpg
Then i used a breakpoin t and on the line:
if (type.StartsWith(#"text/html"))
Its jumping to the MessageBox.Show...
Now i saw that type contain: image/jpeg
I wonder if is that the problem since there is not text/html or something else maybe something with the link ?
EDIT**
I tried to change the method added this:
if (type.StartsWith(#"text/html")|| type.StartsWith(#"image/jpeg"))
Added the part image/jpeg
But then its getting to this method on another class:
private List<string> getLinks(HtmlAgilityPack.HtmlDocument document)
{
List<string> mainLinks = new List<string>();
var linkNodes = document.DocumentNode.SelectNodes("//a[#href]");
if (linkNodes != null)
{
foreach (HtmlNode link in linkNodes)
{
var href = link.Attributes["href"].Value;
if (href.StartsWith("http://") == true || href.StartsWith("https://") == true || href.StartsWith("www") == true) // filter for http
{
mainLinks.Add(href);
}
}
}
return mainLinks;
}
And linkNodes is null all the time.
When the type.StartsWith is image/jpeg then this linkNodes allways null.
When type.StartsWith is text/html then linkNodes is not null.
I can upload my project to my skydrive if needed.

Is there anyway to get the Hotmail email contacts not the email hash in asp.net using C#?

How to get the Hotmail email contacts not the email hash in asp.net using C#?

We can get the email using delegate authentication. Here is the code snippet which I have done.
enter code here
public partial class DefaultPage : System.Web.UI.Page
{
//Comma-delimited list of offers to be used.
const string Offers = "Contacts.View";
//Name of cookie to use to cache the consent token.
const string AuthCookie = "delauthtoken";
// Initialize the WindowsLiveLogin module.
static WindowsLiveLogin wll = new WindowsLiveLogin(true);
protected WindowsLiveLogin.ConsentToken Token;
protected string ConsentUrl;
protected void Page_Load(object sender, EventArgs e)
{
//Get the consent URL for the specified offers.
ConsentUrl = wll.GetConsentUrl(Offers);
HttpRequest req = HttpContext.Current.Request;
HttpCookie authCookie = req.Cookies[AuthCookie];
// If the raw consent token has been cached in a site cookie, attempt to
// process it and extract the consent token.
if (authCookie != null)
{
string t = authCookie.Value;
Token = wll.ProcessConsentToken(t);
getContacts(Token.DelegationToken,Token.LocationID);
if ((Token != null) && !Token.IsValid())
{
Token = null;
}
}
}
private void getContacts(string token, string lId)
{
long intlid = Int64.Parse(lId, System.Globalization.NumberStyles.HexNumber);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(string.Format("https://livecontacts.services.live.com/users/#C#{0}/rest/livecontacts/contacts/", intlid));
request.UserAgent = "Windows Live Data Interactive SDK";
request.ContentType = "application/xml; charset=utf-8";
request.Method = "GET";
request.Headers.Add("Authorization", "DelegatedToken dt=\"" + token + "\"");
using (WebResponse response = request.GetResponse())
{
using (Stream stream = response.GetResponseStream())
{
//docNav = new XPathDocument(stream);
StreamReader reader = new StreamReader(stream);
// var contactStreamReponse = reader.ReadToEnd();
// XmlDocument contacts = new XmlDocument();
// contacts.LoadXml(contactStreamReponse);
//Use the document. For example, display contacts.InnerXml.
// labelXml.Text = contacts.InnerXml;
if (CheckBoxList1.Items.Count < 1)
{
DataSet ds = new DataSet();
ds.ReadXml(reader);
response.Close();
if (ds.Tables["Contact"].Rows.Count > 0)
{
ArrayList al = new ArrayList();
for (int intC = 0; intC < ds.Tables["Email"].Rows.Count; intC++)
{
//Response.Write("FOr instance" + intC);
//Response.Write(ds.Tables["Email"].Rows[intC]["Address"].ToString());
///Response.Write("<br>");
al.Add(ds.Tables["Email"].Rows[intC]["Address"].ToString());
}
CheckBoxList1.DataSource = al;
CheckBoxList1.DataBind();
}
}
else
{
return;
}
}
}
}