Develop Reference

HttpWebRequest.GetResponse always returning "The operation has timed out"

I would like to get the HTML data from specific website ("https://www.justdial.com/Chennai/Silver-Oak-Service-Apartments-Next-to-Vivek-Showroom-Selaiyur/044PXX44-XX44-111215152228-W2G5_BZDET?xid=Q2hlbm5haSBIb3RlbHM=")
Interesting part is, I can able to get the data for other websites like (Google, StackOverFlow, etc)
I'm using below code, But I'm receiving "The operation has timed out" always.
public static string GetPageData(string url)
{
try
{
ServicePointManager.DefaultConnectionLimit = 7000;
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(url);
httpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36";
httpWebRequest.Headers.Add("Accept-Language", "en-US,fr-CA,it-IT;q=0.6");
httpWebRequest.ContentType = "text/html";
httpWebRequest.AllowAutoRedirect = true;
httpWebRequest.Method = "GET";
httpWebRequest.CookieContainer = new CookieContainer();
httpWebRequest.CookieContainer.Add(new Uri("https://justdial.com"), new CookieCollection());
httpWebRequest.KeepAlive = true;
httpWebRequest.MaximumAutomaticRedirections = 200;
httpWebRequest.Timeout = 7000;
using (var response = (HttpWebResponse)httpWebRequest.GetResponse())
{
Stream responseStream = response.GetResponseStream();
if (responseStream == null)
return string.Empty;
StreamReader streamReader = new StreamReader(responseStream);
string end = streamReader.ReadToEnd();
streamReader.Close();
streamReader.Dispose();
responseStream.Close();
responseStream.Dispose();
response.Close();
return end;
}
}
catch (Exception ex)
{
return ex.Message;
}
}

How to implement SSL with HttpWebRequest?

I'm going to get information from https web pages. But the downloaded data is coded and illegible.
\u001f�\b\0\0\0\0\0\0\0�iw۸�0�Y�\u0015\u0018��(y�z��,v���rܝ\u0019�v�N/�s��P\",3�H]��r��\u007f\u007fQ�\b���N���\u0011Il�B��\0\u0014�^�#�ŋ\u0017����\u007f���Gt��\u0003t����S\u001f9k\u001b\u001b\u007fl�76\u000e/\u000fџ�^~>F[��2v��O�(t����'\u000er��t���qww�~��\u001eţ���{�k\v\n�ǵT)��}�\u001eR�\a�^�E��\u0018�.����?���\a�,vGc�A�(Lq�~p�hm�\u000e���a�߇��>�\u001dGA�b\u001f�'~�\u0013���\u0016�\n\bB�/��\K\u001f&X)���t\u0003z����n���×ˣ��\u000e\"�YkjOh\u001f۲��\0�]\���\rY?\u001b\u0015u�����%��\0w�o��{����\u000fo�u��>8��\u001b�$�\b\"��_\lv6�v6�ެ�o\u000e�q��I҇\0'�
How to get the SSL certificate into the HttpWebRequest class?
class MyHttpWebRequest : IDisposable
{
private static int Count = 0;
private HttpWebRequest request;
private Stream dataStream;
private string Status { get; set; }
private string userAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36";
private string accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
private string acceptLanguage = "Accept-Language: en-US,en;q=0.9";
private string acceptEncoding = "Accept-Encoding: gzip, deflate, br";
private string host;
private X509Certificate2 clientCertificates;
private String referer { get; set; } = "";
private CookieCollection cookieCollection { get; set; } = null;
public MyHttpWebRequest()
{
Count++;
clientCertificates = new X509Certificate2(#"mahan.cer");
//clientCertificates = X509Certificate.CreateFromCertFile(#"mahan.cer");//(#"LocalAuthority.crt");
//ServicePointManager.Expect100Continue = true;
//ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3;
//ServicePointManager.Expect100Continue = false;
//ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
//WebRequestHandler handler = new WebRequestHandler();
//X509Certificate2 certificate = GetMyX509Certificate();
//handler.ClientCertificates.Add(certificate);
//HttpClient client = new HttpClient(handler);
}
public MyHttpWebRequest(string host) : this()
{
this.host = host;
}
~MyHttpWebRequest()
{
Count--;
}
public string GetResponse()
{
try
{
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (cookieCollection == null) cookieCollection = new CookieCollection();
cookieCollection.Add(response.Cookies);
this.Status = response.StatusDescription;
dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream);
string responseFromServer = reader.ReadToEnd();
// Clean up the streams.
reader.Close();
//dataStream.Close();
response.Close();
//clientCertificates = new X509Certificate2(request.ServicePoint.Certificate);
this.Status = "Successful";
return responseFromServer;
}
catch (Exception ex)
{
this.Status = ex.Message;
return ex.ToString();
}
}
public string SetRequest(string url)
{
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "GET";
request.UserAgent = userAgent;
request.Accept = accept;
request.Headers.Add(acceptLanguage);
request.Headers.Add(acceptEncoding);
request.KeepAlive = true;
request.CookieContainer = new CookieContainer();
if (cookieCollection != null)
request.CookieContainer.Add(cookieCollection);
request.ProtocolVersion = HttpVersion.Version11;
request.AllowAutoRedirect = false;
request.ContentType = "application/x-www-form-urlencoded";
request.Referer = referer;
referer = url;
request.Host = host;
//request.ClientCertificates.Clear();
request.ClientCertificates.Add(clientCertificates);
request.PreAuthenticate = true;
this.Status = "Successful";
}
catch (Exception ex)
{
this.Status = ex.ToString();
}
return this.ToString();
}
public string SetRequest(string url, string method)
{
try
{
request = (HttpWebRequest)WebRequest.Create(url);
if (method.Equals("GET") || method.Equals("POST"))
{
request.Method = method;
}
else
{
throw new Exception("Invalid Method Type");
}
request.UserAgent = userAgent;
request.Accept = accept;
request.Headers.Add(acceptLanguage);
request.Headers.Add(acceptEncoding);
request.KeepAlive = true;
request.CookieContainer = new CookieContainer();
if (cookieCollection != null)
request.CookieContainer.Add(cookieCollection);
request.ProtocolVersion = HttpVersion.Version11;
request.AllowAutoRedirect = false;
request.ContentType = "application/x-www-form-urlencoded";
request.Referer = referer;
referer = url;
request.Host = host;
//request.ClientCertificates.Clear();
request.ClientCertificates.Add(clientCertificates);
request.PreAuthenticate = true;
this.Status = "Successful";
}
catch (Exception ex)
{
this.Status = ex.ToString();
}
return this.ToString();
}
public string SetRequest(string url, string method, string data)
{
try
{
request = (HttpWebRequest)WebRequest.Create(url);
if (method.Equals("GET") || method.Equals("POST"))
{
request.Method = method;
}
else
{
throw new Exception("Invalid Method Type");
}
request.UserAgent = userAgent;
request.Accept = accept;
request.Headers.Add(acceptLanguage);
request.Headers.Add(acceptEncoding);
request.KeepAlive = true;
request.CookieContainer = new CookieContainer();
if (cookieCollection != null)
request.CookieContainer.Add(cookieCollection);
request.ProtocolVersion = HttpVersion.Version11;
request.AllowAutoRedirect = false;
request.ContentType = "application/x-www-form-urlencoded";
request.Referer = referer;
referer = url;
request.Host = host;
//request.ClientCertificates.Clear();
request.ClientCertificates.Add(clientCertificates);
request.PreAuthenticate = true;
byte[] byteArray = Encoding.UTF8.GetBytes(data);
request.ContentLength = byteArray.Length;
dataStream = request.GetRequestStream();
dataStream.Write(byteArray, 0, byteArray.Length);
dataStream.Close();
this.Status = "Successful";
}
catch (Exception ex)
{
this.Status = ex.Message;
}
return this.Status;
}
public string SetRequest(string url, string method, string data, string contentType)
{
try
{
request = (HttpWebRequest)WebRequest.Create(url);
if (method.Equals("GET") || method.Equals("POST"))
{
request.Method = method;
}
else
{
throw new Exception("Invalid Method Type");
}
request.UserAgent = userAgent;
request.Accept = accept;
request.Headers.Add(acceptLanguage);
request.Headers.Add(acceptEncoding);
request.KeepAlive = true;
request.CookieContainer = new CookieContainer();
if (cookieCollection != null)
request.CookieContainer.Add(cookieCollection);
request.ProtocolVersion = HttpVersion.Version11;
request.AllowAutoRedirect = false;
request.ContentType = contentType;
request.Referer = referer;
referer = url;
request.Host = host;
//request.ClientCertificates.Clear();
request.ClientCertificates.Add(clientCertificates);
request.PreAuthenticate = true;
byte[] byteArray = Encoding.UTF8.GetBytes(data);
request.ContentLength = byteArray.Length;
dataStream = request.GetRequestStream();
dataStream.Write(byteArray, 0, byteArray.Length);
dataStream.Close();
this.Status = "Successful";
}
catch (Exception ex)
{
this.Status = ex.Message;
}
return this.Status;
}
public void Dispose()
{
request.Abort();
request = null;
dataStream.Close();
dataStream.Dispose();
dataStream = null;
}
}

How to get the SSL certificate into the HttpWebRequest class?
Your problem is not related to SSL at all.
private string acceptEncoding = "Accept-Encoding: gzip, deflate, br";
...
request.Headers.Add(acceptEncoding);
With this code you explicitly tell the server that you'll support various content compression algorithms. Only, this is a lie since you don't deal with compression when reading the response.
\u001f�\b\0\0\0\0\0\0\0
This looks the beginning of a gzip data stream. It starts with hex \x1f\x8b but you interpret this wrongly as UTF-8 which it is not.
The easiest way to fix this part is to remove the Accept-Encoding from your request or use Accept-Encoding: identity to signal that you don't accept any compression. This way a well behaving server will send you the body without any compression.

HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://yourwebsite.nld");
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
response.Close();
X509Certificate certV1 = request.ServicePoint.Certificate;
Certificate is a X509Certificate not a X509Certificate2.
To convert add
X509Certificate2 certV2 = new X509Certificate2(certV1);

Why httpwebrequest of C# is returning non compressed response

I am building one auto login application for one of my website. I have developed the auto login tool in C#. I have used httpwebrequest to make post call on the login page. I'm facing issue with the compression of the webpage. In some of the computers I'm not getting compressed response while in others I'm getting the response. Can anyone suggest how I can guaranteed get compressed response. The following is the code I have written for my httpwebrequest:
httpRequest.Connection = "keepalive";
httpRequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
httpRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
httpRequest.UserAgent = "Mozilla/5.0 (Windows T 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36";
httpRequest.UnsafeAuthenticatedConnectionSharing = true;
httpRequest.Headers.Set("Cache-Control", "no-cache");
httpRequest.Method = "POST";
byte[] bytes = Encoding.ASCII.GetBytes(postdata);
httpRequest.ContentLength = bytes.Length;
requestStream = httpRequest.GetRequestStream();
requestStream.Write(bytes, 0, bytes.Length);
requestStream.Flush();
requestStream.Close();
response = (HttpWebResponse)httpRequest.GetResponse();
HttpWebResponse response2 = response;
switch (response2.StatusCode)
{
case HttpStatusCode.OK:
responseStream = response2.GetResponseStream();
if (response2.ContentEncoding.ToLower().Contains("gzip"))
{
str = new StreamReader(new GZipStream(responseStream, CompressionMode.Decompress), Encoding.UTF8).ReadToEnd();
}
else if (response2.ContentEncoding.ToLower().Contains("deflate"))
{
str = new StreamReader(new DeflateStream(responseStream, CompressionMode.Decompress), Encoding.UTF8).ReadToEnd();
}
else
{
str = new StreamReader(responseStream, Encoding.UTF8).ReadToEnd();
}
responseStream.Close();
responseStream = null;
break;
}

401 error when try to connect to Lighthouse API

I'm trying to connect to the Lighthouse api using C# code. This is the php example https://support.lighthouserocks.com/hc/en-gb/articles/201319732-API-The-Basics which it describe how to do this. But I'm fails with it. I was try both NetworkCredentials & send in the Header but still have 401 Unauthorized access to it, here is the code:
public string RequestResponse()
{
HttpWebRequest webRequest = WebRequest.Create(HomeUrl) as HttpWebRequest;
webRequest.Method = "GET";
webRequest.ContentType = "application/json";
webRequest.ServicePoint.Expect100Continue = false;
webRequest.Timeout = 20000;
string auth = CreateAuthorization("domain.lhlive.com", "user", "token");
webRequest.Headers["auth"] = "Basic " + auth;
//webRequest.Credentials = new NetworkCredential("user", "token");
//webRequest.PreAuthenticate = true;
//webRequest.Headers.Add("auth", "user, token");
webRequest.Accept = "application/vnd.lighthouse.v1.hal+json";
Stream responseStream = null;
StreamReader responseReader = null;
string responseData = "";
try
{
WebResponse webResponse = webRequest.GetResponse();
responseStream = webResponse.GetResponseStream();
responseReader = new StreamReader(responseStream);
responseData = responseReader.ReadToEnd();
}
finally
{
if (responseStream != null)
{
responseStream.Close();
responseReader.Close();
}
}
return responseData;
}
public void Test()
{
using (var client = new WebClient())
{
client.Headers["User-Agent"] = "Mozilla/4.0 (Compatible; Windows NT 5.1; MSIE 6.0) (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";
client.Headers["ContentType"] = "application/json";
client.Headers["Accept"] = "application/vnd.lighthouse.v1.hal+json";
//client.Headers["Lighthouse Username"] = "user";
//client.Headers["API Key"] = "token";
client.Headers["WWW-Authenticate"] = "user, token";
byte[] arr = client.DownloadData("https://domain.lhlive.com/contacts");
Console.WriteLine("--- WebClient result ---");
Console.WriteLine(arr.Length);
}
}
anybody know what I should to do?

Hard to say because I can't access Lighthouse but try the following (notice how auth header is set).
var webRequest = WebRequest.Create("http://some.endpoint.com/") as HttpWebRequest;
webRequest.Method = "GET";
webRequest.Accept = "application/vnd.lighthouse.v1.hal+json";
webRequest.ContentType = "application/json";
webRequest.Headers["Authorization"] = string.Format("Basic {0}",
Convert.ToBase64String(Encoding.Default.GetBytes(
string.Format("{0}:{1}", "your username", "your API key"))));
var response = webRequest.GetResponse();
var stream = response.GetResponseStream();
var data = (new StreamReader(stream)).ReadToEnd();
In this case your Authorization header looks like "Basic eW91ciB1c2VybmFtZTp5b3VyIEFQSSBrZXk=".

Reading ajax response from Youtube in C#

I'm trying to post a comment on Youtube and get the xml response (so I can get the comment ID, or if I need to enter captcha or something), but I'm only able to post the comment. For some reason I am not able to read the xml response by using response.GetResponseStream(). When I try to output the response to the console, I get nothing. And, I've sniffed the requests and response my program sends and receives using Wireshark and I can see that the xml is in the response.
Here is the code I'm using to read the response:
using (HttpWebResponse response = MakeRequest(request, cookies, post))
{
using (var reader = new System.IO.StreamReader(response.GetResponseStream(), UTF8Encoding.UTF8))
{
string xml = reader.ReadToEnd();
Console.WriteLine(xml);
}
}
and the MakeRequest function
private static HttpWebResponse MakeRequest(HttpWebRequest request, CookieContainer SessionCookieContainer, Dictionary<string, string> parameters = null)
{
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.52 Safari/536.5Accept: */*";
request.Accept = "text/html,text/xml,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
request.CookieContainer = SessionCookieContainer;
request.AllowAutoRedirect = false;
request.KeepAlive = true;
if (proxy != "") request.Proxy = myproxy;
if (parameters != null) request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
string postData = string.Empty;
if (parameters != null)
{
postData = getPostData(parameters);
byte[] postBuffer = UTF8Encoding.UTF8.GetBytes(postData);
using (Stream postStream = request.GetRequestStream())
{
postStream.Write(postBuffer, 0, postBuffer.Length);
}
}
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
SessionCookieContainer.Add(response.Cookies);
while (response.StatusCode == HttpStatusCode.Found)
{
response.Close();
request = GetNewRequest(response.Headers["Location"], SessionCookieContainer);
response = (HttpWebResponse)request.GetResponse();
SessionCookieContainer.Add(response.Cookies);
}
return response;
}
Any ideas on why this isn't working and how to solve this issue?

I think switching to HttpClient would solve the problem.