HttpWebResponse returns wrong html - c#

For get bitcoin price i did this :
HttpWebRequest req = (HttpWebRequest)WebRequest.Create("https://www.blockchain.com/prices");
req.Method = "GET";
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*;q=0.8";
req.UserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36";
req.ContentType = "text/html; charset=utf-8";
req.Referer = "";
req.KeepAlive = true;
req.Timeout = 25000;
req.AllowAutoRedirect = false;
CookieContainer cookieJar1 = new CookieContainer();
req.CookieContainer = cookieJar1;
HttpWebResponse res = (HttpWebResponse)req.GetResponse();
foreach (Cookie cookie in res.Cookies)
{
cookieJar1.Add(new Cookie(cookie.Name.Trim(), cookie.Value.Trim(), "/", cookie.Domain));
}
Stream Stream = res.GetResponseStream();
StreamReader reader = new StreamReader(Stream);
string reader_str = reader.ReadToEnd();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(reader_str);
HtmlAgilityPack.HtmlNode node = htmlDoc.DocumentNode.SelectSingleNode("//p[#title='Bitcoin']");
string bitcoin_price_str = node.InnerText.Trim().Replace(",", "");
bitcoin_price = double.Parse(bitcoin_price_str);
reader.Close();
Stream.Close();
res.Close();
The reader_str is like this :
https://pastebin.com/jT66GzmA
Why reader_str has wrong html?
How can i fix this problem?

Related

HttpWebRequest - The request was aborted: Could not create SSL/TLS secure channel

I want to try get current DateTime like this :
try {
HttpWebRequest req = (HttpWebRequest) WebRequest.Create("http://www.timestampconvert.com/");
req.Method = "GET";
req.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36";
req.ContentType = "text/html; charset=utf-8";
req.Referer = string.Empty;
req.KeepAlive = true;
req.Timeout = 25000;
//req.Proxy = proxy;
HttpWebResponse res = (HttpWebResponse) req.GetResponse();
Stream Stream = res.GetResponseStream();
StreamReader reader = new StreamReader(Stream);
string reader_str = reader.ReadToEnd();
HtmlAgilityPack.HtmlDocument htmlDoc = new HtmlAgilityPack.HtmlDocument();
htmlDoc.LoadHtml(reader_str);
var divs = from div in htmlDoc.DocumentNode.Descendants("div")
select div;
DateTime dt = DateTime.Now;
foreach(var div in divs) {
if (div.Attributes["class"] != null) {
if (div.Attributes["class"].Value == "remarkrow") {
if (div.InnerText.Contains("Computation based on input date")) {
CultureInfo cultureinfo = new CultureInfo("en-US");
dt = Convert.ToDateTime(div.InnerText.Replace("\n\t*)Computation based on input date ", string.Empty), cultureinfo);
dt = dt.AddHours(2).AddMinutes(30);
break;
}
}
}
}
DateTime dt_ = dt;
}
catch(Exception ex) {
MessageBox.Show(ex.ToString());
}
But it has an exception like below :
The request was aborted: Could not create SSL/TLS secure channel
How can i fix this error?

Sign up instagram Using C#

so im working on an instagram tool. i need to Create an account with HttpWebRequest but somethings wrong with my code and it doesnt work here is the Code:
username = "something";
password = "something";
mail = " something#mail.com";
name = "something";
string postData = "email=" + mail + "&password=" + password + "&enc_password=%23PWD_INSTAGRAM_BROWSER%3A6%3A1583227313%3AAXFQANTXAGE5jwEKNbpJbvot0SGTp%2Bq7a0ckELnuYQnQLIJa2Th6UwIqdknx%2FDa8R7q1%2F2Bt4scBUrh%2B1aFDKL0H%2Fut5cyi3w1sIEPa1keAuNcNj9nAXo1oFzUYMhCGVB7qerse5hQfMjTVjNa4%3D&username=" + username + "&first_name=" + name + "&seamless_login_enabled=1&tos_version=row&opt_into_one_tap=false";
HttpWebRequest req = (HttpWebRequest)WebRequest.Create("https://www.instagram.com/accounts/emailsignup/");
byte[] postBytes = Encoding.ASCII.GetBytes(postData);
req.ContentLength = postBytes.Length;
req.CookieContainer = new CookieContainer();
req.Method = "POST";
req.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36";
req.Accept = "*/*";
req.ContentType = "application/x-www-form-urlencoded";
req.Referer = "https://www.instagram.com/accounts/emailsignup/";
req.Headers["Accept-Language"] = "de,en-US;q=0.7,en;q=0.3";
req.Headers["Accept-Encoding"] = "gzip, deflate";
req.Headers["X-Requested-With"] = "XMLHttpRequest";
Stream dataStream = req.GetRequestStream();
dataStream.Write(postBytes, 0, postBytes.Length);
dataStream.Flush();
dataStream.Close();
HttpWebResponse webResp = (HttpWebResponse)req.GetResponse(); //error in this line---> The remote server returned an error: (403) Forbidden.
Stream datastream = webResp.GetResponseStream();
StreamReader reader = new StreamReader(datastream);
string s = reader.ReadToEnd();
Console.WriteLine(s);

HttpWebRequest - cannot send a content-body with this verb-type

I have an application written in C# that use HttpWebRequest to send the request with body.
To add text to request I use this code:
WebRequest webRequest = WebRequest.Create(responseUri);
((HttpWebRequest)webRequest).Referer = responseUri;
((HttpWebRequest)webRequest).Host = "sts.mycompany.com";
((HttpWebRequest)webRequest).KeepAlive = true;
((HttpWebRequest)webRequest).AllowAutoRedirect = true;
((HttpWebRequest)webRequest).UseDefaultCredentials = true;
((HttpWebRequest)webRequest).UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36";
webRequest.ContentType = "application/x-www-form-urlencoded";
((HttpWebRequest)webRequest).Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
((HttpWebRequest)webRequest).Headers.Add("Cache-Control", "max-age=0");
((HttpWebRequest)webRequest).Headers.Add("Accept-Encoding", "gzip, deflate, br, peerdist");
((HttpWebRequest)webRequest).Headers.Add("Accept-Language", "en-US,en;q=0.5");
//((HttpWebRequest)webRequest).Headers.Add("Accept-Charset", "ISO-8859-2,utf-8;q=0.7,*;q=0.3");
((HttpWebRequest)webRequest).Headers.Add("Upgrade-Insecure-Requests", #"1");
((HttpWebRequest)webRequest).Headers.Add("DNT", #"1");
((HttpWebRequest)webRequest).CookieContainer = CookieContainer;
foreach (var cookie in CookieContainer.GetCookies(new Uri(responseUri)))
{
Console.WriteLine(cookie.ToString());
}
((HttpWebRequest)webRequest).Credentials = GetNetworkCredentials();
((HttpWebRequest)webRequest).AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
webRequest.Method = requestType.ToString();
string msg = String.Format("UserName={0}&Password={1}&AuthMethod=FormsAuthentication", System.Net.WebUtility.UrlEncode(username), System.Net.WebUtility.UrlEncode(password));
webRequest.ContentLength = msg.Length;
Stream reqStream = webRequest.GetRequestStream();
byte[] msgb = System.Text.Encoding.UTF8.GetBytes(msg);
reqStream.Write(msgb, 0, msgb.Length);
reqStream.Close();
var response = (HttpWebResponse)webRequest.GetResponse();
StreamReader sr = new StreamReader(response.GetResponseStream());
string Result = sr.ReadToEnd();
var originalString = response.ResponseUri.OriginalString;
response.Close();
The error is in this line, when I want to add text to body:
byte[] msgb = System.Text.Encoding.UTF8.GetBytes(msg);
reqStream.Write(msgb, 0, msgb.Length);
error: Cannot send a content-body with this verb-type
and any additional informations.

C# - HttpWebRequest Get Data Next Page(s) Issue

Im try to extract data from a website with httpwebrequest. Im able to extract data from the first "Page" but when i try to get data to the next or previous page i get a error.
Next Page Action - Click on the number page
To see the Post parameters i use Fiddler to compare the Post Request Header from my Code and the website.
Post Request - Header 2ºPage
My actual C# code its this:
string URI = "https://www.citius.mj.pt/Portal/consultas/ConsultaVendaAnuncios.aspx";
CookieContainer cc = new CookieContainer();
HttpWebRequest h = (HttpWebRequest)WebRequest.Create(URI);
h.CookieContainer = cc;
HttpWebResponse hr = (HttpWebResponse)h.GetResponse();
using (var s = hr.GetResponseStream())
{
using (var r = new StreamReader(s, Encoding.UTF8))
{
string html = r.ReadToEnd();
Match mVS = Regex.Match(html, "id=\"__VIEWSTATE\" value=\"(.*?)\"", RegexOptions.Singleline);
Match mEV = Regex.Match(html, "id=\"__EVENTVALIDATION\" value=\"(.*?)\"", RegexOptions.Singleline);
///////////////////////////////////////SEARCH///////////////////////////////////////////////////
Dictionary<string, string> param = new Dictionary<string, string>();
param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$ScriptManager1|ctl00$ContentPlaceHolder1$btnSearch");
param.Add("__EVENTTARGET", "");
param.Add("__EVENTARGUMENT", "");
param.Add("__LASTFOCUS", "");
param.Add("__VIEWSTATE", mVS.Groups[1].Value);
param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
param.Add("__ASYNCPOST", "true");
param.Add("ctl00$ContentPlaceHolder1$btnSearch", "Pesquisar");
string post = "";
foreach (var i in param)
{
post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
}
var data = Encoding.UTF8.GetBytes(post);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URI);
request.CookieContainer = cc;
request.Referer = URI;
request.Method = "POST";
request.Headers.Add("Cache-Control", "no-cache");
request.Accept = "*/*";
request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
request.ContentLength = data.Length;
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
request.Headers.Add("Origin", "https://www.citius.mj.pt");
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
request.Headers.Add("X-MicrosoftAjax: Delta=true");
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
int x = 1;
////////////////////////////////////////////NEXT PAGE//////////////////////////////////////////
param.Clear();
param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$upResultados|ctl00$ContentPlaceHolder1$Pager1$lnkNext");
param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
param.Add("__EVENTTARGET", "ctl00$ContentPlaceHolder1$Pager1$lnkNext");
param.Add("__EVENTARGUMENT", "");
param.Add("__LASTFOCUS", "");
param.Add("__VIEWSTATE", mVS.Groups[1].Value);
param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
param.Add("__ASYNCPOST", "true");
post = "";
foreach (var i in param)
{
post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
}
data = Encoding.UTF8.GetBytes(post);
request = (HttpWebRequest)WebRequest.Create(URI);
request.CookieContainer = cc;
request.Referer = URI;
request.Method = "POST";
request.Headers.Add("Cache-Control", "no-cache");
request.Accept = "*/*";
request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
request.ContentLength = data.Length;
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
request.Headers.Add("Origin", "https://www.citius.mj.pt");
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
request.Headers.Add("X-MicrosoftAjax: Delta=true");
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
response = (HttpWebResponse)request.GetResponse();
responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
int y = 1;
}
}
The result i get when i trying to get nextpage data its:
Result from fiddler
Comparing the header between calls from my code and the website the only difference i see its the "_EVENTEVALUATION".
What im doing wrong?

As I suggest, you should split response string.
string URI = "https://www.citius.mj.pt/Portal/consultas/ConsultaVendaAnuncios.aspx";
CookieContainer cc = new CookieContainer();
HttpWebRequest h = (HttpWebRequest)WebRequest.Create(URI);
h.CookieContainer = cc;
HttpWebResponse hr = (HttpWebResponse)h.GetResponse();
using (var s = hr.GetResponseStream())
{
using (var r = new StreamReader(s, Encoding.UTF8))
{
string html = r.ReadToEnd();
Match mVS = Regex.Match(html, "id=\"__VIEWSTATE\" value=\"(.*?)\"", RegexOptions.Singleline);
Match mEV = Regex.Match(html, "id=\"__EVENTVALIDATION\" value=\"(.*?)\"", RegexOptions.Singleline);
///////////////////////////////////////SEARCH///////////////////////////////////////////////////
Dictionary<string, string> param = new Dictionary<string, string>();
param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$ScriptManager1|ctl00$ContentPlaceHolder1$btnSearch");
param.Add("__EVENTTARGET", "");
param.Add("__EVENTARGUMENT", "");
param.Add("__LASTFOCUS", "");
param.Add("__VIEWSTATE", mVS.Groups[1].Value);
param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
param.Add("__ASYNCPOST", "true");
param.Add("ctl00$ContentPlaceHolder1$btnSearch", "Pesquisar");
string post = "";
foreach (var i in param)
{
post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
}
var data = Encoding.UTF8.GetBytes(post);
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(URI);
request.CookieContainer = cc;
request.Referer = URI;
request.Method = "POST";
request.Headers.Add("Cache-Control", "no-cache");
request.Accept = "*/*";
request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
request.ContentLength = data.Length;
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
request.Headers.Add("Origin", "https://www.citius.mj.pt");
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
request.Headers.Add("X-MicrosoftAjax: Delta=true");
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
File.WriteAllText("D:\\tmp1.txt", responseString);
////////////////////////////////////////////NEXT PAGE//////////////////////////////////////////
param.Clear();
string[] split = responseString.Split('|');
for (int i = 0; i < split.Length; i++)
{
if (split[i] == "__VIEWSTATE") param.Add("__VIEWSTATE", split[i + 1]);
if (split[i] == "__EVENTVALIDATION") param.Add("__EVENTVALIDATION", split[i + 1]);
}
param.Add("ctl00$ContentPlaceHolder1$ScriptManager1", "ctl00$ContentPlaceHolder1$upResultados|ctl00$ContentPlaceHolder1$Pager1$lnkNext");
param.Add("ctl00$ContentPlaceHolder1$rbtlTribunais", "False");
param.Add("ctl00$ContentPlaceHolder1$ddlTribunais", "- Todos os Tribunais -");
param.Add("ctl00$ContentPlaceHolder1$rblDias", "todos");
param.Add("__EVENTTARGET", "ctl00$ContentPlaceHolder1$Pager1$lnkNext");
param.Add("__EVENTARGUMENT", "");
param.Add("__LASTFOCUS", "");
//param.Add("__VIEWSTATE", mVS.Groups[1].Value);
//param.Add("__EVENTVALIDATION", mEV.Groups[1].Value);
param.Add("__ASYNCPOST", "true");
post = "";
foreach (var i in param)
{
post += WebUtility.UrlEncode(i.Key) + "=" + WebUtility.UrlEncode(i.Value) + "&";
}
data = Encoding.UTF8.GetBytes(post);
request = (HttpWebRequest)WebRequest.Create(URI);
request.CookieContainer = cc;
request.Referer = URI;
request.Method = "POST";
request.Headers.Add("Cache-Control", "no-cache");
request.Accept = "*/*";
request.Headers.Add("Accept-Language", "pt-PT,pt;q=0.8,en-US;q=0.6,en;q=0.4,es;q=0.2");
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
request.ContentLength = data.Length;
request.ContentType = "application/x-www-form-urlencoded; charset=UTF-8";
request.Headers.Add("Origin", "https://www.citius.mj.pt");
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
request.Headers.Add("X-MicrosoftAjax: Delta=true");
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
response = (HttpWebResponse)request.GetResponse();
responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
File.WriteAllText("D:\\tmp2.txt", responseString);
int y = 1;
}
}

HttpWebRequest SSL Authorization

How login to the resource with SSL? I sent data with help HttpWebRequest on url. In half of cases return redirect to login page and dont't get cookies.
Maybe, make request on 443 port? If yes, how this make with help HttpWebRequest?
ServicePointManager
.ServerCertificateValidationCallback +=
(sender, cert, chain, sslPolicyErrors) => true;
const string root = #"https://****/";
var loginUrl = string.Format(#"{0}csologin/login.jsf", root);
var login = Settings.Default.PacerLogin;
var pass = Settings.Default.PacerPassword;
var postData =
string.Format(
#"login=login&login:loginName={0}&login:password={1}&login:clientCode=&login:j_idt139=&javax.faces.ViewState=stateless",
login, pass);
var request = (HttpWebRequest)WebRequest.Create(loginUrl);
request.CookieContainer = this.cookieContainer;
var ascii = new ASCIIEncoding();
byte[] postBytes = ascii.GetBytes(postData);
request.Proxy = null;
request.Credentials = new NetworkCredential(login, pass);
request = SetHeaders(request, loginUrl);
request.ContentLength = postBytes.Length;
request.Method = WebRequestMethods.Http.Post;
Stream postStream = request.GetRequestStream();
postStream.Write(postBytes, 0, postBytes.Length);
postStream.Flush();
postStream.Close();
And Headers
request.UserAgent =
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.85 Safari/537.36";
request.Headers.Add("Accept-Language", "en-GB,en-US;q=0.8,en;q=0.6,ru;q=0.4");
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
request.Headers.Add("Accept-Encoding", "gzip, deflate");
request.KeepAlive = true;
request.Headers["Cache-Control"] = "max-age=0";
request.AllowAutoRedirect = allowRedirect;
request.Connection =
request.Referer = referer;
request.ContentType = "application/x-www-form-urlencoded";
request.ServicePoint.Expect100Continue = false;

Categories

Resources