I have written some code to check to see if all websites in my database are still hosted and online.
The problem is some of these sites seem to have bot protection and whenever I try to request then via HttpClient they raise an error instead of displaying the page.
I have seen other similar questions that suggest to add in browser headers so I have done this but this does not help. The same sites still reject the HttpClient connection but are perfectly fine when I view them in the browser.
Have I done something wrong with my code or do I need some additional steps?
Here is my code:
public static async Task CheckSite(string url, int id)
{
try
{
using(var db = new PlaceDBContext())
using (HttpClient client = new HttpClient(new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip
}))
using (HttpResponseMessage response = await client.GetAsync(url))
using (HttpContent content = response.Content)
{
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate");
client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:19.0) Gecko/20100101 Firefox/19.0");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", "ISO-8859-1");
var rd = db.RootDomains.Find(id);
string result = await content.ReadAsStringAsync();
if (result != null && result.Length >= 50)
{
Console.WriteLine("fine");
rd.LastCheckOnline = true;
}
else
{
Console.WriteLine("There was empty or short result");
rd.LastCheckOnline = false;
}
db.SaveChanges();
semaphore.Release();
}
}
catch(Exception ex)
{
Console.WriteLine(ex.Message);
using(var db = new PlaceDBContext())
{
var rd = db.RootDomains.Find(id);
rd.LastCheckOnline = false;
db.SaveChanges();
semaphore.Release();
}
}
}
Set the headers before sending the request. You are doing them after already getting a response
public static async Task CheckSite(string url, int id) {
try {
using (var db = new PlaceDBContext())
using (var client = new HttpClient(new HttpClientHandler() {
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip
})) {
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate");
client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:19.0) Gecko/20100101 Firefox/19.0");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Charset", "ISO-8859-1");
using (var response = await client.GetAsync(url))
using (var content = response.Content) {
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml");
var rd = db.RootDomains.Find(id);
string result = await content.ReadAsStringAsync();
if (result != null && result.Length >= 50) {
Console.WriteLine("fine");
rd.LastCheckOnline = true;
} else {
Console.WriteLine("There was empty or short result");
rd.LastCheckOnline = false;
}
db.SaveChanges();
semaphore.Release();
}
}
} catch (Exception ex) {
Console.WriteLine(ex.Message);
using (var db = new PlaceDBContext()) {
var rd = db.RootDomains.Find(id);
rd.LastCheckOnline = false;
db.SaveChanges();
semaphore.Release();
}
}
}
Related
I am trying to make web scraper in C# for NSE. The code works with other sites but when ran on https://www.nseindia.com/ it gives error - An error occurred while sending the request. Unable to read data from the transport connection: Operation timed out.
I have tried with two different approaches Try1() & Try2().
Can anyone please tell what I am missing in my code?
class Program
{
public void Try1() {
HtmlWeb web = new HtmlWeb();
HttpStatusCode statusCode = HttpStatusCode.OK;
web.UserAgent = GetUserAgent();
web.PostResponse = (request, response) =>
{
if (response != null)
{
statusCode = response.StatusCode;
Console.WriteLine("Status Code: " + statusCode);
}
};
Task<HtmlDocument> task = web.LoadFromWebAsync(GetURL());
HtmlDocument document = task.Result;
}
public void Try2() {
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(GetURL());
request.UserAgent = GetUserAgent();
request.Accept= "*/*;";
using (var response = (HttpWebResponse)(request.GetResponse()))
{
HttpStatusCode code = response.StatusCode;
if (code == HttpStatusCode.OK)
{
using (StreamReader streamReader = new StreamReader(response.GetResponseStream(), Encoding.UTF8))
{
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.Load(streamReader);
Console.WriteLine("Document Loaded.");
}
}
}
}
private string GetURL() {
// return "https://html-agility-pack.net/";
return "https://www.nseindia.com/";
}
private string GetUserAgent() {
return "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36";
}
}
Your are lack of headers towards Accept and others so it couldn't response back.
Besides that, I would recommend you using HttpClient instead of HttpWebRequest
public static async Task GetHtmlData(string url)
{
HttpClient httpClient = new HttpClient();
using (var request = new HttpRequestMessage(HttpMethod.Get, new Uri(url)))
{
request.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml, charset=UTF-8, text/javascript, */*; q=0.01");
request.Headers.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate, br");
request.Headers.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36 OPR/67.0.3575.137");
request.Headers.TryAddWithoutValidation("Accept-Charset", "ISO-8859-1");
request.Headers.TryAddWithoutValidation("X-Requested-With", "XMLHttpRequest");
using (var response = await httpClient.SendAsync(request).ConfigureAwait(false))
{
response.EnsureSuccessStatusCode();
using (var responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
using (var decompressedStream = new GZipStream(responseStream, CompressionMode.Decompress))
using (var streamReader = new StreamReader(decompressedStream))
{
var result = await streamReader.ReadToEndAsync().ConfigureAwait(false);
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.OptionFixNestedTags = true;
htmlDoc.LoadHtml(result);
Console.WriteLine(result);
Console.WriteLine("Document Loaded.");
}
}
}
Use it by
await GetHtmlData("https://www.nseindia.com/");
I have written a complex class called DonationListener. This class has a property named onDonation that takes a function that will be called when a donation happens. In a simple console program it works fine, but in the more complex WinForms application - not. I am pretty sure that there is no mistake in the class. So I think the problem is in the usage of it. My class works with WebSockets, so I decided to check with the fiddler if the connection lost after the first donation event, and the answer is no, the connection is fine and sending packages that should invoke onDonation. Here's the code
public string donation_password;
DonationListener dl;
public System.Windows.Forms.PictureBox[,] eliteCards;
public mainForm()
{
InitializeComponent();
initializeDonation();
}
public void initializeDonation()
{
donation_password = db.getPassword();
dl = new DonationListener(donation_password);
dl.OnDonation = donation =>
{
overlay ol = new overlay();
string username = donation["username"].ToString();
User toRide = db.getUserByField(username);
ol.Show();
ol.ride(toRide);
};
Task t = new Task(() => {
dl.DoListen();
});
t.Start();
}
Full class DonationListener(if u need sth from it)
using System;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.WebSockets;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace RacheM
{
public class DonationListener
{
private string addUserTemplate = "69:42[\"add-user\",{{\"token\":\"{0}\",\"type\":\"alert_widget\"}}]";
public Action<JObject> OnDonation = null;
private readonly string _token;
public DonationListener(string token)
{
_token = token;
}
public void DoListen()
{
var result =
DoRequest("GET", "https://socket.donationalerts.ru/socket.io/?EIO=3&transport=polling"); //&t=MYotE0N
var sid = ExtractToken(result);
if (string.IsNullOrEmpty(sid))
{
throw new Exception("Failed to get sid");
}
var cookie = new Cookie("io", sid, "/", "socket.donationalerts.ru");
DoRequest(
"POST",
$"https://socket.donationalerts.ru/socket.io/?EIO=3&transport=polling&sid={sid}",
string.Format(addUserTemplate, _token),
cookie
);
var waiter = DoWebSocket($"wss://socket.donationalerts.ru/socket.io/?EIO=3&transport=websocket&sid={sid}",
cookie);
waiter.GetAwaiter().GetResult();
}
private static string ExtractToken(string strWidthToken)
{
var m = Regex.Match(strWidthToken, "\"sid\":\"(?<token>[^\"]+)\"");
return m.Groups["token"].Value;
}
private static string DoRequest(string method, string url, string data = "", Cookie cookie = null)
{
var httpWebRequest = (HttpWebRequest) WebRequest.Create(url);
httpWebRequest.Method = method;
httpWebRequest.Accept = "*/*";
httpWebRequest.Headers.Add("Origin", "https://www.donationalerts.com");
httpWebRequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate, br");
httpWebRequest.Headers.Add(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.9");
httpWebRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36";
httpWebRequest.Referer = "https://www.donationalerts.com/widget/alerts?group_id=1&token=mcq71m8KVIsojvo5ukFZ";
if (method == "GET")
{
//httpWebRequest.Headers.Remove("Content-Length");
httpWebRequest.ContentLength = 0;
}
else
{
httpWebRequest.ContentType = "text/plain;charset=UTF-8";
}
if (cookie != null)
{
if (httpWebRequest.CookieContainer == null)
{
httpWebRequest.CookieContainer = new CookieContainer();
}
httpWebRequest.CookieContainer.Add(cookie);
}
if (method == "POST" && !string.IsNullOrEmpty(data))
{
using (var streamWriter = new StreamWriter(httpWebRequest.GetRequestStream()))
{
streamWriter.Write(data);
streamWriter.Flush();
streamWriter.Close();
}
}
var httpResponse = (HttpWebResponse) httpWebRequest.GetResponse();
var streamReader = new StreamReader(httpResponse.GetResponseStream());
if(streamReader == null)
{
throw new Exception("Connection error");
}
using (streamReader)
{
return streamReader.ReadToEnd();
}
}
private async Task DoWebSocket(string url, Cookie cookie)
{
using (var ws = new ClientWebSocket())
{
var serverUri = new Uri(url);
ws.Options.Cookies = new CookieContainer();
ws.Options.Cookies.Add(cookie);
await ws.ConnectAsync(serverUri, CancellationToken.None);
var cts = new CancellationTokenSource();
Task.Factory.StartNew(
async () =>
{
while (true)
{
try
{
string srcMessage = string.Empty;
ArraySegment<Byte> buffer = new ArraySegment<byte>(new Byte[8192]);
using (var ms = new MemoryStream())
{
WebSocketReceiveResult result= null;
do
{
result = await ws.ReceiveAsync(buffer, CancellationToken.None);
ms.Write(buffer.Array, buffer.Offset, result.Count);
} while (!result.EndOfMessage);
ms.Seek(0, SeekOrigin.Begin);
if (result.MessageType == WebSocketMessageType.Text)
{
using (var reader = new StreamReader(ms, Encoding.UTF8))
{
srcMessage = reader.ReadToEnd();
}
}
}
if (string.IsNullOrEmpty(srcMessage)) continue;
if (srcMessage.IndexOf("42[\"donation", StringComparison.Ordinal) == 0)
{
OnDonation?.Invoke(ParseDonateMessage(srcMessage));
}
else
{
Console.WriteLine("no donate msg: " + srcMessage);
}
}
catch (Exception ex)
{
Console.WriteLine("Error: " + ex.Message);
}
}
}, cts.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default);
await SendWsMessage(ws, "2probe");
await SendWsMessage(ws, "5");
while (ws.State == WebSocketState.Open)
{
await SendWsMessage(ws, "2");
Thread.Sleep(25000);
}
}
}
private static async Task SendWsMessage(WebSocket ws, string message)
{
var sendBytes = Encoding.UTF8.GetBytes(message);
var cts = new CancellationTokenSource();
var sendBuffer = new ArraySegment<byte>(sendBytes);
await
ws.SendAsync(sendBuffer, WebSocketMessageType.Text, endOfMessage: true,
cancellationToken: cts.Token);
}
private static JObject ParseDonateMessage(string rcvMsg)
{
rcvMsg = rcvMsg.Replace("42[\"donation\",\"", "");
rcvMsg = rcvMsg.Substring(0, rcvMsg.Length - 2);
return (JObject)JsonConvert.DeserializeObject(Regex.Unescape(rcvMsg));
}
}
}
Maybe you should try to make this function repeat itself by:
bool RunFlag = true // can be set to false somewhere else
public void DoListen()
{
while(RunFlag) {
var result =
DoRequest("GET", "https://socket.donationalerts.ru/socket.io/?EIO=3&transport=polling"); //&t=MYotE0N
var sid = ExtractToken(result);
if (string.IsNullOrEmpty(sid))
{
throw new Exception("Failed to get sid");
}
var cookie = new Cookie("io", sid, "/", "socket.donationalerts.ru");
DoRequest(
"POST",
$"https://socket.donationalerts.ru/socket.io/?EIO=3&transport=polling&sid={sid}",
string.Format(addUserTemplate, _token),
cookie
);
var waiter = DoWebSocket($"wss://socket.donationalerts.ru/socket.io/?EIO=3&transport=websocket&sid={sid}",
cookie);
waiter.GetAwaiter().GetResult();
}
}
I'm trying to simulate HTTP post file exactly like the browser (chrome) does:
HTML:
<form name="fnup" method="post" action="action.shtml" enctype="multipart/form-data">
<input name="firmfile" id="firmfile" type="file">
<input type="submit" id="firmbutton" value="Upgrade" class="othsave">
</form>
C#:
public async Task<string> UploadFile(string actionUrl, string filePath)
{
var httpClient = new HttpClient();
//Should I need to add all those headers??? it will help me? what is necessary?
httpClient.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1");
httpClient.DefaultRequestHeaders.Add("Origin", "http://" + ip);
httpClient.DefaultRequestHeaders.Add("Upgrade-Insecure-Requests", "1");
httpClient.DefaultRequestHeaders.Add("Pragma", "no-cache");
httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36");
httpClient.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
httpClient.DefaultRequestHeaders.Add("Referer", "http://" + ip + "/");
httpClient.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
httpClient.DefaultRequestHeaders.Add("Accept-Language", "he,en-US;q=0.9,en;q=0.8");
httpClient.DefaultRequestHeaders.Add("Cookie", "page=DevSet");
FileStream paramFileStream = File.OpenRead(filePath);
HttpContent fileStreamContent = new StreamContent(paramFileStream);
using (var content = new MultipartFormDataContent())
{
content.Add(fileStreamContent, "firmfile", Path.GetFileName(filePath));
HttpResponseMessage response = await httpClient.PostAsync(actionUrl, content);
string res = await response.Content.ReadAsStringAsync();
return await Task.Run(() => res);
}
}
The C# example is not working in my server, and I cannot understand why...
After looking on Wireshark, I've seen some differents:
Chrome:
C#
The questions:
1) How can I delete the "filename*=utf-8''VP445_all_V116.bin" from Content-Disposition
2) Why I cannot see the second line Content-Type: application/octet-stream like in Chrome?
3) Also the Content-length is not the same (even it's the same file)
4) The bottom line is that the C# is not working and chrome is working, and the server side is black-box for me, so I need to guess here what I'm doing wrong in C# post request.
Found the solution.
The problem indeed was the 'Content-Type: application/octet-stream'
Why the HttpClient StreamContent is not added it automatically? I really don't know.
The workaround is added it manually
The main idea is from here:
https://github.com/paulcbetts/ModernHttpClient/issues/92
Here is the correct C# code:
public async Task<string> UploadFile(string actionUrl, string filePath)
{
var httpClient = new HttpClient();
httpClient.Timeout = TimeSpan.FromMilliseconds(FileReqTimeout);
FileStream fileStream = File.OpenRead(filePath);
var streamContent = new StreamContent(fileStream);
streamContent.Headers.ContentDisposition = new ContentDispositionHeaderValue("form-data");
streamContent.Headers.ContentDisposition.Name = "\"firmfile\"";
streamContent.Headers.ContentDisposition.FileName = "\"" + Path.GetFileName(filePath) + "\"";
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
string boundary = Guid.NewGuid().ToString();
var content = new MultipartFormDataContent(boundary);
content.Headers.Remove("Content-Type");
content.Headers.TryAddWithoutValidation("Content-Type", "multipart/form-data; boundary=" + boundary);
content.Add(streamContent);
HttpResponseMessage response = null;
try
{
response = await httpClient.PostAsync(actionUrl, content, cts.Token);
}
catch (WebException ex)
{
// handle web exception
return null;
}
catch (TaskCanceledException ex)
{
if (ex.CancellationToken == cts.Token)
{
// a real cancellation, triggered by the caller
return null;
}
else
{
// a web request timeout (possibly other things!?)
return null;
}
}
try
{
response.EnsureSuccessStatusCode();
}
catch (Exception)
{
return null;
};
string res = await response.Content.ReadAsStringAsync();
return await Task.Run(() => res);
}
}
I am trying to perform a GET request to https://sede.educacion.gob.es/publiventa/catalogo.action?cod=E; with the cod=E parameter, in the browser, the web site open a menu below "Materias de educación", but when I perform the request using C# this menu is not loading and I need it. This is the code I am using to readHtml as string to later parse it with HtmlAgilityPack.
private string readHtml(string urlAddress)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress);
request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0";
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
request.AutomaticDecompression = DecompressionMethods.GZip;
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK)
{
Stream receiveStream = response.GetResponseStream();
StreamReader readStream = null;
if (response.CharacterSet == null)
{
readStream = new StreamReader(receiveStream);
}
else
{
readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
}
string data = readStream.ReadToEnd();
response.Close();
readStream.Close();
return data;
}
return null;
}
The Uri you posted (https://sede.educacion.gob.es/publiventa/catalogo.action?cod=E) uses a Javascript switch to show the Menu content.
When you connect to that Uri (without clicking a menu link), that site shows three different versions of that page.
1) Page with closed menu and proposed new editions
2) Page with closed menu and search engine fields
3) Page with open menu and a selection of the menu content
This switch is based on a internal procedure which records the current session. Unless you click on a menu link (which is connected to an event listener), the Javascript proc shows the page in different states.
I gave it a look; those script are quite long (a whole multi-purpose library) and I had no time to parse it all (may be you can do that) to find out what parameters the event listener is passing.
But, the three-state version switch is constant.
What I mean is you can call that page three times, preserving the Cookie Container: the third time you connect to it, it will stream the whole menu content and its links.
If you request three times the same page, the third time the Html page will
contain all theMaterias de educación links
public async void SomeMethodAsync()
{
string HtmlPage = await GetHttpStream([URI]);
HtmlPage = await GetHttpStream([URI]);
HtmlPage = await GetHttpStream([URI]);
}
This is, more or less, what I used to get that page:
CookieContainer CookieJar = new CookieContainer();
public async Task<string> GetHttpStream(Uri HtmlPage)
{
HttpWebRequest httpRequest;
string Payload = string.Empty;
httpRequest = WebRequest.CreateHttp(HtmlPage);
try
{
httpRequest.CookieContainer = CookieJar;
httpRequest.KeepAlive = true;
httpRequest.ConnectionGroupName = Guid.NewGuid().ToString();
httpRequest.AllowAutoRedirect = true;
httpRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
httpRequest.ServicePoint.MaxIdleTime = 30000;
httpRequest.ServicePoint.Expect100Continue = false;
httpRequest.UserAgent = "Mozilla/5.0 (Windows NT 10; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0";
httpRequest.Accept = "ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
httpRequest.Headers.Add(HttpRequestHeader.AcceptLanguage, "es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3");
httpRequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate;q=0.8");
httpRequest.Headers.Add(HttpRequestHeader.CacheControl, "no-cache");
using (HttpWebResponse httpResponse = (HttpWebResponse)await httpRequest.GetResponseAsync())
{
Stream ResponseStream = httpResponse.GetResponseStream();
if (httpResponse.StatusCode == HttpStatusCode.OK)
{
try
{
//ResponseStream.Position = 0;
Encoding encoding = Encoding.GetEncoding(httpResponse.CharacterSet);
using (MemoryStream _memStream = new MemoryStream())
{
if (httpResponse.ContentEncoding.Contains("gzip"))
{
using (GZipStream _gzipStream = new GZipStream(ResponseStream, System.IO.Compression.CompressionMode.Decompress))
{
_gzipStream.CopyTo(_memStream);
};
}
else if (httpResponse.ContentEncoding.Contains("deflate"))
{
using (DeflateStream _deflStream = new DeflateStream(ResponseStream, System.IO.Compression.CompressionMode.Decompress))
{
_deflStream.CopyTo(_memStream);
};
}
else
{
ResponseStream.CopyTo(_memStream);
}
_memStream.Position = 0;
using (StreamReader _reader = new StreamReader(_memStream, encoding))
{
Payload = _reader.ReadToEnd().Trim();
};
};
}
catch (Exception)
{
Payload = string.Empty;
}
}
}
}
catch (WebException exW)
{
if (exW.Response != null)
{
//Handle WebException
}
}
catch (System.Exception exS)
{
//Handle System.Exception
}
CookieJar = httpRequest.CookieContainer;
return Payload;
}
I am trying to make a web proxy. Here is what I have so far:
IPHostEntry IPHost = Dns.GetHostEntry(sURL);
Console.WriteLine("Resolved:{0}", IPHost.HostName);
string[] aliases = IPHost.Aliases;
IPAddress[] address = IPHost.AddressList;
Console.WriteLine(address[0]);
IPEndPoint sEndpoint = new IPEndPoint(address[0], 80);
Socket IPsocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
IPsocket.Connect(sEndpoint);
if (IPsocket.Connected)
{
Console.WriteLine("Socket OK");
}
NetworkStream ns = new NetworkStream(IPsocket);
StreamWriter sw = new StreamWriter(ns);
StreamReader sr = new StreamReader(ns);
for (int i = 0; i < lista.Count; i++)
{
sw.WriteLine(lista[i]);
Console.WriteLine(lista[i]);
}
sw.Flush();
string response = sr.ReadToEnd();</pre>
And how I read the request:
StreamReader sr = new StreamReader(s);
string plusz = "";
plusz = sr.ReadLine();
while (plusz != "")
{
lista.Add(plusz);
plusz = sr.ReadLine();
}
return lista;
The request looks like this:
GET http://google.com/ HTTP/1.1
Host: google.com
Proxy-Connection: keep-alive
Cache-Control: max-age=0
User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
Accept-Encoding: gzip,deflate,sdch
Accept-Language: hu-HU,hu;q=0.8,en-US;q=0.6,en;q=0.4
Accept-Charset: ISO-8859-2,utf-8;q=0.7,*;q=0.3
Cookie: rememberme=true; NID=54=l
(...)
pY
And as you can see I sent this exactly. The problem is that the program stops at the sr.ReadToEnd() method. It is just waiting for the data to arrive, but nothing happens. If I send a wrong request, then it works, so the browser displays the wrong request page (400).
namespace ProxyTester
{
class Program
{
static void Main(string[] args)
{
var htmlResponse = new StringBuilder();
var RequestPage = BuildHttpRequest("https://google.com/");
GetHttpResponse(RequestPage, htmlResponse);
}
public static HttpWebRequest BuildHttpRequest(string url)
{
try
{
var getPage = (HttpWebRequest)WebRequest.Create(url);
WebProxy proxyHTTP = new WebProxy("201.38.194.50", 53128);
getPage.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
getPage.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.19) Gecko/20110707 Firefox/3.6.19";
getPage.ProtocolVersion = HttpVersion.Version11;
getPage.Method = "GET";
getPage.Proxy = proxyHTTP;
getPage.Timeout = 10000;
getPage.ReadWriteTimeout = 10000;
return getPage;
}
catch (WebException ex)
{
Console.WriteLine(ex.ToString());
}
return null;
}
public static bool GetHttpResponse(HttpWebRequest page, StringBuilder htmlResponse)
{
htmlResponse.Length = 0;
try
{
Console.WriteLine("A");
// var pageResponse = page.GetResponse();
page.Timeout = 10000;
var pageResponse = (HttpWebResponse)page.GetResponse();
Console.WriteLine("5 minutes!");
if (pageResponse.StatusCode == HttpStatusCode.OK)
{
var reader = new StreamReader(pageResponse.GetResponseStream());
htmlResponse.Append(reader.ReadToEnd());
pageResponse.Close();
reader.Close();
return true;
}
Console.WriteLine(pageResponse.StatusCode.ToString());
pageResponse.Close();
return false;
}
catch (WebException ex)
{
Console.WriteLine(ex.ToString());
}
return false;
}
}
}