I'm trying to post a comment on Youtube and get the xml response (so I can get the comment ID, or if I need to enter captcha or something), but I'm only able to post the comment. For some reason I am not able to read the xml response by using response.GetResponseStream(). When I try to output the response to the console, I get nothing. And, I've sniffed the requests and response my program sends and receives using Wireshark and I can see that the xml is in the response.
Here is the code I'm using to read the response:
using (HttpWebResponse response = MakeRequest(request, cookies, post))
{
using (var reader = new System.IO.StreamReader(response.GetResponseStream(), UTF8Encoding.UTF8))
{
string xml = reader.ReadToEnd();
Console.WriteLine(xml);
}
}
and the MakeRequest function
private static HttpWebResponse MakeRequest(HttpWebRequest request, CookieContainer SessionCookieContainer, Dictionary<string, string> parameters = null)
{
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.52 Safari/536.5Accept: */*";
request.Accept = "text/html,text/xml,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
request.CookieContainer = SessionCookieContainer;
request.AllowAutoRedirect = false;
request.KeepAlive = true;
if (proxy != "") request.Proxy = myproxy;
if (parameters != null) request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
string postData = string.Empty;
if (parameters != null)
{
postData = getPostData(parameters);
byte[] postBuffer = UTF8Encoding.UTF8.GetBytes(postData);
using (Stream postStream = request.GetRequestStream())
{
postStream.Write(postBuffer, 0, postBuffer.Length);
}
}
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
SessionCookieContainer.Add(response.Cookies);
while (response.StatusCode == HttpStatusCode.Found)
{
response.Close();
request = GetNewRequest(response.Headers["Location"], SessionCookieContainer);
response = (HttpWebResponse)request.GetResponse();
SessionCookieContainer.Add(response.Cookies);
}
return response;
}
Any ideas on why this isn't working and how to solve this issue?
I think switching to HttpClient would solve the problem.
Related
The error says:
"The remote server returned an error:
(http://www.tgv.com.my/movies/man-city-v-arsenal-HO00005174)
Forbidden"
below is my code:
string url = https://translate.google.com/translate_a/single?client=t&sl=en&tl=vi&hl=vi&dt=bd&dt=ex&dt=ld&dt=md&dt=qca&dt=rw&dt=rm&dt=ss&dt=t&dt=at&ie=UTF-8&oe=UTF-8&otf=1&srcrom=1&ssel=0&tsel=0&kc=5&tk=520987|10880&q=" + keyword;
var request = (HttpWebRequest)WebRequest.Create(url);
WebProxy proxy = (WebProxy)WebProxy.GetDefaultProxy();
if (proxy.Address != null)
{
proxy.Credentials = proxy.Credentials = new NetworkCredential("username", "pw");
WebRequest.DefaultWebProxy = new System.Net.WebProxy(proxy.Address, proxy.BypassProxyOnLocal, proxy.BypassList, proxy.Credentials);
}
request.Proxy = proxy;
var postData = "";
var data = Encoding.ASCII.GetBytes(postData);
request.Method = "POST";
request.ContentType = "text/html; charset=UTF-8";
request.ContentLength = data.Length;
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36";
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
using (var stream = request.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
var response = (HttpWebResponse)request.GetResponse();
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
return responseString;
Thanks!
Using your Url, it's not a POST request, it's a GET request and could be done like this:
string url = "https://translate.google.com/translate_a/single?client=t&sl=de&tl=en&hl=de&dt=at&dt=bd&dt=ex&dt=ld&dt=md&dt=qca&dt=rw&dt=rm&dt=ss&dt=t&ie=UTF-8&oe=UTF-8&otf=2&ssel=0&tsel=0&kc=4&tk=767774.885916&q=hallo%20du";
var request = (HttpWebRequest)WebRequest.Create(url);
var response = (HttpWebResponse)request.GetResponse();
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
But if your q should have another value, other values must be changed too, otherwise you will have the Error 403 whitch tells you, that you do not have permission to do your request.
Using Google Translate, consider having a look at the Google Translate API.
There you can do your request like that:
https://www.googleapis.com/language/translate/v2?key=YOUR_API_KEY&q=hello%20world&source=en&target=de
But this is a payd service...
When making a specific GET request, it seems there is a chunk of html missing from the center of the html response. I can't seem to figure out how its being removed. If you run the sample code and record it with Wireshark. You'll see that Wireshark has more html data than the result in the sample has. (Search for 'french' in both; you'll notice that it is missing from the sample yet it shows up in wireshark).
static string ReadHTML(string urlAddress)
{
urlAddress = "http://www.ebay.com/sch/Dress-Shirts-/57991/i.html?_fln=1&_dmd=1&_ipg=200&_from=R40&_dcat=57991&LH_ItemCondition=3000&_nkw=&LH_Complete=1&LH_Sold=1";
HttpWebRequest request = WebRequest.Create(urlAddress) as HttpWebRequest;
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
request.UserAgent = "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1";
request.Host = "www.ebay.com";
request.Headers.Add("Accept-Encoding", "gzip, deflate, sdch");
request.Headers.Add("Accept-Language", "en-US,*");
//request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
request.CookieContainer = new CookieContainer();
request.CookieContainer.Add(new System.Net.Cookie("JSESSIONID", "989EDC0DF3234BF32A2544E4FA96C314") { Domain = request.Host });//JSESSIONID=989EDC0DF3234BF32A2544E4FA96C314
request.CookieContainer.Add(new System.Net.Cookie("npii", "btguid/4948662a1540a1c448711d0effdd49bb58fe33af^cguid/4948bab11540a56603d04eb0fd0d7c0c58fe33af^") { Domain = request.Host });//npii=btguid/4948662a1540a1c448711d0effdd49bb58fe33af^cguid/4948bab11540a56603d04eb0fd0d7c0c58fe33af^
request.CookieContainer.Add(new System.Net.Cookie("ebay", "%5Esbf%3D%2320000000000000000000210%5E") { Domain = request.Host });//ebay=%5Esbf%3D%2320000000000000000000210%5E
request.CookieContainer.Add(new System.Net.Cookie("ns1", "=BAQAAAVQr0QFlAAaAANgASVkAJ2pjNjl8NjAxXjE0NjE1MTgzNjIxNjleXjFeM3wyfDV8NHw3XjFeMl40XjNeMTJeMTJeMl4xXjFeMF4xXjBeMV42NDQyNDU5MDc17HliWNBjA7nLq7R3ubPYrU6hFOk*") { Domain = request.Host });
request.CookieContainer.Add(new System.Net.Cookie("dp1", "bu1p/QEBfX0BAX19AQA**5900276a^bl/US5ae15aea^") { Domain = request.Host });
request.CookieContainer.Add(new System.Net.Cookie("nonsession", "CgADLAAFXHvryMwDKACBghPVqNDk0ODY2MmExNTQwYTFjNDQ4NzExZDBlZmZkZDQ5YmI2iA27") { Domain = request.Host });
request.CookieContainer.Add(new System.Net.Cookie("s", "CgAD4ACBXIEVqNTBkYjAwODExNTQwYTU2MTc4NDcxNWIyZmZjMzkyNWIA7gCVVyBFajMGaHR0cDovL3d3dy5lYmF5LmNvbS9zY2gvRHJlc3MtU2hpcnRzLS81Nzk5MS9pLmh0bWw/X2Zsbj0xJl9kbWQ9MSZfaXBnPTIwMCZfZnJvbT1SNDAmX2RjYXQ9NTc5OTEmTEhfSXRlbUNvbmRpdGlvbj0zMDAwJl9ua3c9JkxIX0NvbXBsZXRlPTEmTEhfU29sZD0xGTda/A**") { Domain = request.Host });
if (request == null)
return "";
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
if (response == null)
return "";
if (response.StatusCode != HttpStatusCode.OK)
return "";
Stream s2 = response.GetResponseStream();
if (response.ContentEncoding.ToLower().Contains("gzip"))
s2 = new GZipStream(s2, CompressionMode.Decompress);
else if (response.ContentEncoding.ToLower().Contains("deflate"))
s2 = new DeflateStream(s2, CompressionMode.Decompress);
StreamReader sr = null;
if (response.CharacterSet == null)
{
sr = new StreamReader(s2);
}
else
{
sr = new StreamReader(s2, Encoding.GetEncoding(response.CharacterSet));
}
string data = sr.ReadToEnd();
response.Close();
return data;
}
You can also simulate the GET request through PostMan (plugin for chrome) and see the exact same html response that Wireshark is getting from the request which is different from the result in the sample code.
Looks like the 'Text Visualizer' in Visual Studio was showing an incomplete response. The data was always there. Just the debugger was wrong.
Now I know not to trust it.
I'm trying to download this json file to read out some data:
http://ddragon.leagueoflegends.com/cdn/5.7.2/data/en_US/item.json
My problem is now that somehow always the values in the tags attribute is missing.
I already used different methods to retrieve them used different encodings but nothing helped. Also the content length is less than the one which I receive in the browser.
Below are some codes I already tried. As you can see I used the WebClient to download this as a string. Also tried to download it as a file, same result.
Used HttpWebRequest with all header attributes.
Used the HEAD method to receive the content length which is less than in the browser.
I also tried using the MemoryStream and the StreamReader to get the correct data. Nothing worked.
String url = "http://ddragon.leagueoflegends.com/cdn/" + version + "/data/en_US/item.json";
//String json = new WebClient().DownloadString(url);
String json = "";
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36";
request.ContentType = "text/plain; charset=utf-8 ";
request.Accept = "*/*";
request.Method = "GET";
//Get the headers associated with the request.
WebHeaderCollection myWebHeaderCollection = request.Headers;
myWebHeaderCollection.Add("DNT", "1");
myWebHeaderCollection.Add("Accept-Encoding", "gzip, deflate, sdch");
myWebHeaderCollection.Add("Accept-Language", "de-DE,de;q=0.8,en-US;q=0.6,en;q=0.4");
HttpWebRequest requestHeader = ((HttpWebRequest) WebRequest.Create(url));
requestHeader.Method = "HEAD";
using (WebResponse resp = requestHeader.GetResponse())
{
Console.WriteLine(resp.ContentLength);
}
Stream responseStream;
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
using (responseStream = response.GetResponseStream())
{
if (responseStream != null)
{
Console.WriteLine(response.ContentLength);
Console.WriteLine(response.ContentType);
StreamReader readStream = null;
if (string.IsNullOrEmpty(response.CharacterSet))
{
readStream = new StreamReader(responseStream);
}
else
{
readStream = new StreamReader(responseStream, Encoding.GetEncoding(response.CharacterSet));
}
json = readStream.ReadToEnd();
readStream.Close();
//using (var memoryStream = new MemoryStream())
//{
// responseStream.CopyTo(memoryStream);
// json = Encoding.UTF8.GetString(memoryStream.ToArray());
//}
}
}
Edit:
Sample Outputs:
Browser: http://pastebin.com/nQBj64L0
C# Client: http://pastebin.com/7a0Gxvcg
I got a problem on posting data by using HttpWebRequest.
There is a string(ie. key1=value1&key2=value2&key3=value3) and I have post it to a site (ie. www.*.com/edit), but ,I don't know why that sometimes it's nothing wrong , but sometimes ,the first key=value1 will be missing, only key2=value&key3=value3 that can find in HttpAnalyzer.
public static string SubmitData(string Url, string FormData, CookieContainer _Cc, string ContentType)
{
Stream RequestStream = null, ResponseStream = null; StreamReader Sr = null;
HttpWebRequest HRequest = (HttpWebRequest)WebRequest.Create(Url);
try
{
HRequest.CookieContainer = _Cc;
HRequest.Method = "POST";
HRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)";
HRequest.ContentType = ContentType;
HRequest.ContentLength = FormData.Length;
//byte[] BFromData = new ASCIIEncoding().GetBytes(FormData);
byte[] BFromData = Encoding.ASCII.GetBytes(FormData);
BFromData = Encoding.Convert(Encoding.ASCII, Encoding.UTF8, BFromData);//ascii → utf8
RequestStream = HRequest.GetRequestStream();
RequestStream.Write(BFromData, 0, BFromData.Length);
//RequestStream.Write(utf8Bytes,0,utf8Bytes.Length );
HttpWebResponse HResponse = (HttpWebResponse)HRequest.GetResponse();
ResponseStream = HResponse.GetResponseStream();
Sr = new StreamReader(ResponseStream, Encoding.UTF8);
return Sr.ReadToEnd();
}
catch
{
return "";
}
finally
{
if (null != RequestStream) RequestStream.Close();
if (null != ResponseStream) ResponseStream.Close();
if (null != Sr) Sr.Close();
}
}
Use Fiddler to see how the request looks like when you click on the form then try using this approach and modify what you need for your request.
public static void PostDataAndDoSomething()
{
string URI = "http://www.something.com";
//make your request payload
string requestBody = String.Format("{{'param1': {0}, 'param2': {1}}}",value1, value2); //json format
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(URI); //make request
// set request headers as you need
request.ContentType = "application/json; charset=UTF-8";
request.Accept = "application/json, text/javascript;
request.Method = "POST";
request.UserAgent = "";
request.Headers.Add("X-Requested-With", "XMLHttpRequest");
using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write(requestBody); //write your request payload
}
WebResponse response = request.GetResponse();
string jsonData = String.Empty;
using (var reader = new StreamReader(response.GetResponseStream()))
{
jsonData = reader.ReadToEnd();
}
response.Close();
//do something with your data, deserialize, Regex etc....
}
I'm fairly new to using WebRequests in C# and would like some help logging into YouTube and storing cookies into a cookie container. Any help would be much appreciated.
If you are using the .NET client library (http://code.google.com/p/google-gdata/) then it will take care of authenticating for you when needed using the credentials you provided:
https://developers.google.com/youtube/2.0/developers_guide_dotnet#Authentication
GData in C#
When using the GData API there's no explicit logout method, you can invalidate your token but it will also expire after some time if you don't use it. Specific details on how to invalidate the token differ according to the authentication mechanism adopted (OAuth, AuthSub, ClientLogin).
You can also refer to this article on CodeProject, Manage YouTube using C# and Youtube API 1.6
YouTubeService service = new YouTubeService();
service.setUserCredentials(txtUser.Text , txtPassword.Text);
try { service.QueryClientLoginToken(); }
catch(System.Net.WebException e) { MessageBox.Show(e.Message); }
ADDED: I have tweaked below code to match your requirement.
class YouTube
{
public void Login()
{
HttpWebRequest request = GetNewRequest("https://accounts.google.com/ServiceLoginAuth", cookies);
request.Referer = "https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Faction_handle_signin%3Dtrue%26feature%3Dsign_in_button%26nomobiletemp%3D1%26hl%3Den_US%26next%3D%252F&uilel=3&hl=en_US&service=youtube";
request.Host = "accounts.google.com";
Dictionary<string, string> parameters = new Dictionary<string, string>{
{"continue","https%3A%2F%2Fwww.youtube.com%2Fsignin%3Faction_handle_signin%3Dtrue%26feature%3Dsign_in_button%26nomobiletemp%3D1%26hl%3Den_US%26next%3D%252F"},
{"service","youtube"},{"uilel","3"},{"dsh","157212168103955870"},{"hl","en_US"},
{"GALX","PTqcwpZb2aE"},{"pstMsg","1"},{"dnConn",""}, {"checkConnection","youtube%3A248%3A1"},
{"checkedDomains","youtube"}, {"timeStmp",""}, {"secTok",""}, {"Email","username"}, {"Passwd","password"},
{"signIn","Sign+in"}, {"PersistentCookie","yes"}, {"rmShown","1"}};
HttpWebResponse response = MakeRequest(request, cookies, parameters);
response.Close();
}
private static CookieContainer cookies = new CookieContainer();
private static HttpWebRequest GetNewRequest(string targetUrl, CookieContainer SessionCookieContainer)
{
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(targetUrl);
request.CookieContainer = SessionCookieContainer;
request.AllowAutoRedirect = false;
return request;
}
private static HttpWebResponse MakeRequest(HttpWebRequest request, CookieContainer SessionCookieContainer, Dictionary<string, string> parameters = null)
{
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.52 Safari/536.5Accept: */*";
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
request.CookieContainer = SessionCookieContainer;
request.AllowAutoRedirect = false;
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
string postData = string.Empty;
foreach (KeyValuePair<String, String> parametro in parameters)
{
if (postData.Length == 0)
postData += String.Format("{0}={1}", parametro.Key, parametro.Value);
else
postData += String.Format("&{0}={1}", parametro.Key, parametro.Value);
}
byte[] postBuffer = UTF8Encoding.UTF8.GetBytes(postData);
using (Stream postStream = request.GetRequestStream())
{
postStream.Write(postBuffer, 0, postBuffer.Length);
}
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
SessionCookieContainer.Add(response.Cookies);
while (response.StatusCode == HttpStatusCode.Found)
{
response.Close();
request = GetNewRequest(response.Headers["Location"], SessionCookieContainer);
response = (HttpWebResponse)request.GetResponse();
SessionCookieContainer.Add(response.Cookies);
}
return response;
}
}
Reference: http://social.msdn.microsoft.com/Forums/pl-PL/ncl/thread/40d249b5-a9ad-4068-8853-629fb20584a0