Develop Reference

How to get html content from amazon using HttpWebRequest

I am trying to get HTML content from the amazon website. Here is my code to create request, response, and get string:
public static HttpWebResponse GetHttpWebResponse(string url)
{
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(url);
webRequest.ContentType = "text/xml";
try
{
return (HttpWebResponse)webRequest.GetResponse();
}
catch (WebException e)
{
if (e.Response == null)
throw new Exception("Cannot get response");
return (HttpWebResponse)e.Response;
}
}
public static string GetString(HttpWebResponse response)
{
Encoding encoding = Encoding.UTF8;
using (var reader = new StreamReader(response.GetResponseStream(), encoding))
{
string responseText = reader.ReadToEnd();
return responseText;
}
}
It is working fine with other web sites. However, when I try to get content from amazon, for example:
https://www.amazon.com/gp/product/B00AEISSHA/ref=ppx_yo_dt_b_asin_title_o00_s00?ie=UTF8&psc=1
I am seeing encoded content:
I tried to change Encoding and used HttpUtility.HtmlDecode(html); but it couldn't help. Is there any simple way to get content from Amazon?

You're not catering for compression. If you update your webrequest like this, it should do the trick.
public static HttpWebResponse GetHttpWebResponse(string url)
{
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(url);
webRequest.ContentType = "text/xml";
webRequest.AutomaticDecompression = DecompressionMethods.GZip;
try
{
return (HttpWebResponse)webRequest.GetResponse();
}
catch (WebException e)
{
if (e.Response == null)
throw new Exception("Cannot get response");
return (HttpWebResponse)e.Response;
}
}

How to get http content using an given api key like this in C#?

http://ssw.com/profile/?apikey = skdwkdkfkkdj
I tried to use
public async Task<string> GetFromUriAsync(string requestUri, string token)
{
var client = new HttpClient();
client.BaseAddress = new Uri(BaseUri);
client.DefaultRequestHeaders.Authorization =
new AuthenticationHeaderValue("apikey", "=" + token);
HttpResponseMessage response = await client.GetAsync(requestUri);
response.EnsureSuccessStatusCode();
string responseBody = await response.Content.ReadAsStringAsync();
return responseBody;
}
Then it returns null
Am I missing something or is it just totally wrong?
Thanks

You're trying to pass the API key in the header information of your HTTP request. What you need to do is just pass that whole URL without any additional header information.
IE: use "http://ssw.com/profile?apikey=abcdef" as the requestUri and send token as null. Also, remove the setting of the client.DefaultRequestHeaders.Authorization property. Authorization was meant to be a user/pass system and not a token-based system.
To test this, download Fiddler 4 (https://www.telerik.com/download/fiddler). Once you have fiddler installed, on the "Composer" tab, you can test different queries you need by putting the URL directly into the URL box and clicking "Execute". You'll then be able to use the inspectors to see the responses and figure out where you need to go from there.
Here are the classes I use for HTTP GET and POST operations:
public static string HTTPGET(string url)
{
try
{
HttpWebRequest request = (HttpWebRequest) WebRequest.Create(url);
request.Timeout = 100000;
HttpWebResponse response = (HttpWebResponse) request.GetResponse();
Stream responseStream = response.GetResponseStream();
if (responseStream != null)
using (StreamReader resStream = new StreamReader(responseStream))
return resStream.ReadToEnd();
return null;
}
catch (Exception e)
{
Console.WriteLine(url);
Console.WriteLine(e);
return null;
}
}
public static string HTTPPOST(string url, string postData)
{
try
{
HttpWebRequest webRequest = (HttpWebRequest) WebRequest.Create(url);
webRequest.Method = "POST";
webRequest.ContentType = "x-www-form-urlencoded";
byte[] byteArray = Encoding.UTF8.GetBytes(postData);
using (Stream requestStream = webRequest.GetRequestStream())
requestStream.Write(byteArray, 0, byteArray.Length);
using (Stream responseStream = webRequest.GetResponse().GetResponseStream())
if (responseStream != null)
using (StreamReader responseReader = new StreamReader(responseStream))
return responseReader.ReadToEnd();
return null;
}
catch (Exception e)
{
Console.WriteLine(url);
Console.WriteLine(postData);
Console.WriteLine(e);
return null;
}
}

Sending xml using Apache HttpComponents

I am attempting to replicate the following C# code in Java. This code is a helper class that sends a request containing xml, and reads a response.
internal static String Send(String url, String body)
{
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
try
{
// create the new httpwebrequest with the uri
request.ContentLength = 0;
// set the method to POST
request.Method = "POST";
if (!String.IsNullOrEmpty(body))
{
request.ContentType = "application/xml; charset=utf-8";
byte[] postData = Encoding.Default.GetBytes(body);
request.ContentLength = postData.Length;
using (Stream s = request.GetRequestStream())
{
s.Write(postData, 0, postData.Length);
}
}
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
{
String responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
if (response.StatusCode != HttpStatusCode.OK)
{
throw new ResponseException(((int)response.StatusCode),
response.StatusCode.ToString(), request.RequestUri.ToString(),
responseString);
}
return responseString;
}
}
catch (WebException e)
{
using (WebResponse response = e.Response)
{
HttpWebResponse httpResponse = response as HttpWebResponse;
if (httpResponse != null)
{
using (Stream data = response.GetResponseStream())
{
data.Position = 0;
throw new ResponseException(((int)httpResponse.StatusCode),
httpResponse.StatusCode.ToString(), request.RequestUri.ToString(),
new StreamReader(data).ReadToEnd()
);
}
}
else
{
throw;
}
After reading other threads I determined that the Apache HttpComponents library would be my best bet to get the same functionality. After reading the documentation and following the example here:
http://hc.apache.org/httpcomponents-client-ga/quickstart.html
I am unable to figure out how to send the body string as xml. When I attempt to set the entity for the request it requires that I declare a BasicNameValuePair, and I do not understand what this is, or how I would format the body string to meet this specification.
Below is what I have currently done.
protected static String Send(String url, String body)
{
HttpPost request = new HttpPost(url);
try
{
request.setHeader("ContentType", "application/xml; charset=utf=8");
// Encode the body if needed
request.setEntity(new UrlEncodedFormEntity());
//get the response
// if the response code is not valid throw a ResponseException
// else return the response string.
} finally {
request.releaseConnection();
}
return null;
}
EDIT : or should I use a StringEntity and do the following
protected static String SendToJetstream(String url, String body)
{
HttpPost request = new HttpPost(url);
try
{
StringEntity myEntity = new StringEntity(body,
ContentType.create("application/xml", "UTF-8"));
// Encode the body if needed
request.setEntity(myEntity);
//get the response
// if the response code is not valid throw a ResponseException
// else return the response string.
} finally {
request.releaseConnection();
}
return null;
}

Use a FileEntity
File file = new File("somefile.xml");
FileEntity entity = new FileEntity(file, ContentType.create("application/xml", "UTF-8"));
Lots of good examples here: http://hc.apache.org/httpcomponents-client-ga/tutorial/html/fundamentals.html#d5e165

Get real-time data from a http long lived web service

I have a http long lived web service. If it has new data it will push to client using http GET. How can I receive real-time data from http long lived web service with HttpWebRequest c#?

If you wanted to get data using Get, you can use this (the response is synchronous when you use GetResponse):
public string GetMessageViaGet(string endPoint)
{
HttpWebRequest request = CreateWebRequest(endPoint);
using (var response = (HttpWebResponse)request.GetResponse())
{
var responseValue = string.Empty;
if (response.StatusCode != HttpStatusCode.OK)
{
string message = String.Format("Request failed. Received HTTP {0}", response.StatusCode);
throw new ApplicationException(message);
}
// grab the response
using (var responseStream = response.GetResponseStream())
{
using (var reader = new StreamReader(responseStream))
{
responseValue = reader.ReadToEnd();
}
}
return responseValue;
}
}
private HttpWebRequest CreateWebRequest(string endPoint)
{
var request = (HttpWebRequest)WebRequest.Create(endPoint);
request.Method = "GET";
request.ContentLength = 0;
request.ContentType = "text/xml";
return request;
}

If you want to get data via post, do this
public string GetMessageViaPost(string endPoint, string paramtersJson)
{
string responseValue;
byte[] bytes = Encoding.UTF8.GetBytes(paramtersJson);
HttpWebRequest request = CreateWebRequest(endPoint, bytes.Length);
using (var requestStream = request.GetRequestStream())
{
requestStream.Write(bytes, 0, bytes.Length);
}
using (var response = (HttpWebResponse)request.GetResponse())
{
if (response.StatusCode != HttpStatusCode.OK)
{
string message = String.Format("POST failed. Received HTTP {0}", response.StatusCode);
throw new ApplicationException(message);
}
// grab the response
using (var responseStream = response.GetResponseStream())
{
using (var reader = new StreamReader(responseStream))
{
responseValue = reader.ReadToEnd();
}
}
}
return responseValue;
}
private HttpWebRequest CreateWebRequest(string endPoint, Int32 contentLength)
{
var request = (HttpWebRequest)WebRequest.Create(endPoint);
request.Method = "POST";
request.ContentLength = contentLength;
request.ContentType = "application/json";// "application/x-www-form-urlencoded";
return request;
}

Login to website and use cookie to get source for another page

I am trying to login to the TV Rage website and get the source code of the My Shows page. I am successfully logging in (I have checked the response from my post request) but then when I try to perform a get request on the My Shows page, I am re-directed to the login page.
This is the code I am using to login:
private string LoginToTvRage()
{
string loginUrl = "http://www.tvrage.com/login.php";
string formParams = string.Format("login_name={0}&login_pass={1}", "xxx", "xxxx");
string cookieHeader;
WebRequest req = WebRequest.Create(loginUrl);
req.ContentType = "application/x-www-form-urlencoded";
req.Method = "POST";
byte[] bytes = Encoding.ASCII.GetBytes(formParams);
req.ContentLength = bytes.Length;
using (Stream os = req.GetRequestStream())
{
os.Write(bytes, 0, bytes.Length);
}
WebResponse resp = req.GetResponse();
cookieHeader = resp.Headers["Set-cookie"];
String responseStream;
using (StreamReader sr = new StreamReader(resp.GetResponseStream()))
{
responseStream = sr.ReadToEnd();
}
return cookieHeader;
}
I then pass the cookieHeader into this method which should be getting the source of the My Shows page:
private string GetSourceForMyShowsPage(string cookieHeader)
{
string pageSource;
string getUrl = "http://www.tvrage.com/mytvrage.php?page=myshows";
WebRequest getRequest = WebRequest.Create(getUrl);
getRequest.Headers.Add("Cookie", cookieHeader);
WebResponse getResponse = getRequest.GetResponse();
using (StreamReader sr = new StreamReader(getResponse.GetResponseStream()))
{
pageSource = sr.ReadToEnd();
}
return pageSource;
}
I have been using this previous question as a guide but I'm at a loss as to why my code isn't working.

Here's a simplified and working version of your code using a WebClient:
class Program
{
static void Main()
{
var shows = GetSourceForMyShowsPage();
Console.WriteLine(shows);
}
static string GetSourceForMyShowsPage()
{
using (var client = new WebClientEx())
{
var values = new NameValueCollection
{
{ "login_name", "xxx" },
{ "login_pass", "xxxx" },
};
// Authenticate
client.UploadValues("http://www.tvrage.com/login.php", values);
// Download desired page
return client.DownloadString("http://www.tvrage.com/mytvrage.php?page=myshows");
}
}
}
/// <summary>
/// A custom WebClient featuring a cookie container
/// </summary>
public class WebClientEx : WebClient
{
public CookieContainer CookieContainer { get; private set; }
public WebClientEx()
{
CookieContainer = new CookieContainer();
}
protected override WebRequest GetWebRequest(Uri address)
{
var request = base.GetWebRequest(address);
if (request is HttpWebRequest)
{
(request as HttpWebRequest).CookieContainer = CookieContainer;
}
return request;
}
}