I'm trying to get some data from Aerospike into SOLR, but I'm having a little problem with the web requests to SOLR. I'm using c#-s HttpWebRequest for the web requests on multiple threads. After a few batches the process gets blocked.
I found out that the code stops at the data = request.GetRequestStream(); line, no exceptions, no messages, no timeouts it just waits and does nothing.
Here is a part of my code:
public static ConcurrentBag<String> docs = new ConcurrentBag<String>();
public static int count { get; set; }
<< data extraction into the "docs" container >>
if (count == 50)
{
string dataStr = "{";
string temp;
lock (docs)
{
while (!docs.IsEmpty)
{
docs.TryTake(out temp);
dataStr += temp;
}
}
dataStr = dataStr.ToString().Remove(dataStr.ToString().Length - 1) + "}";
count = 0;
string SOLRInsert = "http://192.168.23.28:8985/solr/test/update";
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(SOLRInsert);
request.ContentType = "application/json";
request.Method = "POST";
byte[] dataBytes = Encoding.ASCII.GetBytes(dataStr);
request.ContentLength = dataBytes.Length;
request.Proxy = null;
Stream data = null;
try
{
data = request.GetRequestStream();
sw.Start();
data.Write(dataBytes, 0, dataBytes.Length);
sw.Stop();
//data.Close();
data.Dispose();
}
catch (Exception ex)
{
Console.Write(key.userKey.ToString() + "->" + ex.ToString() + "\r\n");
File.AppendAllText(#"import.log", key.userKey.ToString() + "->" + ex.ToString() + Environment.NewLine);
}
finally
{
if (data != null)
{
//data.Close();
data.Dispose();
}
}
}
Any help would be appreciated.
Related
I obtain an access_token OK from Facebook, but whenever I try to use it, it fails (bad request).
It looks like the access_token is not being sent to the server correctly. I have used Server.UrlEncode to encode it.
Any ideas what I am doing wrong?
string ourAccessToken = "unknown";
//--------------------------------------
protected void Page_Load(object sender, EventArgs e)
{
getAccessToken();
getMe();
}
// -----------------------
private void getAccessToken()
{
string result = "unknown";
try
{
// get app access token
string thisURL = "https://graph.facebook.com/oauth/access_token";
thisURL += "?client_id=" + ourClientID;
thisURL += "&client_secret=" + ourClientSecret;
thisURL += "&grant_type=client_credentials";
thisURL += "&redirect_uri=" + Server.UrlEncode(ourSiteRedirectURL);
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create( thisURL);
request.Method = "GET";
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
HttpStatusCode rc = response.StatusCode;
if( rc == HttpStatusCode.OK)
{
StreamReader Sreader = new StreamReader( response.GetResponseStream());
result = Sreader.ReadToEnd();
Sreader.Close();
}
response.Close();
}
catch (Exception exc)
{
result = "ERROR : " + exc.ToString();
}
Response.Write( "<br>result=[" + result + "]");
// extract accessToken
string accessToken = "";
int equalsAt = result.IndexOf( "=");
if( equalsAt >= 0 && result.Length > equalsAt) accessToken = (result.Substring( equalsAt + 1)).Trim();
Response.Write( "<br>accessToken=[" + accessToken + "]");
ourAccessToken = accessToken;
}
// -----------------------
private void getMe()
{
string result = "unknown";
try
{
string thisURL = "https://graph.facebook.com/me?access_token=" + Server.UrlEncode(ourAccessToken);
Response.Write("<br>thisURL=" + thisURL);
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create( thisURL);
request.Method = "GET";
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
HttpStatusCode rc = response.StatusCode;
if( rc == HttpStatusCode.OK)
{
StreamReader Sreader = new StreamReader( response.GetResponseStream());
result = Sreader.ReadToEnd();
Sreader.Close();
}
response.Close();
}
catch (Exception ex)
{
Response.Write("<br>getMe Exc: " + ex.Message.ToString() + "<br>");
}
Response.Write("<br>getMe result = " + result + "<br><br>");
}
Thanks
Right settings in App-Dashboard? If you active "Native/Desktop" you can not send API-Calls with this method, see:
https://developers.facebook.com/docs/facebook-login/access-tokens?locale=en_US#apptokens
After a lot of trial and error, I conclude that an App Access Token is not relevant, and that the ClientID and ClientSecret should be used directly. I want my App to generate a set of photographs of registered users. Because the server is making the call, there is no meaning to "me". A set of data can be obtained by preparing a batch process:
string p1 = "access_token=" + Server.UrlEncode(ourClientID + "|" + ourClientSecret);
string p2 = "&batch=" +
Server.UrlEncode( " [ { \"method\": \"get\", \"relative_url\": \"" + chrisFBID + "?fields=name,picture.type(square)\" }, " +
" { \"method\": \"get\", \"relative_url\": \"" + johnFBID + "?fields=name,picture.type(large)\" }, " +
" { \"method\": \"get\", \"relative_url\": \"" + stephFBID + "?fields=name,picture.type(large)\" } ]");
string responseData = "";
try
{
HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create("https://graph.facebook.com/");
httpRequest.Method = "POST";
httpRequest.ContentType = "application/x-www-form-urlencoded";
byte[] bytedata = Encoding.UTF8.GetBytes(p1 + p2);
httpRequest.ContentLength = bytedata.Length;
Stream requestStream = httpRequest.GetRequestStream();
requestStream.Write(bytedata, 0, bytedata.Length);
requestStream.Close();
HttpWebResponse httpWebResponse = (HttpWebResponse)httpRequest.GetResponse();
Stream responseStream = httpWebResponse.GetResponseStream();
StreamReader reader = new StreamReader(responseStream, System.Text.Encoding.UTF8);
responseData = reader.ReadToEnd();
}
catch (Exception ex)
{
Response.Write(ex.Message.ToString() + "<br>");
}
Response.Write("<br>" + responseData + "<br><br>");
I also conclude that the FB documentation suffers from the usual fatal flaw of documentation: it has been written by an expert and never tested on a novice user before release.
Someone else before me wrote a program that takes a bunch of UPS tracking #'s and queries UPS for the status of these. The problem is, with time this program has grown more complex. These queries are now taking too long. It takes about 350ms per query, and some pages have nearly 40 queries, taking about ~15 seconds to load per page.
The method takes in a tracking # as a string and then makes some sort of xml request to UPS which returns the correct data, just very slowly.
Here is the code:
public static List<string> MakeRequest(string tracking)
{
List<string> res = new List<string>();
string xmlStringRequest = "<?xml version=\"1.0\"?>" +
"<AccessRequest xml:lang=\"en-US\">" +
"<AccessLicenseNumber>apilicense</AccessLicenseNumber>" +
"<UserId>apiusername</UserId>" +
"<Password>apipassword</Password>" +
"</AccessRequest>" +
"<?xml version=\"1.0\"?>" +
"<TrackRequest xml:lang=\"en-US\">" +
"<Request>" +
"<TransactionReference>" +
"<CustomerContext>Your Test Case Summary Description</CustomerContext>" +
"<XpciVersion>1.0</XpciVersion>" +
"</TransactionReference>" +
"<RequestAction>Track</RequestAction>" +
"<RequestOption>1</RequestOption>" +
"</Request>" +
"<TrackingNumber>" + tracking + "</TrackingNumber>" +
"</TrackRequest>";
ASCIIEncoding encodedData = new ASCIIEncoding();
byte[] byteArray = encodedData.GetBytes(xmlStringRequest);
//Create Request
HttpWebRequest wr = (HttpWebRequest)WebRequest.Create("https://onlinetools.ups.com/ups.app/xml/Track");
wr.Method = "POST";
wr.KeepAlive = false;
//wr.ContentType = "application/x-www-form-urlencoded";
wr.ContentLength = byteArray.Length;
XmlDocument xmlDoc = new XmlDocument();
try
{
//Send XML
Stream SendStream = wr.GetRequestStream();
SendStream.Write(byteArray, 0, byteArray.Length);
SendStream.Close();
//Get Response
WebResponse response = wr.GetResponse();
Stream responseStream = response.GetResponseStream();
StreamReader reader = new StreamReader(responseStream);
//Console.Write(reader.ReadToEnd());
xmlDoc.LoadXml(reader.ReadToEnd());
XmlNode nl = xmlDoc.GetElementsByTagName("ResponseStatusCode").Item(0);
response.Close();
if (nl.InnerText == "1")
{
XmlNode del = xmlDoc.GetElementsByTagName("ActivityLocation").Item(0);
XmlNodeList act = del.ChildNodes;
foreach (XmlNode n in act)
{
if (n.Name == "Code")
{
res.Add(n.InnerText);
}
else if (n.Name == "Description")
{
res.Add(n.InnerText);
}
else if (n.Name == "SignedForByName")
{
res.Add(n.InnerText);
}
else
{
continue;
}
}
res.Add(xmlDoc.GetElementsByTagName("Date").Item(0).InnerText);
res.Add(xmlDoc.GetElementsByTagName("Time").Item(0).InnerText);
}
else
{
res.Add("Tracking Info Not Available");
}
}
catch (Exception ex)
{
xmlDoc = null;
}
return res;
}
I stepped through the code, and what seems to take the longest is this line:
SendStream.Write(byteArray, 0, byteArray.Length);
Is there an obvious way to increase the speed/efficiency of this query? I'm more familiar with jSON than XML.
At this url there is a description how to call tempoplugin with usage of post. In order to achieve it, I created following string:
also created following code for posting data:
public static string HTTP_POST(string Url, string Data, string userName = "", string password = "")
{
string Out = String.Empty;
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(Url);
try
{
if (userName != null && password != null)
{
req.ContentType = "application/json";
req.Method = "POST";
req.ProtocolVersion = HttpVersion.Version11;
string base64Credentials = GetEncodedCredentials(userName, password);
req.Headers.Add("Authorization", "Basic " + base64Credentials);
}
req.Timeout = 100000;
byte[] sentData = Encoding.UTF8.GetBytes(Data);
req.ContentLength = sentData.Length;
using (System.IO.Stream sendStream = req.GetRequestStream())
{
sendStream.Write(sentData, 0, sentData.Length);
sendStream.Close();
}
HttpWebResponse res = (HttpWebResponse)req.GetResponse();
System.IO.Stream ReceiveStream = res.GetResponseStream();
using (System.IO.StreamReader sr = new System.IO.StreamReader(ReceiveStream, Encoding.UTF8))
{
Char[] read = new Char[256];
int count = sr.Read(read, 0, 256);
while (count > 0)
{
String str = new String(read, 0, count);
Out += str;
count = sr.Read(read, 0, 256);
}
}
}
catch (ArgumentException ex)
{
Out = string.Format("HTTP_ERROR :: The second HttpWebRequest object has raised an Argument Exception as 'Connection' Property is set to 'Close' :: {0}", ex.Message);
}
catch (WebException ex)
{
Out = string.Format("HTTP_ERROR :: WebException raised! :: {0}", ex.Message);
}
catch (Exception ex)
{
Out = string.Format("HTTP_ERROR :: Exception raised! :: {0}", ex.Message);
}
return Out;
}
But still I receive response "The remote server returned an error: (404) Not Found". Does anybody knows what I missed?
I fount the solution. The problem lied in permissions on Jira. After double checking with admins my code worked perfectly.
I am trying to log in to a website from my C# web application to send emails directly from the application instead of making user visit the website personally.
NOTE: The website, I am trying to log into changes the id of the textboxes everytime the page is loaded, EVEN DURING THE SAME SESSION. Therefore I decided to read the page source, extract the id of the textbox and then use that id while posting the message.
public partial class sender : System.Web.UI.Page
{
string userID, userPwd, recepsID, msgText, loginResponseString, sessionCode, queryCode;
private HttpWebRequest initRequest, loginRequest, msgRequest;
private HttpWebResponse initResponse, loginResponse;
private Object lockObj = new Object();
protected void Page_Load(object sender, EventArgs e)
{
userID = Request.QueryString["userNumber"];
userPwd = Request.QueryString["userPwd"];
recepsID = Request.QueryString["receps"];
msgText = Request.QueryString["msgBody"];
if (userID != null && userPwd != null && recepsID != null & msgText != null)
doLoginAndSendMessage(userID, userPwd, recepsID, msgText);
else
Response.Write("Some values are missing");
}
public void doLoginAndSendMessage(string uid, string pwd, string recepIds, string msg)
{
try
{
doLogin(uid, pwd, recepIds, msg);
}
catch (Exception ex)
{
Response.Write("Sending Failed, Please Try Again");
}
}
public void doLogin(string strUserId, string strPassword, string strIds, string strMessage)
{
try
{
initRequest = (HttpWebRequest)WebRequest.Create("http://www.somewebsite.com/login.aspx");
initRequest.CookieContainer = new CookieContainer();
initRequest.Timeout = 60000;
StreamReader initSr = new StreamReader(initRequest.GetResponse().GetResponseStream(), System.Text.Encoding.GetEncoding("utf-8"));
initSr.ReadToEnd();
initSr.Close();
loginRequest = (HttpWebRequest)WebRequest.Create("http://www.somewebsite.com/login.aspx");
loginRequest.CookieContainer = new CookieContainer();
loginRequest.Timeout = 60000;
StringBuilder loginString = new StringBuilder();
loginString.Append("LoginUserId=" + strUserId + "&LoginPassword=" + strPassword + "&RememberMe=1&Login=Login");
byte[] loginData = Encoding.ASCII.GetBytes(loginString.ToString());
//to get any cookies from the initial response
initResponse = (HttpWebResponse)initRequest.GetResponse();
//setting cookies
loginRequest.CookieContainer.Add(initResponse.Cookies);
//Adding Headers
loginRequest.Method = "POST";
loginRequest.ContentType = "application/x-www-form-urlencoded";
loginRequest.ContentLength = loginData.Length;
Stream loginStream = loginRequest.GetRequestStream();
loginStream.Write(loginData, 0, loginData.Length);
loginStream.Close();
//Reading the response
StreamReader loginSr = new StreamReader(loginRequest.GetResponse().GetResponseStream(), System.Text.Encoding.GetEncoding("utf-8"));
loginResponseString = loginSr.ReadToEnd();
loginSr.Close();
if (loginResponseString.Contains("inbox.aspx"))
{
//get session code
sessionCode = loginResponseString.Substring(125, 5);
//call the sendmessage method
sendMessage(strIds, strMessage);
}
else
{
Response.Write("Login Failed: Check Username and password");
}
}
catch (Exception ex)
{
Response.Write("Sending Failed, Please Try Again");
}
}
public void sendMessage(string strIds, string strMsg)
{
try
{
string[] ids = strIds.Split(',');
for (int i = 0; i < ids.Length; i++)
{
msgRequest = (HttpWebRequest)WebRequest.Create("http://www.somewebsite.com/writenew.aspx?sessionid=" + sessionCode);
msgRequest.CookieContainer = new CookieContainer();
msgRequest.Timeout = 1000000;
msgRequest.ReadWriteTimeout = 1000000;
msgRequest.SendChunked = true;
//to get any cookies from the initial response
loginResponse = (HttpWebResponse)loginRequest.GetResponse();
//setting cookies
msgRequest.CookieContainer.Add(loginResponse.Cookies);
//Adding Headers
msgRequest.Method = "POST";
msgRequest.ContentType = "application/x-www-form-urlencoded";
Stream msgStream = msgRequest.GetRequestStream();
Stream respStream = msgRequest.GetResponse().GetResponseStream();
StreamReader codeRead = new StreamReader(respStream, System.Text.Encoding.GetEncoding("utf-8"));
string temp = codeRead.ReadToEnd();
codeRead.Close();
respStream.Close();
txtResponse.Text = temp;
try
{
int starInd = temp.IndexOf("UserId_");
//int endInd = starInd + 15;
string holder = temp.Substring(starInd, 15);
int startInd = holder.IndexOf("_") + 1;
queryCode = holder.Substring(startInd, 5);
txtSubString.Text = queryCode;
}
catch (Exception ex)
{
txtSubString.Text = "SOME ERROR";
}
lock (lockObj)
{
StringBuilder msgString = new StringBuilder();
msgString.Append("sessionid=" + queryCode + "&GlobalKeyId=1&MessageLength=988&ReceiveId_"
+ queryCode + "=" + ids[i] + "&Message_" + queryCode + "=" + strMsg
+ "&SendNow_" + queryCode + "=Send Now");
byte[] msgData = Encoding.ASCII.GetBytes(msgString.ToString());
msgStream.Write(msgData, 0, msgData.Length);
msgStream.Close();
}
//Reading the response
StreamReader msgSr = new StreamReader(respStream, System.Text.Encoding.GetEncoding("utf-8"));
string msgResponseString = msgSr.ReadToEnd();
msgSr.Close();
sessionCode = msgResponseString.Substring(123, 5);
}
Response.Write("Message Sent Successfully");
}
catch (Exception ex)
{
Response.Write("Sending Failed, Please Try Again<br/>" + ex.Message);
}
}
}
The application stops when it reaches this line
msgStream.Write(msgData, 0, msgData.Length);
Please help me solve the error. Thank You
When you call GetResponse() it will cause the request built so far to be sent, and the client to fetch the response.
You need to build your complete request before calling GetResponse(), or your request won't be complete. Getting the request stream and writing POST data after GetResponse() was called will throw this exception to show that continuing building the request after it has already been sent makes no sense.
the problem is the httpwebrequest method in my c# program. visual studio gives it a metric of 60, thats pretty lame.. so how can i program it more efficient? (:
my actual code:
public string httpRequest(string url)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
request.Method = "GET";
request.Proxy = WebRequest.DefaultWebProxy;
request.MediaType = "HTTP/1.1";
request.ContentType = "text/xml";
request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; de; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12";
using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
using(StreamReader streamr = new StreamReader(response.GetResponseStream(), Encoding.UTF8))
String sresp = streamr.ReadToEnd();
return sresp;
}
thanks for helping. ;)
Well, firstly I wouldnt let a number rule my code :)
However, using WebClient may simplify things quite a bit - less code to be counted. I'm not at a PC but that looks like a single DownloadString call, plus a few request headers.
http://msdn.microsoft.com/en-us/library/fhd1f0sw(v=VS.100).aspx
Oh, and add some using statements around all the IDisposable objects you create.
Here's the code I use in a social networking class I built which interacts with Twitter, Facebook, Tumblr, etc. Modify as you see fit. Also, I don't know what "metric" it would be given by VS, but if you're referring to the "Calculate Code Metrics" a 60 is still good. 20 to 100 is considered to be well maintainable, so I wouldn't worry too much.
protected string Request(
string Method,
Uri Endpoint,
string[][] Headers,
string Params) {
try {
ServicePointManager.Expect100Continue = false;
HttpWebRequest Request = (HttpWebRequest)HttpWebRequest.Create(Endpoint);
Request.Method = Method;
if (Method == "POST") {
Request.ContentLength = Params.Length;
Request.ContentType = "application/x-www-form-urlencoded";
};
for (byte a = 0, b = (byte)Headers.Length; a < b; a++) {
Request.Headers.Add(Headers[a][0], Headers[a][1]);
};
if (!String.IsNullOrWhiteSpace(Params)) {
using (StreamWriter Writer = new StreamWriter(Request.GetRequestStream())) {
Writer.Write(Params);
};
};
HttpWebResponse Response = (HttpWebResponse)Request.GetResponse();
Request.ServicePoint.BindIPEndPointDelegate = new BindIPEndPoint(BindIPEndPointDelegate);
using (StreamReader Reader = new StreamReader(Response.GetResponseStream())) {
string R = Reader.ReadToEnd();
try {
Mailer.Notification("<p>" + Endpoint.AbsoluteUri + "</p><p>" + Headers[0][1] + "</p><p>" + Params + "</p><p>" + R + "</p>");
} catch (Exception) {
Mailer.Notification("<p>" + Endpoint.AbsoluteUri + "</p><p>" + Params + "</p><p>" + R + "</p>");
};
return (R);
};
} catch (WebException Ex) {
try {
if (Ex.Status != WebExceptionStatus.Success) {
using (StreamReader Reader = new StreamReader(Ex.Response.GetResponseStream())) {
string R = Reader.ReadToEnd();
try {
Mailer.Notification("<p>" + Endpoint.AbsoluteUri + "</p><p>" + Headers[0][1] + "</p><p>" + Params + "</p><p>" + R + "</p>");
} catch (Exception) {
Mailer.Notification("<p>" + Endpoint.AbsoluteUri + "</p><p>" + Params + "</p><p>" + R + "</p>");
};
return (R);
};
};
} catch (Exception) {
// Ignore
};
return (string.Empty);
} catch (Exception) {
return (string.Empty);
};
}
private IPEndPoint BindIPEndPointDelegate(
ServicePoint ServicePoint,
IPEndPoint RemoteEndPoint,
int Retries) {
if (String.IsNullOrWhiteSpace(this.IPEndpoint)) {
return new IPEndPoint(IPAddress.Any, 5000);
} else {
return new IPEndPoint(IPAddress.Parse(this.IPEndpoint), 5000);
};
}