Develop Reference

How to create TCP message framing for the stream

Here is how my Client connects to the server:
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using System.Net.Sockets;
using System.IO;
using System;
using System.Text.RegularExpressions;
using UnityEngine.SceneManagement;
using Newtonsoft.Json;
using System.Linq;
public class ClientWorldServer : MonoBehaviour {
public bool socketReady;
public static TcpClient socket;
public static NetworkStream stream;
public static StreamWriter writer;
public static StreamReader reader;
public void ConnectToWorldServer()
{
if (socketReady)
{
return;
}
//Default host and port values;
string host = "127.0.0.1";
int port = 8080;
try
{
socket = new TcpClient(host, port);
stream = socket.GetStream();
writer = new StreamWriter(stream);
reader = new StreamReader(stream);
socketReady = true;
}
catch (Exception e)
{
Debug.Log("Socket error : " + e.Message);
}
}
}
Here is how i send data to the server using my Send function:
public void Send(string header, Dictionary<string, string> data)
{
if (stream.CanRead)
{
socketReady = true;
}
if (!socketReady)
{
return;
}
JsonData SendData = new JsonData();
SendData.header = "1x" + header;
foreach (var item in data)
{
SendData.data.Add(item.Key.ToString(), item.Value.ToString());
}
SendData.connectionId = connectionId;
string json = JsonConvert.SerializeObject(SendData);
var howManyBytes = json.Length * sizeof(Char);
writer.WriteLine(json);
writer.Flush();
Debug.Log("Client World:" + json);
}
As you can see i'm sending the data to the Stream like a string not like a byte array. As far as i know i should send the data as byte array prepending the size of the message and following the message. On the server side i have no clue how i can read that data.
Here is how i read it now(it works for now, but it will not work if i try to send more messages at once):
class WorldServer
{
public List<ServerClient> clients = new List<ServerClient>();
public List<ServerClient> disconnectList;
public List<CharactersOnline> charactersOnline = new List<CharactersOnline>();
public int port = 8080;
private TcpListener server;
private bool serverStarted;
private int connectionIncrementor;
private string mysqlConnectionString = #"server=xxx;userid=xxx;password=xxx;database=xx";
private MySqlConnection mysqlConn = null;
private MySqlDataReader mysqlReader;
static void Main(string[] args)
{
WorldServer serverInstance = new WorldServer();
Console.WriteLine("Starting World Server...");
try
{
serverInstance.mysqlConn = new MySqlConnection(serverInstance.mysqlConnectionString);
serverInstance.mysqlConn.Open();
Console.WriteLine("Connected to MySQL version: " + serverInstance.mysqlConn.ServerVersion + "\n");
}
catch (Exception e)
{
Console.WriteLine("MySQL Error: " + e.ToString());
}
finally
{
if (serverInstance.mysqlConn != null)
{
serverInstance.mysqlConn.Close();
}
}
serverInstance.clients = new List<ServerClient>();
serverInstance.disconnectList = new List<ServerClient>();
try
{
serverInstance.server = new TcpListener(IPAddress.Any, serverInstance.port);
serverInstance.server.Start();
serverInstance.StartListening();
serverInstance.serverStarted = true;
Console.WriteLine("Server has been started on port: " + serverInstance.port);
}
catch (Exception e)
{
Console.WriteLine("Socket error: " + e.Message);
}
new Thread(() =>
{
Thread.CurrentThread.IsBackground = true;
/* run your code here */
while (true)
{
string input = Console.ReadLine();
string[] commands = input.Split(':');
if (commands[0] == "show online players")
{
Console.WriteLine("Showing connections\n");
foreach (CharactersOnline c in serverInstance.charactersOnline)
{
Console.WriteLine("Character name: " + c.characterName + "Character ID: " + c.characterId + "Connection id: " + c.connectionId + "\n");
}
}
continue;
}
}).Start();
while (true)
{
serverInstance.Update();
}
}
private void Update()
{
//Console.WriteLine("Call");
if (!serverStarted)
{
return;
}
foreach (ServerClient c in clients.ToList())
{
// Is the client still connected?
if (!IsConnected(c.tcp))
{
c.tcp.Close();
disconnectList.Add(c);
Console.WriteLine(c.connectionId + " has disconnected.");
CharacterLogout(c.connectionId);
continue;
//Console.WriteLine("Check for connection?\n");
}
else
{
// Check for message from Client.
NetworkStream s = c.tcp.GetStream();
if (s.DataAvailable)
{
StreamReader reader = new StreamReader(s, true);
string data = reader.ReadLine();
if (data != null)
{
OnIncomingData(c, data);
}
}
//continue;
}
}
for (int i = 0; i < disconnectList.Count - 1; i++)
{
clients.Remove(disconnectList[i]);
disconnectList.RemoveAt(i);
}
}
private void OnIncomingData(ServerClient c, string data)
{
Console.WriteLine(data);
dynamic json = JsonConvert.DeserializeObject(data);
string header = json.header;
//Console.WriteLine("Conn ID:" + json.connectionId);
string connId = json.connectionId;
int.TryParse(connId, out int connectionId);
string prefix = header.Substring(0, 2);
if (prefix != "1x")
{
Console.WriteLine("Unknown packet: " + data + "\n");
}
else
{
string HeaderPacket = header.Substring(2);
switch (HeaderPacket)
{
default:
Console.WriteLine("Unknown packet: " + data + "\n");
break;
case "004":
int accountId = json.data["accountId"];
SelectAccountCharacters(accountId, connectionId);
break;
case "005":
int characterId = json.data["characterId"];
getCharacterDetails(characterId, connectionId);
break;
case "006":
int charId = json.data["characterId"];
SendDataForSpawningOnlinePlayers(charId, connectionId);
break;
case "008":
Dictionary<string, string> dictObj = json.data.ToObject<Dictionary<string, string>>();
UpdateCharacterPosition(dictObj, connectionId);
break;
}
}
private bool IsConnected(TcpClient c)
{
try
{
if (c != null && c.Client != null && c.Client.Connected)
{
if (c.Client.Poll(0, SelectMode.SelectRead))
{
return !(c.Client.Receive(new byte[1], SocketFlags.Peek) == 0);
}
return true;
}
else
{
return false;
}
}
catch
{
return false;
}
}
private void StartListening()
{
server.BeginAcceptTcpClient(OnConnection, server);
}
private void OnConnection(IAsyncResult ar)
{
connectionIncrementor++;
TcpListener listener = (TcpListener)ar.AsyncState;
clients.Add(new ServerClient(listener.EndAcceptTcpClient(ar)));
clients[clients.Count - 1].connectionId = connectionIncrementor;
StartListening();
//Send a message to everyone, say someone has connected!
Dictionary<string, string> SendDataBroadcast = new Dictionary<string, string>();
SendDataBroadcast.Add("connectionId", clients[clients.Count - 1].connectionId.ToString());
Broadcast("001", SendDataBroadcast, clients, clients[clients.Count - 1].connectionId);
Console.WriteLine(clients[clients.Count - 1].connectionId + " has connected.");
}
And this is how the server send back data to the client:
private void Send(string header, Dictionary<string, string> data, int cnnId)
{
foreach (ServerClient c in clients.ToList())
{
if (c.connectionId == cnnId)
{
try
{
//Console.WriteLine("Sending...");
StreamWriter writer = new StreamWriter(c.tcp.GetStream());
if (header == null)
{
header = "000";
}
JsonData SendData = new JsonData();
SendData.header = "0x" + header;
foreach (var item in data)
{
SendData.data.Add(item.Key.ToString(), item.Value.ToString());
}
SendData.connectionId = cnnId;
string JSonData = JsonConvert.SerializeObject(SendData);
writer.WriteLine(JSonData);
writer.Flush();
//Console.WriteLine("Trying to send data to connection id: " + cnnId + " data:" + sendData);
}
catch (Exception e)
{
Console.WriteLine("Write error : " + e.Message + " to client " + c.connectionId);
}
}
}
}
Here is my ServerClient class:
public class ServerClient
{
public TcpClient tcp;
public int accountId;
public int connectionId;
public ServerClient(TcpClient clientSocket)
{
tcp = clientSocket;
}
}
Can you please show me how i should modify my Send function on the client to send the data as byte array so i can create "TCP Message Framing" and how should i change my the following part on the server:
foreach (ServerClient c in clients.ToList())
{
// Is the client still connected?
if (!IsConnected(c.tcp))
{
c.tcp.Close();
disconnectList.Add(c);
Console.WriteLine(c.connectionId + " has disconnected.");
CharacterLogout(c.connectionId);
continue;
//Console.WriteLine("Check for connection?\n");
}
else
{
// Check for message from Client.
NetworkStream s = c.tcp.GetStream();
if (s.DataAvailable)
{
StreamReader reader = new StreamReader(s, true);
string data = reader.ReadLine();
if (data != null)
{
OnIncomingData(c, data);
}
}
//continue;
}
}
which is responsible for receving data on the server ?
Is it possible to change only these parts from the Client and on the Server and make it continue to work but this time properly with TCP Message Framing ?
Of course the listener on the client and the Send function of the server i'll remake once i understand how this framing should look like.

Your frames are already defined by cr/lf - so that much already exists; what you need to do is to keep a back buffer per stream - something like a MemoryStream might be sufficient, depending on how big you need to scale; then essentially what you're looking to do is something like:
while (s.DataAvailable)
{
// try to read a chunk of data from the inbound stream
int bytesRead = s.Read(someBuffer, 0, someBuffer.Length);
if(bytesRead > 0) {
// append to our per-socket back-buffer
perSocketStream.Position = perSocketStream.Length;
perSocketStream.Write(someBuffer, 0, bytesRead);
int frameSize; // detect any complete frame(s)
while((frameSize = DetectFirstCRLF(perSocketStream)) >= 0) {
// decode it as text
var backBuffer = perSocketStream.GetBuffer();
string message = encoding.GetString(
backBuffer, 0, frameSize);
// remove the frame from the start by copying down and resizing
Buffer.BlockCopy(backBuffer, frameSize, backBuffer, 0,
(int)(backBuffer.Length - frameSize));
perSocketStream.SetLength(backBuffer.Length - frameSize);
// process it
ProcessMessage(message);
}
}
}

C# Socket: Client Mishandle 'a' as the Client's id

There are two programs that I made that didn't work. There are the server and the client. The server accepts many client by giving a user an ID (starting from 0). The server sends out the command to the specific client based up the server's id. (Example: 200 client are connected to 1 server. The server's selected id is '5', so the server will send the command to all of the client, and the client will ask the server what ID he wants to execute his command on, and if it's '5', that client will execute and send data to the server). The client has many commands, but to create the smallest code with the same error, I only use 1 command (dir). Basically, the server sends the command to the client and if it matches with the client current id and the server current id, it will process the command. By default, the server's current ID is 10. Here are the list of the commands to help the people who wants to answer:
Server Command:
list (Shows all of the users ID connected and the server's current ID) --> Happens on server
dir (request the client to send its dir listing) --> Sent by the client, read by the Server
set (set the server's current id to any number) (example: 'set 4')
Client:
using System;
using System.Speech.Synthesis;
using System.Windows.Forms;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Net.Sockets;
using System.Net;
namespace clientControl
{
class Program
{
public static string directory = #"C:\";
public static int id = -10;
public static Socket sck = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
static void Main(string[] args)
{
Connect();
getSession();
ReadResponse(sck);
}
static byte[] readResponseFunc()
{
long fileSize = 0;
string fileSizeInString = null;
byte[] fileSizeInByteArray = new byte[1024];
int fileSizeLength = sck.Receive(fileSizeInByteArray);
for (int i = 0; i < fileSizeLength; i++)
{
fileSizeInString = fileSizeInString + (char)fileSizeInByteArray[i];
}
try
{
fileSize = Convert.ToInt64(fileSizeInString);
}
catch { Console.WriteLine(fileSizeInString); }
sck.Send(Encoding.ASCII.GetBytes("a"));
byte[] responseUnknown = new byte[1];
sck.Receive(responseUnknown);
if (Encoding.ASCII.GetString(responseUnknown) == "b")
{
byte[] dataInByteArray = new byte[fileSize];
int dataLength = sck.Receive(dataInByteArray);
return dataInByteArray;
}
return Encoding.ASCII.GetBytes("ERROR RESPONSE FUNC");
}
static void getSession()
{
byte[] message_1 = Encoding.ASCII.GetBytes("make_id");
sck.Send(Encoding.ASCII.GetBytes(message_1.Length.ToString()));
byte[] responseUnknown = new byte[1];
if (Encoding.ASCII.GetString(responseUnknown) == "a")
{
sck.Send(Encoding.ASCII.GetBytes("b"));
sck.Send(message_1);
}
byte[] receivedID = readResponseFunc();
id = Convert.ToInt32(Encoding.ASCII.GetString(receivedID));
}
static bool SocketConnected(Socket s)
{
bool part1 = s.Poll(1000, SelectMode.SelectRead);
bool part2 = (s.Available == 0);
if (part1 && part2)
return false;
else
return true;
}
static void ReadResponse(Socket sck)
{
while (true)
{
if (SocketConnected(sck) == true)
{
try
{
string response = Encoding.ASCII.GetString(readResponseFunc());
byte[] message_1 = Encoding.ASCII.GetBytes("get_id");
sck.Send(Encoding.ASCII.GetBytes(message_1.Length.ToString()));
byte[] responseUnknown = new byte[1];
if (Encoding.ASCII.GetString(responseUnknown) == "a")
{
sck.Send(Encoding.ASCII.GetBytes("b"));
sck.Send(message_1);
}
byte[] response_2InByteArray = readResponseFunc();
string response_2 = Encoding.ASCII.GetString(response_2InByteArray);
if (Convert.ToInt32(response_2) == id)
{
if (response == "dir")
{
string resultOfDirring = "Current Directory: " + directory + "\n\n";
string[] folderListingArray = Directory.GetDirectories(directory);
foreach (string dir in folderListingArray)
{
string formed = "DIRECTORY: " + Path.GetFileName(dir);
resultOfDirring = resultOfDirring + formed + Environment.NewLine;
}
string[] fileListingArray = Directory.GetFiles(directory);
foreach (var file in fileListingArray)
{
FileInfo fileInfo = new FileInfo(file);
string formed = "FILE: " + Path.GetFileName(file) + " - FILE SIZE: " + fileInfo.Length + " BYTES";
resultOfDirring = resultOfDirring + formed + Environment.NewLine;
}
byte[] message_11 = Encoding.ASCII.GetBytes(resultOfDirring);
sck.Send(Encoding.ASCII.GetBytes(message_11.Length.ToString()));
byte[] responseUnknown_11 = new byte[1];
if (Encoding.ASCII.GetString(responseUnknown_11) == "a")
{
sck.Send(Encoding.ASCII.GetBytes("b"));
sck.Send(message_11);
}
}
}
else { }
}
catch { if (SocketConnected(sck) == false) { Console.WriteLine("Client Disconnected: " + sck.RemoteEndPoint); break; }; }
}
else if (SocketConnected(sck) == false) { Console.WriteLine("Client Disconnected: " + sck.RemoteEndPoint); break; }
}
}
static void Connect()
{
while (true)
{
try
{
sck.Connect(IPAddress.Parse("127.0.0.1"), 80);
break;
}
catch { }
}
}
}
}
Server:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.IO;
using System.Text;
using System.Threading;
namespace serverControl
{
class Program
{
public static int ftpNum = 1;
public static List<int> listOfClient = new List<int>();
public static TcpListener server = new TcpListener(IPAddress.Parse("127.0.0.1"), 80);
public static string message = null;
public static int id = 0;
public static int selected_id = 10;
static void Main(string[] args)
{
server.Start();
Thread startHandlingClientThread = new Thread(startHandlingClient);
startHandlingClientThread.Start();
while (true)
{
Console.Write(":> ");
string rawmessage = Console.ReadLine();
if (rawmessage == "list")
{
Console.WriteLine("SELECTED ID: " + selected_id);
Console.WriteLine("List of Clients ID:");
for (int i = 0; i < listOfClient.Count; i++)
{
Console.WriteLine(listOfClient[i]);
}
message = rawmessage+"PREVENT_REPETITION_IN_COMMAND";
}
else if (rawmessage.Contains("set ")) { int wantedChangeId = Convert.ToInt32(rawmessage.Replace("set ", "")); selected_id = wantedChangeId; message = rawmessage+ "PREVENT_REPETITION_IN_COMMAND"; }
else
{
message = rawmessage;
}
}
}
static byte[] readResponseFunc(Socket sck)
{
long fileSize = 0;
string fileSizeInString = null;
byte[] fileSizeInByteArray = new byte[1024];
int fileSizeLength = sck.Receive(fileSizeInByteArray);
for (int i = 0; i < fileSizeLength; i++)
{
fileSizeInString = fileSizeInString + (char)fileSizeInByteArray[i];
}
fileSize = Convert.ToInt64(fileSizeInString);
sck.Send(Encoding.ASCII.GetBytes("a"));
byte[] responseUnknown = new byte[1];
sck.Receive(responseUnknown);
if (Encoding.ASCII.GetString(responseUnknown) == "b")
{
byte[] dataInByteArray = new byte[fileSize];
int dataLength = sck.Receive(dataInByteArray);
return dataInByteArray;
}
return Encoding.ASCII.GetBytes("ERROR RESPONSE FUNC");
}
static void startHandlingClient()
{
while (true)
{
handleClient(server);
}
}
static void handleClient(TcpListener clientToAccept)
{
Socket sck = clientToAccept.AcceptSocket();
Thread myNewThread = new Thread(() => ReadResponse(sck));
myNewThread.Start();
}
static bool SocketConnected(Socket s)
{
bool part1 = s.Poll(1000, SelectMode.SelectRead);
bool part2 = (s.Available == 0);
if (part1 && part2)
return false;
else
return true;
}
static void ReadResponse(Socket sck)
{
Thread myNewThread = new Thread(() => SendtoClient(sck));
myNewThread.Start();
Thread.Sleep(2000);
while (true)
{
if (SocketConnected(sck) == true)
{
try
{
byte[] dataInByteArray = readResponseFunc(sck);
string response = Encoding.ASCII.GetString(dataInByteArray);
Console.WriteLine("res: " + response);
if (response != "make_id" && response != "get_id") { Console.WriteLine(response); }
if (response == "make_id")
{
Console.WriteLine("Someone wants an ID");
byte[] message_1 = Encoding.ASCII.GetBytes(id.ToString());
listOfClient.Add(id);
// START
sck.Send(Encoding.ASCII.GetBytes(message_1.Length.ToString()));
byte[] responseUnknown = new byte[1];
if (Encoding.ASCII.GetString(responseUnknown) == "a")
{
sck.Send(Encoding.ASCII.GetBytes("b"));
sck.Send(message_1);
}
id++;
}
if (response == "get_id")
{
byte[] message_1 = Encoding.ASCII.GetBytes(selected_id.ToString());
sck.Send(Encoding.ASCII.GetBytes(message_1.Length.ToString()));
byte[] responseUnknown = new byte[1];
if (Encoding.ASCII.GetString(responseUnknown) == "a")
{
sck.Send(Encoding.ASCII.GetBytes("b"));
sck.Send(message_1);
}
}
}
catch { if (SocketConnected(sck) == false) { Console.WriteLine("Client Disconnected: " + sck.RemoteEndPoint); break; }; }
}
else if (SocketConnected(sck) == false) { Console.WriteLine("Client Disconnected: " + sck.RemoteEndPoint); break; }
}
}
static void SendtoClient(Socket sck)
{
string tempmessage = null;
while (true)
{
if (SocketConnected(sck) == true)
{
if (tempmessage != message)
{
if (!message.Contains("PREVENT_REPETITION_IN_COMMAND"))
{
byte[] message_1 = Encoding.ASCII.GetBytes(message);
sck.Send(Encoding.ASCII.GetBytes(message_1.Length.ToString()));
byte[] responseUnknown = new byte[1];
if (Encoding.ASCII.GetString(responseUnknown) == "a")
{
sck.Send(Encoding.ASCII.GetBytes("b"));
sck.Send(message_1);
}
}
tempmessage = message;
}
}
else if (SocketConnected(sck) == false)
{ Console.WriteLine("Client Disconnected: " + sck.RemoteEndPoint); break; }
}
}
}
}
Problem:
The problem is within the GetSession or the ReadResponseFunc function. The client thinks that his ID received by the server is 'a' (it's suppose to be an integer). I don't want a solution that suggest me to use other libs or
the TcpClient class
Bounty:
I'll put up a bounty with no expiry time to those who solve the problem.

The logic in your code is very confusing. My question to you: Why are you sending 'a' and 'b' back and forth between the server and client? Is it some sort of confirmation that the message has been received?
Anyways, throughout the quick tests I've done just now, it seems that the problem is Line 59 of your server:
sck.Send(Encoding.ASCII.GetBytes("a"));
Here's what I figured out during testing:
Server executes.
Client executes.
Client sends server the length of "make_id" (Line 51 of client)
Client waits for a response to return.
Server reads the value and sends back 'a' (Line 59 of server)
You may want to spend some time to straighten out your protocol so it's less confusing and more organized. That would help people like me and you spot bugs much more easily.

The user 'Bobby' has already found your problem. Credits go to him.
I further suggest that you use less threads, as thread synchronisation needs some effort when doing it right: all data that is accessed from different threads must be secured by locks, so that no outdated values remain in the CPU caches. Use .net Monitor from the "threading synchronisation primitives" for that job.
About the threads themselves:
You should have only one thread in the server. This thread takes all clients from a list (secured by Monitor), in which they were added when connection attempts were received. On each client it checks for incoming messages, evaluates them and replies with own messages if needed.
The client also has just one thread, that will loop (dont forget a sleep in the loop or you will have 100% usage of the used CPU core), send messages when desired and wait for replies when messages were sent.
About the messages:
I already gave some proposals in a comment to Bobby's answer. I suggest you create a CMessage class that has a Serialize() and Deserialze() to create a byte array to send from the CMessage members (serializing the content) or vice versa filling the members from the received bytes. You then may use this class in both programs and you'll have common solution.

How can i handle and how should i handle exception The remote server returned an error: (500) Internal Server Error?

The exception is happen most of the times but there are times it's working fine.
The exception is always the same
The remote server returned an error: (500) Internal Server Error.
Also the stacktrace is the same:
at System.Net.WebClient.DownloadDataInternal(Uri address, WebRequest& request)
at System.Net.WebClient.DownloadData(Uri address)
at System.Net.WebClient.DownloadData(String address)
at SatelliteImages.ExtractImages.ExtractDateAndTime(String baseAddress) in D:\C-Sharp\SatelliteImages\SatelliteImages\SatelliteImages\ExtractImages.cs:line 120
Line 120 is:
var temp = wc.DownloadData("/en");
The method:
public void ExtractDateAndTime(string baseAddress)
{
try
{
var wc = new WebClient();
wc.BaseAddress = baseAddress;
HtmlDocument doc = new HtmlDocument();
var temp = wc.DownloadData("/en");
doc.Load(new MemoryStream(temp));
var secTokenScript = doc.DocumentNode.Descendants()
.Where(e =>
String.Compare(e.Name, "script", true) == 0 &&
String.Compare(e.ParentNode.Name, "div", true) == 0 &&
e.InnerText.Length > 0 &&
e.InnerText.Trim().StartsWith("var region")
).FirstOrDefault().InnerText;
var securityToken = secTokenScript;
securityToken = securityToken.Substring(0, securityToken.IndexOf("arrayImageTimes.push"));
securityToken = secTokenScript.Substring(securityToken.Length).Replace("arrayImageTimes.push('", "").Replace("')", "");
var dates = securityToken.Trim().Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
var scriptDates = dates.Select(x => new ScriptDate { DateString = x });
foreach (var date in scriptDates)
{
DatesAndTimes.Add(date.DateString);
}
}
catch(WebException wex)
{
if (wex.Response != null)
{
using (var errorResponse = (HttpWebResponse)wex.Response)
{
using (var reader = new StreamReader(errorResponse.GetResponseStream()))
{
string error = reader.ReadToEnd();
}
}
}
countriescodes = new List<string>();
countriesnames = new List<string>();
DatesAndTimes = new List<string>();
imagesUrls = new List<string>();
this.Init();
}
}
When I use a breakpoint on the line:
string error = reader.ReadToEnd();
I see html content and in the content I see the text:
error occurred while processing your request. Return to the homepage of Sat24.com
What I want to do is somehow when the exception happen to start over again and try the download try the method ExtractDateAndTime.
I think using a timer some how and count back showing the user something like 30 seconds and try again. Will 30 seconds try will be consider as spam/flooding in the server site ?
This is the full class code but the exception is on this method ExtractDateAndTime.
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Net;
using System.Xml;
using HtmlAgilityPack;
using System.ComponentModel;
namespace SatelliteImages
{
class ExtractImages
{
static WebClient client;
static string htmltoextract;
public static List<string> countriescodes = new List<string>();
public static List<string> countriesnames = new List<string>();
public static List<string> DatesAndTimes = new List<string>();
public static List<string> imagesUrls = new List<string>();
static string firstUrlPart = "http://www.sat24.com/image2.ashx?region=";
static string secondUrlPart = "&time=";
static string thirdUrlPart = "&ir=";
public class ProgressEventArgs : EventArgs
{
public int Percentage { get; set; }
public string StateText { get; set; }
}
public event EventHandler<ProgressEventArgs> ProgressChanged;
public void Init()
{
object obj = null;
int index = 0;
ExtractCountires();
foreach (string cc in countriescodes)
{
// raise event here
ProgressChanged?.Invoke(obj,new ProgressEventArgs{ Percentage = 100 * index / countriescodes.Count, StateText = cc });
ExtractDateAndTime("http://www.sat24.com/image2.ashx?region=" + cc);
index +=1;
}
ImagesLinks();
}
public void ExtractCountires()
{
try
{
htmltoextract = "http://sat24.com/en/?ir=true";//"http://sat24.com/en/";// + regions;
client = new WebClient();
client.DownloadFile(htmltoextract, #"c:\temp\sat24.html");
client.Dispose();
string tag1 = "<li><a href=\"/en/";
string tag2 = "</a></li>";
string s = System.IO.File.ReadAllText(#"c:\temp\sat24.html");
s = s.Substring(s.IndexOf(tag1));
s = s.Substring(0, s.LastIndexOf(tag2) + tag2.ToCharArray().Length);
s = s.Replace("\r", "").Replace("\n", "").Replace(" ", "");
string[] parts = s.Split(new string[] { tag1, tag2 }, StringSplitOptions.RemoveEmptyEntries);
string tag3 = "<li><ahref=\"/en/";
for (int i = 0; i < parts.Length; i++)
{
if (i == 17)
{
//break;
}
string l = "";
if (parts[i].Contains(tag3))
l = parts[i].Replace(tag3, "");
string z1 = l.Substring(0, l.IndexOf('"'));
if (z1.Contains("</ul></li><liclass="))
{
z1 = z1.Replace("</ul></li><liclass=", "af");
}
countriescodes.Add(z1);
countriescodes.GroupBy(n => n).Any(c => c.Count() > 1);
string z2 = parts[i].Substring(parts[i].LastIndexOf('>') + 1);
if (z2.Contains("&"))
{
z2 = z2.Replace("&", " & ");
}
countriesnames.Add(z2);
countriesnames.GroupBy(n => n).Any(c => c.Count() > 1);
}
}
catch (Exception e)
{
if (countriescodes.Count == 0)
{
countriescodes = new List<string>();
countriesnames = new List<string>();
DatesAndTimes = new List<string>();
imagesUrls = new List<string>();
Init();
}
}
}
public void ExtractDateAndTime(string baseAddress)
{
try
{
var wc = new WebClient();
wc.BaseAddress = baseAddress;
HtmlDocument doc = new HtmlDocument();
var temp = wc.DownloadData("/en");
doc.Load(new MemoryStream(temp));
var secTokenScript = doc.DocumentNode.Descendants()
.Where(e =>
String.Compare(e.Name, "script", true) == 0 &&
String.Compare(e.ParentNode.Name, "div", true) == 0 &&
e.InnerText.Length > 0 &&
e.InnerText.Trim().StartsWith("var region")
).FirstOrDefault().InnerText;
var securityToken = secTokenScript;
securityToken = securityToken.Substring(0, securityToken.IndexOf("arrayImageTimes.push"));
securityToken = secTokenScript.Substring(securityToken.Length).Replace("arrayImageTimes.push('", "").Replace("')", "");
var dates = securityToken.Trim().Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries);
var scriptDates = dates.Select(x => new ScriptDate { DateString = x });
foreach (var date in scriptDates)
{
DatesAndTimes.Add(date.DateString);
}
}
catch(WebException wex)
{
if (wex.Response != null)
{
using (var errorResponse = (HttpWebResponse)wex.Response)
{
using (var reader = new StreamReader(errorResponse.GetResponseStream()))
{
string error = reader.ReadToEnd();
}
}
}
countriescodes = new List<string>();
countriesnames = new List<string>();
DatesAndTimes = new List<string>();
imagesUrls = new List<string>();
this.Init();
}
}
public class ScriptDate
{
public string DateString { get; set; }
public int Year
{
get
{
return Convert.ToInt32(this.DateString.Substring(0, 4));
}
}
public int Month
{
get
{
return Convert.ToInt32(this.DateString.Substring(4, 2));
}
}
public int Day
{
get
{
return Convert.ToInt32(this.DateString.Substring(6, 2));
}
}
public int Hours
{
get
{
return Convert.ToInt32(this.DateString.Substring(8, 2));
}
}
public int Minutes
{
get
{
return Convert.ToInt32(this.DateString.Substring(10, 2));
}
}
}
public void ImagesLinks()
{
int cnt = 0;
foreach (string countryCode in countriescodes)
{
cnt++;
for (; cnt < DatesAndTimes.Count(); cnt++)
{
string imageUrl = firstUrlPart + countryCode + secondUrlPart + DatesAndTimes[cnt] + thirdUrlPart + "true";
imagesUrls.Add(imageUrl);
if (cnt % 10 == 0) break;
}
}
}
}
}
What i want is in case of the exception happen to start over clean over again the whole class operation.
In Form1 i start the class operation first time once:
In top:
ExtractImages ei = new ExtractImages();
Then in constructor:
ei.Init();
The problem is this exception that sometimes happen.

You could go with something like Polly or the Transient Fault Handling Application Block to apply a retry strategy to your code.
Both packages provide multiple out-of-the-box components for various scenarios and you can always develop you own. Some of the included retry policies:
Incremental
Fixed interval
Exponential back-off
Retry
Retry for ever
Retry and wait
Wait and retry for ever
...

stm32f4 discovery sample of using USB to communicate with pc or read/write pendrive using net micro framework

I cannot find any sample of using USB port of STM32F4 Discovery board in .NET Micro Framework. I'm trying to learn how to use USB port to send or write data. Where can I find such examples?
I've tried to make sample application based on https://guruce.com/blogpost/communicating-with-your-microframework-application-over-usb but it doesn't work.
This is part of my code sample:
using Microsoft.SPOT;
using Microsoft.SPOT.Hardware;
using Microsoft.SPOT.Hardware.UsbClient;
using System;
using System.Text;
using System.Threading;
namespace MFConsoleApplication1
{
public class Program
{
private const int WRITE_EP = 1;
private const int READ_EP = 2;
private static bool ConfigureUSBController(UsbController usbController)
{
bool bRet = false;
// Create the device descriptor
Configuration.DeviceDescriptor device = new Configuration.DeviceDescriptor(0xDEAD, 0x0001, 0x0100);
device.bcdUSB = 0x110;
device.bDeviceClass = 0xFF; // Vendor defined class
device.bDeviceSubClass = 0xFF; // Vendor defined subclass
device.bDeviceProtocol = 0;
device.bMaxPacketSize0 = 8; // Maximum packet size of EP0
device.iManufacturer = 1; // String #1 is manufacturer name (see string descriptors below)
device.iProduct = 2; // String #2 is product name
device.iSerialNumber = 3; // String #3 is the serial number
// Create the endpoints
Configuration.Endpoint writeEP = new Configuration.Endpoint(WRITE_EP, Configuration.Endpoint.ATTRIB_Bulk | Configuration.Endpoint.ATTRIB_Write);
writeEP.wMaxPacketSize = 64;
writeEP.bInterval = 0;
Configuration.Endpoint readEP = new Configuration.Endpoint(READ_EP, Configuration.Endpoint.ATTRIB_Bulk | Configuration.Endpoint.ATTRIB_Read);
readEP.wMaxPacketSize = 64;
readEP.bInterval = 0;
Configuration.Endpoint[] usbEndpoints = new Configuration.Endpoint[] { writeEP, readEP };
// Set up the USB interface
Configuration.UsbInterface usbInterface = new Configuration.UsbInterface(0, usbEndpoints);
usbInterface.bInterfaceClass = 0xFF; // Vendor defined class
usbInterface.bInterfaceSubClass = 0xFF; // Vendor defined subclass
usbInterface.bInterfaceProtocol = 0;
// Create array of USB interfaces
Configuration.UsbInterface[] usbInterfaces = new Configuration.UsbInterface[] { usbInterface };
// Create configuration descriptor
Configuration.ConfigurationDescriptor config = new Configuration.ConfigurationDescriptor(500, usbInterfaces);
// Create the string descriptors
Configuration.StringDescriptor manufacturerName = new Configuration.StringDescriptor(1, Consts.MANUFACTURER);
Configuration.StringDescriptor productName = new Configuration.StringDescriptor(2, Consts.PRODUCT_NAME);
Configuration.StringDescriptor serialNumber = new Configuration.StringDescriptor(3, Consts.SERIAL_NO);
Configuration.StringDescriptor displayName = new Configuration.StringDescriptor(4, Consts.DISPLAYED_NAME);
Configuration.StringDescriptor friendlyName = new Configuration.StringDescriptor(5, Consts.FRENDLY_NAME);
// Create the final configuration
Configuration configuration = new Configuration();
configuration.descriptors = new Configuration.Descriptor[]
{
device,
config,
manufacturerName,
productName,
serialNumber,
displayName,
friendlyName
};
try
{
// Set the configuration
usbController.Configuration = configuration;
if (UsbController.ConfigError.ConfigOK != usbController.ConfigurationError)
throw new ArgumentException();
// If all ok, start the USB controller.
bRet = usbController.Start();
}
catch (ArgumentException)
{
Debug.Print(Consts.CONFIG_ERR + usbController.ConfigurationError.ToString());
}
return bRet;
}
private static string GetCommand(byte[] data)
{
return new string(Encoding.UTF8.GetChars(data));
}
private static byte[] GetDataToSend(Reading data, JsonSerializer serializer)
{
return Encoding.UTF8.GetBytes(serializer.Serialize(data));
}
private static void UsbMainLoop(UsbStream usbStream, AnalogInput supplyInput, AnalogInput dataInput, OutputPort recievingDataIndicator, OutputPort seindingDataIndicator, JsonSerializer dataSerializer)
{
byte[] readData = new byte[100];
for (;;)
{
recievingDataIndicator.Write(true);
int bytesRead = usbStream.Read(readData, 0, readData.Length);
recievingDataIndicator.Write(false);
if (bytesRead > 0)
{
string command = GetCommand(readData);
switch (command)
{
case "READ":
seindingDataIndicator.Write(true);
byte[] dataToSend = GetDataToSend(Reading.GetReading(supplyInput, dataInput), dataSerializer);
usbStream.Write(dataToSend, 0, dataToSend.Length);
seindingDataIndicator.Write(false);
break;
}
}
}
}
private static UsbController CheckSupportAndAccessibility(UsbController[] usbControllers, out string message)
{
UsbController usbController = null;
if (0 == usbControllers.Length)
{
message = Consts.USB_NOT_SUPPORTED;
}
else
{
bool foundedFreeUsb = false;
foreach (UsbController controller in usbControllers)
{
if (UsbController.PortState.Stopped == controller.Status)
{
usbController = controller;
foundedFreeUsb = true;
break;
}
}
if (foundedFreeUsb)
{
message = string.Empty;
}
else
{
message = Consts.NO_FREE_USB;
}
}
return usbController;
}
private static void OtherMainProgramLoop()
{
AnalogInput temperature0 = new AnalogInput(Cpu.AnalogChannel.ANALOG_0);
AnalogInput supply0 = new AnalogInput(Cpu.AnalogChannel.ANALOG_1);
double t0, r0, z0;
while (true)
{
z0 = supply0.Read() * 3;
r0 = Reading.GetRt(z0, temperature0.Read() * z0);
t0 = Reading.GetTemp(r0);
Debug.Print("Supply:" + z0.ToString() + "[V]");
Debug.Print("Rt:" + r0.ToString() + "[Ohm]");
Debug.Print("Temperature:" + t0.ToString() + "[^C]");
Thread.Sleep(1000);
}
}
public static void Main()
{
UsbController[] controllers = UsbController.GetControllers();
string msg;
UsbController usbController = CheckSupportAndAccessibility(controllers, out msg);
if (null == usbController)
{
Debug.Print(msg);
OtherMainProgramLoop();
//return;
}
else
{
UsbStream usbStream = null;
try
{
if (ConfigureUSBController(usbController))
usbStream = usbController.CreateUsbStream(WRITE_EP, READ_EP);
else
throw new Exception();
}
catch (Exception)
{
Debug.Print(Consts.USB_CREATE_STREAM_ERR + usbController.ConfigurationError.ToString());
OtherMainProgramLoop();
//return;
}
AnalogInput temperatureSource = new AnalogInput(Cpu.AnalogChannel.ANALOG_0);
AnalogInput supplySource = new AnalogInput(Cpu.AnalogChannel.ANALOG_1);
OutputPort ledGreen = new OutputPort((Cpu.Pin)60, false);
OutputPort ledYellow = new OutputPort((Cpu.Pin)61, false);
UsbMainLoop(usbStream, supplySource, temperatureSource, ledYellow, ledGreen, new JsonSerializer());
usbStream.Close();
}
}
}
}
Thanks for any help.

C# Multi threading not starting worker/job?

For some reason my threads are ignoring my job, it looks like anyone have any sort of idea why it is doing this? Help is very welcome and extremely appreciated, thank you. What this program is, is a ProxyChecker, because i've bought a bunch and will continue to do so of proxies with different user/passes etc, however some have expired
static List<String> user = new List<String>();
static List<String> pass = new List<String>();
static List<String> ips = new List<String>();
static Random rnd = new Random();
static void Main(string[] args)
{
int threads = 4;
loadips();
Console.WriteLine("Enter the amount of threads to run (4 default):");
threads = Int32.Parse(Console.ReadLine());
Console.WriteLine("Starting on " + threads + " threads...");
for (int i = 0; i < threads; i++)
{
ThreadPool.QueueUserWorkItem(new WaitCallback(CheckProxy), i);
}
//Console.ReadLine();
}
public class MyIP
{
public string IP { get; set; }
public bool AcceptsConnection { get; set; }
}
private static void CheckProxy(object state)
{
var u = user[0];
var p = pass[0];
var l = new List<MyIP>();
Parallel.ForEach(l.ToArray(), (item) =>
{
string ip = getip();
try
{
using (var client = new ProxyClient(ip, u, p))
{
Console.WriteLine(ip, user, pass);
client.Connect();
item.AcceptsConnection = client.IsConnected;
}
}
catch
{
l.Remove(item);
}
});
foreach (var item in l)
{
if (item.AcceptsConnection == true)
{
WriteToFile(user[0], pass[0]);
}
Console.WriteLine(item.IP + " is " + (item.AcceptsConnection) + " accepts connections" + " doesn not accept connections");
}
}
private static void loadips()
{
using (TextReader tr = new StreamReader("ips.txt"))
{
string line = null;
while ((line = tr.ReadLine()) != null)
{
ips.Add(line);
}
}
}

You didn't understand my code example... You should add the IP's to the list l, instead of a getip() method... So lose the getip() method
private static void CheckProxy(object state)
{
var u = user[0];
var p = pass[0];
var l = new List<MyIP>();
l.Add(new MyIP { IP = "192.168.1.1" });
l.Add(new MyIP { IP = "192.168.1.2" });
l.Add(new MyIP { IP = "192.168.1.3" });
Parallel.ForEach(l.ToArray(), (ip_item) =>
{
try
{
using (var client = new ProxyClient(ip_item, u, p))
{
Console.WriteLine(ip_item, user, pass);
client.Connect();
item.AcceptsConnection = client.IsConnected;
}
}
catch
{
lock(l)
l.Remove(item);
}
});
foreach (var item in l)
{
if (item.AcceptsConnection == true)
{
WriteToFile(user[0], pass[0]);
}
Console.WriteLine(item.IP + " is " + (item.AcceptsConnection) + " accepts connections" + " doesn not accept connections");
}
}