Develop Reference

NetMQ Unity Pub/Sub

I'm running into some issues trying to get NetMQ to work. My code compiles fine but I'm not achieving my desired outcome. Testing both Pub and Sub side from one PC.
I have a UI Handler that is linked to four buttons, start/stop pub/sub.
Would love if someone could shed some light into the issue.
Desired outcome: To be able to transfer test.wav from "Sending Folder" to "Receiving Folder"
Publisher Logic
using System.Threading;
using NetMQ;
using NetMQ.Sockets;
using System.IO;
public class Publisher
{
private readonly Thread _publisherThread;
private readonly Thread _subscriberThread;
public Publisher()
{
_publisherThread = new Thread(PublisherWork);
_publisherThread.Start();
_subscriberThread = new Thread(SubscriberWork);
_subscriberThread.Start();
}
public void Start()
{
using (var pubSocket = new PublisherSocket())
{
pubSocket.Bind("tcp://*:5557");
string filePath = "C:/Users/XXXXX/Desktop/0MQ Demo/Sending Folder/test.wav";
byte[] fileBytes = File.ReadAllBytes(filePath);
pubSocket.SendMoreFrame("File").SendFrame(fileBytes);
}
}
private void PublisherWork()
{
using (var pubSocket = new PublisherSocket())
{
pubSocket.Bind("tcp://*:5557"); // [C] Port
string filePath = "C:/Users/XXXXX/Desktop/0MQ Demo/Sending Folder/test.wav"; // [C] Filepath
byte[] fileBytes = File.ReadAllBytes(filePath);
while (true)
{
pubSocket.SendMoreFrame("File").SendFrame(fileBytes); // [C] Topic
Thread.Sleep(1000);
}
}
}
private void SubscriberWork()
{
using (var subSocket = new SubscriberSocket())
{
subSocket.Connect("tcp://localhost:5557"); // [C] Port
subSocket.Subscribe("File"); // [C] Topic
while (true)
{
var topic = subSocket.ReceiveFrameString();
var fileBytes = subSocket.ReceiveFrameBytes();
SaveWavFile(fileBytes);
}
}
}
private void SaveWavFile(byte[] fileBytes)
{
string filePath = "C:/Users/XXXXX/Desktop/0MQ Demo/Receiving Folder/test.wav";
File.WriteAllBytes(filePath, fileBytes);
}
}
Subscriber Logic
using System.Threading;
using NetMQ;
using NetMQ.Sockets;
using UnityEngine;
using UnityEngine.UI;
using System.IO;
public class Subscriber : MonoBehaviour
{
public Button startButton;
public Button stopButton;
private Thread _publisherThread;
private Thread _subscriberThread;
private bool _isRunning;
private void Start()
{
startButton.onClick.AddListener(StartThreads);
stopButton.onClick.AddListener(StopThreads);
}
private void StartThreads()
{
_isRunning = true;
_publisherThread = new Thread(PublisherWork);
_publisherThread.Start();
_subscriberThread = new Thread(SubscriberWork);
_subscriberThread.Start();
}
private void StopThreads()
{
_isRunning = false;
_publisherThread.Join();
_subscriberThread.Join();
}
private void PublisherWork()
{
using (var pubSocket = new PublisherSocket())
{
pubSocket.Bind("tcp://*:5557"); // [C] Port
string filePath = "C:/Users/XXXXX/Desktop/0MQ Demo/Sending Folder/test.wav"; // [C] Filepath
byte[] fileBytes = File.ReadAllBytes(filePath);
while (_isRunning)
{
pubSocket.SendMoreFrame("File").SendFrame(fileBytes); // [C] Topic
Thread.Sleep(1000);
}
}
NetMQConfig.Cleanup();
}
private void SubscriberWork()
{
using (var subSocket = new SubscriberSocket())
{
subSocket.Connect("tcp://localhost:5557"); // [C] Port
subSocket.Subscribe("File"); // [C] Topic
while (_isRunning)
{
var topic = subSocket.ReceiveFrameString();
var fileBytes = subSocket.ReceiveFrameBytes();
SaveWavFile(fileBytes);
}
}
NetMQConfig.Cleanup();
}
private void SaveWavFile(byte[] fileBytes)
{
string filePath = "C:/Users/XXXXX/Desktop/0MQ Demo/Receiving Folder/test.wav"; // [C]
File.WriteAllBytes(filePath, fileBytes);
}
}
UI Handler
using System.Threading;
using NetMQ;
using NetMQ.Sockets;
using System.IO;
using UnityEngine;
using UnityEngine.UI;
public class UIHandler : MonoBehaviour
{
public Button startSubscriberButton;
public Button stopSubscriberButton;
public Button startPublisherButton;
public Button stopPublisherButton;
public Text pubStartStatus;
public Text pubStopStatus;
public Text subStartStatus;
public Text subStopStatus;
private Publisher _publisher;
private readonly Thread _publisherThread;
private readonly Thread _subscriberThread;
private bool _publisherRunning = false;
private bool _subscriberRunning = false;
public UIHandler()
{
// Initializing the threads
_publisherThread = new Thread(PublisherWork);
_subscriberThread = new Thread(SubscriberWork);
}
private void Start()
{
_publisher = new Publisher();
startPublisherButton.onClick.AddListener(StartPublisher);
stopPublisherButton.onClick.AddListener(StopPublisher);
startSubscriberButton.onClick.AddListener(StartSubscriber);
stopSubscriberButton.onClick.AddListener(StopSubscriber);
}
public void StartPublisher()
{
// Starting the publisher thread
_publisherThread.Start();
_publisherRunning = true;
pubStartStatus.text = "pubStartStatus: Started";
}
public void StopPublisher()
{
// Stopping the publisher thread
_publisherRunning = false;
_publisherThread.Join();
pubStopStatus.text = "pubStartStatus: Stopped";
}
public void StartSubscriber()
{
// Starting the subscriber thread
_subscriberThread.Start();
_subscriberRunning = true;
subStartStatus.text = "subStartStatus: Stopped";
}
public void StopSubscriber()
{
// Stopping the subscriber thread
_subscriberRunning = false;
_subscriberThread.Join();
subStopStatus.text = "subStartStatus: Stopped";
}
private void PublisherWork()
{
// Creating a publisher socket and binding it to the specified address
using (var pubSocket = new PublisherSocket())
{
pubSocket.Bind("tcp://*:5557");
// Reading the file bytes from the specified location
string filePath = "C:/Users/XXXXX/Desktop/0MQ Demo/Sending Folder/test.wav";
byte[] fileBytes = File.ReadAllBytes(filePath);
// Publishing the file bytes continuously
while (_publisherRunning)
{
pubSocket.SendMoreFrame("File").SendFrame(fileBytes);
Thread.Sleep(1000);
}
}
NetMQConfig.Cleanup();
}
private void SubscriberWork()
{
// Creating a subscriber socket and connecting it to the specified address
using (var subSocket = new SubscriberSocket())
{
subSocket.Connect("tcp://localhost:5557");
subSocket.Subscribe("File");
// Receiving and saving the file bytes continuously
while (_subscriberRunning)
{
var topic = subSocket.ReceiveFrameString();
var fileBytes = subSocket.ReceiveFrameBytes();
SaveWavFile(fileBytes);
}
}
NetMQConfig.Cleanup();
}
private void SaveWavFile(byte[] fileBytes)
{
// Saving the received file bytes to the specified location
string filePath = "C:/Users/XXXXX/Desktop/0MQ Demo/Receiving Folder/test.wav";
File.WriteAllBytes(filePath, fileBytes);
}
}

How to send a picture to the mail using mime and ssl

I'm trying to send a letter with a picture to the mail, my code works with html files, pictures are sent, but when opened, it gives an error. I'm doing my lab work and on assignment I can't use MailMessage for example. I set the sending address from my windows forms.
using System;
using System.IO;
using System.Net.Mail;
using System.Net.Mime;
namespace NetworksNeLaba2
{
public class Program
{
private const string AddressFrom = "MYGM";
private const string PasswordFrom = "MYPS";
private static readonly Emailbox _emailbox = new Emailbox(AddressFrom, PasswordFrom);
private const string HostName = "smtp.gmail.com";
private const int Port = 465;
private static readonly ConnectionInstaller _connectionInstaller = new ConnectionInstaller(HostName, Port);
public static void Main(string[] args)
{
}
public void InitializeComponents(string address, string subject)
{
var direction = new MessageDirection(_emailbox.Address, address);
var commands = new []
{
$"EHLO {HostName}",
"AUTH LOGIN",
_emailbox.Address,
_emailbox.Password,
$"MAIL FROM:<{direction.From}>",
$"RCPT TO:<{direction.To}>",
"DATA"
};
var fileName = "cat.jpeg";
var ct = new ContentType();
ct.MediaType = MediaTypeNames.Image.Jpeg;
var data = new Attachment(fileName, ct.MediaType);
var s = data.ContentStream;
var reader = new StreamReader(s);
var letterContent = new []
{
$"Subject:{subject}\r\n",
$"From:Roma {AddressFrom}\r\n",
$"To:{address}\r\n",
$"Content-Type:{data.ContentType}\r\n",
$"Content-Disposition:{data.ContentDisposition}\r\n",
$"Content:{reader.ReadToEnd()}\r\n",
//$"Date:{DateTime.Now}\r\n",
".\r\n"
};
var letter = String.Concat(letterContent);
_connectionInstaller.SendMessage(commands, letter);
}
}
}
The first part of the code is what commands and the content of the letter I set, below I attached the code that processes the commands.
using System;
using System.Net.Security;
using System.Net.Sockets;
using System.Text;
namespace NetworksNeLaba2
{
public class ConnectionInstaller
{
private readonly TcpClient _tcpClient = new TcpClient();
private readonly SslStream _sslStream;
private readonly byte[] _buffer = new byte[512];
public ConnectionInstaller(string hostName, int port)
{
_tcpClient.Connect(hostName, port);
_sslStream = new SslStream(_tcpClient.GetStream());
_sslStream.AuthenticateAsClient(hostName);
}
public void SendMessage(string[] commands,string letterContent)
{
foreach (var command in commands)
{
WriteBuffer(command + "\r\n");
}
WriteBuffer(letterContent);
}
private void WriteBuffer(string str)
{
_sslStream.Write(Encoding.ASCII.GetBytes(str));
_sslStream.Read(_buffer, 0, _buffer.Length);
Console.WriteLine(Encoding.ASCII.GetString(_buffer));
Array.Clear(_buffer, 0, _buffer.Length);
}
}
}

C# Sending value from WebSocket Server Thread to Main()

I am trying to write a program to control home automation devices. This involves a WebSocket server that runs infinitely in a background thread listening for clients. The client sends JSON packets that contain settings. When the server gets a JSON packet, I want it to read and change any settings as needed in the Main()
I have a working WebSocket server and I have tried to use the PropertyChangedEvent but it never seems to see the handler. I suspect this occurs because they exist in different threads.
I have attached my code below as well as the example I have been working off of. There is a chance I am way off the mark here but this is the best I could find with the reading I have been doing.
Any help would be appreciated!
This is the Main():
using System;
using System.Threading;
using System.ComponentModel;
namespace HomeAutomation
{
class Controller
{
static void Main(string[] args)
{
bool passGo = false;
Thread SocketThread = new Thread(Socket.WebSocketServer);
SocketThread.IsBackground = true;
SocketThread.Start();
//This line lets the user know the Socket Thread is running in the background
Console.WriteLine("Socket Thread is running in the background: {0}", SocketThread.IsBackground);
do {
char input = Console.ReadKey().KeyChar;
if (input == 'x')
{
passGo = true;
}
} while (passGo == false);
Console.ReadLine();
/*
Settings s = new Settings();
s.PropertyChanged += new PropertyChangedEventHandler(S_PropertyChanged);
while (true)
{
string str = Console.ReadLine();
s.State = str;
}*/
}
public static void S_PropertyChanged(object sender, PropertyChangedEventArgs e)
{
Settings RecSettings = (Settings)sender;
Console.WriteLine("The {0} has changed to {1}", e.PropertyName, RecSettings.State);
}
}
public class Settings : INotifyPropertyChanged
//The Class Object 'Settings' is used to recieve the variable(s) from the client software
{
#region INotifyPropertyChanged Members
public event PropertyChangedEventHandler PropertyChanged;
#endregion
string state = string.Empty;
public string State
{
get { return state; }
set
{
state = value;
if (PropertyChanged != null)
{
PropertyChanged(this, new PropertyChangedEventArgs("State"));
Console.WriteLine("Event has been called");
}
}
}
}
}
This is my webSocket:
using System;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using Newtonsoft.Json;
namespace HomeAutomation
{
class Socket
{
public static void WebSocketServer()
{
//Declaring variables used in the program
IPAddress ipAddress = Dns.GetHostEntry("localhost").AddressList[0];
TcpListener server = new TcpListener(ipAddress, 8080);
try
{
//Starts the server
server.Start();
Console.WriteLine("Starting Server...");
}
catch (Exception ex)
{
//If the server cannot start the error will be caught and printed to console
Console.WriteLine(ex.ToString());
Console.ReadLine();
}
while (true)
{
TcpClient client = server.AcceptTcpClient();
MakeNewConnection(client);
}
}
public static void MakeNewConnection(TcpClient client)
{
var thread = new Thread(NewClient);
thread.Start(client);
}
public static void NewClient(object data)
{
var client = (TcpClient)data;
Settings RecSettings = new Settings();
//Lets you know the address of the connected client
string address = client.Client.AddressFamily.ToString();
Console.WriteLine("{0} has connected!", address);
//creates a network stream for information to flow through
NetworkStream stream = client.GetStream();
byte[] receivedBuffer = new byte[100];
stream.Read(receivedBuffer, 0, receivedBuffer.Length);
StringBuilder msg = new StringBuilder();
foreach (byte b in receivedBuffer)
{
if (b.Equals(00))
{
break;
}
else
{
msg.Append(Convert.ToChar(b).ToString());
}
}
Console.WriteLine("Client Says: {0}", msg.ToString());
RecSettings = JsonConvert.DeserializeObject<Settings>(msg.ToString());
Console.WriteLine(RecSettings.State.ToString());
RecSettings.State = "Off";
int byteCount = Encoding.ASCII.GetByteCount("Thank you");
byte[] sendData = new byte[byteCount];
sendData = Encoding.ASCII.GetBytes("Thank you");
stream.Write(sendData, 0, sendData.Length);
}
}
}
This is the event handler example I am using:
using System;
using System.ComponentModel;
namespace HomeAutomation
{
class Program
{
static void Main(string[] args)
{
TestClass sample = new TestClass();
sample.PropertyChanged += new PropertyChangedEventHandler(sample_PropertyChanged);
while (true)
{
string str = Console.ReadLine();
int val;
if (int.TryParse(str, out val))
sample.TestValue = val;
}
}
static void sample_PropertyChanged(object sender, PropertyChangedEventArgs e)
{
TestClass sample = (TestClass)sender;
/*
* Use expression behind if you have more the one property instead sample.TestValue
* typeof(TestClass).GetProperty(e.PropertyName).GetValue(sample, null)*/
Console.WriteLine("Value of property {0} was changed! New value is {1}", e.PropertyName, sample.TestValue);
}
}
public class TestClass : INotifyPropertyChanged
{
#region INotifyPropertyChanged Members
public event PropertyChangedEventHandler PropertyChanged;
#endregion
int testValue = 0;
public int TestValue
{
get { return testValue; }
set
{
testValue = value;
if (PropertyChanged != null)
PropertyChanged(this, new PropertyChangedEventArgs("TestValue"));
}
}
}
}

You may want to wrap the contents of your WebSocket response in code that invokes it in the Dispatcher thread.
System.Windows.Application.Current.Dispatcher.Invoke(new Action(() =>
{
... response handler code
}

"The call is ambiguous between the following methods or properties" in identical code

I have two nearly identical blocks of code in two separate projects; the second project was created to use the same ConnectClass as the first.
However, in the new project (the second code block shown below), the compiler gives the following error message regarding the error handler static void client_ErrorEvents(object sender, a.b.c.ErrorEventArgs e):
The call is ambiguous between the following methods or properties: 'ProjectName.ConnectClass.client_ErrorEvents(object, a.b.c.ErrorEventArgs)' and 'ProjectName.ConnectClass.client_ErrorEvents(object, a.b.c.ErrorEventArgs)'
HERE IS THE FIRST CODE BLOCK (IN RELEVANT PART)
public partial class ConnectClass
{
public static a.b.c.Client client = new a.b.c.Client();
public static string DriveLetter;
public static string CurrentDate;
//_____________________
public static void StoreVars(Form1 frm)
{
DriveLetter = frm.textBox4.Text;
CurrentDate = frm.textBox5.Text;
}
public static string Connect(Form1 frm, int Call)
{
string host = "127.0.0.1";
int port = 7496;
int clientId = 0;
string Result = "Connected";
try
{
client.Connect(host, port, clientId);
}
catch (SocketException e)
{
Result = e.ToString();
}
//if (Call == 0)
client.Error += new EventHandler<a.b.c.ErrorEventArgs>(client_ErrorEvents);
return Result;
}
static void client_ErrorEvents(object sender, a.b.c.ErrorEventArgs e)
{
int ErrorCode;
string path = DriveLetter + "/ThisPath/File.txt";
FileStream ThisFile = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);
StreamWriter sw = new StreamWriter(ThisFile);
DateTime dtNow = DateTime.Now;
ErrorCode = (int)e.ErrorCode;
sw.BaseStream.Seek(0, SeekOrigin.End);
sw.Write(dtNow);
sw.Write(" ");
sw.Write(e.ErrorCode);
sw.Write(" ");
sw.Write(e.ErrorMsg);
sw.Write(sw.NewLine);
sw.Close();
ThisFile.Close();
}
HERE IS SECOND CODE BLOCK (IN RELEVANT PART)
public partial class ConnectClass
{
public static a.b.c.Client client = new a.b.c.Client();
public static string DriveLetter;
public static string CurrentDate;
//_____________________
public static void StoreVars(Form1 frm)
{
DriveLetter = frm.TBx_Drive.Text;
CurrentDate = frm.TBx_Date.Text;
}
public static string Connect(Form1 frm)
{
string host = "127.0.0.1";
int port = 7496;
int clientId = 10;
string Result = "Connected";
try
{
client.Connect(host, port, clientId);
}
catch (SocketException e)
{
Result = e.ToString();
}
client.Error += new EventHandler<a.b.c.ErrorEventArgs>(client_ErrorEvents);
return Result;
}
// THIS IS WHERE THE COMPILER SIGNALS AN ERROR:
static void client_ErrorEvents(object sender, a.b.c.ErrorEventArgs e)
{
string path = DriveLetter + "/ThisPath/File.txt";
FileStream ThisFile = new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.ReadWrite);
StreamWriter sw = new StreamWriter(ThisFile);
DateTime dtNow = DateTime.Now;
ErrorCode = (int)e.ErrorCode;
sw.BaseStream.Seek(0, SeekOrigin.End);
sw.Write(dtNow);
sw.Write(" ");
sw.Write(e.ErrorCode);
sw.Write(" ");
sw.Write(e.ErrorMsg);
sw.Write(sw.NewLine);
sw.Close();
ThisFile.Close();
}
Any ideas?
Thanks very much.

You have single class split between 2 files (partial) and both implement the same function.
Either make them non-partial (you'll probably have name conflict on classes), or don't have duplicate methods in the same class.
See details about using partial in Partial Classes and Methods in C# article on MSDN.

C# Async WebRequests: Perform Action When All Requests Are Completed

I have this basic scraping console application in C# that Asynchronously uses WebRequest to get html from a list of sites. It works fine, but how do I set up a trigger that goes off when every site in the list has been processed?
I've spent a couple hours researching various solutions online, including the MS docs, but none of them provide a straight forward answer via code. I've read about the IAsyncResult.AsyncWaitHandle but I have no clue how to integrate it into my code. I'd just like to call a custom function when all threads complete processing or timeout.
One trick is that I never know ahead of time how many sites are in my list (it's user defined), so I need a solution that's robust enough to wait for 5 events for 100,000 events to complete.
Thanks. Working code below:
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Net;
using System.Threading;
namespace AsyncApp_01
{
class Program
{
static void Main(string[] args)
{
ArrayList alSites = new ArrayList();
alSites.Add("http://www.google.com");
alSites.Add("http://www.lostspires.com");
ScanSites(alSites);
Console.Read();
}
private static void ScanSites(ArrayList sites)
{
foreach (string uriString in sites)
{
WebRequest request = HttpWebRequest.Create(uriString);
request.Method = "GET";
object data = new object(); //container for our "Stuff"
// RequestState is a custom class to pass info to the callback
RequestState state = new RequestState(request, data, uriString);
IAsyncResult result = request.BeginGetResponse(new AsyncCallback(UpdateItem), state);
//Register the timeout callback
ThreadPool.RegisterWaitForSingleObject(result.AsyncWaitHandle, new WaitOrTimerCallback(ScanTimeoutCallback), state, (30 * 1000), true);
}
}
private static void UpdateItem(IAsyncResult result)
{
// grab the custom state object
RequestState state = (RequestState)result.AsyncState;
WebRequest request = (WebRequest)state.Request;
// get the Response
HttpWebResponse response = (HttpWebResponse)request.EndGetResponse(result);
Stream s = (Stream)response.GetResponseStream();
StreamReader readStream = new StreamReader(s);
// dataString will hold the entire contents of the requested page if we need it.
string dataString = readStream.ReadToEnd();
response.Close();
s.Close();
readStream.Close();
Console.WriteLine(dataString);
}
private static void ScanTimeoutCallback(object state, bool timedOut)
{
if (timedOut)
{
RequestState reqState = (RequestState)state;
if (reqState != null)
{
reqState.Request.Abort();
}
Console.WriteLine("aborted- timeout");
}
}
class RequestState
{
public WebRequest Request; // holds the request
public object Data; // store any data in this
public string SiteUrl; // holds the UrlString to match up results (Database lookup, etc).
public RequestState(WebRequest request, object data, string siteUrl)
{
this.Request = request;
this.Data = data;
this.SiteUrl = siteUrl;
}
}
}
}
Bonus points for anyone who can also tell me how to limit the number of concurrent threads. For example, if I have 100 sites to process, how do I set it up so that 10 sites get processed at a time, but not more. I don't want to open 100 threads.

Here's a quick sample I threw together. I removed the WebClient implementation, as it seems like you're using the WebRequest one. I'm also making use of .Net 4's ConcurrentBag:
public class Scraper
{
private readonly IEnumerable<string> _sites;
private readonly ConcurrentBag<string> _data;
private volatile int _count;
private readonly int _total;
public Scraper(IEnumerable<string> sites)
{
_sites = sites;
_data = new ConcurrentBag<string>();
_total = sites.Count();
}
public void Start()
{
foreach (var site in _sites)
{
ScrapeSite(site);
}
}
private void ScrapeSite(string site)
{
var req = WebRequest.Create(site);
req.BeginGetResponse(AsyncCallback, req);
}
private void AsyncCallback(IAsyncResult ar)
{
Interlocked.Increment(ref _count);
var req = ar.AsyncState as WebRequest;
var result = req.EndGetResponse(ar);
var reader = new StreamReader(result.GetResponseStream());
var data = reader.ReadToEnd();
this.OnSiteScraped(req.RequestUri.AbsoluteUri, data);
_data.Add(data);
if (_count == _total)
{
OnScrapingComplete();
}
}
private void OnSiteScraped(string site, string data)
{
var handler = this.SiteScraped;
if (handler != null)
{
handler(this, new SiteScrapedEventArgs(site, data));
}
}
private void OnScrapingComplete()
{
var handler = this.ScrapingComplete;
if (handler != null)
{
handler(this, new ScrapingCompletedEventArgs(_data));
}
}
public event EventHandler<SiteScrapedEventArgs> SiteScraped;
public event EventHandler<ScrapingCompletedEventArgs> ScrapingComplete;
}
public class SiteScrapedEventArgs : EventArgs
{
public string Site { get; private set; }
public string Data { get; private set; }
public SiteScrapedEventArgs(string site, string data)
{
this.Site = site;
this.Data = data;
}
}
OK, I created some basic classes, and this should do the trick. If this isn't enough, I'm sorry, I simply can't help you:
public class RankedPage
{
public int Rank { get; set; }
public string Site { get; set; }
}
public class WebRequestData
{
public WebRequest WebRequest { get; set; }
public RankedPage Page { get; set; }
}
public class Scraper
{
private readonly IEnumerable<RankedPage> _sites;
private readonly ConcurrentBag<KeyValuePair<RankedPage,string>> _data;
private volatile int _count;
private readonly int _total;
public Scraper(IEnumerable<RankedPage> sites)
{
_sites = sites;
_data = new ConcurrentBag<KeyValuePair<RankedPage, string>>();
_total = sites.Count();
}
public void Start()
{
foreach (var site in _sites)
{
ScrapeSite(site);
}
}
private void ScrapeSite(RankedPage site)
{
var req = WebRequest.Create(site.Site);
req.BeginGetResponse(AsyncCallback, new WebRequestData{ Page = site, WebRequest = req});
}
private void AsyncCallback(IAsyncResult ar)
{
Interlocked.Increment(ref _count);
var webRequestData = ar.AsyncState as WebRequestData;
var req = webRequestData.WebRequest;
var result = req.EndGetResponse(ar);
var reader = new StreamReader(result.GetResponseStream());
var data = reader.ReadToEnd();
this.OnSiteScraped(webRequestData.Page, data);
_data.Add(new KeyValuePair<RankedPage, string>(webRequestData.Page,data));
if (_count == _total)
{
OnScrapingComplete();
}
}
private void OnSiteScraped(RankedPage page, string data)
{
var handler = this.SiteScraped;
if (handler != null)
{
handler(this, new SiteScrapedEventArgs(page, data));
}
}
private void OnScrapingComplete()
{
var handler = this.ScrapingComplete;
if (handler != null)
{
handler(this, new ScrapingCompletedEventArgs(_data));
}
}
public event EventHandler<SiteScrapedEventArgs> SiteScraped;
public event EventHandler<ScrapingCompletedEventArgs> ScrapingComplete;
}
public class SiteScrapedEventArgs : EventArgs
{
public RankedPage Site { get; private set; }
public string Data { get; private set; }
public SiteScrapedEventArgs(RankedPage site, string data)
{
this.Site = site;
this.Data = data;
}
}
public class ScrapingCompletedEventArgs : EventArgs
{
public IEnumerable<KeyValuePair<RankedPage,string >> SiteData { get; private set; }
public ScrapingCompletedEventArgs(IEnumerable<KeyValuePair<RankedPage, string>> siteData)
{
this.SiteData = siteData;
}
}