I am trying to scrape results from google search, the tool need to browse pages one by one. However the issue is its not taking the all the list from the listbox. Its just working for the first line of the list box.
Startbtn Code
foreach (string url in urlList.Items)
{
webBrowser1.Navigate("https://www.google.com/search?q=" + url);
await PageLoad(30, 5);
MessageBox.Show("sdsaD3");
string pageSource = webBrowser1.DocumentText;
Scrape(pageSource);
}
--
Scrape Method
private async void Scrape(string pageSource)
{
string regexExpression = "(?<=><div class=\"rc\"><div class=\"r\"><a href=\")(.*?)(?=\" onmousedown=)";
Regex match = new Regex(regexExpression, RegexOptions.Singleline);
MatchCollection collection = Regex.Matches(pageSource, regexExpression);
for (int i = 0; i < collection.Count; i++)
{
CommonCodes.WriteToTxt(collection[i].ToString(), "googlescrapedurls.txt");
if (i == collection.Count - 1)
{
var elementid = webBrowser1.Document.GetElementById("pnnext");
if (elementid != null)
{
for (int w = 0; w < 1; w++)
{
BackgroundWorker worker = new BackgroundWorker();
worker.DoWork += new DoWorkEventHandler(backgroundWorker1_DoWork);
worker.RunWorkerAsync(w);
}
}
else if(webBrowser1.Document.GetElementById("pnnext") == null)
{
for(int pg=0; pg< urlList.Items.Count; pg++)
{
webBrowser1.Navigate("https://www.google.com/search?q=" + urlList.Items[pg+1]);
CommonCodes.WaitXSeconds(10);
//await PageLoad(30, 5);
Scrape(webBrowser1.DocumentText);
}
}
}
}
--
Background worker code:
BackgroundWorker backgroundWorker = sender as BackgroundWorker;
webBrowser1.Invoke(new Action(() => { gCaptcha(); }));
webBrowser1.Invoke(new Action(() => { webBrowser1.Document.GetElementById("pnnext").InvokeMember("Click"); }));
await PageLoad(30, 5);
webBrowser1.Invoke(new Action(() => { Scrape(webBrowser1.DocumentText); }));
pageload code
try
{
TaskCompletionSource<bool> PageLoaded = null;
PageLoaded = new TaskCompletionSource<bool>();
int TimeElapsed = 0;
webBrowser1.DocumentCompleted += (s, e) =>
{
if (webBrowser1.ReadyState != WebBrowserReadyState.Complete) return;
if (PageLoaded.Task.IsCompleted) return; PageLoaded.SetResult(true);
};
//
while (PageLoaded.Task.Status != TaskStatus.RanToCompletion)
{
await Task.Delay(delay * 1000);//interval of 10 ms worked good for me
TimeElapsed++;
if (TimeElapsed >= TimeOut * 100) PageLoaded.TrySetResult(true);
}
}
catch (Exception ex)
{
CommonCodes.WriteLog(ex.ToString());
MessageBox.Show(ex.Message);
}
--
The main problem is when I have 5 lines in listbox, for the first line only it is going to every page and scraping urls but for the other lines its not working properly. I don't understand the problem with in the code. Some how the code
MessageBox.Show("sdsaD3");
executing multiple time(If 5 lines in listbox then this msg boxpoping up 5 times). Thanks for the help.
EDit: I found the issue, it seems the issue is with await PageLoad(30, 5); but I am not sure how to invoke async method. Any one have idea?
So I have this function, designed to download YouTube videos from the internet.
However whenever I run it, it does not catch any exception. Even an explicitly thrown new exception().
It's a WPF program so before anyone mentions bad coding etiquette, please don't. I'm more concerned with getting my program working at the moment.
Here's the code that creates the thread:
void button1_Click (object sender, RoutedEventArgs e)
{
this.IsEnabled = false;
this.downloadProgressText.Text = "Beginning....";
int selectedIndex = this.queueListView.SelectedIndex;
Thread newThread = new Thread (() => Download_Handler(classContainer, 0, selectedIndex, 0));
newThread.Start();
}
And the threaded function itself:
private void Download_Handler (ClassContainer classCont, int curVidPosition, int selectedIndex, int retryCount)
{
ObservableCollection<Video> finishedUrls = new ObservableCollection<Video> ();
ObservableCollection<Video> urlList = videoQueue.Items;
for (int position = curVidPosition, urlListCount = urlList.Count; position < urlListCount; position++)
{
try
{
downloadProgressText.Dispatcher.Invoke(new UpdateSelectedListItemCallback (this.UpdateSelectedListItem), new object[] {
position
});
Video vid = urlList [position];
if (classCont.DownloadingCode.DownloadVideo(vid, this, position) != null)
{
finishedUrls.Add(vid);
}
if (finishedUrls != null && finishedUrls.Count > 0)
{
classCont.IOHandlingCode.WriteUrlsToFile(finishedUrls, true);
}
}
catch (Exception ex)
{
var exceptionMessage = ex.Message;
if (retryCount <= 4)
{
downloadProgressText.Dispatcher.Invoke(new UpdateProgressBarCallback (this.UpdateProgressBar), new object[] {
string.Format(CultureInfo.InstalledUICulture, "URL {0}: {1}. Retrying.... ({2}/{3})", position + 1, classContainer.ConversionCode.Truncate(exceptionMessage, 50), retryCount.ToString(CultureInfo.CurrentCulture), "3"),
-1
});
Thread.Sleep(850);
Download_Handler(classCont, position, selectedIndex, retryCount + 1);
}
else
{
if (finishedUrls.Count < 10)
{
finishedUrls.Clear();
}
downloadProgressText.Dispatcher.Invoke(new UpdateProgressBarCallback (this.UpdateProgressBar), new object[] {
classCont.ConversionCode.Truncate(exceptionMessage, 100),
-1
});
}
}
}
videoQueue.Items = finishedUrls.Count > 0 ? (ObservableCollection<Video>)urlList.Where(video => finishedUrls.All(item => item != video)).Select(video => video) : urlList;
if (this.queueListView.Items.Count > 0)
{
this.RefreshQueue(videoQueue.Items, selectedIndex < this.queueListView.Items.Count ? 0 : selectedIndex);
}
this.IsEnabled = true;
}
This coding scheme worked just fine in WinForms (well, with slight differences), but since the move to WPF I just cannot figure it out!
EDIT:
Including additional code:
public Video DownloadVideo (Video video, MainWindow MainForm, int position)
{
MainForm.Dispatcher.Invoke(new MainWindow.UpdateProgressBarCallback (MainForm.UpdateProgressBar), new object[] {
0,
string.Format(CultureInfo.InstalledUICulture, "Beginning download from '{0}'", video.Location)
});
/*
* Get the available video formats.
* We'll work with them in the video and audio download examples.
*/
IEnumerable<VideoInfo> videoInfos = DownloadUrlResolver.GetDownloadUrls(video.Location, false);
if ((video.Format != VideoType.Mp4 && videoInfos.Any(info => (info.Resolution == video.Resolution && info.VideoType == video.Format)) || video.Format == VideoType.Mp4 && video.Resolution == 360))
{
VideoInfo currentVideo = videoInfos.First(info => info.VideoType == video.Format && info.Resolution == video.Resolution);
MainForm.Dispatcher.Invoke(new MainWindow.UpdateProgressBarCallback (MainForm.UpdateProgressBar), new object[] {
-1,
string.Format(CultureInfo.InstalledUICulture, "Downloading '{0}{1}' at {2}p resolution", Conversion.Truncate(currentVideo.Title, 56), currentVideo.VideoExtension, currentVideo.Resolution)
});
//DownloadAudio(videoInfos);
this.Download_Actual(videoInfos, MainForm, video.Resolution, position, video.Format);
return video;
}
if (videoInfos.Where(info => info.VideoType == video.Format).All(info => info.Resolution != video.Resolution) || (video.Format == VideoType.Mp4 && video.Resolution != 360))
{
List<int> resolutionsEstablished = new List<int> ();
List<VideoType> formatsEstablished = new List<VideoType> ();
using (StreamWriter outfile = new StreamWriter ("Acceptable Options.txt"))
{
outfile.Write(string.Format(CultureInfo.InstalledUICulture, "This file will show you all formats available for the current URL, as well as the resolutions that are acceptable for that URL.\n\n{0}:\n", video.Location));
foreach (VideoType format in videoInfos.Where(info => info.VideoType != VideoType.Unknown && formatsEstablished.All(format => info.VideoType != format)).Select(info => info.VideoType))
{
if (format == VideoType.Mp4)
{
outfile.Write(string.Format(CultureInfo.InstalledUICulture, "Format: {0} | Resolution: {1}p\n", format, "360"));
}
else
{
foreach (int resolution in videoInfos.Where(info => info.Resolution >= 144 && info.Resolution < 720 && resolutionsEstablished.All(res => info.Resolution != res) && info.VideoType == format).Select(info => info.Resolution))
{
outfile.Write(string.Format(CultureInfo.InstalledUICulture, "Format: {0} | Resolution: {1}p\n", format, resolution));
resolutionsEstablished.Add(resolution);
}
}
resolutionsEstablished.Clear();
formatsEstablished.Add(format);
}
}
throw new NotSupportedException("An acceptable options file has been exported to the program's root folder. Check there for more information.");
}
return null;
}
private void Download_Actual (IEnumerable<VideoInfo> videoInfos, MainWindow MainForm, int resolution, int position, VideoType format)
{
/*
* Select the first .mp4 video with 360p resolution
*/
VideoInfo video = videoInfos
.First(info => info.VideoType == format && info.Resolution == resolution);
/*
* If the video has a decrypted signature, decipher it
*/
if (video.RequiresDecryption)
{
DownloadUrlResolver.DecryptDownloadUrl(video);
}
/*
* Create the video downloader.
* The first argument is the video to download.
* The second argument is the path to save the video file.
*/
Settings settings = Storage.ReadFromRegistry();
var videoName = RemoveIllegalPathCharacters(video.Title) + video.VideoExtension;
var videoPath = Path.Combine(settings.TemporarySaveLocation, videoName);
var finalPath = Path.Combine(settings.MainSaveLocation, videoName);
if (!File.Exists(finalPath))
{
var videoDownloader = new VideoDownloader (video, videoPath);
// Register the ProgressChanged event and print the current progress
videoDownloader.DownloadProgressChanged += (
(sender, args) => MainForm.downloadProgressBar.Dispatcher.Invoke(new MainWindow.UpdateProgressBarCallback (MainForm.UpdateProgressBar), new object[] {
(int)args.ProgressPercentage,
null
})
);
/*
* Execute the video downloader.
* For GUI applications note, that this method runs synchronously.
*/
videoDownloader.Execute();
File.Move(videoPath, finalPath);
}
else
{
MainForm.downloadProgressText.Dispatcher.Invoke(new MainWindow.UpdateProgressBarCallback (MainForm.UpdateProgressBar), new object[] {
0,
null
});
throw new FieldAccessException("{0}({1}) already exists! Download process has been aborted and considered successful.");
}
}
I've stripped your code back to basics and works fine for me. Would probably need a complete code sample
private void ButtonBase_OnClick(object sender, RoutedEventArgs e)
{
Thread newThread = new Thread(Download_Handler);
newThread.Start();
}
private void Download_Handler()
{
try
{
throw new Exception();
}
catch (Exception)
{
}
}
I figured it out. It was really stupid of a mistake....
So I call public delegate void UpdateProgressBarCallback (int progress, string message); to update text and progress bar values.
In this case, in the catch block, I was attempting to pass as the other way around, IE public delegate void UpdateProgressBarCallback (string message, int progress);. THAT'S why it was being weird. It all works now.
It's from Windows Phone project. I am trying to invoke few handlers, handler by handler to receive information about GPS / Reverse position. I wonder why it won't run correctly.
When I setup only 1 coordinate it's ok. I have message with Street etc. But when there is more coordinates my handler isn't invoke.
private async void SimulationResults()
{
done = new AutoResetEvent(true);
Geolocator geolocator = new Geolocator();
geolocator.DesiredAccuracy = PositionAccuracy.High;
myCoordinate = new GeoCoordinate(51.751985, 19.426515);
if (myMap.Layers.Count() > 0) myMap.Layers.Clear();
mySimulation = new List<SimulationItem>();
mySimulation = Simulation.SimulationProcess(myCoordinate, 120); // Odległość
for(int i = 0; i<2; i++)
{
done.WaitOne();
if (mySimulation.ElementAt(i).Id == 1 | mySimulation.ElementAt(i).Id == -1)
{
// Oczekiwanie, ponieważ obiekt jest zasygnalizowany od razu wejdziemy
// do sekcji krytycznej
AddMapLayer(mySimulation.ElementAt(i).Coordinate, Colors.Yellow, false);
myReverseGeocodeQuery_1 = new ReverseGeocodeQuery();
myReverseGeocodeQuery_1.GeoCoordinate = mySimulation.ElementAt(i).Coordinate;
myReverseGeocodeQuery_1.QueryCompleted += ReverseGeocodeQuery_QueryCompleted_1;
// Sekcja krytyczna
done.Reset(); // Hey I'm working, wait!
myReverseGeocodeQuery_1.QueryAsync();
}
}
MessageBox.Show("Skonczylem");
}
private void ReverseGeocodeQuery_QueryCompleted_1(object sender, QueryCompletedEventArgs<IList<MapLocation>> e)
{
done.Set();
if (e.Error == null)
{
if (e.Result.Count > 0)
{
MapAddress address = e.Result[0].Information.Address;
MessageBox.Show("Wykonano "+address.Street);
}
}
}
What's happening here is that you are blocking on your AutoResetEvent on the UI thread, but that's the same thread that the ReverseGeocodeQuery is trying to run on. Since it's blocked it can't run and it also can't invoke your callback.
A very quick fix that doesn't change your flow too much and assumes some sort of "on everything complete do X" requirement is below. I triggered the whole thing on a background thread with:
new Thread(new ThreadStart(() =>
{
SimulationResults();
})).Start();
Since all of the below is on a background thread I needed to add some Dispatcher.BeginInvoke() calls around anything that called into the UI thread, but this way the thread that is blocked is your background thread and not your UI thread.
AutoResetEvent done;
int remaining;
private async void SimulationResults()
{
done = new AutoResetEvent(true);
Geolocator geolocator = new Geolocator();
geolocator.DesiredAccuracy = PositionAccuracy.High;
var myCoordinate = new GeoCoordinate(51.751985, 19.426515);
var mySimulation = new List<GeoCoordinate>()
{
new GeoCoordinate(51.751985, 19.426515),
new GeoCoordinate(2, 2)
};
//mySimulation = Simulation.SimulationProcess(myCoordinate, 120); // Odległość
remaining = mySimulation.Count;
for (int i = 0; i < mySimulation.Count; i++)
{
done.WaitOne();
//if (mySimulation.ElementAt(i).Id == 1 | mySimulation.ElementAt(i).Id == -1)
//{
// Oczekiwanie, ponieważ obiekt jest zasygnalizowany od razu wejdziemy
// do sekcji krytycznej
//AddMapLayer(mySimulation.ElementAt(i).Coordinate, Colors.Yellow, false);
var tempI = i;
Dispatcher.BeginInvoke(() =>
{
var myReverseGeocodeQuery_1 = new ReverseGeocodeQuery();
myReverseGeocodeQuery_1.GeoCoordinate = mySimulation.ElementAt(tempI);
myReverseGeocodeQuery_1.QueryCompleted += ReverseGeocodeQuery_QueryCompleted_1;
// Sekcja krytyczna
done.Reset(); // Hey I'm working, wait!
myReverseGeocodeQuery_1.QueryAsync();
});
//}
}
}
private void ReverseGeocodeQuery_QueryCompleted_1(object sender, QueryCompletedEventArgs<IList<MapLocation>> e)
{
done.Set();
remaining--;
if (e.Error == null)
{
if (e.Result.Count > 0)
{
MapAddress address = e.Result[0].Information.Address;
Dispatcher.BeginInvoke(() =>
{
MessageBox.Show("Wykonano " + address.Street);
});
}
}
if (remaining == 0)
{
// Do all done code
Dispatcher.BeginInvoke(() =>
{
MessageBox.Show("Skonczylem");
});
}
}
Alternatively you could also make this a proper async method that awaits on different events to progress.
I'm trying to do a TTS app for Windows Phone, but i have a little problem.
I'v make a function to call Google TTS service and play it on a MediaElement, that works perfect.
That's the function i'v wrote
public void Say(string phrase)
{
mediaElement1.Source = new Uri("http://translate.google.com/translate_tts?tl=en&q=" + phrase, UriKind.Absolute));
mediaElement1.Play();
}
But if i call the function a lot of times, for example, in a for loop, it just say the last item of loop. For example
for (int i = 0; i < 10; i++)
{
Say(Convert.ToString(i));
}
Then it just say 'Nine', nope '1,2,3,4,...'
I'v tried to do a 'Playlist' of words, that the function will play.
This is my code.
bool first = true;
List<Uri> uriTask = new List<Uri>();
public void Say(string phrase)
{
uriTask.Add(new Uri("http://translate.google.com/translate_tts?tl=es&q=" + phrase, UriKind.Absolute));
if (mediaElement1.CurrentState != MediaElementState.Playing)
{
if (first)
{
CHECK();
first = false;
}
else
{
mediaElement1.MediaEnded += delegate
{
mediaElement1.Position = TimeSpan.Zero;
CHECK();
};
}
}
}
private void CHECK()
{
if (uriTask.Count > 0)
{
mediaElement1.Source = uriTask[0];
mediaElement1.Play();
uriTask.RemoveAt(0);
}
}
Thanks.
Greetings
In my application I have three asynchronous events.
After all of them are complete I need to call some Method1().
How can I implement this logic?
Update
Here is one of my asynchronous events:
public static void SetBackground(string moduleName, Grid LayoutRoot)
{
var feedsModule = FeedHandler.GetInstance().ModulesSetting.Where(type => type.ModuleType == moduleName).FirstOrDefault();
if (feedsModule != null)
{
var imageResources = feedsModule.getResources().getImageResource("Background") ??
FeedHandler.GetInstance().MainApp.getResources().getImageResource("Background");
if (imageResources != null)
{
//DownLoad Image
Action<BitmapImage> onDownloaded = bi => LayoutRoot.Background = new ImageBrush() { ImageSource = bi, Stretch = Stretch.Fill };
CacheImageFile.GetInstance().DownloadImageFromWeb(new Uri(imageResources.getValue()), onDownloaded);
}
}
}
Bit field (or 3 booleans) set by each event handler. Each event handler checks that the condition is met then calls Method1()
tryMethod1()
{
if (calledEvent1 && calledEvent2 && calledEvent3) {
Method1();
calledEvent1 = false;
calledEvent2 = false;
calledEvent3 = false;
}
}
eventHandler1() {
calledEvent1 = true;
// do stuff
tryMethod1();
}
Not given any other information, what will work is to use a counter. Just an int variable that is initialized to be 3, decremented in all handlers and checked for equality to 0 and that case go on.
You should use WaitHandles for this. Here is a quick example, but it should give you the basic idea:
List<ManualResetEvent> waitList = new List<ManualResetEvent>() { new ManualResetEvent(false), new ManualResetEvent(false) };
void asyncfunc1()
{
//do work
waitList[0].Set();
}
void asyncfunc2()
{
//do work
waitList[1].Set();
}
void waitFunc()
{
//in non-phone apps you would wait like this:
//WaitHandle.WaitAll(waitList.ToArray());
//but on the phone 'Waitall' doesn't exist so you have to write your own:
MyWaitAll(waitList.ToArray());
}
void MyWaitAll(WaitHandle[] waitHandleArray)
{
foreach (WaitHandle wh in waitHandleArray)
{
wh.WaitOne();
}
}