Develop Reference

'Unable to deserialize content' when updating Microsoft calendar events in batch on retry

I'm working on an application based on .NET Framework 4.8. I'm using Microsoft Batching API. The below are code snippets
public async Task<List<BatchResponse>> UpdateEventsInBatchAsync(string accessToken, Dictionary<int, Tuple<string, OfficeEvent>> absEvents)
{
var httpMethod = new HttpMethod("PATCH");
var batches = GetUpdateRequestBatches(absEvents, httpMethod);
var graphClient = GetGraphClient(accessToken);
var batchResponses = new List<BatchResponse>();
foreach (var batch in batches)
{
try
{
var batchResponseList = await ExecuteBatchRequestAsync(graphClient, batch).ConfigureAwait(false);
batchResponses.AddRange(batchResponseList);
}
catch (ClientException exc)
{
_logService.LogException("Error while processing update batch", exc);
batchResponses.Add(new BatchResponse
{ StatusCode = HttpStatusCode.InternalServerError, ReasonPhrase = exc.Message });
}
catch (Exception exc)
{
_logService.LogException("Error while processing update batch", exc);
batchResponses.Add(new BatchResponse { StatusCode = HttpStatusCode.InternalServerError, ReasonPhrase = exc.Message });
}
}
return batchResponses;
}
The respective methods used in the above code are mentioned below in respective order-
GetUpdateRequestBatches
private IEnumerable<BatchRequestContent> GetUpdateRequestBatches(Dictionary<int, Tuple<string, OfficeEvent>> absEvents, HttpMethod httpMethod)
{
var batches = new List<BatchRequestContent>();
var batchRequestContent = new BatchRequestContent();
const int maxNoBatchItems = 20;
var batchItemsCount = 0;
foreach (var kvp in absEvents)
{
System.Diagnostics.Debug.Write($"{kvp.Key} --- ");
System.Diagnostics.Debug.WriteLine(_serializer.SerializeObject(kvp.Value.Item2));
var requestUri = $"{_msOfficeBaseApiUrl}/me/events/{kvp.Value.Item1}";
var httpRequestMessage = new HttpRequestMessage(httpMethod, requestUri)
{
Content = _serializer.SerializeAsJsonContent(kvp.Value.Item2)
};
var requestStep = new BatchRequestStep(kvp.Key.ToString(), httpRequestMessage);
batchRequestContent.AddBatchRequestStep(requestStep);
batchItemsCount++;
// Max number of 20 request per batch. So we need to send out multiple batches.
if (batchItemsCount > 0 && batchItemsCount % maxNoBatchItems == 0)
{
batches.Add(batchRequestContent);
batchRequestContent = new BatchRequestContent();
batchItemsCount = 0;
}
}
if (batchRequestContent.BatchRequestSteps.Count < maxNoBatchItems)
{
batches.Add(batchRequestContent);
}
if (batches.Count == 0)
{
batches.Add(batchRequestContent);
}
return batches;
}
GetGraphClient
private static GraphServiceClient GetGraphClient(string accessToken)
{
var graphClient = new GraphServiceClient(new DelegateAuthenticationProvider(requestMessage =>
{
requestMessage
.Headers
.Authorization = new AuthenticationHeaderValue("Bearer", accessToken);
return Task.FromResult(0);
}));
return graphClient;
}
ExecuteBatchRequestAsync
private async Task<List<BatchResponse>> ExecuteBatchRequestAsync(IBaseClient graphClient, BatchRequestContent batch)
{
BatchResponseContent response = await graphClient.Batch.Request().PostAsync(batch);
Dictionary<string, HttpResponseMessage> responses = await response.GetResponsesAsync();
var batchResponses = new List<BatchResponse>();
var failedReqKeys = new Dictionary<string, TimeSpan>();
foreach (var key in responses.Keys)
{
using (HttpResponseMessage httpResponseMsg = await response.GetResponseByIdAsync(key))
{
var responseContent = await httpResponseMsg.Content.ReadAsStringAsync();
string eventId = null;
var reasonPhrase = httpResponseMsg.ReasonPhrase;
if (!string.IsNullOrWhiteSpace(responseContent))
{
var eventResponse = JObject.Parse(responseContent);
eventId = (string)eventResponse["id"];
// If still null, then might error have occurred
if (eventId == null)
{
var errorResponse = _serializer.DeserializeObject<ErrorResponse>(responseContent);
var error = errorResponse?.Error;
if (error != null)
{
if (httpResponseMsg.StatusCode == (HttpStatusCode)429)
{
System.Diagnostics.Debug.WriteLine($"{httpResponseMsg.StatusCode} {httpResponseMsg.Content}");
var executionDelay = httpResponseMsg.Headers.RetryAfter.Delta ?? TimeSpan.FromSeconds(5);
failedReqKeys.Add(key, executionDelay);
continue;
}
reasonPhrase = $"{error.Code} - {error.Message}";
}
}
}
var batchResponse = new BatchResponse
{
Key = key,
EventId = eventId,
StatusCode = httpResponseMsg.StatusCode,
ReasonPhrase = reasonPhrase
};
batchResponses.Add(batchResponse);
}
}
if (failedReqKeys.Count == 0) return batchResponses;
return await HandleFailedRequestsAsync(graphClient, failedReqKeys, batch, batchResponses).ConfigureAwait(false);
}
HandleFailedRequestsAsync
private async Task<List<BatchResponse>> HandleFailedRequestsAsync(IBaseClient graphClient, Dictionary<string, TimeSpan> failedReqKeys, BatchRequestContent batch, List<BatchResponse> batchResponses)
{
// Sleep for the duration as suggested in RetryAfter
var sleepDuration = failedReqKeys.Values.Max();
Thread.Sleep(sleepDuration);
var failedBatchRequests = batch.BatchRequestSteps.Where(b => failedReqKeys.Keys.Contains(b.Key)).ToList();
var failedBatch = new BatchRequestContent();
foreach (var kvp in failedBatchRequests)
{
failedBatch.AddBatchRequestStep(kvp.Value);
}
var failedBatchResponses = await ExecuteBatchRequestAsync(graphClient, failedBatch);
batchResponses.AddRange(failedBatchResponses);
return batchResponses;
}
I'm getting an error as on the first line in method ExecuteBatchRequestAsync as
Microsoft.Graph.ClientException: Code: invalidRequest
Message: Unable to deserialize content.
---> System.ObjectDisposedException: Cannot access a closed Stream.
Can anyone nudge me where I'm doing wrong?

Async /Await call gets stuck in web API

We have a web API application which runs on .net4.6.1. We have tried several times to figure out the root cause where it is getting deadlock, but failed. Below is the code snippet. We are hitting this API endpoint every 1 minute. It will pick 300 transaction at a time for processing from the DB. We have observed that it get stuck when there are no files to process from the DB. Not sure though. It would be helpful if someone can help us.TIA
public class TaxEngineIntegratorController : ApiController
{
public async Task Get(int id)
{
try
{
await MainFileMethod();
}
catch (Exception Ex)
{
SerilogMethods.LogError(log, Ex, "Get");
}
}
public async Task MainFileMethod()
{
List<FileTransaction> lstFTtoLock = new List<FileTransaction>();
try
{
List<int> lstStatusIds = new List<int>();
lstStatusIds.Add(objStatusManager.GetStatusIdbyName(Status.ConversionToXmlSucceded));
lstStatusIds.Add(objStatusManager.GetStatusIdbyName(Status.Reprocess));
//Getting the serviceURL of TRTaxEngine
string seriviceURL = objConfigManager.GetConfigurationdbyKey(ConfigurationList.TRTaxEngineURL);
//Getting the output path for the file to be placed after processing
string outputfilePath = objConfigManager.GetConfigurationdbyKey(ConfigurationList.TRTaxOutputXMLFolder);
FileMasterManager objFileMasterManager = new FileMasterManager();
TRTaxXMLOperations objxmlresp = new TRTaxXMLOperations();
//Getting all the files list for proccessing from the DB
List<FileTransaction> lstFiletoProcess = await objTransManager.GetFileListforProcessingAsync(lstStatusIds, true);
lstFTtoLock = lstFiletoProcess;
if (lstFiletoProcess.Count == 0)
return;
if (lstFiletoProcess.Count > 0)
{
var tasks = new List<Task<string>>();
using (HttpClient httpClnt = new HttpClient())
{
httpClnt.Timeout = TimeSpan.FromMilliseconds(-1);
foreach (FileTransaction item in lstFiletoProcess)
{
TRXMLResponseModel objRespModel = new TRXMLResponseModel();
objRespModel.strxmlResponse = string.Empty;
string fullFileName = item.FilePath + item.ConvertedName;
objRespModel.outputFilename = outputfilePath + item.ConvertedName;
FileMaster fileMaster = objFileMasterManager.GetById(item.FileId);
//Proccessing the file and getting the output filedata
Task<string> t = objxmlresp.GetXMLResponse(seriviceURL, fullFileName, fileMaster.CountryId.GetValueOrDefault(), httpClnt, objFileOperation, objRespModel.outputFilename, item);
tasks.Add(t);
objRespModel.strxmlResponse = await t;
}
var result = await Task.WhenAll(tasks);
}
SerilogMethods.LogCustomException(log, "Http Client Destroyed in Tax Engine", "GetXMLResponse");
}
}
catch (Exception Ex)
{
if (lstFTtoLock != null && lstFTtoLock.Count > 0)
{
objTransManager.UpdateFileTransactionIsPickedtoFalse(lstFTtoLock);
}
throw Ex;
}
}
}
//Getting all the files list for proccessing from the DB
public async Task<List<FileTransaction>> GetFileListforProcessingAsync(List<int> lstStatusList, bool IsActive)
{
try
{
List<FileTransaction> lstFTList = new List<FileTransaction>();
using (SUTBACDEVContext db = new SUTBACDEVContext())
{
//DataTable dtFileTransactions = GetFileTransactionListAsync(lstStatusList, IsActive);
string connectionString = db.Database.GetDbConnection().ConnectionString;
var conn = new SqlConnection(connectionString);
string query = #"[SUTGITA].[GetFileListforProcessing]";
using (var sqlAdpt = new SqlDataAdapter(query, conn))
{
sqlAdpt.SelectCommand.CommandType = CommandType.StoredProcedure;
sqlAdpt.SelectCommand.Parameters.AddWithValue("#StatusId", string.Join(",", lstStatusList.Select(n => n.ToString()).ToArray()));
sqlAdpt.SelectCommand.Parameters.AddWithValue("#IsActive", IsActive);
sqlAdpt.SelectCommand.CommandTimeout = 60000;
DataTable dtFileTransactions = new DataTable();
sqlAdpt.Fill(dtFileTransactions);
if (dtFileTransactions != null && dtFileTransactions.Rows.Count > 0)
{
IEnumerable<long> ids = dtFileTransactions.AsEnumerable().ToList().Select(p => p["id"]).ToList().OfType<long>();
lstFTList = await db.FileTransaction.Include(x => x.File.Country).Where(x => ids.Contains(x.Id)).OrderBy(x => x.Id).ToListAsync();
}
}
}
return lstFTList;
}
catch (Exception ex)
{
throw ex;
}
}
public async Task<string> GetXMLResponse(string baseUrl, string fullFileName, int countryId, HttpClient client, FileOperations objFileOperation, string outputfilePath, FileTransaction item)
{
try
{
var fileData = new StringBuilder(objFileOperation.ReadFile(fullFileName));
using (HttpContent content = new StringContent(TransformToSOAPXml(fileData, countryId), Encoding.UTF8, "text/xml"))
{
using (HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, baseUrl))
{
request.Headers.Add("SOAPAction", "");
request.Content = content;
using (HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead))
{
response.EnsureSuccessStatusCode();
if (response.IsSuccessStatusCode)
{
using (Stream streamToReadFrom = await response.Content.ReadAsStreamAsync())
{
using (Stream streamToWriteTo = File.Open(outputfilePath, FileMode.Create))
{
await streamToReadFrom.CopyToAsync(streamToWriteTo);
}
}
var transactionEntry = new FileTransaction
{
FileId = item.FileId,
FilePath = outputfilePath,
ConvertedName = item.ConvertedName,
ActionedBy = Process.Process3,
TimeStamp = DateTime.UtcNow,
StatusId = objStatusManager.GetStatusIdbyName(Status.OutputXmlReceived),
IsActive = true,
CreatedBy = Others.Scheduler,
CreatedOn = DateTime.UtcNow,
ModifiedBy = Others.Scheduler,
ModifiedOn = DateTime.UtcNow
};
//Inserting the new record and Updating isActive filed of previous record in Tranasaction table(Calling updateDataonTRSuccess method of TRTaxXMLOperations class)
await updateDataonTRSuccessAsync(item, transactionEntry);
return "Success";
}
else
{
SerilogMethods.LogCustomException(log, "Error occured in Tax Engine", "GetXMLResponse");
//Log the SOAP response when the SOAP fails with an error message
if (response.Content != null)
{
throw new Exception(await response.Content.ReadAsStringAsync());
}
return null;
}
}
}
}
}
catch (Exception ex)
{
SerilogMethods.LogError(log, ex, "GetXMLResponse");
return null;
}
}
The following changes I have done to make it work to this specific method.
Removal of this line : objRespModel.strxmlResponse = await t;
and added configureawait(false) to this line :List lstFiletoProcess = await objTransManager.GetFileListforProcessingAsync(lstStatusIds, true).ConfigureAwait(false); Below is the working code
public async Task MainFileMethod()
{
List<FileTransaction> lstFTtoLock = new List<FileTransaction>();
try
{
List<int> lstStatusIds = new List<int>();
lstStatusIds.Add(objStatusManager.GetStatusIdbyName(Status.ConversionToXmlSucceded));
lstStatusIds.Add(objStatusManager.GetStatusIdbyName(Status.Reprocess));
//Getting the serviceURL of TRTaxEngine
string seriviceURL = objConfigManager.GetConfigurationdbyKey(ConfigurationList.TRTaxEngineURL);
//Getting the output path for the file to be placed after processing
string outputfilePath = objConfigManager.GetConfigurationdbyKey(ConfigurationList.TRTaxOutputXMLFolder);
FileMasterManager objFileMasterManager = new FileMasterManager();
TRTaxXMLOperations objxmlresp = new TRTaxXMLOperations();
//Getting all the files list for proccessing from the DB
List<FileTransaction> lstFiletoProcess = await objTransManager.GetFileListforProcessingAsync(lstStatusIds, true).ConfigureAwait(false);
lstFTtoLock = lstFiletoProcess;
if (lstFiletoProcess.Count == 0)
return;
if (lstFiletoProcess.Count > 0)
{
var tasks = new List<Task<string>>();
using (HttpClient httpClnt = new HttpClient())
{
httpClnt.Timeout = TimeSpan.FromMilliseconds(-1);
//Getting the files for processing
foreach (FileTransaction item in lstFiletoProcess)
{
TRXMLResponseModel objRespModel = new TRXMLResponseModel();
objRespModel.strxmlResponse = string.Empty;
string fullFileName = item.FilePath + item.ConvertedName;
objRespModel.outputFilename = outputfilePath + item.ConvertedName;
FileMaster fileMaster = objFileMasterManager.GetById(item.FileId);
//Proccessing the file and getting the output filedata
Task<string> t = objxmlresp.GetXMLResponse(seriviceURL, fullFileName, fileMaster.CountryId.GetValueOrDefault(), httpClnt, objFileOperation, objRespModel.outputFilename, item, objTransManager);
tasks.Add(t);
//objRespModel.strxmlResponse = await t;
}
var result = await Task.WhenAll(tasks);
}
}
}
catch (Exception Ex)
{
if (lstFTtoLock != null && lstFTtoLock.Count > 0)
{
objTransManager.UpdateFileTransactionIsPickedtoFalse(lstFTtoLock);
}
throw Ex;
}
}

My Recommendation:
The method "Get(int id)" is somewhat confusing. first, it takes "id" and does nothing with it. Also it return nothing so it is not a "Get" method. It is basically asking for all transactions with status "Status.ConversionToXmlSucceded" & "Status.Reprocess" and are active to be gotten and processed via the "objxmlresp.GetXMLResponse" method... You Dont Have To Await the "MainFileMethod();" in "Get(int id)" just return the task or return Ok(); and allow all the process to go on in the background. You can experiment with reducing the "sqlAdpt.SelectCommand.CommandTimeout = 60000;".

C# async method does not return to caller as expected

I have a series of asynchronous functions that are cascaded, but when it finishes executing the last, it does not return the values ​​to the previous ones.
This is where my first call is run. This is a WebAPI function and always comes to an end.
[HttpGet]
[Route("integracao/iniciar")]
public IHttpActionResult FazerIntegrar()
{
try
{
Integrar objIntegrar = new Integrar();
return Ok(objIntegrar.Integra());
}
catch (Exception ex)
{
return InternalServerError(ex);
}
}
so my function is in my library is called. Within my function I have a for executing only once. The flow is never resumed by it to continue the loop
public async Task<bool> Integra()
{
var files = Directory.GetFiles(#"C:\inetpub\wwwroot\Atendup\Arquivos\Backup\");
bool retorno = false;
if (files != null)
{
foreach (var item in files)
{
retorno = false;
using (StreamReader sr = new StreamReader(item))
{
if (sr != null)
{
while (sr.EndOfStream == false)
{
string line = await sr.ReadLineAsync();
string[] grupo = line.Split(new[] { "#*#" }, StringSplitOptions.None);
procData objProc = new procData();
objProc.proc = grupo[0];
objProc.name = JsonConvert.DeserializeObject<List<string>>(grupo[1]);
objProc.valor = JsonConvert.DeserializeObject<List<object>>(grupo[2]);
objProc.tipo = JsonConvert.DeserializeObject<List<Type>>(grupo[3]);
_context = new IntegrarRepository("online_pedidopizza");
retorno = await _context.IntegrarAsync(objProc);
//retorno = await _context.IntegrarAsync(objProc);
}
}
}
if (retorno == true)
{
await DeleteAsync(item);
}
}
}
return retorno;
}
I have a third function just to mediate with the repository
public async Task<bool> IntegrarAsync(procData objProc)
{
return await this.SendIntegrarAsync(objProc);
}
And finally, communication with the database, all code is executed correctly. Debugging you can get to the end of this fourth function but then the debug stop and does not go back to the beginning
protected async Task<bool> SendIntegrarAsync(procData parametro)
{
bool retorno = false;
using (SqlConnection conn = new SqlConnection(""))
{
using (SqlCommand cmd = new SqlCommand(parametro.proc, conn))
{
cmd.CommandType = CommandType.StoredProcedure;
if (parametro != null)
{
for (int i = 0; i < parametro.name.Count; i++)
{
AdicionaParametro(cmd, parametro.name[i], parametro.valor[i], parametro.tipo[i]);
}
}
try
{
cmd.CommandTimeout = 300;
await conn.OpenAsync().ConfigureAwait(false);
var resultado = await cmd.ExecuteScalarAsync().ConfigureAwait(false);
if (resultado != null)
{
retorno = Convert.ToBoolean(resultado);
}
}
catch (Exception ex)
{
Logs objLog = new Logs()
{
metodo = MethodBase.GetCurrentMethod().Name,
classe = this.GetType().Name,
dados = parametro,
data = DateTime.Now,
mensagem = ex.Message,
exception = ex.InnerException == null ? "" : ex.InnerException.ToString()
};
objLog.Adiciona();
string name = DateTime.Now.ToBinary().ToString();
using (StreamWriter sw = new StreamWriter(#"C:\inetpub\wwwroot\Atendup\Arquivos\Backup\" + name + ".txt"))
{
string line = "";
line += parametro.proc + "#*#";
line += JsonConvert.SerializeObject(parametro.name) + "#*#";
line += JsonConvert.SerializeObject(parametro.valor) + "#*#";
line += JsonConvert.SerializeObject(parametro.tipo) + "#*#";
sw.WriteLine(line);
}
}
}
}
return retorno;
}
What should I have to do? Thanks

Your Web Api call is not async try changing it to:
[HttpGet]
[Route("integracao/iniciar")]
public async Task<IHttpActionResult> FazerIntegrar()
{
try
{
Integrar objIntegrar = new Integrar();
return Ok(await objIntegrar.Integra());
}
catch (Exception ex)
{
return InternalServerError(ex);
}
}

Connect C Sharp (login) to a online PHP plugin (MySQL database) through JSON

Can anyone help me with the correct notation of receiving a JSON POST request in PHP and a working (and aligned) C Sharp script for sending the POST request. I want a decentral login on my PC (with C Sharp software) which uses a JSON POST request to connect to my MySQL database (through a PHP 7.1.1 plugin) on my webspace. I can see the Exceptions (so the JSON feedback is received properly) which the script returns but the PHP POST superglobal keeps coming up empty (on the request side).
My C Sharp script:
using Newtonsoft.Json;
using System;
using System.IO;
using System.Net;
using System.Windows.Forms;
namespace HBI_Statistics_UI
{
public partial class Login : Form
{
public class API_Response
{
public bool IsError { get; set; }
public string ErrorMessage { get; set; }
public string ResponseData { get; set; }
}
public Login()
{
InitializeComponent();
}
private void buttonLogin_Click(object sender, EventArgs e)
{
try
{
// request params
var apiUrl = "http://www.home-business-intelligence.net/plugin/HBI-Statistics/login.php";
var httpWebRequest = (HttpWebRequest)WebRequest.Create(apiUrl);
httpWebRequest.ContentType = "application/json";
httpWebRequest.Method = "POST";
using (var streamWriter = new StreamWriter(httpWebRequest.GetRequestStream()))
{
string json = "json=" + "{\"Email\":\"" + textBox1.Text + "\"," +
"\"Pwd\":\"" + textBox2.Text + "\"}";
streamWriter.Write(json);
streamWriter.Flush();
streamWriter.Close();
}
var httpResponse = (HttpWebResponse)httpWebRequest.GetResponse();
using (var streamReader = new StreamReader(httpResponse.GetResponseStream()))
{
var result = streamReader.ReadToEnd();
API_Response r = JsonConvert.DeserializeObject<API_Response>(result);
// check response
if (r.ResponseData == "Success")
{
this.Hide();
ProgramMainUI pr = new ProgramMainUI();
pr.Show();
}
else
{
MessageBox.Show(r.ErrorMessage, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
}
catch (System.Net.WebException ex)
{
MessageBox.Show(ex.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
catch (Newtonsoft.Json.JsonReaderException ne)
{
MessageBox.Show(ne.ToString(), "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
}
}
My PHP Script:
include ("library.php");
try
{
if ($_SERVER["REQUEST_METHOD"] == "POST")
{
// Post Request
$api_data = strip_tags(isset($_POST['json']) ? $_POST['json'] : '');
// Validate Request
if (empty($api_data))
{
throw new Exception('Invalid request!! Please provide login details.' . print_r($api_data));
}
if (!empty($api_data))
{
// Decode json
$data = json_decode($api_data, true);
// Set connection
$conn = Connection::Conn_2();
// Sanitize data for query
$pwd = mysqli_real_escape_string($conn, $data->json->Pwd);
$email = mysqli_real_escape_string($conn, $data->json->Email);
$pwd2 = VanillaSpecial::Hash($pwd); // Hashed Password
// Insert Query of SQL
$result = mysqli_query($conn, $sql = "SELECT * FROM `Management-employees` WHERE email='$email' AND password = '$pwd2'") or die(json_encode(array(
'IsError' => 'true',
'ErrorMessage' => 'Invalid Request!! Oops, something went wrong. Please try again!!'
)));
if (mysqli_num_rows($result) == 1)
{
// output data of each row
while ($row = mysqli_fetch_assoc($result))
{
$functionCategory = $row[functionCategoryID];
}
switch ($functionCategory)
{
case "Management":
// echo "You have got clearance for this page!";
exit(json_encode(array(
'IsError' => 'false',
'ResponseData' => 'Success'
)));
break;
default:
throw new Exception('Invalid clearance!!');
}
}
else
{
throw new Exception('ERROR: Could not be able to execute ' . $sql . ' - ' . mysqli_error($conn));
}
}
}
else
{
throw new Exception('Invalid access method!!');
}
}
catch(Exception $e)
{
exit(json_encode(array(
'IsError' => 'true',
'ErrorMessage' => $e->getMessage()
)));
}

using (var client = new HttpClient())
{
var values = new Dictionary<string, string>
{
{ "thing1", "hello" },
{ "thing2", "world" }
};
var content = new FormUrlEncodedContent(values);
var response = await client.PostAsync("http://www.example.com/recepticle.aspx", content);
var responseString = await response.Content.ReadAsStringAsync();
}

Trying to download files from a website but it does nothing and doesn't throw errors

In general the code is working when im crawling to sites like www.cnn.com or www.foxnews.com
But now i tried this site: https://github.com/jasonwupilly/Obsidian/tree/master/Obsidian
And i wanted to download files for example when its crawling there is a link like this:
https://github.com/jasonwupilly/Obsidian/blob/master/Obsidian/Obsidian.sln
So the file on my hard disk should be Obsidian.sln
This is my crawling method:
public List<string> webCrawler(string mainUrl, int levels)
{
List<string> csFiles = new List<string>();
wc = new System.Net.WebClient();
HtmlWeb hw = new HtmlWeb();
List<string> webSites;
csFiles.Add("temp string to know that something is happening in level = " + levels.ToString());
csFiles.Add("current site name in this level is : " + mainUrl);
try
{
HtmlAgilityPack.HtmlDocument doc = TimeOut.getHtmlDocumentWebClient(mainUrl, false, "", 0, "", "");
if (doc == null)
{
failed = true;
wccfg.failedUrls++;
failed = false;
}
else
{
done = true;
// progress should be reported here I guess
Object[] temp_arr = new Object[8];
temp_arr[0] = csFiles;
temp_arr[1] = mainUrl;
temp_arr[2] = levels;
temp_arr[3] = currentCrawlingSite;
temp_arr[4] = sitesToCrawl;
temp_arr[5] = done;
temp_arr[6] = wccfg.failedUrls;
temp_arr[7] = failed;
OnProgressEvent(temp_arr);
currentCrawlingSite.Add(mainUrl);
webSites = getLinks(doc);
removeDupes(webSites);
removeDuplicates(webSites, currentCrawlingSite);
removeDuplicates(webSites, sitesToCrawl);
if (wccfg.removeext == true)
{
for (int i = 0; i < webSites.Count; i++)
{
webSites.Remove(removeExternals(webSites, mainUrl, wccfg.localy));
}
}
if (wccfg.downloadcontent == true)
{
retwebcontent.retrieveImages(mainUrl);
}
if (wccfg.downloadallfiles == true)
{
retwebcontent.retrieveFiles(mainUrl);
}
if (levels > 0)
sitesToCrawl.AddRange(webSites);
webSites = FilterJunkLinks(webSites);
if (levels == 0)
{
return csFiles;
}
else
{
for (int i = 0; i < webSites.Count(); i++)//&& i < 20; i++)
{
if (wccfg.toCancel == true)
{
return new List<string>();
}
if (pause == true)
{
_busy.Reset();
}
else
{
_busy.Set();
string t = webSites[i];
if ((t.StartsWith("http://") == true) || (t.StartsWith("https://") == true))
{
csFiles.AddRange(webCrawler(t, levels - 1));
}
}
}
return csFiles;
}
}
return csFiles;
}
catch (WebException ex)
{
failed = true;
wccfg.failedUrls++;
return csFiles;
}
catch (Exception ex)
{
failed = true;
wccfg.failedUrls++;
throw;
}
}
In the beginning of this method i have this line:
HtmlAgilityPack.HtmlDocument doc = TimeOut.getHtmlDocumentWebClient(mainUrl, false, "", 0, "", "");
I used a breakpoint on this line and enter it to the class:
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using HtmlAgilityPack;
using System.IO;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using System.Net;
using System.Web;
using System.Threading;
using DannyGeneral;
namespace GatherLinks
{
class TimeOut
{
static HtmlAgilityPack.HtmlDocument doc;
public TimeOut()
{
}
class MyClient : WebClient
{
public bool HeadOnly { get; set; }
protected override WebRequest GetWebRequest(Uri address)
{
WebRequest req = base.GetWebRequest(address);
if (HeadOnly && req.Method == "GET")
{
req.Method = "HEAD";
}
return req;
}
}
public static HtmlAgilityPack.HtmlDocument getHtmlDocumentWebClient(string url, bool useProxy, string proxyIp, int proxyPort, string usename, string password)
{
try
{
doc = null;
using (MyClient clients = new MyClient())
{
clients.HeadOnly = true;
byte[] body = clients.DownloadData(url);
// note should be 0-length
string type = clients.ResponseHeaders["content-type"];
clients.HeadOnly = false;
// check 'tis not binary... we'll use text/, but could
// check for text/html
if (type == null)
{
return null;
}
else
{
if (type.StartsWith(#"text/html"))
{
string text = clients.DownloadString(url);
doc = new HtmlAgilityPack.HtmlDocument();
WebClient client = new WebClient();
//client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
client.Credentials = CredentialCache.DefaultCredentials;
client.Proxy = WebRequest.DefaultWebProxy;
if (useProxy)
{
//Proxy
if (!string.IsNullOrEmpty(proxyIp))
{
WebProxy p = new WebProxy(proxyIp, proxyPort);
if (!string.IsNullOrEmpty(usename))
{
if (password == null)
password = string.Empty;
NetworkCredential nc = new NetworkCredential(usename, password);
p.Credentials = nc;
}
}
}
doc.Load(client.OpenRead(url));
}
}
}
}
catch (Exception err)
{
}
return doc;
}
private static string GetUrl(string url)
{
string startTag = "Url: ";
string endTag = " ---";
int startTagWidth = startTag.Length;
int endTagWidth = endTag.Length;
int index = 0;
index = url.IndexOf(startTag, index);
int start = index + startTagWidth;
index = url.IndexOf(endTag, start + 1);
string g = url.Substring(start, index - start);
return g;
}
}
}
Its doing the firstl ines in getHtmlDocumentWebClient method and when its getting to the line:
byte[] body = clients.DownloadData(url);
Its jumping to the method : GetWebRequest
After doing the line: return req; it does nothing its returning to the program and nothing happen.
And thats only if the link is: https://github.com/jasonwupilly/Obsidian/tree/master/Obsidian
If the link is www.cnn.com for example its downloading images no problems and also crawling no problems.
THe code is some long but it's all connected so i had to add it all.
What could be the problem here ?