Develop Reference

MongoDB - Error connecting to 192.xx.xx.xx : Too many open files

I've setup my mongodb (v4.2.8) sharded cluster on ubuntu 18.04 and using c#.net for my application. I have 2 mongoS in my cluster and below is my code to connect to my mongodb shard
public sealed class MongoDBConnection
{
private static MongoClient client1 = null;
public static MongoClient Connection1
{
get
{
if (client1 == null)
{
lock (InstanceLock)
{
if (client1 == null)
{
TimeSpan connectionTime = new TimeSpan(1, 0, 0);
MongoClientSettings settingShard = new MongoClientSettings
{
MaxConnectionPoolSize = 60000,
MinConnectionPoolSize = 30000,
ConnectTimeout = connectionTime,
Server = new MongoServerAddress("xxx.xxx.xx.xx", 27017)
};
client1 = new MongoClient(settingShard);
}
}
}
return client1;
}
}
}
this class contains all settings to my connection and below is my class to perform DB operatoins
public static class MongoDBOperations
{
private static IMongoClient consDB1 = MongoDBConnection.Connection1;
private static IMongoDatabase MyDB1 = consDB1.GetDatabase("DBEmp");
private static IMongoCollection<EmpDetails> Mycollection1= MyDB1.GetCollection<EmpDetails>("tblEmployees");
public static Queue<string> returnEmployeeList(int limit)
{
Queue<string> empQ = new Queue<string>();
try
{
var filter = Builders<EmpDetails>.Filter.Eq("InfoUpdated", 0);
var emp1 = Mycollection1.Find(filter).Limit(limit).ToList();
if (emp1.Count > 0)
{
for (int i = 0; i < emp1.Count; i++)
{
var _Emp = Builders<EmpDetails>.Filter.Eq("EmailAddress", emp1[i].EmailAddress);
var UpdateDef = Builders<EmpDetails>.Update.Set(s => s.InfoUpdated, 1).Set(s => s.InfoUpdatedOn, DateTime.Now.Date);
var result = Mycollection1.UpdateManyAsync(_Emp, UpdateDef);
empQ.Enqueue(emp[i].EmailAddress.ToString());
}
}
else
{
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("No Emp found to update : ");
Console.ForegroundColor = ConsoleColor.White;
}
}
catch (Exception ex)
{
Console.WriteLine("Error in Getting emp from DB: " + ex.Message);
}
return empQ;
}
}
so what I am actually doing is, I am returning a queue from my class and do some stuff on that list. As soon as the records are fetched from database, I update there status "InfoUpdated" to 1 so they are not picked agian.
I am doing this in my multithreaded application and below is the code for that
class Program
{
static Dictionary<string, Queue<string>> Qs = new Dictionary<string, Queue<string>>();
static int elementsInQueue = 100;
static int totalQueues = 200;
static void Main(string[] args)
{
new Program().FillUpQueuesOnStart();
}
void FillUpQueuesOnStart()
{
for (int i = 1; i <= totalQueues; i++)
{
if (!Qs.ContainsKey("q" + i))
Qs.Add("q" + i, new Queue<string>());
Qs["q" + i] = MongoDBOperations.returnEmployeeList(elementsInQueue);
Console.WriteLine("q" + i + " filled on start");
}
}
}
when I run my program, it works fine initially and fills up to 35 to 30 queues, but then gives the following error
Error connecting to xxx.xxx.xx.xxx:27017 :: caused by :: Too many open files
where xxx.xxx.xx.xxx is the IP of one of my shards. I see this error on one of my mongoS' terminal in my cluster. I've searched a lot on this to no help. I even changed the ulimit -n in all of my nodes in my cluster, still to no luck.
Any help would really be greatly appreciated.

Azure Devops - Get release definitions by agent pool ID

I'm trying to find all the builds and releases that are configured to use a specific agent pool, using the .NET Client Libraries.
Assuming agentPoolId, I can get all the build definitions like this:
// _connection is of type VssConnection
using (var buildClient = _connection.GetClient<BuildHttpClient>())
{
List<BuildDefinitionReference> allBuilds = await buildClient.GetDefinitionsAsync(projectName, top: 1000, queryOrder: DefinitionQueryOrder.DefinitionNameAscending);
List<BuildDefinitionReference> builds = allBuilds.Where(x => HasAgentPoolId(x, agentPoolId)).ToList();
}
private bool HasAgentPoolId(BuildDefinitionReference buildDefinition, int agentPoolId)
{
TaskAgentPoolReference pool = buildDefinition?.Queue?.Pool;
if (pool == null)
{
return false;
}
return pool.Id.Equals(agentPoolId);
}
But I couldn't find a way to find the release definitions that have one or more environments configured to use a particular agent. Any suggestion?

I was manged to get all releases by Agent Pool ID via Rest Api and not via NET Client Libraries.Hope that helps.
C# Code snippet:
public class ReleaseResponse
{
[JsonProperty("value")]
public List<ReleaseItem> Value { get; set; }
}
public class ReleaseItem
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("Id")]
public int Id { get; set; }
}
static void Main(string[] args)
{
string tfsURL = "TFS URL";
string releaseDefurl = $"{tfsURL}/_apis/release/definitions?$expand=artifacts&api-version=3.2-preview.3";
const int agentPoolID = "AGENT Pool ID";
List<string> relevantReleases = new List<string>();
WebClient client = new WebClient();
client.UseDefaultCredentials = true;
client.Headers.Add("Content-Type", "application/json");
var releaseList = client.DownloadString(releaseDefurl);
var allReleases = JsonConvert.DeserializeObject<ReleaseResponse>(releaseList).Value;
foreach (var release in allReleases)
{
string releaseInfoApi = $"{tfsURL}/_apis/Release/definitions/{release.Id}";
var getReleseInfo = client.DownloadString(releaseInfoApi);
var releaseInfo = JsonConvert.DeserializeObject<TFSLogic.RootObject>(getReleseInfo);
var deploymentAgents = releaseInfo.environments.ToList().Where(e => e.deployPhases.FirstOrDefault().deploymentInput.queueId == agentPoolID).Count();
if (deploymentAgents > 0)
{
relevantReleases.Add(release.Name);
}
}
}
Find TFSLogic here : https://codebeautify.org/online-json-editor/cb7aa0d9
Powershell Code snippet:
$tfsUrl = "TFS URL"
$releaseDefurl = $tfsUrl + '/_apis/release/definitions?$expand=artifacts&api-version=3.2-preview.3'
$agentPoolID = "Agent Pool ID"
$relevantReleases = #();
$allReleasesID = (Invoke-RestMethod -Uri ($releaseDefurl) -Method Get -UseDefaultCredentials).value.id
function getReleaseByAgentPoolID($releaseID,$agentPoolID)
{
$ReleaseInfo = Invoke-RestMethod -Uri "$tfsUrl/_apis/Release/definitions/$releaseID" -Method Get -UseDefaultCredentials
$deploymentAgents = $ReleaseInfo.environments | % {$_.deployPhases.deploymentInput.queueId} | where {$_ -eq $agentPoolID}
if($deploymentAgents.Count -gt 0)
{
return $ReleaseInfo.name
}
}
foreach ($releaseID in $allReleasesID)
{
$relevantReleases += getReleaseByAgentPoolID -releaseID $releaseID -agentPoolID $agentPoolID
}
UPDATE :
It took me some time,But i was able to achieve that with azure-devops-dotnet-samples
I hope this example is finally what you are looking for.
using Microsoft.VisualStudio.Services.WebApi;
using System;
using System.Linq;
using Microsoft.VisualStudio.Services.ReleaseManagement.WebApi.Clients;
using Microsoft.VisualStudio.Services.ReleaseManagement.WebApi.Contracts;
using Microsoft.VisualStudio.Services.Common;
using System.Collections.Generic;
namespace FindReleaseByAgentPoolID
{
class Program
{
const int agentPoolID = 999;
static void Main(string[] args)
{
var relevantReleases = new List<string>();
VssCredentials c = new VssCredentials(new WindowsCredential(System.Net.CredentialCache.DefaultNetworkCredentials));
var tfsURL = new Uri("TFS URL");
var teamProjectName = "PROJECT";
using (var connection = new VssConnection(tfsURL, c))
using (var rmClient = connection.GetClient<ReleaseHttpClient2>())
{
var releases = rmClient
.GetReleaseDefinitionsAsync(teamProjectName, string.Empty, ReleaseDefinitionExpands.Environments)
.Result.ToArray();
foreach (var release in releases)
{
var r = rmClient.GetReleaseDefinitionAsync(teamProjectName, release.Id);
var deploymentAgents = r.Result.Environments.SelectMany(e =>
e.DeployPhases.Select(dp =>
dp.GetDeploymentInput()).Cast<DeploymentInput>()).Where(di =>
di.QueueId == agentPoolID).Count();
if (deploymentAgents > 0)
{
relevantReleases.Add(release.Name);
}
}
}
}
}
}

Found a solution, many thanks to #amit-baranes for pointing me in the right direction.
I've changed his code sample to use the await keyword instead of using .Result, and use .OfType<DeploymentInput>() instead of .Cast<DeploymentInput>() (it was throwing some exceptions).
Oh, and the most important thing I've learned: agent pool ID and queue ID are different things!!! If you intend to use the agent pool ID to get the release definitions you'll need to get the correspondent agent queue.
Code sample:
// set agent pool Id and project name
int agentPoolId = 123456;
string teamProjectName = ".....";
// _connection is of type VssConnection
using (var taskAgentClient = _connection.GetClient<TaskAgentHttpClient>())
using (var releaseClient = _connection.GetClient<ReleaseHttpClient2>())
{
// please note: agent pool Id != queue Id
// agent pool id is used to get the build definitions
// queue Id is used to get the release definitions
TaskAgentPool agentPool = await taskAgentClient.GetAgentPoolAsync(agentPoolId);
List<TaskAgentQueue> queues = await taskAgentClient.GetAgentQueuesByNamesAsync(teamProjectName, queueNames: new[] { agentPool.Name });
TaskAgentQueue queue = queues.FirstOrDefault();
List<ReleaseDefinition> definitions = await releaseClient.GetReleaseDefinitionsAsync(teamProjectName, string.Empty, ReleaseDefinitionExpands.Environments);
foreach (ReleaseDefinition definition in definitions)
{
var fullDefinition = await releaseClient.GetReleaseDefinitionAsync(teamProjectName, definition.Id);
bool hasReleasesWithPool = fullDefinition.Environments.SelectMany(GetDeploymentInputs)
.Any(di => di.QueueId == queue.Id);
if (hasReleasesWithPool)
{
Debug.WriteLine($"{definition.Name}");
}
}
}
private IEnumerable<DeploymentInput> GetDeploymentInputs(ReleaseDefinitionEnvironment environment)
{
return environment.DeployPhases.Select(dp => dp.GetDeploymentInput())
.OfType<DeploymentInput>();
}

Akka.NET Cluster Client

I try to create application using Akka.NET.
The main goal is to make a server that can handle many client connections and requests at the same time. I chose Akka.NET for this. I have a cluster, which now consists only 1 node.
I also have cluster clients (ClusterClient) that are simultaneously starting to connect to the server. The logic of the client is simple: it connects to the server and subscribe an actor there. There are no problems with publish yet, customers are getting everything, of course, if the connection is stable. Somewhere near ​​~4000-5000 client connections, reconnects begin and the connection is lost accordingly. I have tried to add a second node to the cluster and make 3000 connections for each node, but this was unsuccessful.
The question is how to make a server on AKKA.Net, which would hold a large number of connections (for example, 100 000 - 1 000 000). And could I use cluster for this purpose?
Server
using Akka.Actor;
using Akka.Configuration;
using System;
using System.Configuration;
using Akka.Cluster.Tools.Client;
using System.Threading;
namespace CoreSPServer
{
class Program
{
static void Main(string[] args)
{
Config config = ConfigurationFactory.ParseString(ConfigurationManager.AppSettings["ClusterConfig"]);
ActorSystem system = ActorSystem.Create("ClusterSystem", config);
var publisher = system.ActorOf(Props.Create(() => new Publisher()), "Publisher");
var clientSub = system.ActorOf(Props.Create(() => new ClientSubscriber()), "Sub");
ClusterClientReceptionist.Get(system).RegisterService(clientSub);
if (Console.ReadLine() == "start")
{
publisher.Tell("test");
Thread.Sleep(10);
}
Console.ReadKey();
}
}
}
ClientSubscriber and publisher
class ClientSubscriber : ReceiveActor
{
public ClientSubscriber()
{
var mediator = DistributedPubSub.Get(Context.System).Mediator;
Receive<IActorRef>(senderToSub =>
{
mediator.Tell(new Subscribe("content", senderToSub));
});
}
}
public class Publisher : ReceiveActor
{
public Publisher()
{
var mediator = DistributedPubSub.Get(Context.System).Mediator;
Receive<string>(str =>
{
var upperCase = str.ToUpper();
mediator.Tell(new Publish("content", upperCase));
});
}
}
Client
static void Main(string[] args)
{
var config = ConfigurationFactory.ParseString(ConfigurationManager.AppSettings["ClientConf"]);
ActorSystem Sys = ActorSystem.Create("ClusterClient", config);
//Connection path to Cluster Node
var initialContacts = new List<ActorPath>(){
ActorPath.Parse("akka.tcp://ClusterSystem#localhost:5001/system/receptionist"),
}.ToImmutableHashSet();
var settings = ClusterClientSettings.Create(Sys).WithInitialContacts(initialContacts);
for(int i = 0; i < 5000; i++)
{
IActorRef c = Sys.ActorOf(ClusterClient.Props(settings), "client" + Path.GetFileNameWithoutExtension(Path.GetRandomFileName()));
var asd = Sys.ActorOf(Props.Create<Subscriber>(), "clientSub" + Path.GetFileNameWithoutExtension(Path.GetRandomFileName()));
c.Tell(new ClusterClient.Send("/user/Sub5001", asd));
Thread.Sleep(1/10);
}
Console.ReadKey();
}
Server config
akka {
extensions = ["Akka.Cluster.Tools.Client.ClusterClientReceptionistExtensionProvider, Akka.Cluster.Tools"]
actor.provider = cluster
remote {
dot-netty.tcp {
port = 5001
public-hostname = localhost
auto-down-unreachable-after = off
}
}
cluster {
seed-nodes = ["akka.tcp://ClusterSystem#localhost:5001"]
}
}

check internet connection available or not in c#

I am using below method to check internet connection available or not in c#
and I was using it from What is the best way to check for Internet connectivity using .NET?
public static bool CheckForInternetConnection()
{
try
{
using (var client = new WebClient())
using (var stream = client.OpenRead("http://www.google.com"))
{
return true;
}
}
catch
{
return false;
}
}
above method works but I am facing problem, some time it takes a long time to rerun value, may be internet speed, but when I open Google.com in web browser then link open in a second, so why it is taking time to get result from C#

You can check whether internet is available or not like this:
ConnectionProfile internetConnectionProfile = Windows.Networking.Connectivity.NetworkInformation.GetInternetConnectionProfile();
if (internetConnectionProfile == null)
{
//logic ....
}
if (internetConnectionProfile != null)
{
this.IsInternetAvailable = internetConnectionProfile.GetNetworkConnectivityLevel() ==
NetworkConnectivityLevel.InternetAccess;
if (internetConnectionProfile.NetworkAdapter.IanaInterfaceType != 71)// Connection is not a Wi-Fi connection.
{
var isRoaming = internetConnectionProfile.GetConnectionCost().Roaming;
//user is Low on Data package only send low data.
var isLowOnData = internetConnectionProfile.GetConnectionCost().ApproachingDataLimit;
//User is over limit do not send data
var isOverDataLimit = internetConnectionProfile.GetConnectionCost().OverDataLimit;
IsWifiConnected = true;
}
else //Connection is a Wi-Fi connection. Data restrictions are not necessary.
{
IsWifiConnected = true;
}
}

I think your TimeOut is not working because of Domain Name Resolution. I you are proxy behind you must configure it on app.config file.
Check that example, I hope it helps you. I did it using .Net 4.0.
static bool CheckForInternetConnection(int timeOut = 3000)
{
var task = CheckForInternetConnectionTask(timeOut);
return task.Wait(timeOut) && task.Result;
}
static Task<bool> CheckForInternetConnectionTask(int timeOut = 3000)
{
return Task.Factory.StartNew
(() =>
{
try
{
var client = (HttpWebRequest) WebRequest.Create("http://google.com/");
client.Method = "HEAD";
client.Timeout = timeOut;
using (var response = client.GetResponse())
using (response.GetResponseStream())
{
return true;
}
}
catch
{
return false;
}
});
}
And a sample call:
Console.WriteLine("CheckForInternetConnection -> {0}", CheckForInternetConnection());
If you are proxy behind, do not forget to add proxy configuration on app.config file.
<?xml version="1.0" encoding="utf-8" ?>
<configuration>
<system.net>
<defaultProxy enabled="true">
<proxy proxyaddress="http://127.0.0.1:1111" usesystemdefault="True"/>
</defaultProxy>
</system.net>
</configuration>

public static bool ConnectionTest()
{
try
{
string site = "http://www.google.com/";
TcpClient client = TcpClient(site, 80);
client.Close();
return true;
}
catch (Exception)
{
return false;
}
}
Something like this should be faster.

Retrieve process network usage

How can I get a process send/receive bytes? the preferred way is doing it with C#.
I've searched this a lot and I didn't find any simple solution for this. Some solutions suggested to install the WinPCap on the machine and to work with this lib.
Like this guy asked: Need "Processes with Network Activity" functionality in managed code - Like resmon.exe does it
I don't want the overhead of the lib.
Is there a simple solution for this?
Actually I want the exactly data that the Resource Monitor of Windows gives under the "Processes with Network Activity" tab:
How does the Resource Monitor of Windows gets this information?
Any example?
Also, tried to use the counter method which is mentioned over here:
Missing network sent/received
but with no success - as not every process is shown under this counter.
And again I'm wondering how the Resource Monitor gets this information even without using this counter...

Resource monitor uses ETW - thankfully, Microsoft have created a nice nuget .net wrapper to make it easier to use.
I wrote something like this recently to report back my process's network IO:
using System;
using System.Diagnostics;
using System.Threading.Tasks;
using Microsoft.Diagnostics.Tracing.Parsers;
using Microsoft.Diagnostics.Tracing.Session;
namespace ProcessMonitoring
{
public sealed class NetworkPerformanceReporter : IDisposable
{
private DateTime m_EtwStartTime;
private TraceEventSession m_EtwSession;
private readonly Counters m_Counters = new Counters();
private class Counters
{
public long Received;
public long Sent;
}
private NetworkPerformanceReporter() { }
public static NetworkPerformanceReporter Create()
{
var networkPerformancePresenter = new NetworkPerformanceReporter();
networkPerformancePresenter.Initialise();
return networkPerformancePresenter;
}
private void Initialise()
{
// Note that the ETW class blocks processing messages, so should be run on a different thread if you want the application to remain responsive.
Task.Run(() => StartEtwSession());
}
private void StartEtwSession()
{
try
{
var processId = Process.GetCurrentProcess().Id;
ResetCounters();
using (m_EtwSession = new TraceEventSession("MyKernelAndClrEventsSession"))
{
m_EtwSession.EnableKernelProvider(KernelTraceEventParser.Keywords.NetworkTCPIP);
m_EtwSession.Source.Kernel.TcpIpRecv += data =>
{
if (data.ProcessID == processId)
{
lock (m_Counters)
{
m_Counters.Received += data.size;
}
}
};
m_EtwSession.Source.Kernel.TcpIpSend += data =>
{
if (data.ProcessID == processId)
{
lock (m_Counters)
{
m_Counters.Sent += data.size;
}
}
};
m_EtwSession.Source.Process();
}
}
catch
{
ResetCounters(); // Stop reporting figures
// Probably should log the exception
}
}
public NetworkPerformanceData GetNetworkPerformanceData()
{
var timeDifferenceInSeconds = (DateTime.Now - m_EtwStartTime).TotalSeconds;
NetworkPerformanceData networkData;
lock (m_Counters)
{
networkData = new NetworkPerformanceData
{
BytesReceived = Convert.ToInt64(m_Counters.Received / timeDifferenceInSeconds),
BytesSent = Convert.ToInt64(m_Counters.Sent / timeDifferenceInSeconds)
};
}
// Reset the counters to get a fresh reading for next time this is called.
ResetCounters();
return networkData;
}
private void ResetCounters()
{
lock (m_Counters)
{
m_Counters.Sent = 0;
m_Counters.Received = 0;
}
m_EtwStartTime = DateTime.Now;
}
public void Dispose()
{
m_EtwSession?.Dispose();
}
}
public sealed class NetworkPerformanceData
{
public long BytesReceived { get; set; }
public long BytesSent { get; set; }
}
}

You can use PerformanceCounter. Sample code:
//Define
string pn = "MyProcessName.exe";
var readOpSec = new PerformanceCounter("Process","IO Read Operations/sec", pn);
var writeOpSec = new PerformanceCounter("Process","IO Write Operations/sec", pn);
var dataOpSec = new PerformanceCounter("Process","IO Data Operations/sec", pn);
var readBytesSec = new PerformanceCounter("Process","IO Read Bytes/sec", pn);
var writeByteSec = new PerformanceCounter("Process","IO Write Bytes/sec", pn);
var dataBytesSec = new PerformanceCounter("Process","IO Data Bytes/sec", pn);
var counters = new List<PerformanceCounter>
{
readOpSec,
writeOpSec,
dataOpSec,
readBytesSec,
writeByteSec,
dataBytesSec
};
// get current value
foreach (PerformanceCounter counter in counters)
{
float rawValue = counter.NextValue();
// display the value
}
And this is to get performance counters for the Network card. Note it is not process specific
string cn = "get connection string from WMI";
var networkBytesSent = new PerformanceCounter("Network Interface", "Bytes Sent/sec", cn);
var networkBytesReceived = new PerformanceCounter("Network Interface", "Bytes Received/sec", cn);
var networkBytesTotal = new PerformanceCounter("Network Interface", "Bytes Total/sec", cn);
Counters.Add(networkBytesSent);
Counters.Add(networkBytesReceived);
Counters.Add(networkBytesTotal);

Have a look at the IP Helper API. There is an implementation in C# by Simon Mourier that sums transferred bytes per process: https://stackoverflow.com/a/25650933/385513
It would be interesting to know how this compares with Event Tracing for Windows (ETW)...