Develop Reference

how to get Task id,Feature id,Complete hrs by WIQL tfs by date?

how to get Task,Feature id,completed hours by date.lets say there is a
task 123 in which was created on a sprint which start date is 1st July and end at 10th July
task 123 effort hours is 5 completed hrs is 0 from 1-7-2018
and on 5th July effort is 3 hrs completed 2 hrs
and on 10th July effort is 1 hr and completed is 4 hrs
so how can i find task id,Feature id (by Tree WIQL) of date from 1st July to 5th July.

Based on your description, seems you want to get the history (revisions) for a specific work item.
If it is, then you can use the REST API to retrieve that: See Get a list of work items revisions
GET https://{instance}/DefaultCollection/_apis/wit/workitems/{id}/revisions?api-version={version}[&$top={int}&$skip={int}&$expand={enum{relations}
Try below sample to filter the list by date:
Param(
[string]$collectionurl = "http://172.17.16.115:8080/tfs/DefaultCollection",
[string]$witid = "12",
[string]$user = "username",
[string]$token = "password"
)
# Base64-encodes the Personal Access Token (PAT) appropriately
$base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f $user,$token)))
$baseUrl = "$collectionurl/_apis/wit/workitems/$witid/revisions?"+"$"+"expand=all"
$response = (Invoke-RestMethod -Uri $baseUrl -Method Get -Headers #{Authorization=("Basic {0}" -f $base64AuthInfo)}).value
# Filter the revisions changed between the date '2018-07-01'and '2018-07-05' and remaining work >=O
$revisions = $response | where({$_.fields.'System.ChangedDate' -ge '2018-07-01' -and $_.fields.'System.ChangedDate' -le '2018-07-06' -and $_.fields.'Microsoft.VSTS.Scheduling.RemainingWork' -ge '0' } )
$witrevisions = #()
foreach($revision in $revisions){
$customObject = new-object PSObject -property #{
"WorkItemType" = $revision.fields.'System.WorkItemType'
"WorkItemID" = $revision.fields.'System.Id'
"Revid" = $revision.rev
"RemainingWork" = $revision.fields.'Microsoft.VSTS.Scheduling.RemainingWork'
"Relations" = $revision.relations.url
"ChangeddDate" = $revision.fields.'System.ChangedDate'
}
$witrevisions += $customObject
}
$witrevisions | Select-Object `
WorkItemID,
WorkItemType,
Revid,
RemainingWork,
Relations,
ChangeddDate #| export-csv -Path E:\filename.csv -NoTypeInformation

I have created test solution. It based on rest api. It requires nugate packages:
Microsoft.TeamFoundationServer.Client
Microsoft.VisualStudio.Services.InteractiveClient
And you have to update them after install because I had the error: Inheritance security rules violated by type
Test source code:
using Microsoft.TeamFoundation.WorkItemTracking.WebApi;
using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models;
using Microsoft.VisualStudio.Services.Common;
using Microsoft.VisualStudio.Services;
using Microsoft.VisualStudio.Services.WebApi;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.VisualStudio.Services.Client;
namespace GetFTTaskCompleteForDates
{
class Program
{
public class FTTaskComleted
{
public int FTID = 0;
public int TaskID = 0;
public int CompletedOnDate = 0;
public int CompletedFieldValue = 0;
public DateTime ReportDT;
}
static public class WIQLS
{
static public string ChangedTasksFromDate = "SELECT [System.Id] FROM WorkItems WHERE [System.TeamProject] = '{0}' AND [System.WorkItemType] = 'Task' AND [System.ChangedDate] >= '{1}-{2}-{3}T00:00:00.0000000' ORDER BY [System.Id]";
static public string GetTopParentFeature = "SELECT [System.Id] FROM WorkItemLinks WHERE ([Source].[System.TeamProject] = '{0}' AND [Source].[System.WorkItemType] = 'Feature') And ([System.Links.LinkType] = 'System.LinkTypes.Hierarchy-Forward') And ([Target].[System.Id] = {1}) ORDER BY [System.Id] mode(Recursive,ReturnMatchingChildren)";
}
static string ServiceUrl = "http://tfs-srv:8080/tfs/DefaultCollection";
static string TeamProject = "VSTSAgile";
static WorkItemTrackingHttpClient WiClient = null;
static VssConnection ServiceConnection = null;
static string CompletedWorkFieldRef = "Microsoft.VSTS.Scheduling.CompletedWork";
static string ChangedDateFieldRef = "System.ChangedDate";
static void Main(string[] args)
{
DateTime _startDate = new DateTime(2018, 06, 19);
DateTime _finishDate = DateTime.Today; // set to DateTime.MinValue if only one date
//DateTime _finishDate = DateTime.MinValue;
List<FTTaskComleted> _lstReport = new List<FTTaskComleted>();
if (!ConnectToService()) return;
Dictionary<int, int> _dctTasksFeatures = GetTaskAndFeatures(_startDate);
if (_dctTasksFeatures.Keys.Count > 0) FillReportList(_dctTasksFeatures, _lstReport, _startDate, _finishDate);
foreach (FTTaskComleted _item in _lstReport)
Console.WriteLine("DATE:{0} -- FT:{1} -- TSK:{2} -- HRS -- {3}", _item.ReportDT.ToShortDateString(), _item.FTID, _item.TaskID, _item.CompletedOnDate);
}
//Fill list with hours for each date and task
private static void FillReportList(Dictionary<int, int> pDctTasksFT, List<FTTaskComleted> pLstReport, DateTime pStartDate, DateTime pFinishtDate)
{
foreach (int _taskId in pDctTasksFT.Keys)
{
List<WorkItem> _revs = WiClient.GetRevisionsAsync(_taskId).Result;
DateTime _lastDate = DateTime.MinValue; //last processed date for revisions of work item
int _lastHours = int.MinValue; //last processed value of compteted work for revisions of work item
foreach (WorkItem _rev in _revs)
{
if (!_rev.Fields.Keys.Contains(CompletedWorkFieldRef) || !_rev.Fields.Keys.Contains(ChangedDateFieldRef)) continue;
DateTime _changedDate;
if (!DateTime.TryParse(_rev.Fields[ChangedDateFieldRef].ToString(), out _changedDate)) continue;
bool _inscope = false;
// calculate hours based on previous revision
int _completedValue, _completedDiff;
if (!int.TryParse(_rev.Fields[CompletedWorkFieldRef].ToString(), out _completedValue)) continue;
if (_lastHours == int.MinValue) _completedDiff = _completedValue;
else _completedDiff = _completedValue - _lastHours;
_lastHours = _completedValue;
// check for date of revision between needed dates
if (pFinishtDate == DateTime.MinValue) { if (_changedDate.Date == pStartDate.Date) _inscope = true; }
else if (_changedDate.Date >= pStartDate.Date && _changedDate.Date <= pFinishtDate.Date) _inscope = true;
if (_inscope && _completedDiff != 0)
{
if (_lastDate.Date == _changedDate.Date && pLstReport.Count > 0)
{
//update existing item if several changes in one day
pLstReport[pLstReport.Count - 1].CompletedOnDate += _completedDiff;
pLstReport[pLstReport.Count - 1].CompletedFieldValue = _completedValue;
}
else // add new report item
pLstReport.Add(
new FTTaskComleted
{
FTID = pDctTasksFT[_taskId],
TaskID = _taskId, ReportDT = _changedDate,
CompletedOnDate = _completedDiff,
CompletedFieldValue = _completedValue }
);
_lastDate = _changedDate;
}
}
}
}
static Dictionary<int, int> GetTaskAndFeatures(DateTime pStartDate)
{
Dictionary<int, int> _taskft = new Dictionary<int, int>();
List<int> _ids = GetTasksIdsForPeriod(pStartDate);
if (_ids.Count > 0)
{
foreach(int _wiId in _ids)
if (!AddFeaturesToTask(_wiId, _taskft)) break;
}
return _taskft;
}
//add top level parent to task id and exclude tasks without parent features
static bool AddFeaturesToTask(int pId, Dictionary<int, int> pWiDict)
{
try
{
Wiql _wiql = new Wiql();
_wiql.Query = String.Format(WIQLS.GetTopParentFeature, TeamProject, pId);
WorkItemQueryResult _res = WiClient.QueryByWiqlAsync(_wiql).Result;
if (_res.WorkItemRelations.Count() > 1)
pWiDict.Add(pId, _res.WorkItemRelations.ElementAt(0).Target.Id); //first is a top parent
}
catch (Exception ex)
{
Console.WriteLine("Query problem:");
Console.WriteLine(ex.Message);
return false;
}
return true;
}
//Connect to tfs/vsts
static public bool ConnectToService()
{
try
{
VssCredentials creds = new VssClientCredentials();
creds.Storage = new VssClientCredentialStorage();
ServiceConnection = new VssConnection(new Uri(ServiceUrl), creds);
ServiceConnection.ConnectAsync().Wait();
WiClient = ServiceConnection.GetClient<WorkItemTrackingHttpClient>();
}
catch(Exception ex)
{
Console.WriteLine("Connection problem:");
Console.WriteLine(ex.Message);
return false;
}
return true;
}
//find all tasks changed after start date. beacouse we can not select revisions between dates.
static public List<int> GetTasksIdsForPeriod(DateTime pStart)
{
List<int> _ids = new List<int>();
try
{
Wiql _wiql = new Wiql();
_wiql.Query = String.Format(WIQLS.ChangedTasksFromDate, TeamProject, pStart.Year, pStart.Month, pStart.Day);
WorkItemQueryResult _res = WiClient.QueryByWiqlAsync(_wiql).Result;
if (_res.WorkItems.Count() > 0)
foreach (WorkItemReference _wi in _res.WorkItems) _ids.Add(_wi.Id);
}
catch (Exception ex)
{
Console.WriteLine("Query problem:");
Console.WriteLine(ex.Message);
}
return _ids;
}
}
}

Using ExecuteNonQueryAsync and Reporting Progress

I thought I was trying to do something very simple. I just want to report a running number on the screen so the user gets the idea that the SQL Stored Procedure that I'm executing is working and that they don't get impatient and start clicking buttons.
The problem is that I can't figure out how to actually call the progress reporter for the ExecutNonQueryAsync command. It gets stuck in my reporting loop and never executes the command but, if I put it after the async command, it will get executed and result will never not equal zero.
Any thoughts, comments, ideas would be appreciated. Thank you so much!
int i = 0;
lblProcessing.Text = "Transactions " + i.ToString();
int result = 0;
while (result==0)
{
i++;
if (i % 500 == 0)
{
lblProcessing.Text = "Transactions " + i.ToString();
lblProcessing.Refresh();
}
}
// Yes - I know - the code never gets here - that is the problem!
result = await cmd.ExecuteNonQueryAsync();

The simplest way to do this is to use a second connection to monitor the progress, and report on it. Here's a little sample to get you started:
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.Text;
using System.Threading.Tasks;
namespace Microsoft.Samples.SqlServer
{
public class SessionStats
{
public long Reads { get; set; }
public long Writes { get; set; }
public long CpuTime { get; set; }
public long RowCount { get; set; }
public long WaitTime { get; set; }
public string LastWaitType { get; set; }
public string Status { get; set; }
public override string ToString()
{
return $"Reads {Reads}, Writes {Writes}, CPU {CpuTime}, RowCount {RowCount}, WaitTime {WaitTime}, LastWaitType {LastWaitType}, Status {Status}";
}
}
public class SqlCommandWithProgress
{
public static async Task ExecuteNonQuery(string ConnectionString, string Query, Action<SessionStats> OnProgress)
{
using (var rdr = await ExecuteReader(ConnectionString, Query, OnProgress))
{
rdr.Dispose();
}
}
public static async Task<DataTable> ExecuteDataTable(string ConnectionString, string Query, Action<SessionStats> OnProgress)
{
using (var rdr = await ExecuteReader(ConnectionString, Query, OnProgress))
{
var dt = new DataTable();
dt.Load(rdr);
return dt;
}
}
public static async Task<SqlDataReader> ExecuteReader(string ConnectionString, string Query, Action<SessionStats> OnProgress)
{
var mainCon = new SqlConnection(ConnectionString);
using (var monitorCon = new SqlConnection(ConnectionString))
{
mainCon.Open();
monitorCon.Open();
var cmd = new SqlCommand("select ##spid session_id", mainCon);
var spid = Convert.ToInt32(cmd.ExecuteScalar());
cmd = new SqlCommand(Query, mainCon);
var monitorQuery = #"
select s.reads, s.writes, r.cpu_time, s.row_count, r.wait_time, r.last_wait_type, r.status
from sys.dm_exec_requests r
join sys.dm_exec_sessions s
on r.session_id = s.session_id
where r.session_id = #session_id";
var monitorCmd = new SqlCommand(monitorQuery, monitorCon);
monitorCmd.Parameters.Add(new SqlParameter("#session_id", spid));
var queryTask = cmd.ExecuteReaderAsync( CommandBehavior.CloseConnection );
var cols = new { reads = 0, writes = 1, cpu_time =2,row_count = 3, wait_time = 4, last_wait_type = 5, status = 6 };
while (!queryTask.IsCompleted)
{
var firstTask = await Task.WhenAny(queryTask, Task.Delay(1000));
if (firstTask == queryTask)
{
break;
}
using (var rdr = await monitorCmd.ExecuteReaderAsync())
{
await rdr.ReadAsync();
var result = new SessionStats()
{
Reads = Convert.ToInt64(rdr[cols.reads]),
Writes = Convert.ToInt64(rdr[cols.writes]),
RowCount = Convert.ToInt64(rdr[cols.row_count]),
CpuTime = Convert.ToInt64(rdr[cols.cpu_time]),
WaitTime = Convert.ToInt64(rdr[cols.wait_time]),
LastWaitType = Convert.ToString(rdr[cols.last_wait_type]),
Status = Convert.ToString(rdr[cols.status]),
};
OnProgress(result);
}
}
return queryTask.Result;
}
}
}
}
Which you would call something like this:
class Program
{
static void Main(string[] args)
{
Run().Wait();
}
static async Task Run()
{
var constr = "server=localhost;database=tempdb;integrated security=true";
var sql = #"
set nocount on;
select newid() d
into #foo
from sys.objects, sys.objects o2, sys.columns
order by newid();
select count(*) from #foo;
";
using (var rdr = await SqlCommandWithProgress.ExecuteReader(constr, sql, s => Console.WriteLine(s)))
{
if (!rdr.IsClosed)
{
while (rdr.Read())
{
Console.WriteLine("Row read");
}
}
}
Console.WriteLine("Hit any key to exit.");
Console.ReadKey();
}
}
Which outputs:
Reads 0, Writes 0, CPU 1061, RowCount 0, WaitTime 0, LastWaitType SOS_SCHEDULER_YIELD, Status running
Reads 0, Writes 0, CPU 2096, RowCount 0, WaitTime 0, LastWaitType SOS_SCHEDULER_YIELD, Status running
Reads 0, Writes 0, CPU 4553, RowCount 11043136, WaitTime 198, LastWaitType CXPACKET, Status suspended
Row read
Hit any key to exit.

You're not going to be able to get ExecuteNonQueryAsync to do what you want here. To do what you're looking for, the result of the method would have to be either row by row or in chunks incremented during the SQL call, but that's not how submitting a query batch to SQL Server works or really how you would want it to work from an overhead perspective. You hand a SQL statement to the server and after it is finished processing the statement, it returns the total number of rows affected by the statement.

Do you just want to let the user know that something is happening, and you don't actually need to display current progress?
If so, you could just display a ProgressBar with its Style set to Marquee.
If you want this to be a "self-contained" method, you could display the progress bar on a modal form, and include the form code in the method itself.
E.g.
public void ExecuteNonQueryWithProgress(SqlCommand cmd) {
Form f = new Form() {
Text = "Please wait...",
Size = new Size(400, 100),
StartPosition = FormStartPosition.CenterScreen,
FormBorderStyle = FormBorderStyle.FixedDialog,
MaximizeBox = false,
ControlBox = false
};
f.Controls.Add(new ProgressBar() {
Style = ProgressBarStyle.Marquee,
Dock = DockStyle.Fill
});
f.Shown += async (sender, e) => {
await cmd.ExecuteNonQueryAsync();
f.Close();
};
f.ShowDialog();
}

That is an interesting question. I have had to implement similar things in the past. In our case the priority was to:
Keep client side responsive in case the user doesn't want to stick around and wait.
Update the user of action and progress.
What I would do is use threading to run the process in the background like:
HostingEnvironment.QueueBackgroundWorkItem(ct => FunctionThatCallsSQLandTakesTime(p, q, s));
Then using a way to estimate work time I would increment a progress bar from client side on a clock. For this, query your data for a variable that gives you a linear relationship to the work time needed by FunctionThatCallsSQLandTakesTime.
For example; the number of active users this month drives the time FunctionThatCallsSQLandTakesTime takes. For each 10000 user it takes 5 minutes. So you can update your progress bar accordingly.

I'm wondering if this might be a reasonable approach:
IAsyncResult result = cmd2.BeginExecuteNonQuery();
int count = 0;
while (!result.IsCompleted)
{
count++;
if (count % 500 == 0)
{
lblProcessing.Text = "Transactions " + i.ToString();
lblProcessing.Refresh();
}
// Wait for 1/10 second, so the counter
// does not consume all available resources
// on the main thread.
System.Threading.Thread.Sleep(100);
}

How can i extract the date and time from the html page file?

In the class i have:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Net;
namespace SatelliteImages
{
class ExtractImages
{
static WebClient client;
List<string> imagesUrls = new List<string>();
static string htmltoextract;
static string link;
static string text;
public static List<string> countriescodes = new List<string>();
public static List<string> countriesnames = new List<string>();
private static int lastsatimage = 0;
private static string Next_Sat_File;
private static string temp_sat_dir;
public void Init()
{
ExtractCountires();
}
public static void ExtractCountires()
{
try
{
htmltoextract = "http://sat24.com/en/?ir=true";
client = new WebClient();
client.DownloadFile(htmltoextract, #"c:\temp\sat24.html");
client.Dispose();
string tag1 = "<li><a href=\"/en/";
string tag2 = "</a></li>";
string s = System.IO.File.ReadAllText(#"c:\temp\sat24.html");
s = s.Substring(s.IndexOf(tag1));
s = s.Substring(0, s.LastIndexOf(tag2) + tag2.ToCharArray().Length);
s = s.Replace("\r", "").Replace("\n", "").Replace(" ", "");
string[] parts = s.Split(new string[] { tag1, tag2 }, StringSplitOptions.RemoveEmptyEntries);
string tag3 = "<li><ahref=\"/en/";
for (int i = 0; i < parts.Length; i++)
{
if (i == 17)
{
break;
}
string l = "";
if (parts[i].Contains(tag3))
l = parts[i].Replace(tag3, "");
string z1 = l.Substring(0, l.IndexOf('"'));
countriescodes.Add(z1);
string z2 = parts[i].Substring(parts[i].LastIndexOf('>') + 1);
countriesnames.Add(z2);
}
}
catch (Exception e)
{
}
}
}
}
I found that when reading the sat24.html file in the file the dates and times are in this part in the html file:
var arrayImageTimes = [];
arrayImageTimes.push('201612271810');arrayImageTimes.push('201612271825');arrayImageTimes.push('201612271840');arrayImageTimes.push('201612271855');arrayImageTimes.push('201612271910');arrayImageTimes.push('201612271925');arrayImageTimes.push('201612271940');arrayImageTimes.push('201612271955');arrayImageTimes.push('201612272010');arrayImageTimes.push('201612272025');
Then what i want to do is to exctract the dates and times and to add them to two lists: 201612271810 So the first list will be of this format.
The second list to be of date and time not sure what format but: Year = 2016 month = 12 day = 27 hours = 18 minutes = 10
What i want to do later is to build a new link for each date and time the new links should be in this format: If we take for example the date and time 201612271810. Then: http://www.sat24.com/image2.ashx?region=is&time=201612271810&ir=true The question is how can i extract the date and time and then using my ExtractCountries method to build the links.
In the ExtractCountries method i'm getting two lists one the countries codes and one the countries names.
What i need to build the links is the country/ies codes and the country/ies date and time.
So then i can use the list of codes with a date time for example:
The first list will be the dates and times as they are in the format in the html file: For example the first date and time is:
http://www.sat24.com/image2.ashx?region=is&time=201612271810&ir=true
Where is = Israel and 201612271810 is the date time for this image link.
Or for example
http://www.sat24.com/image2.ashx?region=tu&time=201612271810&ir=true
Region tu is turkey
so what i need is to get list of links built from all countries codes and all dates and times per region(countrey) so i can later download the images.
So in the list for example uri string type or string list type will be something like in first 10 indexs:
http://www.sat24.com/image2.ashx?region=tu&time=201612271825&ir=true
http://www.sat24.com/image2.ashx?region=tu&time=201612271840&ir=true
http://www.sat24.com/image2.ashx?region=tu&time=201612271855&ir=true
http://www.sat24.com/image2.ashx?region=tu&time=201612271910&ir=true
.
.
.
.
.
.
http://www.sat24.com/image2.ashx?region=is&time=201612271810&ir=true
http://www.sat24.com/image2.ashx?region=is&time=201612271825&ir=true
http://www.sat24.com/image2.ashx?region=is&time=201612271840&ir=true
http://www.sat24.com/image2.ashx?region=is&time=201612271910&ir=true
Ofcourse dpending on each region/country extracted date and time from the sat24 page.
Then when finished creating the list to download the images.Each link for example: http://www.sat24.com/image2.ashx?region=is&time=201612271910&ir=true should be downloaded and saved as image.

This is an example using the agility pack to extract info from an html document.
using System;
using System.Linq;
using System.IO;
using System.Xml;
using System.Net;
using HtmlAgilityPack;
public class Program
{
public static void Main()
{
var wc = new WebClient();
wc.BaseAddress = "http://sat24.com/";
HtmlDocument doc = new HtmlDocument();
var temp = wc.DownloadData("/en");
doc.Load(new MemoryStream(temp));
var secTokenScript = doc.DocumentNode.Descendants()
.Where(e =>
String.Compare(e.Name, "script", true) == 0 &&
String.Compare(e.ParentNode.Name, "div", true) == 0 &&
e.InnerText.Length > 0 &&
e.InnerText.Trim().StartsWith("var region")
).FirstOrDefault().InnerText;
var securityToken = secTokenScript;
securityToken = securityToken.Substring(0, securityToken.IndexOf("arrayImageTimes.push"));
securityToken = secTokenScript.Substring(securityToken.Length).Replace("arrayImageTimes.push('", "").Replace("')", "");
var dates = securityToken.Trim().Split(new string[] { ";"}, StringSplitOptions.RemoveEmptyEntries);
var scriptDates = dates.Select(x => new ScriptDate { DateString = x });
foreach(var date in scriptDates)
{
Console.WriteLine("Date String: '" + date.DateString + "'\tYear: '" + date.Year + "'\t Month: '" + date.Month + "'\t Day: '" + date.Day + "'\t Hours: '" + date.Hours + "'\t Minutes: '" + date.Minutes + "'");
}
}
public class ScriptDate
{
public string DateString {get;set;}
public int Year
{
get
{
return Convert.ToInt32(this.DateString.Substring(0, 4));
}
}
public int Month
{
get
{
return Convert.ToInt32(this.DateString.Substring(4, 2));
}
}
public int Day
{
get
{
return Convert.ToInt32(this.DateString.Substring(6, 2));
}
}
public int Hours
{
get
{
return Convert.ToInt32(this.DateString.Substring(8, 2));
}
}
public int Minutes
{
get
{
return Convert.ToInt32(this.DateString.Substring(10, 2));
}
}
}
}
I have also created a .Net Fiddle showing that this works

Memory leak with DataAdapter.Fill to DataSet

I have a simple program that grab data from the database and stored it in a DataSet through DataAdapter.Fill().
The problem I am facing is that the program's memory size keep increasing. Using process explorer, I monitor the virtual size of the program. The processes are run in a standalne build(not in unity editor).
2mins after the process is launch, the program sits at 824,444K virtual size, but leaving the program running for 30 mins, the program's virtual size increased to 1,722,340K.
(can't upload screenshot) https://drive.google.com/open?id=0B0DwzunTEqfKcDhHcXRmV2twUEE
The program only consist of 2 simple script: A helper class SQLController which manage the loading from database to dataset and a monobehavior script DataProducer that poll the database at regular interval for "update" using SQLController object.
SQLController:
using UnityEngine;
using System.Collections;
using System.Data;
using System.Data.Sql;
using System.Data.SqlClient;
using UnityEngine.UI;
using System.Threading;
public class SQLController {
string connectionString;
string dataSource, catalog, uid, pwd;
DataSet dat_set;
public SQLController(string dataSource, string catalog, string uid, string pwd)
{
this.dataSource = dataSource;
this.catalog = catalog;
this.uid = uid;
this.pwd = pwd;
}
/// <summary>
/// Open a connectio to the database and query it for data with the statement input
/// </summary>
/// <param name="statement"> The query statement to be used to query the server</param>
/// <returns></returns>
public DataSet Load(string statement, string name)
{
connectionString = string.Format("data source={0};initial catalog={1};uid={2};pwd={3}", dataSource, catalog, uid, pwd);
using (SqlConnection dbcon = new SqlConnection(connectionString))
using (SqlDataAdapter dataAdapter = new SqlDataAdapter(statement, dbcon))
{
dat_set = new System.Data.DataSet();
dbcon.Open();
dataAdapter.Fill(dat_set, name);
}
return dat_set;
}
}
DataProducer
using UnityEngine;
using System.Collections;
using System.Threading;
using System.Data;
using UnityEngine.UI;
using SimpleJSON;
public class DataProducer : MonoBehaviour {
public GameObject textObj;
private string[] _configData;
private string _outputText;
private SQLController _sqlController;
private bool _toggle = true;
private bool _updating = false;
private DataSet _dataSetCache;
private Thread _dataGrabThread;
// Use this for initialization
void Start () {
_configData = new string[5];
if (LoadFromConfigFile())
{
StartCoroutine(LoadFromDB());
}
}
// Update is called once per frame
void Update () {
textObj.GetComponent<Text>().text = _outputText;
}
public void OnDisable()
{
// stop any running thread
if (null != _dataGrabThread && _dataGrabThread.IsAlive)
{
_dataGrabThread.Abort();
}
}
IEnumerator LoadFromDB()
{
while (true)
{
if (_updating)
{
Debug.Log("Data System Poll DataBase ignored");
}
else
{
_updating = true;
_dataGrabThread = new Thread(Load);
_dataGrabThread.IsBackground = true;
_dataGrabThread.Start();
}
yield return new WaitForSeconds(10f);
}
}
void Load()
{
string statement;
if (_toggle)
{
_toggle = !_toggle;
statement = "SELECT TOP 100000 [AIIDX],[LASTATTACKDATE],[LASTRECEIVEDDATE],[DEVICEID],[INSTANCES],[ATTACKTYPE],[SEVERITY],[STATUS] FROM AI (NOLOCK)";
}
else
{
_toggle = !_toggle;
statement = "SELECT TOP 100000 [AIIDX],[LASTATTACKDATE],[LASTRECEIVEDDATE],[DEVICEID],[SEVERITY],[STATUS] FROM AI (NOLOCK)";
}
_sqlController = new SQLController(_configData[0], _configData[1], _configData[2], _configData[3]);
_outputText = "Loading";
_dataSetCache = _sqlController.Load(statement, "TestObject");
PrintDataSet();
_updating = false;
}
/// <summary>
/// Convert datatable into string and print it out through a text object
/// </summary>
void PrintDataSet()
{
if (null == _dataSetCache)
{
return;
}
DataTable dt = _dataSetCache.Tables["TestObject"];
if (null == dt)
{
return;
}
System.Text.StringBuilder builder = new System.Text.StringBuilder();
for (int i = 0; i < 20; ++i)
{
builder.AppendFormat("{0,-5}", (i + 1) + ".");
//comp.text += string.Format("{0,-5}", (i + 1) + ".");
DataRow dr = dt.Rows[i];
for (int j = 0; j < dt.Columns.Count; ++j)
{
builder.AppendFormat("{0, -30}", dr[dt.Columns[j]].ToString());
}
builder.Append("\n");
}
_outputText = builder.ToString();
builder = null;
}
bool LoadFromConfigFile()
{
string line;
using (System.IO.StreamReader file = new System.IO.StreamReader("./config.txt"))
{
if (file == null)
{
return false;
}
int index = 0;
while ((line = file.ReadLine()) != null)
{
if (index > _configData.Length)
{
Debug.LogError("Invalid Config file");
return false;
}
_configData[index++] = line;
}
//if the config file does not consist of 5 data
if (index < _configData.Length)
{
Debug.LogError("Invalid Config file");
return false;
}
return true;
}
}
}
I am not very sure what exactly causes the memory leak but when I changed the loading from threaded
_dataGrabThread = new Thread(Load);
to running in the main thread,
Load()
the process virtual size, though still increment, but at a slower rate. From 828,312K at 2 min to 1,083,908K at 40mins.

Using an ObjectDataSource with a GridView in a dynamic scenario

I have a search page that is tasked with searching 3.5 million records for individuals based on their name, customer ID, address, etc. The queries range from complex to simple.
Currently, this code relies on a SqlDataSource and a GridView. When a user types a serach term in and presses enter, the TextBoxChanged even runs a Search(term, type) function that changes the query that the SqlDataSource uses, adds the parameters, and rebinds the GridView.
It works well, but I've become obsessed with rewriting the code more efficiently. I want the paging to be done by SQL Server instead of the inefficiencies of a SqlDataSource in DataSet mode.
Enter the ObjectDataSource. Caveat: I have never used one before today.
I have spent the better part of the day putting together this class:
using System;
using System.Data;
using System.Data.SqlClient;
using System.Configuration;
using System.Text;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Web.UI.HtmlControls;
/// <summary>
/// Summary description for MultiSearchData
/// </summary>
public class MultiSearchData
{
private string _connectionString = string.Empty;
private string _sortColumns = string.Empty;
private string _selectQuery = string.Empty;
private int _lastUpdate;
private int _lastRowCountUpdate;
private int _lastRowCount;
private SqlParameterCollection _sqlParams;
public MultiSearchData()
{
}
private void UpdateDate()
{
_lastUpdate = (int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds;
}
private string ReplaceFirst(string text, string search, string replace)
{
int pos = text.IndexOf(search);
if (pos < 0)
{
return text;
}
return text.Substring(0, pos) + replace + text.Substring(pos + search.Length);
}
public string SortColumns
{
get { return _sortColumns; }
set { _sortColumns = value; }
}
public SqlParameterCollection SqlParams
{
get { return _sqlParams; }
set { _sqlParams = value; }
}
public string ConnectionString
{
get { return _connectionString; }
set { _connectionString = value; }
}
public string SelectQuery
{
get { return _selectQuery; }
set
{
if (value != _selectQuery)
{
_selectQuery = value;
UpdateDate();
}
}
}
public DataTable GetFullDataTable()
{
return GetDataTable(AssembleSelectSql());
}
public DataTable GetPagedDataTable(int startRow, int pageSize, string sortColumns)
{
if (sortColumns.Length > 0)
_sortColumns = sortColumns;
return GetDataTable(AssemblePagedSelectSql(startRow, pageSize));
}
public int GetRowCount()
{
if (_lastRowCountUpdate == _lastUpdate)
{
return _lastRowCount;
}
else
{
string strCountQuery = _selectQuery.Remove(7, _selectQuery.IndexOf("FROM") - 7);
strCountQuery = strCountQuery.Replace("SELECT FROM", "SELECT COUNT(*) FROM");
using (SqlConnection conn = new SqlConnection(_connectionString))
{
conn.Open();
using (SqlCommand cmd = new SqlCommand(strCountQuery, conn))
{
if (_sqlParams.Count > 0)
{
foreach (SqlParameter param in _sqlParams)
{
cmd.Parameters.Add(param);
}
}
_lastRowCountUpdate = _lastUpdate;
_lastRowCount = (int)cmd.ExecuteScalar();
return _lastRowCount;
}
}
}
}
public DataTable GetDataTable(string sql)
{
DataTable dt = new DataTable();
using (SqlConnection conn = new SqlConnection(_connectionString))
{
using (SqlCommand GetCommand = new SqlCommand(sql, conn))
{
conn.Open();
if (_sqlParams.Count > 0)
{
foreach (SqlParameter param in _sqlParams)
{
GetCommand.Parameters.Add(param);
}
}
using (SqlDataReader dr = GetCommand.ExecuteReader())
{
dt.Load(dr);
conn.Close();
return dt;
}
}
}
}
private string AssembleSelectSql()
{
StringBuilder sql = new StringBuilder();
sql.Append(_selectQuery);
return sql.ToString();
}
private string AssemblePagedSelectSql(int startRow, int pageSize)
{
StringBuilder sql = new StringBuilder();
string originalQuery = ReplaceFirst(_selectQuery, "FROM", ", ROW_NUMBER() OVER (ORDER BY " + _sortColumns + ") AS ResultSetRowNumber FROM");
sql.Append("SELECT * FROM (");
sql.Append(originalQuery);
sql.Append(") AS PagedResults");
sql.AppendFormat(" WHERE ResultSetRowNumber > {0} AND ResultSetRowNumber <= {1}", startRow.ToString(), (startRow + pageSize).ToString());
return sql.ToString();
}
}
I don't know if it's pretty. It works. I give it a query in the ObjectCreating method:
protected void dataMultiSearchData_ObjectCreating(object sender, ObjectDataSourceEventArgs e)
{
MultiSearchData info;
info = Cache["MultiSearchDataObject"] as MultiSearchData;
if (null == info)
{
info = new MultiSearchData();
}
info.SortColumns = "filteredcontact.fullname";
info.ConnectionString = "Data Source=SERVER;Initial Catalog=TheDatabase;Integrated Security=sspi;Connection Timeout=60";
info.SelectQuery = #"SELECT filteredcontact.contactid,
filteredcontact.new_libertyid,
filteredcontact.fullname,
'' AS line1,
filteredcontact.emailaddress1,
filteredcontact.telephone1,
filteredcontact.birthdateutc AS birthdate,
filteredcontact.gendercodename
FROM filteredcontact
WHERE fullname LIKE 'Griffin%' AND filteredcontact.statecode = 0";
e.ObjectInstance = info;
}
protected void dataMultiSearchData_ObjectDisposing(object sender, ObjectDataSourceDisposingEventArgs e)
{
MultiSearchData info = e.ObjectInstance as MultiSearchData;
MultiSearchData temp = Cache["MultiSearchDataObject"] as MultiSearchData;
if (null == temp)
{
Cache.Insert("MultiSearchDataObject", info);
}
e.Cancel = true;
}
Once the class has the query, it wraps it in paging friendly SQL and we're off to the races. I've implemented caching so that it can skip some expensive queries. Etc.
My problem is, this completely breaks my pretty little Search(term, type) world. Having ot set the query in the ObjectCreating method is completely harshing my vibe.
I've been trying to think of a better way to do this all day, but I keep ending up with a really messy...do it all in ObjectCreating model that just turns my stomach.
How would you do this? How can I keep the efficiency of this new method whilst have the organizational simplicity of my former model?
Am I being too OCD?

I determined that it can't be done. Furthermore, after benchmarking this class I found it performed no better than a SqlDataSource but was much more difficult to maintain.
Thus I abandoned this project. I hope someone finds this code useful at some point though.