Develop Reference

SSIS Google Analytics api call

I've got some code that I have used to pull Google Analytics data with a c# console application and it works great. Whenever I try to use that same code in an SSIS script task I get the error "Error deserializing JSON credential data.". I get the error when running locally and when deployed. I've got all the libraries added to the GAC and I'm using the same version libraries and .net Framework as the console app. Anyone have any ideas?
public void Main()
{
string SQL_Script = null;
string ErrorMessage = string.Empty;
string ExceptionMessage = "No error";
// Declare the variables that you'll be pulling from Google Analytics into the database
DateTime GA_Session_Date = new DateTime();
DateTime GA_End_Date = new DateTime();
GA_End_Date = DateTime.Today.AddDays(-1);
string GA_TransactionId = null;
string GA_ChannelGrouping = null;
string GA_Source = null;
string GA_Medium = null;
string GA_Keyword = null;
string GA_Campaign = null;
string GA_Device_Category = null;
string GA_Region = null;
int GA_Transactions = 0;
/*
* Get the last SessionDate loaded
*/
GA_Session_Date = Convert.ToDateTime(GetMaxSessionnDate());
GA_Session_Date = GA_Session_Date.AddDays(-1);
/*
* Delete the last SessionDate loaded from the table
*
* The free version of Google Analytics takes up to 24 hours to bake
* so reloading the last day will ensure that we get all of the data.
*/
SQL_Script = "DELETE FROM OmniChannelAnalytics.GoogleAnalytics.Transactions WHERE SessionDate >= '" + GA_Session_Date.ToString() + "';";
ErrorMessage = ExecuteSQL(SQL_Script);
/*
* Create the DataTable and DataSet to house the data from GA until
* it is bulk loaded into SQL
*/
DataSet dataSet = new DataSet();
DataTable sessionTable = new DataTable();
sessionTable.TableName = "Sessions";
// Add the columns to the Sessions table
sessionTable.Columns.Add("SessionDate", typeof(string));
sessionTable.Columns.Add("TransactionId", typeof(string));
sessionTable.Columns.Add("ChannelGrouping", typeof(string));
sessionTable.Columns.Add("Source", typeof(string));
sessionTable.Columns.Add("Medium", typeof(string));
sessionTable.Columns.Add("Keyword", typeof(string));
sessionTable.Columns.Add("Campaign", typeof(string));
sessionTable.Columns.Add("DeviceCategory", typeof(string));
sessionTable.Columns.Add("Region", typeof(string));
sessionTable.Columns.Add("Transactions", typeof(int));
sessionTable.Columns.Add("LoadDate", typeof(string));
dataSet.Tables.Add(sessionTable);
while (GA_Session_Date <= GA_End_Date)
{
try
{
var credential = Google.Apis.Auth.OAuth2.GoogleCredential.FromFile(GlobalVariables.GA_ClientSecretFileLocation)
.CreateScoped(new[] { Google.Apis.AnalyticsReporting.v4.AnalyticsReportingService.Scope.AnalyticsReadonly });
using (var analytics = new Google.Apis.AnalyticsReporting.v4.AnalyticsReportingService(new Google.Apis.Services.BaseClientService.Initializer
{
HttpClientInitializer = credential
}))
{
var request = analytics.Reports.BatchGet(new GetReportsRequest
{
ReportRequests = new[] {
new ReportRequest{
DateRanges = new[] { new DateRange{ StartDate = GA_Session_Date.ToString("yyyy-MM-dd"), EndDate = GA_Session_Date.ToString("yyyy-MM-dd") } },
Dimensions = new[] {
new Dimension{ Name = "ga:transactionId" }
, new Dimension { Name = "ga:channelGrouping" }
, new Dimension { Name = "ga:sourceMedium" }
, new Dimension { Name = "ga:keyword" }
, new Dimension { Name = "ga:campaign" }
, new Dimension { Name = "ga:deviceCategory" }
, new Dimension { Name = "ga:region" }
},
Metrics = new[] { new Metric{ Expression = "ga:transactions", Alias = "Transactions"}},
ViewId = GlobalVariables.GA_View_ID
}
}
});
var response = request.Execute();
foreach (var row in response.Reports[0].Data.Rows)
{
GA_TransactionId = row.Dimensions[0];
GA_ChannelGrouping = row.Dimensions[1];
GA_Source = row.Dimensions[2].Substring(0, row.Dimensions[2].IndexOf("/")).Trim().Replace("'", "''");
GA_Medium = row.Dimensions[2].Substring(row.Dimensions[2].IndexOf("/") + 1, row.Dimensions[2].Length - row.Dimensions[2].IndexOf("/") - 1).Trim().Replace("'", "''");
GA_Keyword = row.Dimensions[3];
GA_Campaign = row.Dimensions[4];
GA_Device_Category = row.Dimensions[5];
GA_Region = row.Dimensions[6];
foreach (var metric in row.Metrics)
{
GA_Transactions = Convert.ToInt32(metric.Values[0]);
}
// Populate the data table to hold until everything is bulk loaded into SQL
DataRow newRow = sessionTable.NewRow();
newRow["SessionDate"] = GA_Session_Date;
newRow["TransactionId"] = GA_TransactionId;
newRow["ChannelGrouping"] = GA_ChannelGrouping;
newRow["Source"] = GA_Source;
newRow["Medium"] = GA_Medium;
newRow["Keyword"] = GA_Keyword;
newRow["Campaign"] = GA_Campaign;
newRow["DeviceCategory"] = GA_Device_Category;
newRow["Region"] = GA_Region;
newRow["Transactions"] = GA_Transactions;
newRow["LoadDate"] = DateTime.Now;
sessionTable.Rows.Add(newRow);
} // foreach (var row in rows)
}
} // try
catch (Exception ex)
{
ExceptionMessage = ex.Message;
}
finally
{
// Import the current day's Session data
foreach (DataTable table in dataSet.Tables)
{
ImportTable(table);
}
sessionTable.Clear();
}
// Iterate the session date to import by 1
GA_Session_Date = GA_Session_Date.AddDays(1);
} // while (GA_Session_Date <= GA_End_Date)
Dts.TaskResult = (int)ScriptResults.Success;
}

Creating pivot table using ClosedXML

I'm trying to create a Pivot table using ClosedXML V0.91.1, but I keep getting problems with my excel file having unreadable content and then the excel workbook removing my pivot table when clicking Yes below.
Below is that shows when I hit Yes. It's removing my pivot table.
My Pivot Table is getting data from a table that is created from a history of TFS Changesets. The changesets are set into a IEnumerable<Changeset> which is then converted into a DataTable object which include the column headings.
A table is then created from the DataTable which is the source of the PivotTable. This is the code that I'm using:
public bool CreateReport(IEnumerable<Changeset> changesets)
{
workbook = new XLWorkbook();
var sumSheet= workbook.Worksheets.Add("Summary");
// Converting IEnumerable<Changeset> into DataTable
DataTable changesetTable = ConvertToDataTable(changesets);
// Table
var sumTable = sumSheet.Cell(1, 1).InsertTable(changesetTable.AsEnumerable(), "SummaryTable", true);
// Table - Formatting table
tableWithData.Column("A").Cells(firstRow, lastRow).DataType = XLDataType.Number;
tableWithData.Column("C").Cells(firstRow, lastRow).DataType = XLDataType.DateTime;
tableWithData.Column("C").Cells(firstRow, lastRow).Style.DateFormat.Format = "d-MMM-yyy";
sumSheet.Columns().AdjustToContents();
// Pivot Table
var header = sumTable.Range(1, 1, 1, 6); // A1 to F1
var range = sumTable.DataRange;
var dataRange = sumSheet.Range(header.FirstCell(), range.LastCell());
var pivotSheet = workbook.Worksheets.Add("History Report");
var pivotTable = pivotSheet.PivotTables.AddNew("PivotTable", pivotSheet.Cell(1, 1), dataRange);
// Pivot Table - Formatting table
pivotTable.ShowPropertiesInTooltips = false;
pivotTable.RepeatRowLabels = false;
pivotTable.ShowGrandTotalsColumns = false;
pivotTable.ShowGrandTotalsRows = false;
pivotTable.ShowEmptyItemsOnRows = true;
pivotTable.ShowEmptyItemsOnColumns = true;
pivotTable.ShowExpandCollapseButtons = false;
pivotTable.Layout = XLPivotLayout.Tabular;
pivotTable.RowLabels.Add("Changeset");
pivotTable.RowLabels.Add("Committer");
pivotTable.RowLabels.Add("Date");
pivotTable.RowLabels.Add("Comment");
pivotTable.RowLabels.Add("File Changes");
pivotTable.RowLabels.Add("Source");
// Go off and save the workbook.
bool saved = SaveFile();
return saved;
}
I believe the problem is with how I am selecting the dataRange for the source of the Pivot Table.
var header = sumTable.Range(1, 1, 1, 6); // A1 to F1
var range = sumTable.DataRange;
var dataRange = sumSheet.Range(header.FirstCell(), range.LastCell());
I followed the example found on their wiki, but it gives those errors with my implementation. The only difference between my problem and the example, is that I am getting my source for the Pivot Table from a DataTable, and I am only inserting values into RowLabels in my Pivot Table.
If it helps, this is how I convert the IEnumerable<Changeset> to a DataTable
private DataTable ConvertToDataTable(IEnumerable<Changeset> changesets)
{
DataTable table = new DataTable();
table.Columns.Add("Changeset", typeof(int));
table.Columns.Add("Committer", typeof(string));
table.Columns.Add("Date", typeof(DateTime));
table.Columns.Add("Comment", typeof(string));
table.Columns.Add("File Changes", typeof(string));
table.Columns.Add("Source", typeof(string));
foreach(Changeset changeset in changesets) // Loop over all changesets
{
int changesetID = changeset.ChangesetId;
string committer = changeset.CommitterDisplayName;
DateTime creationDate = changeset.CreationDate;
string comment = changeset.Comment;
foreach(Change change in changeset.Changes) // Loop over all Changes in changeset
{
string filename = change.Item.ServerItem.Substring(change.Item.ServerItem.LastIndexOf("/") + 1);
table.Rows.Add(changesetID, committer, creationDate, comment, filename, change.Item.ServerItem);
}
}
return table;
}

If I recall correctly, a pivot table in ClosedXML should have at least one value field.
pivotTable.Values.Add("somefield");

I know I am a bit late on this one, but in case anyone else tries to do this in future, my solution was as follows.
IXLPivotTable xlPvTable = TargetWorksheet.PivotTables.Add(
"PivotTable123",
TargetWorksheet.Cell(1, 1),
SourceWorksheet.Range(1, 1, MainData.Rows.Count, MainData.Columns.Count)
);
Then add the ColumnLabel and RowLabels etc..

How to Save an array of Objects (just the fields) to a DB using DataTable?

I have a webservice (SOAP), from where I get an ArrayOfHumans (Array of Objects), I can get the array but I am having a hard time, saving it to the DB, something as simple (or maybe it is not so simple and I am just ignorant!) as taking an object from the array, and then getting the value of one property and saving that in a table column. I have the following code:
public JsonResult returnAllHumans()
{
TestService.TestServiceSoapClient soapClient = new TestService.TestServiceSoapClient();
object[] humansfromWS = soapClient.GetAllHumans();
System.Data.DataTable table = new System.Data.DataTable();
table.Columns.Add("FirstName", typeof(string));
table.Columns.Add("LastName", typeof(string));
table.Columns.Add("Gender", typeof(string));
table.Columns.Add("Age", typeof(int));
for (int i = 0; i < humansfromWS.Length; i++){
table.Rows.Add(new object[] {
humansfromWS[i].Age(); (And my problem is here!)
I have no idea what am I doing wrong to get this out of my array?, can someone point me in the right direction?

humansfromWS[i] is an object, it has no Age method. You have to cast it to the right type.

Please look at the documentation on the Add method. There are two versions of Add. One takes a DataRow, the other takes an array of values for a single row.
public JsonResult returnAllHumans()
{
TestService.TestServiceSoapClient soapClient = new TestService.TestServiceSoapClient();
object[] humansfromWS = soapClient.GetAllHumans();
System.Data.DataTable table = new System.Data.DataTable();
table.Columns.Add("FirstName", typeof(string));
table.Columns.Add("LastName", typeof(string));
table.Columns.Add("Gender", typeof(string));
table.Columns.Add("Age", typeof(int));
for (int i = 0; i < humansfromWS.Length; i++){
var humanFromWS = humansfromWS[i];
table.Rows.Add(new object[] {
humansFromWS.FirstName,
humansFromWS.LastName,
humansFromWS.Gender,
humanFromWS.Age});
}
or
for (int i = 0; i < humansfromWS.Length; i++){
var humanFromWS = humansfromWS[i];
var row = table.NewRow();
row["Age"] = humanFromWS.Age;
row["FirstName"] = humanFromWS.FirstName;
row["LastName"] = humanFromWS.LastName;
row["Gender"] = humanFromWS.Gender;
table.Rows.Add(row);
}

error in binding datatable to database by linq

i exacly copy from msdn but the following code gives me error
The type 'AnonymousType#1' cannot be used as type parameter 'T' in the generic type or method
'System.Data.DataTableExtensions.CopyToDataTable<T> (System.Collections.Generic.IEnumerable<T>,
System.Data.DataTable, System.Data.LoadOption)'. There is no implicit reference conversion from
'AnonymousType#1' to 'System.Data.DataRow'.
my code:
Item[] items = new Item[]
{
new Book{Id = 1, Price = 13.50, Genre = "Comedy", Author = "Gustavo Achong"},
new Book{Id = 2, Price = 8.50, Genre = "Drama", Author = "Jessie Zeng"},
new Movie{Id = 1, Price = 22.99, Genre = "Comedy", Director = "Marissa Barnes"},
new Movie{Id = 1, Price = 13.40, Genre = "Action", Director = "Emmanuel Fernandez"}
};
// Create a table with a schema that matches that of the query results.
DataTable dt1 = new DataTable();
dt1.Columns.Add("Price", typeof(int));
dt1.Columns.Add("Genre", typeof(string));
var query = from i in items
where i.Price > 9.99
orderby i.Price
select new { i.Price, i.Genre };
query.CopyToDataTable(dt1, LoadOption.PreserveChanges);
how to make it workabe?

Try this code, place it in a static helper class and this will allow you to call ToDataTable(); on the items.
public static DataTable ToDataTable<T>(this IEnumerable<T> data)
{
DataTable dt = new DataTable();
foreach (var prop in data.First().GetType().GetProperties())
{
dt.Columns.Add(prop.Name);
}
foreach (T entry in data)
{
List<object> newRow = new List<object>();
foreach (DataColumn dc in dt.Columns)
{
var val = entry.GetType().GetProperty(dc.ColumnName).GetValue(entry, null);
newRow.Add(val);
}
dt.Rows.Add(newRow.ToArray());
}
return dt;
}

I exacly copy from msdn...
Are you trying to implement Microsofts ObjectShredder<T>-class which allows to use CopyToDataTable with any type?
Then try to rename the extension(f.e. CopyAnyToDataTable), it could be a naming conflict with the DataTableExtensions.CopyToDataTable extension method which allows only DataRows.
Some time ago i also had that issue, here's a similar question:
Exception using CopyToDataTable with "new {..}" LINQ query

DataRows coming out null despite assigning values to them

I am trying to learn how to join two data tables into one using Linq. My linq query is working correctly and I can see expected values in it. However, when I loop the linq results, and assign the values to a newly created data row and add that row to a new data table, the rows come out empty.
Here is my code:
private void btnCombine_Click(object sender, EventArgs e)
{
var innerJoinQuery =
from strRow in StrDataTable.AsEnumerable()
join intRow in IntDataTable.AsEnumerable()
on strRow.Field<int>("IntID") equals intRow.Field<int>("ID")
select new {
IntOne = intRow.Field<int>("FirstNum"),
IntTwo = intRow.Field<int>("SecondNum"),
StrOne = strRow.Field<string>("FirstStr"),
StrTwo = strRow.Field<string>("SecondStr"),
StrThree = strRow.Field<string>("SecondStr")
};
DataTable newTable = new DataTable();
newTable.Columns.Add("IntOne");
newTable.Columns.Add("IntTwo");
newTable.Columns.Add("FirstStr");
newTable.Columns.Add("SecondStr");
newTable.Columns.Add("ThirdStr");
newTable.Columns["IntOne"].DataType = System.Type.GetType("System.String");
newTable.Columns["IntTwo"].DataType = System.Type.GetType("System.String");
newTable.Columns["FirstStr"].DataType = System.Type.GetType("System.String");
newTable.Columns["SecondStr"].DataType = System.Type.GetType("System.String");
newTable.Columns["ThirdStr"].DataType = System.Type.GetType("System.String");
foreach (var row in innerJoinQuery)
{
DataRow rowToAdd = newTable.NewRow();
rowToAdd.ItemArray[0] = row.IntOne.ToString();
rowToAdd.ItemArray[1] = row.IntTwo.ToString();
rowToAdd.ItemArray[2] = row.StrOne.ToString();
rowToAdd.ItemArray[3] = row.StrTwo.ToString();
rowToAdd.ItemArray[4] = row.StrThree.ToString();
newTable.Rows.Add(rowToAdd);
}
dataGridView3.DataSource = newTable;
}

Using DataRow.ItemArray property with individual values doesn't work - instead, create the object[] array and then set the whole thing to the .ItemArray property. See this MSDN page for additional examples.
foreach (var row in innerJoinQuery)
{
DataRow rowToAdd = newTable.NewRow();
object[] items = new object[] {
row.IntOne.ToString(),
row.IntTwo.ToString(),
row.StrOne.ToString(),
row.StrTwo.ToString(),
row.StrThree.ToString()
};
rowToAdd.ItemArray = items;
newTable.Rows.Add(rowToAdd);
}
Alternately, use the DataRow indexer directly, which works with individual columns:
rowToAdd[0] = row.IntOne.ToString();
rowToAdd[1] = row.IntTwo.ToString();
rowToAdd[2] = row.StrOne.ToString();
rowToAdd[3] = row.StrTwo.ToString();
rowToAdd[4] = row.StrThree.ToString();
Additionally, when creating columns, there is a constructor that takes the type which can save you some code. Your first two column types are mismatched.
newTable.Columns.Add("IntOne", typeof(int));
newTable.Columns.Add("FirstStr", typeof(string));

The first two values appear to be Integers:
IntOne = intRow.Field<int>("FirstNum"),
IntTwo = intRow.Field<int>("SecondNum"),
But the DataType you assign the columns to is String:
newTable.Columns["IntOne"].DataType = System.Type.GetType("System.String");
newTable.Columns["IntTwo"].DataType = System.Type.GetType("System.String");
Update those to int's and see if that resolves it:
newTable.Columns["IntOne"].DataType = System.Type.GetType("System.Int32");
newTable.Columns["IntTwo"].DataType = System.Type.GetType("System.Int32");
rowToAdd.ItemArray[0] = row.IntOne;
rowToAdd.ItemArray[1] = row.IntTwo;
You may also need to supply the DataPropertyName for the columns:
newTable.Columns["IntOne"].DataPropertyName = "FirstNum";
newTable.Columns["IntTwo"].DataPropertyName = "SecondNum";
...
And ensure that the AutoGenerateColumns value is set to false
dataGridView3.AutoGenerateColumns = false;