This is the page I'm using for documentation https://lichess.org/api#operation/player
I want to get player usernamename, rating, and title.
My code.
public class Player {
public string username;
public double rating;
public string title;
}
HttpClient client = new HttpClient();
client.BaseAddress = new Uri("https://lichess.org/");
HttpResponseMessage response = client.GetAsync("player/top/200/bullet").Result;
Here I'm getting response, But I have no clue how to take only properties that I need and store it in a list of players.
After a discussion with you on this problem, it was found that the response that you are receiving is a HTML string, therefore you need to deal with this case differently. I was playing around with the HTML that you have posted in the comments and I was able to parse the string with HTML Agility Pack which can be found here. You can also download this pack from the Nuget Package Manager in Visual Studio.
I am giving you a very basic example of the parsing process that I tried out:
public class ProcessHtml()
{
List<Player> playersList = new List<Player>();
//Get your HTML loaded from a URL. Giving me SSL exceptions so took a different route
//var url = "https://lichess.org/player/top/200/bullet";
//var web = new HtmlWeb();
//var doc = web.Load(url);
//Get your HTML loaded as a file in my case
var doc = new HtmlDocument();
doc.Load("C:\\Users\\Rahul\\Downloads\\CkBsZtvf.html", Encoding.UTF8);
foreach (HtmlNode table in doc.DocumentNode.SelectNodes("//tbody"))
{
foreach (HtmlNode row in table.SelectNodes("tr"))
{
int i = 0;
Player player = new Player();
//Since there are 4 rounds per tr, hence get only what is required based on loop condition
foreach (HtmlNode cell in row.SelectNodes("th|td"))
{
if(i==1)
{
player.username = cell.InnerText;
}
if(i==2)
{
player.rating = Convert.ToDouble(cell.InnerText);
}
if(i==3)
{
player.title = cell.InnerText;
}
i++;
}
playersList.Add(player);
}
}
var finalplayerListCopy = playersList;
}
public class Player
{
public string username;
public double rating;
public string title;
}
After running this, your finalplayerListCopy has a count of 200 and an example data would look like:
Obviously, you would have to play with the data and tailor it as per your need. I hope this helps you out.
Cheers!
from what Ive read from the documentation
async Task<Player> getPlayerAsync(string path)
{
Player player= null;
HttpResponseMessage response = await client.GetAsync(path);
if (response.IsSuccessStatusCode)
{
player = await response.Content.ReadAsAsync<Player>();
}
return player;
}
getPlayerAsync("https://lichess.org/player/top/200/bullet");
Related
I've been working on a webscraper as a Windows Forms application in C#. The user enter a search term and the term and the program will then split the search string for each individual words and look up the amount of search results through Yahoo and Google.
My issue lies with the orientation of the huge HTML document. I've tried multiple approaches such as
iterating recursively and comparing ids aswell as with lamba and the Where statements. Both results in null. I also manually looked into the html document to make sure the id of the div I want exist in the document.
The id I'm looking for is "resultStats" but it is suuuuuper nested. My code looks like this:
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace WebScraper2._0
{
public class Webscraper
{
private string Google = "http://google.com/#q=";
private string Yahoo = "http://search.yahoo.com/search?p=";
private HtmlWeb web = new HtmlWeb();
private HtmlDocument GoogleDoc = new HtmlDocument();
private HtmlDocument YahooDoc = new HtmlDocument();
public Webscraper()
{
Console.WriteLine("Init");
}
public int WebScrape(string searchterms)
{
//Console.WriteLine(searchterms);
string[] ssize = searchterms.Split(new char[0]);
int YahooMatches = 0;
int GoogleMatches = 0;
foreach (var term in ssize)
{
//Console.WriteLine(term);
var y = web.Load(Yahoo + term);
var g = web.Load(Google + term + "&cad=h");
YahooMatches += YahooFilter(y);
GoogleMatches += GoogleFilter(g);
}
Console.WriteLine("Yahoo found " + YahooMatches.ToString() + " matches");
Console.WriteLine("Google found " + GoogleMatches.ToString() + " matches");
return YahooMatches + GoogleMatches;
}
//Parse to get correct info
public int YahooFilter(HtmlDocument doc)
{
//Look for node with correct ID
IEnumerable<HtmlNode> nodes = doc.DocumentNode.Descendants().Where(n => n.HasClass("mw-jump-link"));
foreach (var item in nodes)
{
// displaying final output
Console.WriteLine(item.InnerText);
}
//TODO: Return search resultamount.
return 0;
}
int testCounter = 0;
string toReturn = "";
bool foundMatch = false;
//Parse to get correct info
public int GoogleFilter(HtmlDocument doc)
{
if (doc == null)
{
Console.WriteLine("Null");
}
foreach (var node in doc.DocumentNode.ChildNodes)
{
toReturn += Looper(node, testCounter, toReturn, foundMatch);
}
Console.WriteLine(toReturn);
/*
var stuff = doc.DocumentNode.Descendants("div")
.Where(node => node.GetAttributeValue("id", "")
.Equals("extabar")).ToList();
IEnumerable<HtmlNode> nodes = doc.DocumentNode.Descendants().Where(n => n.HasClass("appbar"));
*/
return 0;
}
public string Looper(HtmlNode node, int counter, string returnstring, bool foundMatch)
{
Console.WriteLine("Loop started" + counter.ToString());
counter++;
Console.WriteLine(node.Id);
if (node.Id == "resultStats")
{
returnstring += node.InnerText;
}
foreach (HtmlNode n in node.Descendants())
{
Looper(n, counter, returnstring, foundMatch);
}
return returnstring;
}
}
}
I made an google HTML Scraper a few weeks ago, a few things to consider
First: Google don't like when you try to Scrape their Search HTML, while i was running a list of companies trying to get their addresses and phone number, Google block my IP from accessing their website for a little bit (Which cause a hilarious panic in the office)
Second: Google will change the HTML (Id names and etc) of the page so using ID's won't work, on my case i used the combination of HTML Tags and specific information to parse the response and extract the information that i wanted.
Third: It's better to just use their API to grab the information you need, just make sure you respect their free tier query limit and you should be golden.
Here is the Code i used.
public static string getBetween(string strSource, string strStart, string strEnd)
{
int Start, End;
if (strSource.Contains(strStart) && strSource.Contains(strEnd))
{
Start = strSource.IndexOf(strStart, 0) + strStart.Length;
End = strSource.IndexOf(strEnd, Start);
return strSource.Substring(Start, End - Start);
}
else
{
return "";
}
}
public void SearchResult()
{
//Run a Google Search
string uriString = "http://www.google.com/search";
string keywordString = "Search String";
WebClient webClient = new WebClient();
NameValueCollection nameValueCollection = new NameValueCollection();
nameValueCollection.Add("q", keywordString);
webClient.QueryString.Add(nameValueCollection);
string result = webClient.DownloadString(uriString);
string search = getBetween(result, "Address", "Hours");
rtbHtml.Text = getBetween(search, "\">", "<");
}
On my case i used the String Address and Hours to limit what information i wanted to extract.
Edit: Fixed the Logic and added the Code i used.
Edit2: forgot to add the GetBetween Class. (sorry it's my first Answer)
The aim of the program below is to get a list of Reports built in our database and find out how many of these reports use the field NameFirst within them.
I'm able to make an API call and, at GetReports, get a list of the ReportIDs.
However, I'm unable to move forward with calling the list I created at GetReports in the next method, GetNameFirst. I was wondering if someone could please help me out with this.
For the script below, I get a red underline for the variable values. This is understandable because I didn't know where and how to tell my code to bind the list output for GetReports to the variable values in GetNameFirst.
Also, if I could get some help in finding out which reports have the field NameFirst in them once I accomplish calling the list from the first method to the second, I'd appreciate that also. I'm currently heading in the direction of using a foreach, but I'm unsure if that's the best path to take.
Main Program
namespace NameFirstSearch
{
class Program
{
static void Main(string[] args)
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12 | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls;
const string username = "Username";
const string password = "Password";
const string baseUrl = "https://example.com/rest/services/";
const string queryString = "query?q=Select * From Report Where LastRanDate is not null";
const string queryNameFirst = "getreport/";
var client = new HttpClient();
client.BaseAddress = new Uri(baseUrl);
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var auth = Convert.ToBase64String(Encoding.Default.GetBytes(username + ":" + password));
client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", auth);
GetReports(client, queryString).Wait();
GetNameFirst(client, queryNameFirst).Wait();
Console.ReadLine();
}
static async Task<List<Properties>> GetReports(HttpClient client, string queryString)
{
List<Properties> result = new List<Properties>();
var response = await client.GetAsync(queryString);
// Check for a successfull result
if (response.IsSuccessStatusCode)
{
var json = await response.Content.ReadAsStringAsync();
result = JsonConvert.DeserializeObject<List<Properties>>(json);
Console.WriteLine(result.Count());
}
else
{
// Error code returned
Console.WriteLine("No records found on first method.");
}
return result;
}
static async Task GetNameFirst(HttpClient client, string queryNameFirst)
{
string reportType = "JSON";
foreach (var item in values)
{
var output = await client.GetAsync(queryNameFirst + item.ReportID + reportType);
if (output.IsSuccessStatusCode)
{
var allText = await output.Content.ReadAsStringAsync();
var fields = JsonConvert.DeserializeObject<List<NameFirst>>(allText);
}
else
{
// Error code returned
Console.WriteLine("No records found on second method.");
}
}
}
}
Class for report list
class Properties
{
public int ReportID { get; set; }
}
Class for reports' NameFirst property
class NameFirst
{
public string FirstName { get; set; }
}
I thought this was a partial code, but since you've cleared things out.
you'll need to change your code a bit
this :
GetReports(client, queryString).Wait();
do it like this :
var reportsList = GetReports(client, queryString).Result;
now, you'll need to pass the reportsList to the second method GetNameFirst which would be adjusted to this :
static async Task GetNameFirst(HttpClient client, string queryNameFirst, List<Properties> results)
{
string reportType = "JSON";
foreach (var item in results)
{
var output = await client.GetAsync(queryNameFirst + item.ReportID + reportType);
if (output.IsSuccessStatusCode)
{
var allText = await output.Content.ReadAsStringAsync();
var fields = JsonConvert.DeserializeObject<List<NameFirst>>(allText);
}
else
{
// Error code returned
Console.WriteLine("No records found on second method.");
}
}
}
with this adjustment, you'll need to adjust the call as well :
GetNameFirst(client, queryNameFirst, reportsList).Wait();
Few days back i learnt making reports in power BI and it was great experience learning power BI. As i am creating a dashboard for my MVC based web application, i wanted to make look and fill of my dashboard attractive. I am thinking of embedding power BI report with that for that i have used following code inside view :-
<body>
<script type="text/javascript" src="~/Scripts/PowerBI/powerbi.js"></script>
<script type="text/javascript">
window.onload = function () {
var iframe = document.getElementById("iFrameEmbedReport");
iframe.src = "https://app.powerbi.com/reportEmbed?reportId=" + embedReportId;
iframe.onload = postActionLoadReport;
}
function postActionLoadReport() {
var m = {
action: "loadReport",
accessToken: accessToken
};
message = JSON.stringify(m);
iframe = document.getElementById("iFrameEmbedReport");
iframe.contentWindow.postMessage(message, "*");;
}
</script>
<style>
#iFrameEmbedReport {
width: 95%;
height: 95%;
}
</style>
<iframe ID="iFrameEmbedReport"></iframe>
</body>
And code for controller is given below :-
public class DashBoardController : Controller
{
string baseUri = WebConfigurationManager.AppSettings["PowerBiDataset"];
string AccessToken = string.Empty;
// GET: DashBoard
public ActionResult DashBoard()
{
if (Request.Params.Get("code") != null)
{
Session["AccessToken"] = GetAccessToken(
Request.Params.GetValues("code")[0],
WebConfigurationManager.AppSettings["ClientID"],
WebConfigurationManager.AppSettings["ClientSecret"],
WebConfigurationManager.AppSettings["RedirectUrl"]);
Response.Redirect("~/DashBoard/DashBoard");
}
if (Session["AccessToken"] != null)
{
AccessToken = Session["AccessToken"].ToString();
GetReport(0);
}
return View();
}
protected void GetReport(int index)
{
System.Net.WebRequest request = System.Net.WebRequest.Create(
String.Format("{0}/Reports",
baseUri)) as System.Net.HttpWebRequest;
request.Method = "GET";
request.ContentLength = 0;
request.Headers.Add("Authorization", String.Format("Bearer {0}", AccessToken));
using (var response = request.GetResponse() as System.Net.HttpWebResponse)
{
using (var reader = new System.IO.StreamReader(response.GetResponseStream()))
{
PBIReports Reports = JsonConvert.DeserializeObject<PBIReports>(reader.ReadToEnd());
if (Reports.value.Length > 0)
{
var report = Reports.value[index];
ViewData["AccessToken"] = Session["AccessToken"].ToString();
ViewData["EmbedURL"] = report.embedUrl;
ViewData["ReportID"] = report.id;
}
}
}
}
public void GetAuthorizationCode()
{
var #params = new NameValueCollection
{
{"response_type", "code"},
{"client_id", WebConfigurationManager.AppSettings["ClientID"]},
{"resource", WebConfigurationManager.AppSettings["PowerBiAPI"]},
{ "redirect_uri", WebConfigurationManager.AppSettings["RedirectUrl"]}
};
var queryString = HttpUtility.ParseQueryString(string.Empty);
queryString.Add(#params);
Response.Redirect(String.Format(WebConfigurationManager.AppSettings["AADAuthorityUri"] + "?{0}", queryString));
}
public string GetAccessToken(string authorizationCode, string clientID, string clientSecret, string redirectUri)
{
TokenCache TC = new TokenCache();
string authority = WebConfigurationManager.AppSettings["AADAuthorityUri"];
AuthenticationContext AC = new AuthenticationContext(authority, TC);
ClientCredential cc = new ClientCredential(clientID, clientSecret);
return AC.AcquireTokenByAuthorizationCode(
authorizationCode,
new Uri(redirectUri), cc).AccessToken;
}
}
public class PBIReports
{
public PBIReport[] value { get; set; }
}
public class PBIReport
{
public string id { get; set; }
public string name { get; set; }
public string webUrl { get; set; }
public string embedUrl { get; set; }
}
As i think, I am doing everything right but i don't know why it's not able to display report. Please suggest me if i did any mistake in the above given code.
It's not clear where the error is, as you provided lots of code and no specfics on the error itself. Here are few things to note:
To embed Power BI content in your HTML you just need an empty div element
Too much code overall.
You need to perform following steps:
Acquire authentication token by using AuthenticationContext.AcquireTokenAsync with credentials provisioned on Power BI side
Instantiate PowerBIClient with the token you just obtained. That's the token for your application. Never pass it to users. Don't store in Session as it'll expire. PowerBIClient(new Uri(_Context.ApiUrl), new TokenCredentials(authResult.AccessToken, "Bearer"))
Obtain ID(s) of content that's available in Power BI. There are different APIs for different types (dashboards, reports, tiles) as well as for content that's in Groups(workspaces) or not, e.g. client.Dashboards.GetDashboardsInGroupAsync(GroupId). This step could be skipped if you already know what type of content you're getting and its ID. Keep in mind that if EmbedUrl property is empty on returned object(s) you won't be able to render, even if you manually construct such url.
Obtain Embed Token for particular content. There are different methods available, e.g. client.Reports.GenerateTokenInGroupAsync(GroupId, Id-of-content, new GenerateTokenRequest(accessLevel: "view"))
The final step is to apply Embed Token and EmbedUrl on the client-side. Something among following lines:
.
var embedToken = $('#embedToken').val();
var txtEmbedUrl = $('#txtReportEmbed').val();
var txtEmbedReportId = $('#txtEmbedReportId').val();
var models = window['powerbi-client'].models;
var permissions = models.Permissions.All;
var config= {
type: 'report',
tokenType: models.TokenType.Embed,
accessToken: embedToken,
embedUrl: txtEmbedUrl,
id: txtEmbedReportId,
permissions: permissions,
settings: {
filterPaneEnabled: true,
navContentPaneEnabled: true
}
};
var embedContainer = $('#embedContainer')[0];
var report = powerbi.embed(embedContainer, config);
You should be able to test your stuff here. Just plugin your values.
You can also observe sample app here]2. Flow provided above is for "app owns data" case.
There is a method in Wikimedia API that gives a localized title.
Examples:
Cloud:
http://en.wikipedia.org/w/api.php?format=json&action=query&titles=Cloud&prop=langlinks&lllimit=500&lllang=ru&continue=
Rain: http://en.wikipedia.org/w/api.php?format=json&action=query&titles=Rain&prop=langlinks&lllimit=500&lllang=ru&continue=
Cloud response:
{
"batchcomplete":"",
"query":{
"pages":{
"47515":{
"pageid":47515,
"ns":0,
"title":"Cloud",
"langlinks":[
{
"lang":"ru",
"*":"\u041e\u0431\u043b\u0430\u043a\u0430"
}
]
}
}
}
}
Rain response:
{
"batchcomplete":"",
"query":{
"pages":{
"19009110":{
"pageid":19009110,
"ns":0,
"title":"Rain",
"langlinks":[
{
"lang":"ru",
"*":"Дождь"
}
]
}
}
}
}
Important note: integer container under pages (e.g. 19009110) is always different, because it equals page id.
C# code:
dynamic datacontainer_RUname2 = JObject.Parse(cleanJson_string_RUname);
String localizedName = datacontainer_RUname.[HERE SHOULD BE *];
How can I access a key named with asterisk '*'?
string content;
using (var webClient = new WebClient())
{
const string url = "http://en.wikipedia.org/w/api.php?format=json&action=query&titles=Cloud&prop=langlinks&lllimit=500&lllang=ru&continue=";
content = webClient.DownloadString(url);
}
var obj = JObject.Parse(content);
var query = obj["query"];
var pages = query["pages"].Value<JObject>();
var page = pages.PropertyValues().First();
var langLinks = page["langlinks"].Values<JObject>();
var firstLangLink = langLinks.First();
var localizedName = firstLangLink["*"];
See a working demo with live data.
Just use the normal indexing on the object.
string localizedName = obj["*"];
In your case... to get to your object, you can do this query in both cases. To collect all links returned from the query:
var allLinks =
from page in response.SelectToken("query.pages").Values()
from link in page["langlinks"]
select (string)link["*"];
Is there a way to use the SQL Server 2012 Microsoft.SqlServer.Dac Namespace to determine if a database has an identical schema to that described by a DacPackage object? I've looked at the API docs for DacPackage as well as DacServices, but not having any luck; am I missing something?
Yes there is, I have been using the following technique since 2012 without issue.
Calculate a fingerprint of the dacpac.
Store that fingerprint in the target database.
The .dacpac is just a zip file containing goodies like metadata, and
model information.
Here's a screen-grab of what you will find in the .dacpac:
The file model.xml has XML structured like the following
<DataSchemaModel>
<Header>
... developer specific stuff is in here
</Header>
<Model>
.. database model definition is in here
</Model>
</<DataSchemaModel>
What we need to do is extract the contents from <Model>...</Model>
and treat this as the fingerprint of the schema.
"But wait!" you say. "Origin.xml has the following nodes:"
<Checksums>
<Checksum Uri="/model.xml">EB1B87793DB57B3BB5D4D9826D5566B42FA956EDF711BB96F713D06BA3D309DE</Checksum>
</Checksums>
In my experience, this <Checksum> node changes regardless of a schema change in the model.
So let's get to it.
Calculate the fingerprint of the dacpac.
using System.IO;
using System.IO.Packaging;
using System.Security.Cryptography;
static string DacPacFingerprint(byte[] dacPacBytes)
{
using (var ms = new MemoryStream(dacPacBytes))
using (var package = ZipPackage.Open(ms))
{
var modelFile = package.GetPart(new Uri("/model.xml", UriKind.Relative));
using (var streamReader = new System.IO.StreamReader(modelFile.GetStream()))
{
var xmlDoc = new XmlDocument() { InnerXml = streamReader.ReadToEnd() };
foreach (XmlNode childNode in xmlDoc.DocumentElement.ChildNodes)
{
if (childNode.Name == "Header")
{
// skip the Header node as described
xmlDoc.DocumentElement.RemoveChild(childNode);
break;
}
}
using (var crypto = new SHA512CryptoServiceProvider())
{
byte[] retVal = crypto.ComputeHash(Encoding.UTF8.GetBytes(xmlDoc.InnerXml));
return BitConverter.ToString(retVal).Replace("-", "");// hex string
}
}
}
}
With this fingerprint now available, pseudo code for applying a dacpac can be:
void main()
{
var dacpacBytes = File.ReadAllBytes("<path-to-dacpac>");
var dacpacFingerPrint = DacPacFingerprint(dacpacBytes);// see above
var databaseFingerPrint = Database.GetFingerprint();//however you choose to do this
if(databaseFingerPrint != dacpacFingerPrint)
{
DeployDacpac(...);//however you choose to do this
Database.SetFingerprint(dacpacFingerPrint);//however you choose to do this
}
}
Here's what I've come up with, but I'm not really crazy about it. If anyone can point out any bugs, edge cases, or better approaches, I'd be much obliged.
...
DacServices dacSvc = new DacServices(connectionString);
string deployScript = dacSvc.GenerateDeployScript(myDacpac, #"aDb", deployOptions);
if (DatabaseEqualsDacPackage(deployScript))
{
Console.WriteLine("The database and the DacPackage are equal");
}
...
bool DatabaseEqualsDacPackage(string deployScript)
{
string equalStr = string.Format("GO{0}USE [$(DatabaseName)];{0}{0}{0}GO{0}PRINT N'Update complete.'{0}GO", Environment.NewLine);
return deployScript.Contains(equalStr);
}
...
What I really don't like about this approach is that it's entirely dependent upon the format of the generated deployment script, and therefore extremely brittle. Questions, comments and suggestions very welcome.
#Aaron Hudon answer does not account for post script changes. Sometimes you just add a new entry to a type table without changing the model. In our case we want this to count as new dacpac. Here is my modification of his code to account for that
private static string DacPacFingerprint(string path)
{
using (var stream = File.OpenRead(path))
using (var package = Package.Open(stream))
{
var extractors = new IDacPacDataExtractor [] {new ModelExtractor(), new PostScriptExtractor()};
string content = string.Join("_", extractors.Select(e =>
{
var modelFile = package.GetPart(new Uri($"/{e.Filename}", UriKind.Relative));
using (var streamReader = new StreamReader(modelFile.GetStream()))
{
return e.ExtractData(streamReader);
}
}));
using (var crypto = new MD5CryptoServiceProvider())
{
byte[] retVal = crypto.ComputeHash(Encoding.UTF8.GetBytes(content));
return BitConverter.ToString(retVal).Replace("-", "");// hex string
}
}
}
private class ModelExtractor : IDacPacDataExtractor
{
public string Filename { get; } = "model.xml";
public string ExtractData(StreamReader streamReader)
{
var xmlDoc = new XmlDocument() { InnerXml = streamReader.ReadToEnd() };
foreach (XmlNode childNode in xmlDoc.DocumentElement.ChildNodes)
{
if (childNode.Name == "Header")
{
// skip the Header node as described
xmlDoc.DocumentElement.RemoveChild(childNode);
break;
}
}
return xmlDoc.InnerXml;
}
}
private class PostScriptExtractor : IDacPacDataExtractor
{
public string Filename { get; } = "postdeploy.sql";
public string ExtractData(StreamReader stream)
{
return stream.ReadToEnd();
}
}
private interface IDacPacDataExtractor
{
string Filename { get; }
string ExtractData(StreamReader stream);
}