Develop Reference

Superpower: match a string with tokenizer only if it begins a line

When tokenizing in superpower, how to match a string only if it is the first thing in a line (note: this is a different question than this one) ?
For example, assume I have a language with only the following 4 characters (' ', ':', 'X', 'Y'), each of which is a token. There is also a 'Header' token to capture cases of the following regex pattern /^[XY]+:/ (any number of Xs and Ys followed by a colon, only if they start the line).
Here is a quick class for testing (the 4th test-case fails):
using System;
using Superpower;
using Superpower.Parsers;
using Superpower.Tokenizers;
public enum Tokens { Space, Colon, Header, X, Y }
public class XYTokenizer
{
static void Main(string[] args)
{
Test("X", Tokens.X);
Test("XY", Tokens.X, Tokens.Y);
Test("X Y:", Tokens.X, Tokens.Space, Tokens.Y, Tokens.Colon);
Test("X: X", Tokens.Header, Tokens.Space, Tokens.X);
}
public static readonly Tokenizer<Tokens> tokenizer = new TokenizerBuilder<Tokens>()
.Match(Character.EqualTo('X'), Tokens.X)
.Match(Character.EqualTo('Y'), Tokens.Y)
.Match(Character.EqualTo(':'), Tokens.Colon)
.Match(Character.EqualTo(' '), Tokens.Space)
.Build();
static void Test(string input, params Tokens[] expected)
{
var tokens = tokenizer.Tokenize(input);
var i = 0;
foreach (var t in tokens)
{
if (t.Kind != expected[i])
{
Console.WriteLine("tokens[" + i + "] was Tokens." + t.Kind
+ " not Tokens." + expected[i] + " for '" + input + "'");
return;
}
i++;
}
Console.WriteLine("OK");
}
}

I came up with a custom Tokenizer based on the example found here. I added comments throughout the code so you can follow what's happening.
public class MyTokenizer : Tokenizer<Tokens>
{
protected override IEnumerable<Result<Tokens>> Tokenize(TextSpan input)
{
Result<char> next = input.ConsumeChar();
bool checkForHeader = true;
while (next.HasValue)
{
// need to check for a header when starting a new line
if (checkForHeader)
{
var headerStartLocation = next.Location;
var tokenQueue = new List<Result<Tokens>>();
while (next.HasValue && (next.Value == 'X' || next.Value == 'Y'))
{
tokenQueue.Add(Result.Value(next.Value == 'X' ? Tokens.X : Tokens.Y, next.Location, next.Remainder));
next = next.Remainder.ConsumeChar();
}
// only if we had at least one X or one Y
if (tokenQueue.Any())
{
if (next.HasValue && next.Value == ':')
{
// this is a header token; we have to return a Result of the start location
// along with the remainder at this location
yield return Result.Value(Tokens.Header, headerStartLocation, next.Remainder);
next = next.Remainder.ConsumeChar();
}
else
{
// this isn't a header; we have to return all the tokens we parsed up to this point
foreach (Result<Tokens> tokenResult in tokenQueue)
{
yield return tokenResult;
}
}
}
if (!next.HasValue)
yield break;
}
checkForHeader = false;
if (next.Value == '\r')
{
// skip over the carriage return
next = next.Remainder.ConsumeChar();
continue;
}
if (next.Value == '\n')
{
// line break; check for a header token here
next = next.Remainder.ConsumeChar();
checkForHeader = true;
continue;
}
if (next.Value == 'A')
{
var abcStart = next.Location;
next = next.Remainder.ConsumeChar();
if (next.HasValue && next.Value == 'B')
{
next = next.Remainder.ConsumeChar();
if (next.HasValue && next.Value == 'C')
{
yield return Result.Value(Tokens.ABC, abcStart, next.Remainder);
next = next.Remainder.ConsumeChar();
}
else
{
yield return Result.Empty<Tokens>(next.Location, $"unrecognized `AB{next.Value}`");
}
}
else
{
yield return Result.Empty<Tokens>(next.Location, $"unrecognized `A{next.Value}`");
}
}
else if (next.Value == 'X')
{
yield return Result.Value(Tokens.X, next.Location, next.Remainder);
next = next.Remainder.ConsumeChar();
}
else if (next.Value == 'Y')
{
yield return Result.Value(Tokens.Y, next.Location, next.Remainder);
next = next.Remainder.ConsumeChar();
}
else if (next.Value == ':')
{
yield return Result.Value(Tokens.Colon, next.Location, next.Remainder);
next = next.Remainder.ConsumeChar();
}
else if (next.Value == ' ')
{
yield return Result.Value(Tokens.Space, next.Location, next.Remainder);
next = next.Remainder.ConsumeChar();
}
else
{
yield return Result.Empty<Tokens>(next.Location, $"unrecognized `{next.Value}`");
next = next.Remainder.ConsumeChar(); // Skip the character anyway
}
}
}
}
And you can call it like this:
var tokens = new MyTokenizer().Tokenize(input);

Parse WwwAuthenticate challenge string

I am working on a client for a RESTful service, using .NET Core 2.0. The remote service returns challenges like this:
WwwAuthenticate: Bearer realm="https://somesite/auth",service="some site",scope="some scope"
Which need to get turned into token requests like:
GET https://somesite/auth?service=some%20site&scope=some%20scope
Parsing the header to get a scheme and parameter is easy with AuthenticationHeaderValue, but that just gets me the realm="https://somesite/auth",service="some site",scope="some scope" string. How can I easily and reliably parse this to the individual realm, service, and scope components? It's not quite JSON, so deserializing it with NewtonSoft JsonConvert won't work. I could regex it into something that looks like XML or JSON, but that seems incredibly hacky (not to mention unreliable).
Surely there's a better way?

Since I don't see a non-hacky way. Maybe this hacky way may help
string input = #"WwwAuthenticate: Bearer realm=""https://somesite/auth"",service=""some site"",scope=""some, scope""";
var dict = Regex.Matches(input, #"[\W]+(\w+)=""(.+?)""").Cast<Match>()
.ToDictionary(x => x.Groups[1].Value, x => x.Groups[2].Value);
var url = dict["realm"] + "?" + string.Join("&", dict.Where(x => x.Key != "realm").Select(x => x.Key + "=" + WebUtility.UrlEncode(x.Value)));
OUTPUT
url => https://somesite/auth?service=some+site&scope=some%2C+scope
BTW: I added a , in "scope"

Possible duplicate of How to parse values from Www-Authenticate
Using the schema defined in RFC6750 and RFC2616, a slightly more precise parser implementation is included below. This parser takes into account the possibility that strings might contain =, ,, and/or escaped ".
internal class AuthParamParser
{
private string _buffer;
private int _i;
private AuthParamParser(string param)
{
_buffer = param;
_i = 0;
}
public static Dictionary<string, string> Parse(string param)
{
var state = new AuthParamParser(param);
var result = new Dictionary<string, string>();
var token = state.ReadToken();
while (!string.IsNullOrEmpty(token))
{
if (!state.ReadDelim('='))
return result;
result.Add(token, state.ReadString());
if (!state.ReadDelim(','))
return result;
token = state.ReadToken();
}
return result;
}
private string ReadToken()
{
var start = _i;
while (_i < _buffer.Length && ValidTokenChar(_buffer[_i]))
_i++;
return _buffer.Substring(start, _i - start);
}
private bool ReadDelim(char ch)
{
while (_i < _buffer.Length && char.IsWhiteSpace(_buffer[_i]))
_i++;
if (_i >= _buffer.Length || _buffer[_i] != ch)
return false;
_i++;
while (_i < _buffer.Length && char.IsWhiteSpace(_buffer[_i]))
_i++;
return true;
}
private string ReadString()
{
if (_i < _buffer.Length && _buffer[_i] == '"')
{
var buffer = new StringBuilder();
_i++;
while (_i < _buffer.Length)
{
if (_buffer[_i] == '\\' && (_i + 1) < _buffer.Length)
{
_i++;
buffer.Append(_buffer[_i]);
_i++;
}
else if (_buffer[_i] == '"')
{
_i++;
return buffer.ToString();
}
else
{
buffer.Append(_buffer[_i]);
_i++;
}
}
return buffer.ToString();
}
else
{
return ReadToken();
}
}
private bool ValidTokenChar(char ch)
{
if (ch < 32)
return false;
if (ch == '(' || ch == ')' || ch == '<' || ch == '>' || ch == '#'
|| ch == ',' || ch == ';' || ch == ':' || ch == '\\' || ch == '"'
|| ch == '/' || ch == '[' || ch == ']' || ch == '?' || ch == '='
|| ch == '{' || ch == '}' || ch == 127 || ch == ' ' || ch == '\t')
return false;
return true;
}
}

Split string in square brackets from Google translator

I am receiving a data from a Google Language Translator service and need help splitting the data.
void Start()
{
translateText("Hello, This is a test!", "en", "fr");
}
void translateText(string text, string fromLanguage, string toLanguage)
{
string url = "https://translate.googleapis.com/translate_a/single?client=gtx&sl=" + fromLanguage + "&tl=" + toLanguage + "&dt=t&q=" + Uri.EscapeUriString(text);
StartCoroutine(startTranslator(url));
}
IEnumerator startTranslator(string url)
{
UnityWebRequest www = UnityWebRequest.Get(url);
yield return www.Send();
Debug.Log("Raw string Received: " + www.downloadHandler.text);
LanguageResult tempResult = decodeResult(www.downloadHandler.text);
Debug.Log("Original Text: " + tempResult.originalText);
Debug.Log("Translated Text: " + tempResult.translatedText);
Debug.Log("LanguageIso: " + tempResult.languageIso);
yield return null;
}
LanguageResult decodeResult(string result)
{
char[] delims = { '[', '\"', ']', ',' };
string[] arr = result.Split(delims, StringSplitOptions.RemoveEmptyEntries);
LanguageResult tempLang = null;
if (arr.Length >= 4)
{
tempLang = new LanguageResult();
tempLang.translatedText = arr[0];
tempLang.originalText = arr[1];
tempLang.unknowValue = arr[2];
tempLang.languageIso = arr[3];
}
return tempLang;
}
public class LanguageResult
{
public string translatedText;
public string originalText;
public string unknowValue;
public string languageIso;
}
then calling it with translateText("Hello, This is a test!", "en", "fr"); from the Start() function which converts the English sentence to French with ISO 639-1 Code.
The received data looks like this:
[[["Bonjour, Ceci est un test!","Hello, This is a test!",,,0]],,"en"]
I want to split it like this:
Bonjour, Ceci est un test!
Hello, This is a test!
0
en
and put them into a string array in order.
I currently use this:
char[] delims = { '[', '\"', ']', ',' };
string[] arr = result.Split(delims, StringSplitOptions.RemoveEmptyEntries);
This works if there is no comma in the received string. If there is a comma, the splitted values are messed up. What's the best way of splitting this?
EDIT:
With Blorgbeard's solution, the final working code is as below. Hopefully, this will help somebody else. This shouldn't be used for commercial purposes but for personal or school project.
void Start()
{
//translateText("Hello, This is \" / \\ a test !", "en", "fr");
//translateText("Hello, This is , \\ \" a test !", "en", "fr");
translateText("Hello, This is a test!", "en", "fr");
}
void translateText(string text, string fromLanguage, string toLanguage)
{
string url = "https://translate.googleapis.com/translate_a/single?client=gtx&sl=" + fromLanguage + "&tl=" + toLanguage + "&dt=t&q=" + Uri.EscapeUriString(text);
StartCoroutine(startTranslator(url));
}
IEnumerator startTranslator(string url)
{
UnityWebRequest www = UnityWebRequest.Get(url);
yield return www.Send();
Debug.Log("Raw string Received: " + www.downloadHandler.text);
LanguageResult tempResult = decodeResult(www.downloadHandler.text);
displayResult(tempResult);
yield return null;
}
void displayResult(LanguageResult translationResult)
{
Debug.Log("Original Text: " + translationResult.originalText);
Debug.Log("Translated Text: " + translationResult.translatedText);
Debug.Log("LanguageIso: " + translationResult.languageIso);
}
LanguageResult decodeResult(string result)
{
string[] arr = Decode(result);
LanguageResult tempLang = null;
if (arr.Length >= 4)
{
tempLang = new LanguageResult();
tempLang.translatedText = arr[0];
tempLang.originalText = arr[1];
tempLang.unknowValue = arr[2];
tempLang.languageIso = arr[3];
}
return tempLang;
}
public class LanguageResult
{
public string translatedText;
public string originalText;
public string unknowValue;
public string languageIso;
}
private string[] Decode(string input)
{
List<string> finalResult = new List<string>();
bool inToken = false;
bool inString = false;
bool escaped = false;
var seps = ",[]\"".ToArray();
var current = "";
foreach (var chr in input)
{
if (!inString && chr == '"')
{
current = "";
inString = true;
continue;
}
if (inString && !escaped && chr == '"')
{
finalResult.Add(current);
current = "";
inString = false;
continue;
}
if (inString && !escaped && chr == '\\')
{
escaped = true;
continue;
}
if (inString && (chr != '"' || escaped))
{
escaped = false;
current += chr;
continue;
}
if (inToken && seps.Contains(chr))
{
finalResult.Add(current);
current = "";
inToken = false;
continue;
}
if (!inString && chr == '"')
{
inString = true;
current = "";
continue;
}
if (!inToken && !seps.Contains(chr))
{
inToken = true;
current = "";
}
current += chr;
}
return finalResult.ToArray();
}

You could code up a simple parser yourself. Here's one I threw together (could use some cleaning up, but demonstrates the idea):
private static IEnumerable<string> Parse(string input) {
bool inToken = false;
bool inString = false;
bool escaped = false;
var seps = ",[]\"".ToArray();
var current = "";
foreach (var chr in input) {
if (!inString && chr == '"') {
current = "";
inString = true;
continue;
}
if (inString && !escaped && chr == '"') {
yield return current;
current = "";
inString = false;
continue;
}
if (inString && !escaped && chr == '\\') {
escaped = true;
continue;
}
if (inString && (chr != '"' || escaped)) {
escaped = false;
current += chr;
continue;
}
if (inToken && seps.Contains(chr)) {
yield return current;
current = "";
inToken = false;
continue;
}
if (!inString && chr == '"') {
inString = true;
current = "";
continue;
}
if (!inToken && !seps.Contains(chr)) {
inToken = true;
current = "";
}
current += chr;
}
}
Here's a jsfiddle demo.

Using Regex.Split you could do something like this for example:
using System;
using System.Text.RegularExpressions;
public class Example
{
public static void Main()
{
var input ="[[[\"Bonjour, Ceci est un test!\",\"Hello, This is a test!\",,,0]],,\"en\"]";
var parse = Regex.Split(input, "\\[|\\]|[^a-zA-Z ],|\",\"|\"|\"");
foreach(var item in parse) {
bool result = !String.IsNullOrEmpty(item) && (Char.IsLetter(item[0]) || Char.IsDigit(item[0]));
if (result) {
Console.WriteLine(item);
}
}
}
}
Output:
Bonjour, Ceci est un test!
Hello, This is a test!
0
en
If you want everything that was split you can simply remove the bool check for alphacharacters.

Here is a crazy idea - split by " and then by the rest (but won't work if there is " between the "'s)
var s = #"[[[""Bonjour, Ceci est un test!"",""Hello, This is a test!"",,,0]],,""en""]";
var a = s.Split('"').Select((x, i) => (i & 1) > 0 ? new[] { x } : x.Split("[],".ToArray(),
StringSplitOptions.RemoveEmptyEntries)).SelectMany(x => x).ToArray();
Debug.Print(string.Join("|", a)); // "Bonjour, Ceci est un test!|Hello, This is a test!|0|en"

You can try regex for splitting. I tested with the sample you provided. It results like this.
var str="[[[\"Bonjour, Ceci est un test!\",\"Hello, This is a test!\",,,0]],,\"en\"]";
var splitted=Regex.Split(str,#"\[|\]|\,");
foreach(var split in splitted){
Console.WriteLine(split );
}
"Bonjour Ceci est un test!"
"Hello This is a test!"
0
"en"

C# for case in string(easy)

so I have this code. I need to generate a for loop that checks all the characters in the string and checks if they are all valid(So numbers from 0->7). But I don't know how to write it, I tried something but it didn't work. Here are the examples:user enters: 77, code works, user enters 99, code doesn't work, user enters 5., code doesn't work, etc..
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace NALOGA1
{
class Program
{
static string decToOct(int stevilo)//v mojon primere 7
{
string izhod = "";
//7>0 DRŽI
while (stevilo > 0)
{
//izhodi se dodeli ostanek deljenja z 8 keri se spremeni v string
izhod = (stevilo % 8) + izhod;
//7/8;
stevilo /= 8;
}
return izhod;
}
static int Octtodesetisko(string stevilo)
{
double vsota = 0;
for (int i = stevilo.Length - 1; i >= 0; i--)
{
int stevka = stevilo[i] - '0';
vsota += (stevka * Math.Pow(8, i));
}
return (int)vsota;
}
static void Main(string[] args)
{
//3 podprogram-in progress
string prvastevilka = Console.ReadLine();
int prvasprememba = Int32.Parse(prvastevilka);
if (prvasprememba > 0)
{
Console.WriteLine(decToOct(prvasprememba));
}
else
{
Console.WriteLine("Napaka");
}
string drugastevilka = Console.ReadLine();
int drugasprememba = Octtodesetisko(drugastevilka);
foreach (char znak in drugastevilka)
{
if(znak!=1 || znak!=2 || znak!=3 || znak!=4 || znak!=5 || znak!=6 || znak!=7)
{
Console.WriteLine("Napaka");
}
else
{
Console.WriteLine("dela :D");
}
}
Console.ReadKey();
}
}
}

Personally, I would take advantage of the LINQ Enumerable.All method to express this in a very concise and readable way:
if (str.Any() && str.All(c => c >= '0' && c <= '7'))
{
Console.WriteLine("good");
}
else
{
Console.WriteLine("bad");
}
EDIT: No LINQ
It's not hard to translate what the LINQ Enumerable.All method does to a normal loop. It's just more verbose:
bool isValid = true;
foreach (char c in str)
{
if (c < '0' || c > '7')
{
isValid = false;
break;
}
}
if (str.Length != 0 && isValid)
{
Console.WriteLine("good");
}
else
{
Console.WriteLine("bad");
}

Firstly, there seems to be a mistake in the line
if(znak!=1 || znak!=2 || znak!=3 || znak!=4 || znak!=5 || znak!=6 || znak!=7)
I guess it should read
if(znak!='1' || znak!='2' || znak!='3' || znak!='4' || znak!='5' || znak!='6' || znak!='7')
which should be compressed to
if (znak >= '0' && znak <= '7')
You can use linq instead of the for loop here like this:
if (drugastevilka.All(c => c >= '0' && c <= '7')
Console.WriteLine("dela :D");
else
Console.WriteLine("Napaka");
But the best solution is probably to use a regular expression:
Regex regex = new Regex("^[0-7]+$");
if (regex.IsMatch(drugastevilka))
Console.WriteLine("dela :D");
else
Console.WriteLine("Napaka");
Edit: the linq solution shown accepts empty strings, the regex (as shown) needs at least 1 character. Exchange the + with a * and it will accept empty strings, too. But I don't think you want to accept empty strings.

You are messing up with the datatype
Can you try with below code
static string decToOct(int stevilo)//v mojon primere 7
{
int izhod = 0;
//7>0 DRŽI
while (stevilo > 0)
{
//izhodi se dodeli ostanek deljenja z 8 keri se spremeni v string
izhod = (stevilo % 8) + izhod;
//7/8;
stevilo /= 8;
}
return (izhod.ToString());
}

What about something like this?
class Program
{
static void Main(string[] args)
{
string someString = "1234567";
string someOtherString = "1287631";
string anotherString = "123A6F2";
Console.WriteLine(IsValidString(someString));
Console.WriteLine(IsValidString(someOtherString));
Console.WriteLine(IsValidString(anotherString));
Console.ReadLine();
}
public static bool IsValidString(string str)
{
bool isValid = true;
char[] splitString = str.ToCharArray(); //get an array of each character
for (int i = 0; i < splitString.Length; i++)
{
try
{
double number = Char.GetNumericValue(splitString[i]); //try to convert the character to a double (GetNumericValue returns a double)
if (number < 0 || number > 7) //we get here if the character is an int, then we check for 0-7
{
isValid = false; //if the character is invalid, we're done.
break;
}
}
catch (Exception) //this will hit if we try to convert a non-integer character.
{
isValid = false;
break;
}
}
return isValid;
}
}
IsValidString() takes a string, converts it to a Char array, then checks each value as such:
Get the numeric value
Check if the value is between 0-7
GetNumericValue will fail on a non-integer character, so we wrap it in a try/catch - if we hit an exception we know that isValid = false, so we break.
If we get a valid number, and it's not between 0-7 we also know that isValid = false, so we break.
If we make it all the way through the list, the string is valid.
The sample given above returns:
IsValidString(someString) == true
IsValidString(someOtherString) == false
IsValidString(anotherString) == false

Create Space Between Capital Letters and Skip Space Between Consecutive

I get the way to create space "ThisCourse" to be "This Course"
Add Space Before Capital Letter By (EtienneT) LINQ Statement
But i cannot
Create Space Betweeen This "ThisCourseID" to be "This Course ID" without space between "ID"
And Is there a way to do this in Linq ??

Well, if it has to be a single linq statement...
var s = "ThisCourseIDMoreXYeahY";
s = string.Join(
string.Empty,
s.Select((x,i) => (
char.IsUpper(x) && i>0 &&
( char.IsLower(s[i-1]) || (i<s.Count()-1 && char.IsLower(s[i+1])) )
) ? " " + x : x.ToString()));
Console.WriteLine(s);
Output: "This Course ID More X Yeah Y"

var s = "ThisCourseID";
for (var i = 1; i < s.Length; i++)
{
if (char.IsLower(s[i - 1]) && char.IsUpper(s[i]))
{
s = s.Insert(i, " ");
}
}
Console.WriteLine(s); // "This Course ID"
You can improve this using StringBuilder if you are going to use this on very long strings, but for your purpose, as you presented it, it should work just fine.
FIX:
var s = "ThisCourseIDSomething";
for (var i = 1; i < s.Length - 1; i++)
{
if (char.IsLower(s[i - 1]) && char.IsUpper(s[i]) ||
s[i - 1] != ' ' && char.IsUpper(s[i]) && char.IsLower(s[i + 1]))
{
s = s.Insert(i, " ");
}
}
Console.WriteLine(s); // This Course ID Something

You don't need LINQ - but you could 'enumerate' and use lambda to make it more generic...
(though not sure if any of this makes sense)
static IEnumerable<string> Split(this string text, Func<char?, char?, char, int?> shouldSplit)
{
StringBuilder output = new StringBuilder();
char? before = null;
char? before2nd = null;
foreach (var c in text)
{
var where = shouldSplit(before2nd, before, c);
if (where != null)
{
var str = output.ToString();
switch(where)
{
case -1:
output.Remove(0, str.Length -1);
yield return str.Substring(0, str.Length - 1);
break;
case 0: default:
output.Clear();
yield return str;
break;
}
}
output.Append(c);
before2nd = before;
before = c;
}
yield return output.ToString();
}
...and call it like this e.g. ...
static IEnumerable<string> SplitLines(this string text)
{
return text.Split((before2nd, before, now) =>
{
if ((before2nd ?? 'A') == '\r' && (before ?? 'A') == '\n') return 0; // split on 'now'
return null; // don't split
});
}
static IEnumerable<string> SplitOnCase(this string text)
{
return text.Split((before2nd, before, now) =>
{
if (char.IsLower(before ?? 'A') && char.IsUpper(now)) return 0; // split on 'now'
if (char.IsUpper(before2nd ?? 'a') && char.IsUpper(before ?? 'a') && char.IsLower(now)) return -1; // split one char before
return null; // don't split
});
}
...and somewhere...
var text = "ToSplitOrNotToSplitTHEQuestionIsNow";
var words = text.SplitOnCase();
foreach (var word in words)
Console.WriteLine(word);
text = "To\r\nSplit\r\nOr\r\nNot\r\nTo\r\nSplit\r\nTHE\r\nQuestion\r\nIs\r\nNow";
words = text.SplitLines();
foreach (var word in words)
Console.WriteLine(word);
:)