Related
Is there a way to convert any audio file to ogg with .net core?
I figured out how to convert with NAudio.Vorbis from ogg to wav:
using (var vorbis = new VorbisWaveReader(inputFile))
{
WaveFileWriter.CreateWaveFile(outputFile, vorbis);
}
But I could not find a way to convert any audio file to ogg.
Does NAudio provide the functionallity I require or are there any other libraries better suited?
You could try this library:
Vorbis Encoder
(this is on nuget, the linked GitHub project provides the source and the following example of encoding)
using System;
using System.IO;
namespace OggVorbisEncoder.Example
{
public class Encoder
{
private const int SampleSize = 1024;
[STAThread]
private static void Main()
{
var stdin = new FileStream(#"unencoded.raw", FileMode.Open, FileAccess.Read);
var stdout = new FileStream(#"encoded.ogg", FileMode.Create, FileAccess.Write);
// StripWavHeader(stdin);
// Stores all the static vorbis bitstream settings
var info = VorbisInfo.InitVariableBitRate(2, 44100, 0.1f);
// set up our packet->stream encoder
var serial = new Random().Next();
var oggStream = new OggStream(serial);
// =========================================================
// HEADER
// =========================================================
// Vorbis streams begin with three headers; the initial header (with
// most of the codec setup parameters) which is mandated by the Ogg
// bitstream spec. The second header holds any comment fields. The
// third header holds the bitstream codebook.
var headerBuilder = new HeaderPacketBuilder();
var comments = new Comments();
comments.AddTag("ARTIST", "TEST");
var infoPacket = headerBuilder.BuildInfoPacket(info);
var commentsPacket = headerBuilder.BuildCommentsPacket(comments);
var booksPacket = headerBuilder.BuildBooksPacket(info);
oggStream.PacketIn(infoPacket);
oggStream.PacketIn(commentsPacket);
oggStream.PacketIn(booksPacket);
// Flush to force audio data onto its own page per the spec
OggPage page;
while (oggStream.PageOut(out page, true))
{
stdout.Write(page.Header, 0, page.Header.Length);
stdout.Write(page.Body, 0, page.Body.Length);
}
// =========================================================
// BODY (Audio Data)
// =========================================================
var processingState = ProcessingState.Create(info);
var buffer = new float[info.Channels][];
buffer[0] = new float[SampleSize];
buffer[1] = new float[SampleSize];
var readbuffer = new byte[SampleSize*4];
while (!oggStream.Finished)
{
var bytes = stdin.Read(readbuffer, 0, readbuffer.Length);
if (bytes == 0)
{
processingState.WriteEndOfStream();
}
else
{
var samples = bytes/4;
for (var i = 0; i < samples; i++)
{
// uninterleave samples
buffer[0][i] = (short) ((readbuffer[i*4 + 1] << 8) | (0x00ff & readbuffer[i*4]))/32768f;
buffer[1][i] = (short) ((readbuffer[i*4 + 3] << 8) | (0x00ff & readbuffer[i*4 + 2]))/32768f;
}
processingState.WriteData(buffer, samples);
}
OggPacket packet;
while (!oggStream.Finished
&& processingState.PacketOut(out packet))
{
oggStream.PacketIn(packet);
while (!oggStream.Finished
&& oggStream.PageOut(out page, false))
{
stdout.Write(page.Header, 0, page.Header.Length);
stdout.Write(page.Body, 0, page.Body.Length);
}
}
}
stdin.Close();
stdout.Close();
}
/// <summary>
/// We cheat on the WAV header; we just bypass the header and never
/// verify that it matches 16bit/stereo/44.1kHz.This is just an
/// example, after all.
/// </summary>
private static void StripWavHeader(BinaryReader stdin)
{
var tempBuffer = new byte[6];
for (var i = 0; (i < 30) && (stdin.Read(tempBuffer, 0, 2) > 0); i++)
if ((tempBuffer[0] == 'd') && (tempBuffer[1] == 'a'))
{
stdin.Read(tempBuffer, 0, 6);
break;
}
}
}
}
I'm trying to parse a binary file as fastest as possible. So this is what I first tried to do:
using (FileStream filestream = path.OpenRead()) {
using (var d = new GZipStream(filestream, CompressionMode.Decompress)) {
using (MemoryStream m = new MemoryStream()) {
d.CopyTo(m);
m.Position = 0;
using (BinaryReaderBigEndian b = new BinaryReaderBigEndian(m)) {
while (b.BaseStream.Position != b.BaseStream.Length) {
UInt32 value = b.ReadUInt32();
} } } } }
Where BinaryReaderBigEndian class is implemented as it follows:
public static class BinaryReaderBigEndian {
public BinaryReaderBigEndian(Stream stream) : base(stream) { }
public override UInt32 ReadUInt32() {
var x = base.ReadBytes(4);
Array.Reverse(x);
return BitConverter.ToUInt32(x, 0);
} }
Then, I tried to get a performance improvement using ReadOnlySpan instead of MemoryStream. So, I tried doing:
using (FileStream filestream = path.OpenRead()) {
using (var d = new GZipStream(filestream, CompressionMode.Decompress)) {
using (MemoryStream m = new MemoryStream()) {
d.CopyTo(m);
int position = 0;
ReadOnlySpan<byte> stream = new ReadOnlySpan<byte>(m.ToArray());
while (position != stream.Length) {
UInt32 value = stream.ReadUInt32(position);
position += 4;
} } } }
Where BinaryReaderBigEndian class changed in:
public static class BinaryReaderBigEndian {
public override UInt32 ReadUInt32(this ReadOnlySpan<byte> stream, int start) {
var data = stream.Slice(start, 4).ToArray();
Array.Reverse(x);
return BitConverter.ToUInt32(x, 0);
} }
But, unfortunately, I didn't notice any improvement. So, where am I doing wrong?
I did some measurement of your code on my computer (Intel Q9400, 8 GiB RAM, SSD disk, Win10 x64 Home, .NET Framework 4/7/2, tested with 15 MB (when unpacked) file) with these results:
No-Span version: 520 ms
Span version: 720 ms
So Span version is actually slower! Why? Because new ReadOnlySpan<byte>(m.ToArray()) performs additional copy of whole file and also ReadUInt32() performs many slicings of the Span (slicing is cheap, but not free). Since you performed more work, you can't expect performance to be any better just because you used Span.
So can we do better? Yes. It turns out that the slowest part of your code is actually garbage collection caused by repeatedly allocating 4-byte Arrays created by the .ToArray() calls in ReadUInt32() method. You can avoid it by implementing ReadUInt32() yourself. It's pretty easy and also eliminates need for Span slicing. You can also replace new ReadOnlySpan<byte>(m.ToArray()) with new ReadOnlySpan<byte>(m.GetBuffer()).Slice(0, (int)m.Length);, which performs cheap slicing instead of copy of whole file. So now code looks like this:
public static void Read(FileInfo path)
{
using (FileStream filestream = path.OpenRead())
{
using (var d = new GZipStream(filestream, CompressionMode.Decompress))
{
using (MemoryStream m = new MemoryStream())
{
d.CopyTo(m);
int position = 0;
ReadOnlySpan<byte> stream = new ReadOnlySpan<byte>(m.GetBuffer()).Slice(0, (int)m.Length);
while (position != stream.Length)
{
UInt32 value = stream.ReadUInt32(position);
position += 4;
}
}
}
}
}
public static class BinaryReaderBigEndian
{
public static UInt32 ReadUInt32(this ReadOnlySpan<byte> stream, int start)
{
UInt32 res = 0;
for (int i = 0; i < 4; i++)
{
res = (res << 8) | (((UInt32)stream[start + i]) & 0xff);
}
return res;
}
}
With these changes I get from 720 ms down to 165 ms (4x faster). Sounds great, doesn't it? But we can do even better. We can completely avoid MemoryStream copy and inline and further optimize ReadUInt32():
public static void Read(FileInfo path)
{
using (FileStream filestream = path.OpenRead())
{
using (var d = new GZipStream(filestream, CompressionMode.Decompress))
{
var buffer = new byte[64 * 1024];
do
{
int bufferDataLength = FillBuffer(d, buffer);
if (bufferDataLength % 4 != 0)
throw new Exception("Stream length not divisible by 4");
if (bufferDataLength == 0)
break;
for (int i = 0; i < bufferDataLength; i += 4)
{
uint value = unchecked(
(((uint)buffer[i]) << 24)
| (((uint)buffer[i + 1]) << 16)
| (((uint)buffer[i + 2]) << 8)
| (((uint)buffer[i + 3]) << 0));
}
} while (true);
}
}
}
private static int FillBuffer(Stream stream, byte[] buffer)
{
int read = 0;
int totalRead = 0;
do
{
read = stream.Read(buffer, totalRead, buffer.Length - totalRead);
totalRead += read;
} while (read > 0 && totalRead < buffer.Length);
return totalRead;
}
And now it takes less than 90 ms (8x faster then the original!). And without Span! Span is great in situations, where it allows perform slicing and avoid array copy, but it won't improve performance just by blindly using it. After all, Span is designed to have performance characteristics on par with Array, but not better (and only on runtimes that have special support for it, such as .NET Core 2.1).
I have a set of markdown files to be passed to jekyll project , need to find the encoding format of them i.e UTF-8 with BOM or UTF-8 without BOM or ANSI using a program or a API .
if i pass the location of the files , the files have to be listed,read and the encoding should be produced as result .
Is there any Code or API for it ?
i have already tried the sr.CurrentEncoding for stream reader as mentioned in Effective way to find any file's Encoding but the result varies with the result from a notepad++ result .
also tried to use https://github.com/errepi/ude ( Mozilla Universal Charset Detector) as suggested in https://social.msdn.microsoft.com/Forums/vstudio/en-US/862e3342-cc88-478f-bca2-e2de6f60d2fb/detect-encoding-of-the-file?forum=csharpgeneral by implementing the ude.dll in the c# project but the result is not effective as in notepad++ , the file encoding is shown as utf-8 , but from the program , the result is utf-8 with BOM.
but i should get same result from both ways , so where the problem has occurred?
Detecting encoding is always a tricky business, but detecting BOMs is dead simple. To get the BOM as byte array, just use the GetPreamble() function of the encoding objects. This should allow you to detect a whole range of encodings by preamble.
Now, as for detecting UTF-8 without preamble, actually that's not very hard either. See, UTF8 has strict bitwise rules about what values are expected in a valid sequence, and you can initialize a UTF8Encoding object in a way that will fail by throwing an exception when these sequences are incorrect.
So if you first do the BOM check, and then the strict decoding check, and finally fall back to Win-1252 encoding (what you call "ANSI") then your detection is done.
Byte[] bytes = File.ReadAllBytes(filename);
Encoding encoding = null;
String text = null;
// Test UTF8 with BOM. This check can easily be copied and adapted
// to detect many other encodings that use BOMs.
UTF8Encoding encUtf8Bom = new UTF8Encoding(true, true);
Boolean couldBeUtf8 = true;
Byte[] preamble = encUtf8Bom.GetPreamble();
Int32 prLen = preamble.Length;
if (bytes.Length >= prLen && preamble.SequenceEqual(bytes.Take(prLen)))
{
// UTF8 BOM found; use encUtf8Bom to decode.
try
{
// Seems that despite being an encoding with preamble,
// it doesn't actually skip said preamble when decoding...
text = encUtf8Bom.GetString(bytes, prLen, bytes.Length - prLen);
encoding = encUtf8Bom;
}
catch (ArgumentException)
{
// Confirmed as not UTF-8!
couldBeUtf8 = false;
}
}
// use boolean to skip this if it's already confirmed as incorrect UTF-8 decoding.
if (couldBeUtf8 && encoding == null)
{
// test UTF-8 on strict encoding rules. Note that on pure ASCII this will
// succeed as well, since valid ASCII is automatically valid UTF-8.
UTF8Encoding encUtf8NoBom = new UTF8Encoding(false, true);
try
{
text = encUtf8NoBom.GetString(bytes);
encoding = encUtf8NoBom;
}
catch (ArgumentException)
{
// Confirmed as not UTF-8!
}
}
// fall back to default ANSI encoding.
if (encoding == null)
{
encoding = Encoding.GetEncoding(1252);
text = encoding.GetString(bytes);
}
Note that Windows-1252 (US / Western European ANSI) is a one-byte-per-character encoding, meaning everything in it produces a technically valid character, so unless you go for heuristic methods, no further detection can be done on it to distinguish it from other one-byte-per-character encodings.
Necromancing.
First, you check the Byte-Order Mark:
If that doesn't work, you can try to infer the encoding from the text-content with Mozilla Universal Charset Detector C# port.
If that doesn't work, you just return the CurrentCulture/InstalledUiCulture/System-Encoding - or whatever.
if the system-encoding doesn't work, we can either return ASCII or UTF8. Since entries 0-127 of UTF8 are identical to ASCII, we so simply return UTF8.
Example (DetectOrGuessEncoding):
namespace SQLMerge
{
class EncodingDetector
{
public static System.Text.Encoding BomInfo(string srcFile)
{
return BomInfo(srcFile, false);
} // End Function BomInfo
public static System.Text.Encoding BomInfo(string srcFile, bool thorough)
{
byte[] b = new byte[5];
using (System.IO.FileStream file = new System.IO.FileStream(srcFile, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
{
int numRead = file.Read(b, 0, 5);
if (numRead < 5)
System.Array.Resize(ref b, numRead);
file.Close();
} // End Using file
if (b.Length >= 4 && b[0] == 0x00 && b[1] == 0x00 && b[2] == 0xFE && b[3] == 0xFF) // UTF32-BE
return System.Text.Encoding.GetEncoding("utf-32BE"); // UTF-32, big-endian
else if (b.Length >= 4 && b[0] == 0xFF && b[1] == 0xFE && b[2] == 0x00 && b[3] == 0x00) // UTF32-LE
return System.Text.Encoding.UTF32; // UTF-32, little-endian
// https://en.wikipedia.org/wiki/Byte_order_mark#cite_note-14
else if (b.Length >= 4 && b[0] == 0x2b && b[1] == 0x2f && b[2] == 0x76 && (b[3] == 0x38 || b[3] == 0x39 || b[3] == 0x2B || b[3] == 0x2F)) // UTF7
return System.Text.Encoding.UTF7; // UTF-7
else if (b.Length >= 3 && b[0] == 0xEF && b[1] == 0xBB && b[2] == 0xBF) // UTF-8
return System.Text.Encoding.UTF8; // UTF-8
else if (b.Length >= 2 && b[0] == 0xFE && b[1] == 0xFF) // UTF16-BE
return System.Text.Encoding.BigEndianUnicode; // UTF-16, big-endian
else if (b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE) // UTF16-LE
return System.Text.Encoding.Unicode; // UTF-16, little-endian
// Maybe there is a future encoding ...
// PS: The above yields more than this - this doesn't find UTF7 ...
if (thorough)
{
System.Collections.Generic.List<System.Collections.Generic.KeyValuePair<System.Text.Encoding, byte[]>> lsPreambles =
new System.Collections.Generic.List<System.Collections.Generic.KeyValuePair<System.Text.Encoding, byte[]>>();
foreach (System.Text.EncodingInfo ei in System.Text.Encoding.GetEncodings())
{
System.Text.Encoding enc = ei.GetEncoding();
byte[] preamble = enc.GetPreamble();
if (preamble == null)
continue;
if (preamble.Length == 0)
continue;
if (preamble.Length > b.Length)
continue;
System.Collections.Generic.KeyValuePair<System.Text.Encoding, byte[]> kvp =
new System.Collections.Generic.KeyValuePair<System.Text.Encoding, byte[]>(enc, preamble);
lsPreambles.Add(kvp);
} // Next ei
// li.Sort((a, b) => a.CompareTo(b)); // ascending sort
// li.Sort((a, b) => b.CompareTo(a)); // descending sort
lsPreambles.Sort(
delegate (
System.Collections.Generic.KeyValuePair<System.Text.Encoding, byte[]> kvp1,
System.Collections.Generic.KeyValuePair<System.Text.Encoding, byte[]> kvp2)
{
return kvp2.Value.Length.CompareTo(kvp1.Value.Length);
}
);
for (int j = 0; j < lsPreambles.Count; ++j)
{
for (int i = 0; i < lsPreambles[j].Value.Length; ++i)
{
if (b[i] != lsPreambles[j].Value[i])
{
goto NEXT_J_AND_NOT_NEXT_I;
}
} // Next i
return lsPreambles[j].Key;
NEXT_J_AND_NOT_NEXT_I: continue;
} // Next j
} // End if (thorough)
return null;
} // End Function BomInfo
public static System.Text.Encoding DetectOrGuessEncoding(string fileName)
{
return DetectOrGuessEncoding(fileName, false);
}
public static System.Text.Encoding DetectOrGuessEncoding(string fileName, bool withOutput)
{
if (!System.IO.File.Exists(fileName))
return null;
System.ConsoleColor origBack = System.ConsoleColor.Black;
System.ConsoleColor origFore = System.ConsoleColor.White;
if (withOutput)
{
origBack = System.Console.BackgroundColor;
origFore = System.Console.ForegroundColor;
}
// System.Text.Encoding systemEncoding = System.Text.Encoding.Default; // Returns hard-coded UTF8 on .NET Core ...
System.Text.Encoding systemEncoding = GetSystemEncoding();
System.Text.Encoding enc = BomInfo(fileName);
if (enc != null)
{
if (withOutput)
{
System.Console.BackgroundColor = System.ConsoleColor.Green;
System.Console.ForegroundColor = System.ConsoleColor.White;
System.Console.WriteLine(fileName);
System.Console.WriteLine(enc);
System.Console.BackgroundColor = origBack;
System.Console.ForegroundColor = origFore;
}
return enc;
}
using (System.IO.Stream strm = System.IO.File.OpenRead(fileName))
{
UtfUnknown.DetectionResult detect = UtfUnknown.CharsetDetector.DetectFromStream(strm);
if (detect != null && detect.Details != null && detect.Details.Count > 0 && detect.Details[0].Confidence < 1)
{
if (withOutput)
{
System.Console.BackgroundColor = System.ConsoleColor.Red;
System.Console.ForegroundColor = System.ConsoleColor.White;
System.Console.WriteLine(fileName);
System.Console.WriteLine(detect);
System.Console.BackgroundColor = origBack;
System.Console.ForegroundColor = origFore;
}
foreach (UtfUnknown.DetectionDetail detail in detect.Details)
{
if (detail.Encoding == systemEncoding
|| detail.Encoding == System.Text.Encoding.UTF8
)
return detail.Encoding;
}
return detect.Details[0].Encoding;
}
else if (detect != null && detect.Details != null && detect.Details.Count > 0)
{
if (withOutput)
{
System.Console.BackgroundColor = System.ConsoleColor.Green;
System.Console.ForegroundColor = System.ConsoleColor.White;
System.Console.WriteLine(fileName);
System.Console.WriteLine(detect);
System.Console.BackgroundColor = origBack;
System.Console.ForegroundColor = origFore;
}
return detect.Details[0].Encoding;
}
enc = GetSystemEncoding();
if (withOutput)
{
System.Console.BackgroundColor = System.ConsoleColor.DarkRed;
System.Console.ForegroundColor = System.ConsoleColor.Yellow;
System.Console.WriteLine(fileName);
System.Console.Write("Assuming ");
System.Console.Write(enc.WebName);
System.Console.WriteLine("...");
System.Console.BackgroundColor = origBack;
System.Console.ForegroundColor = origFore;
}
return systemEncoding;
} // End Using strm
} // End Function DetectOrGuessEncoding
public static System.Text.Encoding GetSystemEncoding()
{
// The OEM code page for use by legacy console applications
// int oem = System.Globalization.CultureInfo.CurrentCulture.TextInfo.OEMCodePage;
// The ANSI code page for use by legacy GUI applications
// int ansi = System.Globalization.CultureInfo.InstalledUICulture.TextInfo.ANSICodePage; // Machine
int ansi = System.Globalization.CultureInfo.CurrentCulture.TextInfo.ANSICodePage; // User
try
{
// https://stackoverflow.com/questions/38476796/how-to-set-net-core-in-if-statement-for-compilation
#if ( NETSTANDARD && !NETSTANDARD1_0 ) || NETCORE || NETCOREAPP3_0 || NETCOREAPP3_1
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
#endif
System.Text.Encoding enc = System.Text.Encoding.GetEncoding(ansi);
return enc;
}
catch (System.Exception)
{ }
try
{
foreach (System.Text.EncodingInfo ei in System.Text.Encoding.GetEncodings())
{
System.Text.Encoding e = ei.GetEncoding();
// 20'127: US-ASCII
if (e.WindowsCodePage == ansi && e.CodePage != 20127)
{
return e;
}
}
}
catch (System.Exception)
{ }
// return System.Text.Encoding.GetEncoding("iso-8859-1");
return System.Text.Encoding.UTF8;
} // End Function GetSystemEncoding
} // End Class
}
namespace WindowsFormsApp2
{
public partial class Form1 : Form
{
public Form1()
{
InitializeComponent();
}
private void button1_Click(object sender, EventArgs e)
{
List<FilePath> filePaths = new List<FilePath>();
filePaths = GetLstPaths();
}
public static List<FilePath> GetLstPaths()
{
#region Getting Files
DirectoryInfo directoryInfo = new DirectoryInfo(#"C:\Users\Safi\Desktop\ss\");
DirectoryInfo directoryTargetInfo = new DirectoryInfo(#"C:\Users\Safi\Desktop\ss1\");
FileInfo[] fileInfos = directoryInfo.GetFiles("*.txt");
List<FilePath> lstFiles = new List<FilePath>();
foreach (FileInfo fileInfo in fileInfos)
{
Encoding enco = GetLittleIndianFiles(directoryInfo + fileInfo.Name);
string filePath = directoryInfo + fileInfo.Name;
string targetFilePath = directoryTargetInfo + fileInfo.Name;
if (enco != null)
{
FilePath f1 = new FilePath();
f1.filePath = filePath;
f1.targetFilePath = targetFilePath;
lstFiles.Add(f1);
}
}
int count = 0;
lstFiles.ForEach(d =>
{
count++;
});
MessageBox.Show(Convert.ToString(count) + "Files are Converted");
#endregion
return lstFiles;
}
public static Encoding GetLittleIndianFiles(string srcFile)
{
byte[] b = new byte[5];
using (System.IO.FileStream file = new System.IO.FileStream(srcFile, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
{
int numRead = file.Read(b, 0, 5);
if (numRead < 5)
System.Array.Resize(ref b, numRead);
file.Close();
} // End Using file
if (b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE)
return System.Text.Encoding.Unicode; // UTF-16, little-endian
return null;
}
}
public class FilePath
{
public string filePath { get; set; }
public string targetFilePath { get; set; }
}
}
All i need to do is verify the message below but I can not get Bouncy Castle to take the data in and given the public key verify the message. I am happy for it to be some other Lib that is used if it is free. This is to be embedded in my app that receives data over the Internet so i would prefer to keep it all managed code if at all possible.
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
SCI Version: 1.0
SCI Code: 1
SCI Reason: OK
SCI Balance: 0.00050000
-----BEGIN PGP SIGNATURE-----
Version: GnuPG/MBC v1.0
iQEcBAEBAgAGBQJOGSuYAAoJEJ+5g06lAnqFkdsH/0NoqQbNvR8ZPe8D4gL4gvff
6K1t2LOt0sQGj+RSPeEbag7ZnVNI65LiES/yie1N6cXMkFgb9/ttjxi9/wlbxD/j
gSkuZ6mT9Oc5ExLsRZq9ygytvVs7Ol7uQm6oxDzJX1JMs0ls2EwJbmmpTEOHn8Av
dGlxdZeh+3RlqHJmOdssQCJ0cw5VXuj5vfP35OYz2zO2+sNg0eCXdR5Ml+2S7n3U
n9VHPEECg72LvpxF/y/nApopXoHpwECXoBwHgyd9QIIw1IJgalyRLDmAJ2WXdROV
ln2Mkt/km3KtBS3h4QL407wi/KhgZ4tFohZupt7zq2zUwtHWOhbL2KSUu939OKk=
=mIjM
-----END PGP SIGNATURE-----
For those interested i discovered an example for this exact task in the BouncyCastle source code. You need to download the source code not the binary to get the examples and it seems to have examples for all the different OpenPGP use cases.
Following Seer's suggestion to look at this example I finally got message verification running.
I have method VerifyFile which takes signed message and public key and returns the content of the message if the verification passes. For example it can be used like this:
string key = #"-----BEGIN PGP PUBLIC KEY BLOCK-----
Version: BCPG C# v1.6.1.0
mQENBFpc87wBCACK5FG6Z70iovzSzQF7OEB/YbKF7SPS1/GZAQhp/2n2G8x5Lxj5
/CKqR8JLj1+222unuWgfqvfny0fLvttt1r6lAH/kqDYMBg26GTbZy93R5BYatBjd
pzYl/lIyKxc/QwDdZm8zNxeUpDSfoe9jVULOg0MiMDtdQupOf6CanlEioXfyf88F
1BLcJyFSckaYieosBw5hnnI+1cZZ3k+4HpDJJslVzngfTPWRibtN5PKff1CKP55E
ME99XkuPDaNL7XZmu7lZSEUN3jJFVydZQrDkvxddihzV4pTgRI3gDAFoJxxIYZX3
JsQAJItlqq8bBsQ+bKPikgAiMySGcEi+ilI5ABEBAAG0GnNoYWxhbWFub3YubWFy
aW5AZ21haWwuY29tiQEcBBABAgAGBQJaXPO8AAoJEBvHdfmVFHzkvHEH/179VAdH
gWRN5HVprsp4vHP3q1CJV9j+fPlQIZU3JEwrM+INxzpfSqZeN4NwB7yoo2NCdCNP
Ndg8zhiuEYM51hNtqU5cwYBcaAbm1so6TSVo8i4nrfN3+oDYEfYPqglNrd1V233J
oyLriwpGkR6RBYMY2q2Re+EFNR1bxUmeE0wnb8FOodRCSh0Wd3Iy9mvmhv5voHIr
aZzgsuifGw1JilSu9+RoC6b1CHb9jUkWQ/odkTvl5/rxA14TKstgoLoSLHktYQfw
le6B8+lPtmODtagWoDEeR/M0zm/wyCOt5wqjjJCgvaipUaA+oiijIYwCpqUBwfm3
DZ9DStGHGVxQQnc=
=s91O
-----END PGP PUBLIC KEY BLOCK-----
";
string message = #"-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256
test tes tes ts tse tse t
-----BEGIN PGP SIGNATURE-----
Version: OpenPGP.js v1.0.1
Comment: http://openpgpjs.org
wsBcBAEBCAAQBQJaXP5WCRAbx3X5lRR85AAAUcoH/jtcyWcpTVyXyz/ptWLo
Hx+g51EeeA0Hpq7kZCXu4FuyhNn/QvnvKyt9qegxQoRSZhT37ln8t80NW6sS
B4XVFziq8TBkjPuaYBI/ijbLigdPMEi81PsOpIXx3BXKCt27TLmUVHpFTWPa
u2NQUQl3k3Xc0H1sy1A+jmjfvCyqWxTOU1IY4rlzRKHdp+D1oCz5iKfyfUko
ktAZgqOMx5pWL975YhM793MS8aYqhOdQpeuW401fm18xxwE4x6svSSys+qq8
MdkL/i7YVjUKr/M8SIrGPb/IjKwClM7jfpN+sHv0p/GcQ7J1kmXYUdA6AJp5
Z0vYk4aPcMSlrPwdRX21I9w=
=zXfe
-----END PGP SIGNATURE-----
";
MemoryStream messageStream = new MemoryStream(Encoding.ASCII.GetBytes(message));
MemoryStream keyStream = new MemoryStream(Encoding.ASCII.GetBytes(key));
try {
var msg= VerifyFile(messageStream,
PgpUtilities.GetDecoderStream(keyStream));
// verification passes msg is the content of the message
} catch (Exception e) {
// verification fails
}
And here is the implementation:
private static String VerifyFile(Stream inputStream, Stream keyIn)
{
ArmoredInputStream aIn = new ArmoredInputStream(inputStream);
MemoryStream outStr = new MemoryStream(); // File.Create(resultName);
//
// write out signed section using the local line separator.
// note: trailing white space needs to be removed from the end of
// each line RFC 4880 Section 7.1
//
MemoryStream lineOut = new MemoryStream();
int lookAhead = ReadInputLine(lineOut, aIn);
byte[] lineSep = LineSeparator;
if (lookAhead != -1 && aIn.IsClearText())
{
byte[] line = lineOut.ToArray();
outStr.Write(line, 0, GetLengthWithoutSeparatorOrTrailingWhitespace(line));
outStr.Write(lineSep, 0, lineSep.Length);
while (lookAhead != -1 && aIn.IsClearText())
{
lookAhead = ReadInputLine(lineOut, lookAhead, aIn);
line = lineOut.ToArray();
outStr.Write(line, 0, GetLengthWithoutSeparatorOrTrailingWhitespace(line));
outStr.Write(lineSep, 0, lineSep.Length);
}
}
else
{
// a single line file
if (lookAhead != -1)
{
byte[] line = lineOut.ToArray();
outStr.Write(line, 0, GetLengthWithoutSeparatorOrTrailingWhitespace(line));
outStr.Write(lineSep, 0, lineSep.Length);
}
}
outStr.Flush();
//outStr.Close();
PgpPublicKeyRingBundle pgpRings = new PgpPublicKeyRingBundle(keyIn);
PgpObjectFactory pgpFact = new PgpObjectFactory(aIn);
PgpSignatureList p3 = (PgpSignatureList)pgpFact.NextPgpObject();
PgpSignature sig = p3[0];
var key = pgpRings.GetPublicKey(sig.KeyId);
if (key == null)
{
throw new Exception("Can't verify the message signature.");
}
sig.InitVerify(key);
//
// read the input, making sure we ignore the last newline.
//
outStr.Seek(0, SeekOrigin.Begin);
StreamReader reader = new StreamReader(outStr);
string messageContent = reader.ReadToEnd();
outStr.Seek(0, SeekOrigin.Begin);
Stream sigIn = outStr; //File.OpenRead(resultName);
lookAhead = ReadInputLine(lineOut, sigIn);
ProcessLine(sig, lineOut.ToArray());
if (lookAhead != -1)
{
do
{
lookAhead = ReadInputLine(lineOut, lookAhead, sigIn);
sig.Update((byte)'\r');
sig.Update((byte)'\n');
ProcessLine(sig, lineOut.ToArray());
}
while (lookAhead != -1);
}
sigIn.Close();
if (sig.Verify()) {
// signature verified
return messageContent;
} else {
// signature verification failed
throw new Exception("Can't verify the message signature.");
}
}
private static int ReadInputLine(
MemoryStream bOut,
Stream fIn)
{
bOut.SetLength(0);
int lookAhead = -1;
int ch;
while ((ch = fIn.ReadByte()) >= 0)
{
bOut.WriteByte((byte)ch);
if (ch == '\r' || ch == '\n')
{
lookAhead = ReadPassedEol(bOut, ch, fIn);
break;
}
}
return lookAhead;
}
private static int ReadPassedEol(
MemoryStream bOut,
int lastCh,
Stream fIn)
{
int lookAhead = fIn.ReadByte();
if (lastCh == '\r' && lookAhead == '\n')
{
bOut.WriteByte((byte)lookAhead);
lookAhead = fIn.ReadByte();
}
return lookAhead;
}
private static void ProcessLine(
PgpSignature sig,
byte[] line)
{
// note: trailing white space needs to be removed from the end of
// each line for signature calculation RFC 4880 Section 7.1
int length = GetLengthWithoutWhiteSpace(line);
if (length > 0)
{
sig.Update(line, 0, length);
}
}
private static void ProcessLine(
Stream aOut,
PgpSignatureGenerator sGen,
byte[] line)
{
int length = GetLengthWithoutWhiteSpace(line);
if (length > 0)
{
sGen.Update(line, 0, length);
}
aOut.Write(line, 0, line.Length);
}
private static int GetLengthWithoutSeparatorOrTrailingWhitespace(byte[] line)
{
int end = line.Length - 1;
while (end >= 0 && IsWhiteSpace(line[end]))
{
end--;
}
return end + 1;
}
private static bool IsLineEnding(
byte b)
{
return b == '\r' || b == '\n';
}
private static int GetLengthWithoutWhiteSpace(
byte[] line)
{
int end = line.Length - 1;
while (end >= 0 && IsWhiteSpace(line[end]))
{
end--;
}
return end + 1;
}
private static bool IsWhiteSpace(
byte b)
{
return IsLineEnding(b) || b == '\t' || b == ' ';
}
private static int ReadInputLine(
MemoryStream bOut,
int lookAhead,
Stream fIn)
{
bOut.SetLength(0);
int ch = lookAhead;
do
{
bOut.WriteByte((byte)ch);
if (ch == '\r' || ch == '\n')
{
lookAhead = ReadPassedEol(bOut, ch, fIn);
break;
}
}
while ((ch = fIn.ReadByte()) >= 0);
if (ch < 0)
{
lookAhead = -1;
}
return lookAhead;
}
private static byte[] LineSeparator
{
get { return Encoding.ASCII.GetBytes(Environment.NewLine); }
}
using System;
using System.Collections;
using System.IO;
using Org.BouncyCastle.Bcpg.OpenPgp;
namespace Org.BouncyCastle.Bcpg.OpenPgp.Examples
{
/**
* A simple utility class that signs and verifies files.
* <p>
* To sign a file: SignedFileProcessor -s [-a] fileName secretKey passPhrase.<br/>
* If -a is specified the output file will be "ascii-armored".</p>
* <p>
* To decrypt: SignedFileProcessor -v fileName publicKeyFile.</p>
* <p>
* <b>Note</b>: this example will silently overwrite files, nor does it pay any attention to
* the specification of "_CONSOLE" in the filename. It also expects that a single pass phrase
* will have been used.</p>
* <p>
* <b>Note</b>: the example also makes use of PGP compression. If you are having difficulty Getting it
* to interoperate with other PGP programs try removing the use of compression first.</p>
*/
public sealed class SignedFileProcessor
{
private SignedFileProcessor() {}
/**
* verify the passed in file as being correctly signed.
*/
private static void VerifyFile(
Stream inputStream,
Stream keyIn)
{
inputStream = PgpUtilities.GetDecoderStream(inputStream);
PgpObjectFactory pgpFact = new PgpObjectFactory(inputStream);
PgpCompressedData c1 = (PgpCompressedData) pgpFact.NextPgpObject();
pgpFact = new PgpObjectFactory(c1.GetDataStream());
PgpOnePassSignatureList p1 = (PgpOnePassSignatureList) pgpFact.NextPgpObject();
PgpOnePassSignature ops = p1[0];
PgpLiteralData p2 = (PgpLiteralData) pgpFact.NextPgpObject();
Stream dIn = p2.GetInputStream();
PgpPublicKeyRingBundle pgpRing = new PgpPublicKeyRingBundle(PgpUtilities.GetDecoderStream(keyIn));
PgpPublicKey key = pgpRing.GetPublicKey(ops.KeyId);
Stream fos = File.Create(p2.FileName);
ops.InitVerify(key);
int ch;
while ((ch = dIn.ReadByte()) >= 0)
{
ops.Update((byte)ch);
fos.WriteByte((byte) ch);
}
fos.Close();
PgpSignatureList p3 = (PgpSignatureList)pgpFact.NextPgpObject();
PgpSignature firstSig = p3[0];
if (ops.Verify(firstSig))
{
Console.Out.WriteLine("signature verified.");
}
else
{
Console.Out.WriteLine("signature verification failed.");
}
}
/**
* Generate an encapsulated signed file.
*
* #param fileName
* #param keyIn
* #param outputStream
* #param pass
* #param armor
*/
private static void SignFile(
string fileName,
Stream keyIn,
Stream outputStream,
char[] pass,
bool armor,
bool compress)
{
if (armor)
{
outputStream = new ArmoredOutputStream(outputStream);
}
PgpSecretKey pgpSec = PgpExampleUtilities.ReadSecretKey(keyIn);
PgpPrivateKey pgpPrivKey = pgpSec.ExtractPrivateKey(pass);
PgpSignatureGenerator sGen = new PgpSignatureGenerator(pgpSec.PublicKey.Algorithm, HashAlgorithmTag.Sha1);
sGen.InitSign(PgpSignature.BinaryDocument, pgpPrivKey);
foreach (string userId in pgpSec.PublicKey.GetUserIds())
{
PgpSignatureSubpacketGenerator spGen = new PgpSignatureSubpacketGenerator();
spGen.SetSignerUserId(false, userId);
sGen.SetHashedSubpackets(spGen.Generate());
// Just the first one!
break;
}
Stream cOut = outputStream;
PgpCompressedDataGenerator cGen = null;
if (compress)
{
cGen = new PgpCompressedDataGenerator(CompressionAlgorithmTag.ZLib);
cOut = cGen.Open(cOut);
}
BcpgOutputStream bOut = new BcpgOutputStream(cOut);
sGen.GenerateOnePassVersion(false).Encode(bOut);
FileInfo file = new FileInfo(fileName);
PgpLiteralDataGenerator lGen = new PgpLiteralDataGenerator();
Stream lOut = lGen.Open(bOut, PgpLiteralData.Binary, file);
FileStream fIn = file.OpenRead();
int ch = 0;
while ((ch = fIn.ReadByte()) >= 0)
{
lOut.WriteByte((byte) ch);
sGen.Update((byte)ch);
}
fIn.Close();
lGen.Close();
sGen.Generate().Encode(bOut);
if (cGen != null)
{
cGen.Close();
}
if (armor)
{
outputStream.Close();
}
}
public static void Main(
string[] args)
{
// TODO provide command-line option to determine whether to use compression in SignFile
if (args[0].Equals("-s"))
{
Stream keyIn, fos;
if (args[1].Equals("-a"))
{
keyIn = File.OpenRead(args[3]);
fos = File.Create(args[2] + ".asc");
SignFile(args[2], keyIn, fos, args[4].ToCharArray(), true, true);
}
else
{
keyIn = File.OpenRead(args[2]);
fos = File.Create(args[1] + ".bpg");
SignFile(args[1], keyIn, fos, args[3].ToCharArray(), false, true);
}
keyIn.Close();
fos.Close();
}
else if (args[0].Equals("-v"))
{
using (Stream fis = File.OpenRead(args[1]),
keyIn = File.OpenRead(args[2]))
{
VerifyFile(fis, keyIn);
}
}
else
{
Console.Error.WriteLine("usage: SignedFileProcessor -v|-s [-a] file keyfile [passPhrase]");
}
}
}
}
What is the best method to replace sequence of bytes in binary file to the same length of other bytes? The binary files will be pretty large, about 50 mb and should not be loaded at once in memory.
Update: I do not know location of bytes which needs to be replaced, I need to find them first.
Assuming you're trying to replace a known section of the file.
Open a FileStream with read/write access
Seek to the right place
Overwrite existing data
Sample code coming...
public static void ReplaceData(string filename, int position, byte[] data)
{
using (Stream stream = File.Open(filename, FileMode.Open))
{
stream.Position = position;
stream.Write(data, 0, data.Length);
}
}
If you're effectively trying to do a binary version of a string.Replace (e.g. "always replace bytes { 51, 20, 34} with { 20, 35, 15 } then it's rather harder. As a quick description of what you'd do:
Allocate a buffer of at least the size of data you're interested in
Repeatedly read into the buffer, scanning for the data
If you find a match, seek back to the right place (e.g. stream.Position -= buffer.Length - indexWithinBuffer; and overwrite the data
Sounds simple so far... but the tricky bit is if the data starts near the end of the buffer. You need to remember all potential matches and how far you've matched so far, so that if you get a match when you read the next buffer's-worth, you can detect it.
There are probably ways of avoiding this trickiness, but I wouldn't like to try to come up with them offhand :)
EDIT: Okay, I've got an idea which might help...
Keep a buffer which is at least twice as big as you need
Repeatedly:
Copy the second half of the buffer into the first half
Fill the second half of the buffer from the file
Search throughout the whole buffer for the data you're looking for
That way at some point, if the data is present, it will be completely within the buffer.
You'd need to be careful about where the stream was in order to get back to the right place, but I think this should work. It would be trickier if you were trying to find all matches, but at least the first match should be reasonably simple...
My solution :
/// <summary>
/// Copy data from a file to an other, replacing search term, ignoring case.
/// </summary>
/// <param name="originalFile"></param>
/// <param name="outputFile"></param>
/// <param name="searchTerm"></param>
/// <param name="replaceTerm"></param>
private static void ReplaceTextInBinaryFile(string originalFile, string outputFile, string searchTerm, string replaceTerm)
{
byte b;
//UpperCase bytes to search
byte[] searchBytes = Encoding.UTF8.GetBytes(searchTerm.ToUpper());
//LowerCase bytes to search
byte[] searchBytesLower = Encoding.UTF8.GetBytes(searchTerm.ToLower());
//Temporary bytes during found loop
byte[] bytesToAdd = new byte[searchBytes.Length];
//Search length
int searchBytesLength = searchBytes.Length;
//First Upper char
byte searchByte0 = searchBytes[0];
//First Lower char
byte searchByte0Lower = searchBytesLower[0];
//Replace with bytes
byte[] replaceBytes = Encoding.UTF8.GetBytes(replaceTerm);
int counter = 0;
using (FileStream inputStream = File.OpenRead(originalFile)) {
//input length
long srcLength = inputStream.Length;
using (BinaryReader inputReader = new BinaryReader(inputStream)) {
using (FileStream outputStream = File.OpenWrite(outputFile)) {
using (BinaryWriter outputWriter = new BinaryWriter(outputStream)) {
for (int nSrc = 0; nSrc < srcLength; ++nSrc)
//first byte
if ((b = inputReader.ReadByte()) == searchByte0
|| b == searchByte0Lower) {
bytesToAdd[0] = b;
int nSearch = 1;
//next bytes
for (; nSearch < searchBytesLength; ++nSearch)
//get byte, save it and test
if ((b = bytesToAdd[nSearch] = inputReader.ReadByte()) != searchBytes[nSearch]
&& b != searchBytesLower[nSearch]) {
break;//fail
}
//Avoid overflow. No need, in my case, because no chance to see searchTerm at the end.
//else if (nSrc + nSearch >= srcLength)
// break;
if (nSearch == searchBytesLength) {
//success
++counter;
outputWriter.Write(replaceBytes);
nSrc += nSearch - 1;
}
else {
//failed, add saved bytes
outputWriter.Write(bytesToAdd, 0, nSearch + 1);
nSrc += nSearch;
}
}
else
outputWriter.Write(b);
}
}
}
}
Console.WriteLine("ReplaceTextInBinaryFile.counter = " + counter);
}
You can use my BinaryUtility to search and replace one or more bytes without loading the entire file into memory like this:
var searchAndReplace = new List<Tuple<byte[], byte[]>>()
{
Tuple.Create(
BitConverter.GetBytes((UInt32)0xDEADBEEF),
BitConverter.GetBytes((UInt32)0x01234567)),
Tuple.Create(
BitConverter.GetBytes((UInt32)0xAABBCCDD),
BitConverter.GetBytes((UInt16)0xAFFE)),
};
using(var reader =
new BinaryReader(new FileStream(#"C:\temp\data.bin", FileMode.Open)))
{
using(var writer =
new BinaryWriter(new FileStream(#"C:\temp\result.bin", FileMode.Create)))
{
BinaryUtility.Replace(reader, writer, searchAndReplace);
}
}
BinaryUtility code:
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
public static class BinaryUtility
{
public static IEnumerable<byte> GetByteStream(BinaryReader reader)
{
const int bufferSize = 1024;
byte[] buffer;
do
{
buffer = reader.ReadBytes(bufferSize);
foreach (var d in buffer) { yield return d; }
} while (bufferSize == buffer.Length);
}
public static void Replace(BinaryReader reader, BinaryWriter writer, IEnumerable<Tuple<byte[], byte[]>> searchAndReplace)
{
foreach (byte d in Replace(GetByteStream(reader), searchAndReplace)) { writer.Write(d); }
}
public static IEnumerable<byte> Replace(IEnumerable<byte> source, IEnumerable<Tuple<byte[], byte[]>> searchAndReplace)
{
foreach (var s in searchAndReplace)
{
source = Replace(source, s.Item1, s.Item2);
}
return source;
}
public static IEnumerable<byte> Replace(IEnumerable<byte> input, IEnumerable<byte> from, IEnumerable<byte> to)
{
var fromEnumerator = from.GetEnumerator();
fromEnumerator.MoveNext();
int match = 0;
foreach (var data in input)
{
if (data == fromEnumerator.Current)
{
match++;
if (fromEnumerator.MoveNext()) { continue; }
foreach (byte d in to) { yield return d; }
match = 0;
fromEnumerator.Reset();
fromEnumerator.MoveNext();
continue;
}
if (0 != match)
{
foreach (byte d in from.Take(match)) { yield return d; }
match = 0;
fromEnumerator.Reset();
fromEnumerator.MoveNext();
}
yield return data;
}
if (0 != match)
{
foreach (byte d in from.Take(match)) { yield return d; }
}
}
}
public static void BinaryReplace(string sourceFile, byte[] sourceSeq, string targetFile, byte[] targetSeq)
{
FileStream sourceStream = File.OpenRead(sourceFile);
FileStream targetStream = File.Create(targetFile);
try
{
int b;
long foundSeqOffset = -1;
int searchByteCursor = 0;
while ((b=sourceStream.ReadByte()) != -1)
{
if (sourceSeq[searchByteCursor] == b)
{
if (searchByteCursor == sourceSeq.Length - 1)
{
targetStream.Write(targetSeq, 0, targetSeq.Length);
searchByteCursor = 0;
foundSeqOffset = -1;
}
else
{
if (searchByteCursor == 0)
{
foundSeqOffset = sourceStream.Position - 1;
}
++searchByteCursor;
}
}
else
{
if (searchByteCursor == 0)
{
targetStream.WriteByte((byte) b);
}
else
{
targetStream.WriteByte(sourceSeq[0]);
sourceStream.Position = foundSeqOffset + 1;
searchByteCursor = 0;
foundSeqOffset = -1;
}
}
}
}
finally
{
sourceStream.Dispose();
targetStream.Dispose();
}
}