I have an IRandomAccessStream to store data on WindowsPhone 8.1 and I want to process the data in a certain BLOCKSIZE.
Here is the code
in the capture function:
await this.mediaCapture.StartRecordToStreamAsync(recordProfile, this.message);
in the stop function, when done I stop:
await this.mediaCapture.StopRecordAsync();
using (var dataReader = new DataReader(message.GetInputStreamAt(0)))
{
dataReader.UnicodeEncoding = Windows.Storage.Streams.UnicodeEncoding.Utf8;
dataReader.ByteOrder = Windows.Storage.Streams.ByteOrder.LittleEndian;
// await dataReader.LoadAsync((uint)message.Size);
await dataReader.LoadAsync(((uint)message.Size + (int)FREQUENCY) * 10);
byte[] buffer = new byte[((int)message.Size + (int)FREQUENCY) * 10];
dataReader.ReadBytes(buffer);
short[] signal = new short[BLOCKSIZE];
int bufferReadResult = 0;
while (bufferReadResult != buffer.Length)
{
for (int index = 0; index < BLOCKSIZE; index += 2, bufferReadResult++)
{
signal[bufferReadResult] = BitConverter.ToInt16(buffer, index);
}
// .....process the signal[] in BLOCKSIZE,
// keep processing BLOCKSIZE until end of buffer
// so process [BLOCKSIZE][BLOCKSIZE]..............buffer end
}
}
The problem is that when bufferReadResult reaches BLOCKSIZE that's it. It won't continue with a new block unit end of buffer.
What is the best way to do this? I have a buffer or IRandomAccessStream and I want to process the all the data in chunks of short[BLOCKSIZE].
Related
I'm trying to determine how to best implement .Net Core 3 Channels and whether it's a good idea to pass very large objects between tasks. In my example, one task that is very fast can read in a 1GB chunk from a very large file. A number of consumer tasks can read a chunk from the channel and process them in parallel, as processing is much slower and needs parallel (multi-threaded) execution.
In testing my code, there is a massive amount of GC happening and total RAM used far exceeds the sum of all data waiting in one bounded channel and all executing tasks. I've simplified my code down to the most basic example hoping someone can give me some tips on how to better allocate/manage memory or if this approach is a good idea?
using System;
using System.IO;
using System.Threading.Channels;
using System.Threading.Tasks;
namespace MergeSort
{
public class Example
{
private Channel<byte[]> _channelProcessing;
public async Task DoSort(int queueDepth, int parallelTaskCount)
{
// Hard-code some values so we can talk about details
queueDepth = 2;
parallelTasks = 8;
_channelProcessing = Channel.CreateBounded<byte[]>(queueDepth);
Task[] processingTasks = new Task[parallelTaskCount];
int outputBufferSize = 1024 * 1024;
for (int x = 0; x < parallelTaskCount; x++)
{
string outputFile = $"C:\\Output.{x:00000000}.txt";
processingTasks[x] = Task.Run(() => ProcessChunkAsync(outputBufferSize));
}
// Task put unsorted chunks on the channel
string inputFile = "C:\\Input.txt";
int chunkSize = 1024 * 1024 * 1024; // 1GiB
Task inputTask = Task.Run(() => ReadInputAsync(inputFile, chunkSize));
// Wait for all tasks building chunk files to complete before continuing
await inputTask;
await Task.WhenAll(processingTasks);
}
private async Task ReadInputAsync(string inputFile, int chunkSize)
{
int bytesRead = 0;
byte[] chunkBuffer = new byte[chunkSize];
using (FileStream fileStream = File.Open(inputFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
// Read chunks until input EOF
while (fileStream.Position != fileStream.Length)
{
bytesRead = fileStream.Read(chunkBuffer, 0, chunkBuffer.Length);
// Fake code him to simulate the work I need to do showing outBuffer.Length is calculated at runtime
Random rnd = new Random();
int runtimeCalculatedAmount = rnd.Next(100, 600);
byte[] tempBuffer = new byte[runtimeCalculatedAmount];
// Create the buffer with a slightly variable size that needs to be passed to the channel for next task
byte[] outBuffer = new byte[1024 * 1024 * 1024 + runtimeCalculatedAmount];
Array.Copy(chunkBuffer, outBuffer, bytesRead);
Array.Copy(tempBuffer, 0, outBuffer, bytesRead, outBuffer.Length);
await _channelProcessing.Writer.WriteAsync(outBuffer);
outBuffer = null;
}
}
// Not sure if it's safe to .Complete() before consumers have read all data from channel?
_channelProcessing.Writer.Complete();
}
private async Task ProcessChunkAsync(int outputBufferSize)
{
while (await _channelProcessing.Reader.WaitToReadAsync())
{
if (_channelProcessing.Reader.TryRead(out byte[] inBuffer))
{
// myBigThing is also a very large object (result of processing inBuffer and slightly larger)
MyBigThing myBigThing = new MyBigThing(inBuffer);
inBuffer = null;
// Create file and write all rows
using (FileStream fileStream = File.Create("C:\\Output.txt", outputBufferSize, FileOptions.SequentialScan))
{
// Write myBigThing to output file
fileStream.Write(myBigThing.Data);
}
myBigThing = null;
}
}
}
}
}
I have a problem turning a RandomAccessStream into a float array. The float array contains values that are NaN. I can't tell if they come from the stream, the byte array or the float array. Performance & quality are important in this so if there is a better way to do this let me know.
Last count I was getting 122 NaN's.
thanks
private async void button_Click(object sender, RoutedEventArgs e)
{
string text = "this is text";
SpeechSynthesizer synthesizer = new SpeechSynthesizer();
SpeechSynthesisStream synthesisStream = await synthesizer.SynthesizeTextToStreamAsync(text);
Stopwatch watch = new Stopwatch();
watch.Start();
ProcessStream(synthesisStream.CloneStream());
watch.Stop();
// Performance is important
Debug.WriteLine(watch.Elapsed);
}
private async void ProcessStream(IRandomAccessStream stream)
{
// Create a buffer (somewhere to put the stream)
byte[] bytes = new byte[stream.Size];
// Add stream data to buffer (Following or After that) same result
// IBuffer x = await stream.ReadAsync(bytes.AsBuffer(), (uint)stream.Size, InputStreamOptions.None);
using (DataReader reader = new DataReader(stream))
{
await reader.LoadAsync((uint)stream.Size);
reader.ReadBytes(bytes);
}
// Change buffer(in bytes) to a float array
float[] floatArray = MainPage.ConvertByteToFloat(bytes.ToArray());
int nanCount = 0;
for (var index = 0; index < floatArray.Length; index++)
{
float value = floatArray[index];
if (float.IsNaN(value))
{
nanCount++;
}
}
Debug.WriteLine("Nan count: " + nanCount);
}
public static float[] ConvertByteToFloat(byte[] array)
{
float[] floatArr = new float[array.Length / 4];
for (int i = 0; i < floatArr.Length; i++)
{
if (BitConverter.IsLittleEndian)
{
Array.Reverse(array, i * 4, 4);
}
floatArr[i] = BitConverter.ToSingle(array, i * 4);
}
return floatArr;
}
Found the answer At this SO post
Basically I did not know that the 32 bit wav format stored its data in a 16 bit format.
I'm writing a multiple clients per server application in C# using async sockets. Each client on connection sends 10 items. The problem is, when starting lots of clients quickly, it appears that each client sent less than the 10 items and sometimes it sends nothing, the server just logs them connecting only.
The packet structure is the first 4 bytes are an int with size of data in them. Here's a part of server code. Each connected client has its own receive buffer which BeginReceive should write to.
private void Recieve(IAsyncResult iar) //Called when socket receives something.
{
Socket server_conn = (Socket)iar.AsyncState;
if (!SocketConnected(server_conn))
{
server_conn.Close();
logthis("Client Disconnected");
return;
}
int n = server_conn.EndReceive(iar); //Stop Receiving and parse data, n is number of bytes received
ClientData asdf = null;
foreach (ClientData cls in clientlist)
{
if (server_conn.RemoteEndPoint == cls.clientsock.RemoteEndPoint) //Who sent this data
{
asdf = cls; //cls is who sent this data
//Start a new thread and pass received bytes to it in order to be parsed
var t = new Thread(() => parse(cls, n,cls.recvbuffer));
t.Start();
Thread.Sleep(100);
break;
}
}
asdf.recvbuffer = new byte[1024]; //Clear buffer of client
server_conn.BeginReceive(asdf.recvbuffer, 0, asdf.recvbuffer.Length, SocketFlags.None, new AsyncCallback(Recieve), server_conn); //Start receiving again
}
private void parse(ClientData theclient, int nobytesreceived, byte[] bytesreceived)
{
ClientData cls = theclient;
int n = nobytesreceived;
byte[] receivedbytes = bytesreceived;
lock(s)
{
if (!cls.dataphase) //If there's no fragmented packets still waiting to be read
{
cls.dataphase = true;
byte[] sizeinbytes = new byte[4];
for (int i = 0; i < 4; i++)
{
sizeinbytes[i] = receivedbytes[i];
}
int size = BitConverter.ToInt32(sizeinbytes, 0); //Read first four bytes of packet to get size of data
if ((n - 4) == size) //If received number of bytes - 4 is equals to datasize
{
byte[] payload = new byte[size];
Array.Copy(receivedbytes, 4, payload, 0, (n - 4)); //Copy data to separately to payload array to be displayed to user
logthis(cls.clientsock.RemoteEndPoint.ToString());
logthis(Encoding.ASCII.GetString(payload));
cls.dataphase = false; //packet read successfully
}
else if ((n - 4) < size) //If received amount of bytes is less than data size (fragmented data)
{
cls.data = new byte[size];
for (int i = 4; i <= n - 4; i++)
{
cls.data[i - 4] += receivedbytes[i];
}
cls.datasize = size; //And cls.dataphase will remain true so it can be handled correctly the next time we receive something from same client
}
else if((n-4) > size) //If received amount of bytes is bigger than data (lumped packets)
{
byte[] payload = new byte[size];
byte[] otherpacket = new byte[(n - 4) - size];
for(int i = 0; i < size; i++)
{
payload[i] += receivedbytes[i + 4];
}
logthis(cls.clientsock.RemoteEndPoint.ToString());
logthis(Encoding.ASCII.GetString(payload));
Array.Copy(receivedbytes, (size + 4), otherpacket, 0, ((n - 4) - size));
receivedbytes = new byte[(n - 4) - size];
receivedbytes = otherpacket;
cls.dataphase = false;
parse(cls, ((n - 4) - size), receivedbytes); //Send rest of packet to read again
}
}
else
{
//not implemented, supposed to handle fragmented packets
if (n >= cls.datasize)
{
}
else if (n < cls.datasize)
{
}
}
}
}
Your problem comes from
else
{
//not implemented, supposed to handle fragmented packets
I would put money on the fact that you are hitting that else statement and loosing data. As soon as you don't have a complete read of a packet or two packets lumped together returned from your read function (which is a lot more common than you think) your client is now stuck in cls.dataphase = true; and will never get out of it.
My goal is to have a file stream open up a user-chosen file, then, it should stream the files bytes through in chunks (buffers) of about 4mb (this can be changed it's just for fun). As the bytes travel (in chunks) through the stream, I'd like to have a looping if-statement see if the bytes value is contained in an array I have declared elsewhere. (The code below will build a random array for replacing bytes), and the replacement loop could just say something like the bottom for-loop. As you can see I'm fairly fluent in this language but for some reason the editing and rewriting of chunks as they are read from a file to a new one is eluding me. Thanks in advance!
private void button2_Click(object sender, EventArgs e)
{
GenNewKey();
const int chunkSize = 4096; // read the file by chunks of 4KB
using (var file = File.OpenRead(textBox1.Text))
{
int bytesRead;
var buffer = new byte[chunkSize];
while ((bytesRead = file.Read(buffer, 0, buffer.Length)) > 0)
{
byte[] newbytes = buffer;
int index = 0;
foreach (byte b in buffer)
{
for (int x = 0; x < 256; x++)
{
if (buffer[index] == Convert.ToByte(lst[x]))
{
try
{
newbytes[index] = Convert.ToByte(lst[256 - x]);
}
catch (System.Exception ex)
{
//just to show why the error was thrown, but not really helpful..
MessageBox.Show(index + ", " + newbytes.Count().ToString());
}
}
}
index++;
}
AppendAllBytes(textBox1.Text + ".ENC", newbytes);
}
}
}
private void GenNewKey()
{
Random rnd = new Random();
while (lst.Count < 256)
{
int x = rnd.Next(0, 255);
if (!lst.Contains(x))
{
lst.Add(x);
}
}
foreach (int x in lst)
{
textBox2.Text += ", " + x.ToString();
//just for me to see what was generated
}
}
public static void AppendAllBytes(string path, byte[] bytes)
{
if (!File.Exists(path + ".ENC"))
{
File.Create(path + ".ENC");
}
using (var stream = new FileStream(path, FileMode.Append))
{
stream.Write(bytes, 0, bytes.Length);
}
}
Where textbox1 holds the path and name of file to encrypt, textBox2 holds the generated cipher for personal debugging purposes, button two is the encrypt button, and of course I am using System.IO.
Indeed you have a off by one error in newbytes[index] = Convert.ToByte(lst[256 - x])
if x is 0 then you will have lst[256], however lst only goes between 0-255. Change that to 255 should fix it.
The reason it freezes up is your program is EXTREMELY inefficient and working on the UI thread (and has a few more errors like you should only go up to bytesRead in size when processing buffer, but that will just give you extra data in your output that should not be there. Also you are reusing the same array for buffer and newbytes so your inner for loop could modify the same index more than once because every time you do newbytes[index] = Convert.ToByte(lst[256 - x]) you are modifying buffer[index] which will get checked again the next itteration of the for loop).
There is a lot of ways you can improve your code, here is a snippet that does similar to what you are doing (I don't do the whole "find the index and use the opposite location", I just use the byte that is passed in as the index in the array).
while ((bytesRead = file.Read(buffer, 0, buffer.Length)) > 0)
{
byte[] newbytes = new byte[bytesRead];
for(int i = 0; i < newbytes.Length; i++)
{
newbytes[i] = (byte)lst[buffer[i]]))
}
AppendAllBytes(textBox1.Text + ".ENC", newbytes);
}
This may also lead to freezing but not as much, to solve the freeing you should put all of this code in to a BackgroundWorker or similar to run on another thread.
(apologies if this is a duplicate ... i posted but saw no evidence that it actually made it to the forum)
I've been trying to get SlimDX DirectSound working. Here's the code I have. It fills the secondary buffer from a wav file and then, in a thread loop, alternately fills the lower or upper halves of the buffer.
It plays the first load of the buffer fine. The AutoResetEvents fire when they should and the lower half then upper half of the buffer are populated (verified with Debug statements). But playing does not continue after the first load of the buffer. So somehow the repopulation of the buffer doesn't work as it should.
Ideas?
(I'm using DirectSound because it's the only way I've found to set the guid of the audio device that I want to use. Am open to other .NET-friendly approaches.)
private void PlaySound(Guid soundCardGuid, string audioFile) {
DirectSound ds = new DirectSound(soundCardGuid);
ds.SetCooperativeLevel(this.Handle, CooperativeLevel.Priority);
WaveFormat format = new WaveFormat();
format.BitsPerSample = 16;
format.BlockAlignment = 4;
format.Channels = 2;
format.FormatTag = WaveFormatTag.Pcm;
format.SamplesPerSecond = 44100;
format.AverageBytesPerSecond = format.SamplesPerSecond * format.BlockAlignment;
SoundBufferDescription desc = new SoundBufferDescription();
desc.Format = format;
desc.Flags = BufferFlags.GlobalFocus;
desc.SizeInBytes = 8 * format.AverageBytesPerSecond;
PrimarySoundBuffer pBuffer = new PrimarySoundBuffer(ds, desc);
SoundBufferDescription desc2 = new SoundBufferDescription();
desc2.Format = format;
desc2.Flags = BufferFlags.GlobalFocus | BufferFlags.ControlPositionNotify | BufferFlags.GetCurrentPosition2;
desc2.SizeInBytes = 8 * format.AverageBytesPerSecond;
SecondarySoundBuffer sBuffer1 = new SecondarySoundBuffer(ds, desc2);
NotificationPosition[] notifications = new NotificationPosition[2];
notifications[0].Offset = desc2.SizeInBytes / 2 + 1;
notifications[1].Offset = desc2.SizeInBytes - 1; ;
notifications[0].Event = new AutoResetEvent(false);
notifications[1].Event = new AutoResetEvent(false);
sBuffer1.SetNotificationPositions(notifications);
byte[] bytes1 = new byte[desc2.SizeInBytes / 2];
byte[] bytes2 = new byte[desc2.SizeInBytes];
Stream stream = File.Open(audioFile, FileMode.Open);
Thread fillBuffer = new Thread(() => {
int readNumber = 1;
int bytesRead;
bytesRead = stream.Read(bytes2, 0, desc2.SizeInBytes);
sBuffer1.Write<byte>(bytes2, 0, LockFlags.None);
sBuffer1.Play(0, PlayFlags.None);
while (true) {
if (bytesRead == 0) { break; }
notifications[0].Event.WaitOne();
bytesRead = stream.Read(bytes1, 0, bytes1.Length);
sBuffer1.Write<byte>(bytes1, 0, LockFlags.None);
if (bytesRead == 0) { break; }
notifications[1].Event.WaitOne();
bytesRead = stream.Read(bytes1, 0, bytes1.Length);
sBuffer1.Write<byte>(bytes1, desc2.SizeInBytes / 2, LockFlags.None);
}
stream.Close();
stream.Dispose();
});
fillBuffer.Start();
}
}
You haven't set it to loop on the play buffer. Change your code to:
sBuffer1.Play(0, PlayFlags.Looping);