Develop Reference

Transfer files directly from FTP to Azure File Storage without keeping them locally in memory or disk

I have to transfer files from FTP to an Azure File Storage. My code works fine, but I'm transferring those files in memory which is not a best practice. So first I read the stream to an Byte array in memory. Then I upload the output to an Azure file storage.
Now I know it's better to do this asynchronicaly. But I don't know if this is possible and how to do it.
My code:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.WindowsAzure.Storage;
using System.Configuration;
using Microsoft.WindowsAzure.Storage.File;
using System.IO;
using Microsoft.Azure;
using System.Net;
namespace TransferFtpToAzure
{
class Program
{
public static void Main(string[] args)
{
List<FileName> sourceFileList = new List<FileName>();
List<FileName> targetFileList = new List<FileName>();
string targetShareReference = ConfigurationManager.AppSettings["AzureShare"];
string targetDirectoryReference = ConfigurationManager.AppSettings["Environment"] + "/" + Enums.AzureFolders.Mos + "/" + Enums.AzureFolders.In;
string sourceURI = (ConfigurationManager.AppSettings["FtpConnectionString"] + ConfigurationManager.AppSettings["Environment"].ToUpper() +"/"+ Enums.FtpFolders.Mos + "/").Replace("\\","/");
string sourceUser = ConfigurationManager.AppSettings["FtpServerUserName"];
string sourcePass = ConfigurationManager.AppSettings["FtpServerPassword"];
getFileLists(sourceURI, sourceUser, sourcePass, sourceFileList, targetShareReference, targetDirectoryReference, targetFileList);
Console.WriteLine(sourceFileList.Count + " files found!");
CheckLists(sourceFileList, targetFileList);
targetFileList.Sort();
Console.WriteLine(sourceFileList.Count + " unique files on sourceURI" + Environment.NewLine + "Attempting to move them.");
foreach (var file in sourceFileList)
{
try
{
CopyFile(file.fName, sourceURI, sourceUser, sourcePass, targetShareReference, targetDirectoryReference);
}
catch
{
Console.WriteLine("There was move error with : " + file.fName);
}
}
}
public class FileName : IComparable<FileName>
{
public string fName { get; set; }
public int CompareTo(FileName other)
{
return fName.CompareTo(other.fName);
}
}
public static void CheckLists(List<FileName> sourceFileList, List<FileName> targetFileList)
{
for (int i = 0; i < sourceFileList.Count; i++)
{
if (targetFileList.BinarySearch(sourceFileList[i]) > 0)
{
sourceFileList.RemoveAt(i);
i--;
}
}
}
public static void getFileLists(string sourceURI, string sourceUser, string sourcePass, List<FileName> sourceFileList, string targetShareReference, string targetDirectoryReference, List<FileName> targetFileList)
{
string line = "";
/////////Source FileList
FtpWebRequest sourceRequest;
sourceRequest = (FtpWebRequest)WebRequest.Create(sourceURI);
sourceRequest.Credentials = new NetworkCredential(sourceUser, sourcePass);
sourceRequest.Method = WebRequestMethods.Ftp.ListDirectory;
sourceRequest.UseBinary = true;
sourceRequest.KeepAlive = false;
sourceRequest.Timeout = -1;
sourceRequest.UsePassive = true;
FtpWebResponse sourceRespone = (FtpWebResponse)sourceRequest.GetResponse();
//Creates a list(fileList) of the file names
using (Stream responseStream = sourceRespone.GetResponseStream())
{
using (StreamReader reader = new StreamReader(responseStream))
{
line = reader.ReadLine();
while (line != null)
{
var fileName = new FileName
{
fName = line
};
sourceFileList.Add(fileName);
line = reader.ReadLine();
}
}
}
/////////////Target FileList
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
//var test = fileClient.ListShares();
CloudFileShare fileShare = fileClient.GetShareReference(targetShareReference);
if (fileShare.Exists())
{
CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference();
if (rootDirectory.Exists())
{
CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(targetDirectoryReference);
if (customDirectory.Exists())
{
var fileCollection = customDirectory.ListFilesAndDirectories().OfType<CloudFile>();
foreach (var item in fileCollection)
{
var fileName = new FileName
{
fName = item.Name
};
targetFileList.Add(fileName);
}
}
}
}
}
public static void CopyFile(string fileName, string sourceURI, string sourceUser, string sourcePass, string targetShareReference, string targetDirectoryReference)
{
try
{
FtpWebRequest request = (FtpWebRequest)WebRequest.Create(sourceURI + fileName);
request.Method = WebRequestMethods.Ftp.DownloadFile;
request.Credentials = new NetworkCredential(sourceUser, sourcePass);
FtpWebResponse response = (FtpWebResponse)request.GetResponse();
Stream responseStream = response.GetResponseStream();
Upload(fileName, ToByteArray(responseStream), targetShareReference, targetDirectoryReference);
responseStream.Close();
}
catch
{
Console.WriteLine("There was an error with :" + fileName);
}
}
public static Byte[] ToByteArray(Stream stream)
{
MemoryStream ms = new MemoryStream();
byte[] chunk = new byte[4096];
int bytesRead;
while ((bytesRead = stream.Read(chunk, 0, chunk.Length)) > 0)
{
ms.Write(chunk, 0, bytesRead);
}
return ms.ToArray();
}
public static bool Upload(string FileName, byte[] Image, string targetShareReference, string targetDirectoryReference)
{
try
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
//var test = fileClient.ListShares();
CloudFileShare fileShare = fileClient.GetShareReference(targetShareReference);
if (fileShare.Exists())
{
CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference();
if (rootDirectory.Exists())
{
CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(targetDirectoryReference);
if (customDirectory.Exists())
{
var cloudFile = customDirectory.GetFileReference(FileName);
using (var stream = new MemoryStream(Image, writable: false))
{
cloudFile.UploadFromStream(stream);
}
}
}
}
return true;
}
catch
{
return false;
}
}
}
}

If I understand you correctly, you want to avoid storing the file in memory between the download and upload.
For that see:
Azure function to copy files from FTP to blob storage.

Using Azure Storage File Share this is the only way it worked for me without loading the entire ZIP into Memory. I tested with a 3GB ZIP File (with thousands of files or with a big file inside) and Memory/CPU was low and stable. I hope it helps!
var zipFiles = _directory.ListFilesAndDirectories()
.OfType<CloudFile>()
.Where(x => x.Name.ToLower().Contains(".zip"))
.ToList();
foreach (var zipFile in zipFiles)
{
using (var zipArchive = new ZipArchive(zipFile.OpenRead()))
{
foreach (var entry in zipArchive.Entries)
{
if (entry.Length > 0)
{
CloudFile extractedFile = _directory.GetFileReference(entry.Name);
using (var entryStream = entry.Open())
{
byte[] buffer = new byte[16 * 1024];
using (var ms = extractedFile.OpenWrite(entry.Length))
{
int read;
while ((read = entryStream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, read);
}
}
}
}
}
}
}

Extract audio from video using autogen ffmpeg C# in Unity

Hi I'm using ffmpeg autogen to extract audio from video in Unity, but when I following this code, the file write cannot write, it's 0Kb, so what's issue of this or someone have any examples for extract audio using this library, apologize for my English. This is github of library:
https://github.com/Ruslan-B/FFmpeg.AutoGen
unsafe void TestExtractAudio()
{
string inFile = Application.streamingAssetsPath + "/" + strFileName;
string outFile = Application.streamingAssetsPath + "/" + strFileNameAudio;
AVOutputFormat* outFormat = null;
AVFormatContext* inFormatContext = null;
AVFormatContext* outFormatContext = null;
AVPacket packet;
ffmpeg.av_register_all();
inFormatContext = ffmpeg.avformat_alloc_context();
outFormatContext = ffmpeg.avformat_alloc_context();
if (ffmpeg.avformat_open_input(&inFormatContext, inFile, null, null) < 0)
{
throw new ApplicationException("Could not open input file.");
}
if (ffmpeg.avformat_find_stream_info(inFormatContext, null) < 0)
{
throw new ApplicationException("Failed to retrieve input stream info.");
}
ffmpeg.avformat_alloc_output_context2(&outFormatContext, null, null, outFile);
if (outFormatContext == null)
{
throw new ApplicationException("Could not create output context");
}
outFormat = outFormatContext->oformat;
AVStream* inStream = inFormatContext->streams[1];
AVStream* outStream = ffmpeg.avformat_new_stream(outFormatContext, inStream->codec->codec);
if (outStream == null)
{
throw new ApplicationException("Failed to allocate output stream.");
}
if (ffmpeg.avcodec_copy_context(outStream->codec, inStream->codec) < 0)
{
throw new ApplicationException("Couldn't copy input stream codec context to output stream codec context");
}
outFormatContext->audio_codec_id = AVCodecID.AV_CODEC_ID_MP3;
int retcode = ffmpeg.avio_open(&outFormatContext->pb, outFile, ffmpeg.AVIO_FLAG_WRITE);
if (retcode < 0)
{
throw new ApplicationException("Couldn't open output file");
}
int returnCode = ffmpeg.avformat_write_header(outFormatContext, null);
if (returnCode < 0)
{
throw new ApplicationException("Error occurred opening output file.");
}
while (true)
{
if (ffmpeg.av_read_frame(inFormatContext, &packet) < 0)
{
break;
}
if (packet.stream_index == 1)
{
inStream = inFormatContext->streams[1];
outStream = outFormatContext->streams[0];
// TODO: Replicate log packet functionality to print out what's inside the packet.
packet.pts = ffmpeg.av_rescale_q_rnd(packet.pts, inStream->time_base, outStream->time_base,
AVRounding.AV_ROUND_NEAR_INF | AVRounding.AV_ROUND_PASS_MINMAX);
packet.dts = ffmpeg.av_rescale_q_rnd(packet.dts, inStream->time_base, outStream->time_base,
AVRounding.AV_ROUND_NEAR_INF | AVRounding.AV_ROUND_PASS_MINMAX);
packet.duration = ffmpeg.av_rescale_q(packet.duration, inStream->time_base, outStream->time_base);
int returncode = ffmpeg.av_interleaved_write_frame(outFormatContext, &packet);
}
ffmpeg.av_packet_unref(&packet);
}
ffmpeg.av_write_trailer(outFormatContext);
ffmpeg.avformat_close_input(&inFormatContext);
ffmpeg.avformat_free_context(outFormatContext);
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
}
the value returnCode return less than 0, so someone can fix this, thanks so much for that

The problem is here:
inStream = inFormatContext->streams[1];
outStream = outFormatContext->streams[0];
This code:
ffmpeg.av_interleaved_write_frame
does the following validation
static int check_packet(AVFormatContext *s, AVPacket *pkt)
{
if (pkt->stream_index < 0 || pkt->stream_index >= s->nb_streams) {
av_log(s, AV_LOG_ERROR, "Invalid packet stream index: %d\n"
}
you need to change the packet.stream_index from 1 to 0 before calling
int returncode = ffmpeg.av_interleaved_write_frame(outFormatContext, &packet);
See:
if (type == AVMediaType.AVMEDIA_TYPE_AUDIO){
inStream = inFormatContext->streams[1];
outStream = outFormatContext->streams[0];
// TODO: Replicate log packet functionality to print out what's inside the packet.
ffmpeg.av_packet_rescale_ts(&packet, inStream->time_base, outStream->time_base);
packet.stream_index = 0;
int returncode = ffmpeg.av_write_frame(outFormatContext, &packet);
}

How to read excel file data using memory stream?

I want to read Excel file from JSON data which I am sending from ARC, Can anyone help me to sorted out?
public bool ControlAttachment(AttachmentFile file)
{
try
{
if (file != null && file.File != null)
{
string xlsfile = file.File;
string [] xls = {"application/excel","application/vnd.msexcel","xls","xlsx","application/vnd.ms-excel",};
if (xls.ToList().Contains(file.FileType.Trim()))
{
file.FileType = ".xls";
byte[] contents = Convert.FromBase64String(xlsfile);
string LogFilePaths = ConfigurationManager.AppSettings["ExcelMapperPath"];
string fileName = file.FileName.Split('.')[0] + file.FileType;
string LogFile = HttpContext.Current.Server.MapPath(LogFilePaths + file.FileName.Split('.')[0] + file.FileType);
System.IO.File.WriteAllBytes(LogFile, contents);
if (!File.Exists(LogFile))
{
File.Create(LogFile).Dispose();
}
MemoryStream ms = new MemoryStream();
using (var fs = new FileStream(LogFile, FileMode.Open, FileAccess.Write))
{
ms.CopyTo(fs);
ms.Dispose();
}
}
}
return true;
}
catch
{
return false;
}
}

Unzip files in Windows Phone 8 - 'System.OutOfMemoryException'

I used the UnZipper class from this (How to unzip files in Windows Phone 8) post in my app for zips with images, but in some rare cases it gives me this error:
A first chance exception of type 'System.OutOfMemoryException' occurred in System.Windows.ni.dll System.OutOfMemoryException: Exception of type 'System.OutOfMemoryException' was thrown. at System.Windows.Application.GetResourceStreamInternal(StreamResourceInfo zipPackageStreamResourceInfo, Uri resourceUri) at System.Windows.Application.GetResourceStream(StreamResourceInfo zipPackageStreamResourceInfo, Uri uriResource) at ImperiaOnline.Plugins.UnZipper.GetFileStream(String filename) at ImperiaOnline.Plugins.IOHelpers.unzip(String zipFilePath, String zipDestinationPath)
The device has more then twice needed free memory. Can somebody help me with this. Here is my code:
public static void unzip(string zipFilePath,string zipDestinationPath) {
using (IsolatedStorageFile isolatedStorage = IsolatedStorageFile.GetUserStoreForApplication())
{
var dirNames = isolatedStorage.GetDirectoryNames(zipDestinationPath);
bool doesFolderExists = (dirNames.Length > 0) ? true : false;
if (!doesFolderExists)
{
Debug.WriteLine("Folder does not exists");
isolatedStorage.CreateDirectory(zipDestinationPath);
}
try
{
using (IsolatedStorageFileStream zipFile = isolatedStorage.OpenFile(zipFilePath, FileMode.Open, FileAccess.ReadWrite))
{
UnZipper unzip = new UnZipper(zipFile);
bool isModuleFolderDeleted = false;
foreach (string currentFileAndDirectory in unzip.FileNamesInZip())
{
string[] fileLocations = currentFileAndDirectory.Split('/');
string prefix = zipDestinationPath + '/';
int locationsCount = fileLocations.Length;
string fileName = fileLocations.Last();
string currentPath = prefix;
for (int i = 0; i < locationsCount - 1; i++)
{
dirNames = isolatedStorage.GetDirectoryNames(currentPath + fileLocations[i]);
doesFolderExists = (dirNames.Length > 0) ? true : false;
if (!doesFolderExists)
{
isolatedStorage.CreateDirectory(currentPath + fileLocations[i]);
if (i == 2)
{
isModuleFolderDeleted = true;
}
}
else if (i == 2 && !isModuleFolderDeleted)
{
Debug.WriteLine(currentPath + fileLocations[i] + " is deleted and recreated");
DeleteDirectoryRecursively(isolatedStorage, currentPath + fileLocations[i]);
isolatedStorage.CreateDirectory(currentPath + fileLocations[i]);
isModuleFolderDeleted = true;
}
currentPath += fileLocations[i] + '/';
}
var newFileStream = isolatedStorage.CreateFile(currentPath + fileName);
byte[] fileBytes = new byte[unzip.GetFileStream(currentFileAndDirectory).Length];
unzip.GetFileStream(currentFileAndDirectory).Read(fileBytes, 0, fileBytes.Length);
unzip.GetFileStream(currentFileAndDirectory).Close();
try
{
newFileStream.Write(fileBytes, 0, fileBytes.Length);
}
catch (Exception ex)
{
Debug.WriteLine("FILE WRITE EXCEPTION: " + ex);
newFileStream.Close();
newFileStream = null;
zipFile.Close();
unzip.Dispose();
}
newFileStream.Close();
newFileStream = null;
}
zipFile.Close();
unzip.Dispose();
}
}
catch (Exception ex)
{
Debug.WriteLine(ex);
}
isolatedStorage.DeleteFile(zipFilePath);
}
}
This error appears here:
var newFileStream = isolatedStorage.CreateFile(currentPath + fileName);
byte[] fileBytes = new byte[unzip.GetFileStream(currentFileAndDirectory).Length]; unzip.GetFileStream(currentFileAndDirectory).Read(fileBytes, 0, fileBytes.Length);
unzip.GetFileStream(currentFileAndDirectory).Close();
I debugged it and it fails on
byte[] fileBytes = new byte[unzip.GetFileStream(currentFileAndDirectory).Length];
I checked GetFileStream method
public Stream GetFileStream(string filename)
{
if (fileEntries == null)
fileEntries = ParseCentralDirectory(); //We need to do this in case the zip is in a format Silverligth doesn't like
long position = this.stream.Position;
this.stream.Seek(0, SeekOrigin.Begin);
Uri fileUri = new Uri(filename, UriKind.Relative);
StreamResourceInfo info = new StreamResourceInfo(this.stream, null);
StreamResourceInfo stream = System.Windows.Application.GetResourceStream(info, fileUri);
this.stream.Position = position;
if (stream != null)
return stream.Stream;
return null;
}
It throws OutOfMemory exception on this row:
StreamResourceInfo stream = System.Windows.Application.GetResourceStream(info, fileUri);

Netsuite webservices in C#

I am working on C# program to upload image file to netsuite. Can anybody help me how to invoke netsuite script(written in
java script) in C# because I can find upload api only in netsuite script. Is there any webservices or functions
in netsuite to upload image file in netsuite ?

You can upload a file directly with SuiteTalk. Examples below are written in C#.
Call the below methods like this:
uploadFile(#"SERIAL_NUMBERS.csv", "csv", "123456");
Methods:
public static void UploadFile(string filename, string filetype, string folderId)
{
var sFileName = filename;
var sNsFileName = filename;
var sFileType = filetype;
var sFolderId = folderId;
var uploadFile = new com.netsuite.webservices.File { attachFromSpecified = true, attachFrom = FileAttachFrom._computer };
if (sFolderId != null)
{
var folderRef = new RecordRef { internalId = sFolderId };
uploadFile.folder = folderRef;
}
// Specify the NetSuite filename
if (sNsFileName != null)
uploadFile.name = sNsFileName;
uploadFile.fileTypeSpecified = true;
if (sFileType != null)
{
if (sFileType.Trim().ToLower().Equals("plaintext"))
uploadFile.fileType = MediaType._PLAINTEXT;
else if (sFileType.Trim().ToLower().Equals("image"))
uploadFile.fileType = MediaType._IMAGE;
else if (sFileType.Trim().ToLower().Equals("csv"))
uploadFile.fileType = MediaType._CSV;
else
uploadFile.fileType = MediaType._PLAINTEXT;
}
else
uploadFile.fileType = MediaType._PLAINTEXT;
uploadFile.content = LoadFile(sFileName);
// Invoke add() operation to upload the file to NetSuite
var response = Service.add(uploadFile);
// Process the response
if (response.status.isSuccess)
{
Console.WriteLine(
"\nThe file was uploaded successfully:" +
"\nFile Record key=" + ((RecordRef)response.baseRef).internalId +
"\nRenaming file");
}
else
{
Console.WriteLine("The file was not uploaded. Please notify the NetSuite team of the following error:");
DisplayError(response.status.statusDetail);
}
}
private static byte[] LoadFile(String sFileName)
{
byte[] data;
try
{
FileStream inFile;
using (inFile = new FileStream(sFileName, FileMode.Open, FileAccess.Read))
{
data = new Byte[inFile.Length];
inFile.Read(data, 0, (int)inFile.Length);
}
}
catch (Exception ex)
{
// Error creating stream or reading from it.
Console.WriteLine(ex.Message);
return null;
}
return data;
}