Develop Reference

How to generate/retrieve file in datalake using message queue?

I have an Azure function QueueTrigger1 that executes a function executeTemplateProcess to upload a tsv file on Google Drive and update a Jira ticket.
I need to create a Message Queue to generate a tsv file on datalake, run a python code, and then retrieve the tsv file (location) from the datalake and add it to the queue.
I have a basic foundation for the queue today, but i am not sure how to generate the file on the datalake and retrieve its location. We need to pass the file into the python code as input, thats why I am thinking we would need the file location on the datalake to be enqueued, but I am not sure how to perform this.
This is the namespaces for both the QueueTrigger1 and the executeTemplateProcess()
namespace DI
{
public class DIProcess
{
public static void executeTemplateProcess(string jiraKey, string jiraIssueType, string jiraSummary, Component component, string jiraDescription)
{
if (rowCount > 0)
{ //python code would run somewhere here following queue process before jira code executes below
string dfileId = CopyTemplate(component.FileId, sheetName);
// stop process if copy template not sucessfull
if (string.IsNullOrEmpty(dfileId))
return;
jira.AddComment("Google File copied.");
// Update JIRA with the web link
webLink = $"https://docs.google.com/spreadsheets/d/{dfileId}";
jira.AddWebLink(webLink, sheetName);
jira.AddComment("Jira weblink added.");
}
else
{
jira.UpdateStatus("Abandoned");
jira.AddComment("Jira status updated to Abandoned.");
}
}
}
}
namespace companyxyzjira.QueueTrigger1
{
public static class JiraQueueTrigger
{
[FunctionName("QueueTrigger1")]
public static void Run([QueueTrigger("companyxyz-jira-dev-am", Connection = "storageaccountcompanyxyzji42f6_STORAGE")]string myQueueItem
, ILogger log, ExecutionContext context)
{
dynamic jira;
string jiraKey;
string jiraIssueType;
string jiraSummary;
string jiraDescription;
string[] jiraComponentNames;
Component jiraComponent;
log.LogInformation("Queue trigger function processing");
jira = JsonConvert.DeserializeObject(myQueueItem);
jiraKey = jira.issue.key;
jiraIssueType = jira.issue.fields.issuetype.name;
jiraSummary = jira.issue.fields.summary;
jiraDescription = jira.issue.fields.description;
try
{
DIProcess.executeTemplateProcess(jiraKey, jiraIssueType, jiraSummary, jiraComponent, jiraDescription);
}
catch (System.Exception e)
{
log.LogError(e.ToString());
log.LogError(e.Message);
log.LogError(e.StackTrace);
}
}
}
}
I suppose this is my line of thinking but I am not sure how to communicate with the datalake...
[FunctionName("HttpTriggerCSharp")]
public static async Task<IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)]
HttpRequest req, [Queue("companyxyz-jira-dev-pm-mapping-done")] ICollector<string> QueueItem, ILogger log)
{
log.LogInformation("HTTP trigger function processed a request.");
string name = req.Query["name"];
string requestBody = String.Empty;
using (StreamReader streamReader = new StreamReader(req.Body))
{
requestBody = await streamReader.ReadToEndAsync();
QueueItem.Add(requestBody); //i think?
}
dynamic data = JsonConvert.DeserializeObject(requestBody);
name = name ?? data?.name;
return name != null
? (ActionResult)new OkObjectResult($"{name}")
: new BadRequestObjectResult("Please pass a name on the query string or in the request body");
}
datalake snapshot with input/output files (uploaded manually but thats what we want to automate from now on so we need to generate/retrieve these artifacts from/to message queue as described above)

from azure.storage.filedatalake import DataLakeServiceClient
import pandas as pd
connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')
service_client = DataLakeServiceClient.from_connection_string(connect_str)
#Put above code out of the body of function.
file_system_client = service_client.get_file_system_client(file_system="test")
directory_client = file_system_client.get_directory_client("test")
file_client = directory_client.create_file("uploaded-file0316.txt")
#Upload to datalake
head = ["col1" , "col2" , "col3"]
l = [[1 , 2 , 3],[4,5,6] , [8 , 7 , 9]]
df = pd.DataFrame (l , columns = head)
data = df.to_csv(index_label="idx", encoding = "utf-8")
output = data.replace(',', '\t')
print(output)
file_client.append_data(data=output, offset=0, length=len(output))
file_client.flush_data(len(output))
#download from datalake
download = file_client.download_file()
content = download.readall()
print(content)

Add , Edit Metadata in Azure

I recently asked a question here and thanks to Gaurav Mantri I could add Metadata to blob azure .
my Code after editing in AzureBlobStorage class :
public void SaveMetaData(string fileName, string container, string key, string value)
{
var blob = GetBlobReference(fileName, container);
blob.FetchAttributes();
blob.Metadata.Add(key, value);
blob.SetMetadata();
}
and I call it from myController by this :
public JsonResult SaveMetaData(string name, string key, int id)
{
var uploadedFils = _FileStorage.GetUploadedFiles("images", id + "/");
if (!uploadedFils.Any())
_FileStorage.SaveMetaData(name, "images", key, "true");
foreach (var file in uploadedFils)
{
if (name == file.Name)
{
_FileStorage.SaveMetaData(FormatFileName(id, name), "images", key, "true");
}
else
{
_FileStorage.SaveMetaData(FormatFileName(id, file.Name), "images", key, "false");
}
}
return Json("");
}
the code to get uploaded file
public IEnumerable<Attachment> GetUploadedFiles(string container, string blobprefix)
{
if (string.IsNullOrWhiteSpace(container))
container = DefaultBlobContainer;
var storageAccount = CreateStorageAccountFromConnectionString(GetStorageConnectionString());
var blobContainer = GetBlobContainer(storageAccount, container);
var resultList = new List<Attachment>();
try
{
foreach (IListBlobItem item in blobContainer.ListBlobs(blobprefix, false))
{
var blob = (CloudBlockBlob) item;
var file = new Attachment
{
Name = blob.Name.Substring(blob.Name.LastIndexOf('/') + 1),
Size = blob.Properties.Length,
Extension = Path.GetExtension(blob.Name)
};
resultList.Add(file);
}
}
catch (Exception e)
{
}
return resultList;
}
and I call this action when I click on the desired image that I want to set as active .
for first time it works , but I don't know how to edit it for second click ,specially this is first time for me dealing with Azure ?
the logic behind this line that : when the Gallery is empty and the users upload the first image , this image will be set automatically to active:
if (!uploadedFils.Any())
_FileStorage.SaveMetaData(name, "images", key, "true");

According to your description, I checked your code, you need to modify your code as follows:
SaveMetaData method under your AzureBlobStorage class:
public void SaveMetaData(string fileName, string container, string key, string value)
{
var blob = GetBlobReference(fileName, container);
blob.FetchAttributes();
if (blob.Metadata.ContainsKey(key))
{
blob.Metadata[key] = value;
}
else
blob.Metadata.Add(key, value);
blob.SetMetadata();
}
Based on your scenario, your image files would be uploaded to images\{id}\{filename}. And before you invoke the SaveMetaData under your controller, you need to make sure the file with the specific parameters name and id exists in your blob storage. I assumed that you need to remove the following code snippet:
if (!uploadedFils.Any())
FileStorage.SaveMetaData(name, "images", key, "true");
Note: If there has no files, you could not add/update the meta data for it. Also, you just set the name for the parameter fileName without combining the id. Based on my understanding, the SaveMetaData method is used to set meta data for existing files. I recommend you move the above logic to the action for uploading the file and set the default meta data if there has no files.

how to distinguish between inline image and signature and other blank images in email Imap

I'm using Mailkit to fetch email from mailbox and save it to database to display in my MVC application.
I save html email as plain text in database, i can fetch attachments and save it in file system, but when there are inline images in email, i'm having issue as signatures and other blank images are too being saved as attachment in file system.
Is there a way to distinguish between inline attachment and signatures or other blank images?
Thanks in advance

It doesn't matter which IMAP library you use, none of them have a feature that will help you do what you want to do because it's a non-trivial problem to solve that you are going to need to use some ingenuity to solve.
What you can do is start with the HtmlPreviewVisitor sample from the FAQ and modify it every-so-slightly to just split the attachments into 2 lists:
The list of actual attachments
The list of images actually referenced by the HTML (by walking the HTML and tracking which images are referenced)
code:
/// <summary>
/// Visits a MimeMessage and splits attachments into those that are
/// referenced by the HTML body vs regular attachments.
/// </summary>
class AttachmentVisitor : MimeVisitor
{
List<MultipartRelated> stack = new List<MultipartRelated> ();
List<MimeEntity> attachments = new List<MimeEntity> ();
List<MimePart> embedded = new List<MimePart> ();
bool foundBody;
/// <summary>
/// Creates a new AttachmentVisitor.
/// </summary>
public AttachmentVisitor ()
{
}
/// <summary>
/// The list of attachments that were in the MimeMessage.
/// </summary>
public IList<MimeEntity> Attachments {
get { return attachments; }
}
/// <summary>
/// The list of embedded images that were in the MimeMessage.
/// </summary>
public IList<MimePart> EmbeddedImages {
get { return embedded; }
}
protected override void VisitMultipartAlternative (MultipartAlternative alternative)
{
// walk the multipart/alternative children backwards from greatest level of faithfulness to the least faithful
for (int i = alternative.Count - 1; i >= 0 && !foundBody; i--)
alternative[i].Accept (this);
}
protected override void VisitMultipartRelated (MultipartRelated related)
{
var root = related.Root;
// push this multipart/related onto our stack
stack.Add (related);
// visit the root document
root.Accept (this);
// pop this multipart/related off our stack
stack.RemoveAt (stack.Count - 1);
}
// look up the image based on the img src url within our multipart/related stack
bool TryGetImage (string url, out MimePart image)
{
UriKind kind;
int index;
Uri uri;
if (Uri.IsWellFormedUriString (url, UriKind.Absolute))
kind = UriKind.Absolute;
else if (Uri.IsWellFormedUriString (url, UriKind.Relative))
kind = UriKind.Relative;
else
kind = UriKind.RelativeOrAbsolute;
try {
uri = new Uri (url, kind);
} catch {
image = null;
return false;
}
for (int i = stack.Count - 1; i >= 0; i--) {
if ((index = stack[i].IndexOf (uri)) == -1)
continue;
image = stack[i][index] as MimePart;
return image != null;
}
image = null;
return false;
}
// called when an HTML tag is encountered
void HtmlTagCallback (HtmlTagContext ctx, HtmlWriter htmlWriter)
{
if (ctx.TagId == HtmlTagId.Image && !ctx.IsEndTag && stack.Count > 0) {
// search for the src= attribute
foreach (var attribute in ctx.Attributes) {
if (attribute.Id == HtmlAttributeId.Src) {
MimePart image;
if (!TryGetImage (attribute.Value, out image))
continue;
if (!embedded.Contains (image))
embedded.Add (image);
}
}
}
}
protected override void VisitTextPart (TextPart entity)
{
TextConverter converter;
if (foundBody) {
// since we've already found the body, treat this as an
// attachment
attachments.Add (entity);
return;
}
if (entity.IsHtml) {
converter = new HtmlToHtml {
HtmlTagCallback = HtmlTagCallback
};
converter.Convert (entity.Text);
}
foundBody = true;
}
protected override void VisitTnefPart (TnefPart entity)
{
// extract any attachments in the MS-TNEF part
attachments.AddRange (entity.ExtractAttachments ());
}
protected override void VisitMessagePart (MessagePart entity)
{
// treat message/rfc822 parts as attachments
attachments.Add (entity);
}
protected override void VisitMimePart (MimePart entity)
{
// realistically, if we've gotten this far, then we can treat
// this as an attachment even if the IsAttachment property is
// false.
attachments.Add (entity);
}
}
To use it:
var visitor = new AttachmentVisitor ();
message.Accept (visitor);
// Now you can use visitor.Attachments and visitor.EmbeddedImages
An even simpler, although less error-proof (sine it doesn't actually verify whether the image is referenced by the HTML), way of doing it is this:
var embeddedImages = message.BodyParts.OfType<MimePart> ().
Where (x => x.ContentType.IsMimeType ("image", "*") &&
x.ContentDisposition != null &&
x.ContentDisposition.Disposition.Equals ("inline" StringComparison.OrdinalIgnoreCase));
Now that you have your list of embeddedImages, you'll have to figure out a way to determine if they are only used in the signature or used elsewhere in the HTML.
Most likely you'll have to analyze the HTML itself as well.
It is also probably worth noting that some HTML mail will reference images located on the web that are not embedded in the MIME of the message. If you want these images as well, you'll need to modify TryGetImage to fall back to downloading the image from the web if the code I provided fails to locate it within the MIME of the message.
For text/plain messages (which can't use images at all), the common convention to separate the signature from the rest of the message body is a line with only 2 dashes and a space: --.
From my limited experience with HTML messages that have signatures, they do not appear to follow a similar convention. Looking at a few of the HTML messages I receive from co-workers at Microsoft using Outlook, they appear to be within a <table> at the end of the message. However, this assumes that the message is not a reply. Once you start parsing message replies, this <table> ends up in the middle of the message somewhere because the original message being replied to is at the end.
Since everyone's signature is different as well, I'm not sure if this <table> similarity is an Outlook convention or if people are manually constructing their signatures and they are all just using tables out of coincidence (I've also only seen a few, most do not use signatures, so my sample size is very small).

Using https://mailsystem.codeplex.com/:
the class wich read the email:
class readMail:IDisposable
{
public Imap4Client client = new Imap4Client();
public readMail(string mailServer, int port, bool ssl, string login, string password)
{
Pop3Client pop = new Pop3Client();
if (ssl)
{
client.ConnectSsl(mailServer, port);
}
else
client.Connect(mailServer, port);
client.Login(login, password);
}
public IEnumerable<Message> GetAllMails(string mailBox)
{
IEnumerable<Message> ms = GetMails(mailBox, "ALL").Cast<Message>();
return GetMails(mailBox, "ALL").Cast<Message>();
}
protected Imap4Client Client
{
get { return client ?? (client = new Imap4Client()); }
}
private MessageCollection GetMails(string mailBox, string searchPhrase)
{
try
{
MessageCollection messages = new MessageCollection();
Mailbox mails = new Mailbox();
mails = Client.SelectMailbox(mailBox);
messages = mails.SearchParse(searchPhrase);
return messages;
}
catch(Exception ecc)
{
}
}
public void Dispose()
{
throw new NotImplementedException();
}
}
and then:
using (readMail read = new readMail("host.name.information", port, true, username, password) )
{
var emailList = read.GetAllMails(this.folderEmail);
int k = 0;
Mailbox bbb = read.client.SelectMailbox(this.folderEmail);
int[] unseen = bbb.Search("UNSEEN");
foreach (Message email in emailList)
{
/// Contains all parts for which no Content-Disposition header was found. Disposition is left to the final agent.
MimePartCollection im1= email.UnknownDispositionMimeParts;
//Collection containing embedded MIME parts of the message (included text parts)
EmbeddedObjectCollection im2 = email.EmbeddedObjects;
//Collection containing attachments of the message.
AttachmentCollection attach=email.Attachments;
}
}
in my case all the signature's images were in UnknownDispositionMimeParts, but this could be a specific case (different email client and so on)..so for what i know i didn't find any library that separate embedded images from contextual images to signature images

Window Phone 8.1: Confusing about file location and writing data

I'm trying to read and write data with json file.
I created some class.
public class SimpleTask{...}
public class DayTask{...}
public class DataModel
{
...
private async Task GetSimpleTaskAsync()
{
if (_daytask.Count != 0)
return;
string fileName = "a.json";
Uri appUri = new Uri("ms-appx:///"+ fileName);
StorageFile file = await StorageFile.GetFileFromApplicationUriAsync(appUri);
string jsonText = await FileIO.ReadTextAsync(file);
JsonObject jsonObject = JsonObject.Parse(jsonText);
JsonArray jsonArray = jsonObject["DayTasks"].GetArray();
foreach (JsonValue daytaskValue in jsonArray)
{
JsonObject daytaskObject = daytaskValue.GetObject();
ObservableCollection<SimpleTask> simpletask = new ObservableCollection<SimpleTask>();
foreach (JsonValue simpletaskValue in daytaskObject["Tasks"].GetArray())
{
JsonObject simpletaskObject = simpletaskValue.GetObject();
simpletask.Add(new SimpleTask( simpletaskObject["StartTime"].GetString(),
simpletaskObject["EndTime"].GetString(),
simpletaskObject["Description"].GetString()));
}
DayTask daytask = new DayTask(daytaskObject["Day"].GetString(),simpletask);
this.DayTasks.Add(daytask);
}
}
}
As you can see, i have a method that gets data form a.json file. I created a.json file:
In the MainPage.xaml.cs, there is a method which calls GetDayTaskAysnc() method and retrieves data :
private async void ReadData1(object sender, TappedRoutedEventArgs e)
{
string test = String.Empty;
var daytask = await DataModel.GetDayTaskAsync();
foreach (var tasks in daytask)
{
test += String.Format("Day:{0}:\n", tasks.Day);
foreach (var simpletask in tasks.Tasks)
{
test += String.Format("\tStart Time: {0}\n", simpletask.StartTime);
test += String.Format("\tEnd Time: {0}\n", simpletask.EndTime);
test += String.Format("\tDescription Time: {0}\n", simpletask.Description);
}
}
TextBlock.Text = test;
}
It worked fine ! But i want to write data to the same file, so i added data in hardcore way:
private List<DayTask> creatList()
{
List<DayTask> DayTasks = new List<DayTask>();
ObservableCollection<SimpleTask> simpletask1 = new ObservableCollection<SimpleTask>();
simpletask1.Add(new SimpleTask("6AM","7AM","Breakfast"));
simpletask1.Add(new SimpleTask("8AM", "9AM", "Game"));
ObservableCollection<SimpleTask> simpletask2 = new ObservableCollection<SimpleTask>();
simpletask2.Add(new SimpleTask("6AM", "7AM", "Sleep"));
simpletask2.Add(new SimpleTask("8AM", "9AM", "School"));
DayTasks.Add(new DayTask ("3/8/2014",simpletask1));
DayTasks.Add(new DayTask("4/8/2014", simpletask2));
return DayTasks;
}
private async void WriteData(object sender, TappedRoutedEventArgs e)
{
string json = "a.json";
List<DayTask> daytasks = creatList();
var serializer = new DataContractJsonSerializer(typeof(List<DayTask>));
var stream = await ApplicationData.Current.LocalFolder.OpenStreamForWriteAsync(json, CreationCollisionOption.ReplaceExisting);
using (stream)
{
serializer.WriteObject(stream, daytasks);
}
TextBlock.Text = "Write to Json file succeeded";
}
When i ran my app with window phone emulator, Firstly, it wrote to the file. Then i clicked read data button to ensure data written correctly, the emulator showed data from a.json file without being modified by WriteData() method. I continued to creat the second read data method:
private async void ReadData2(object sender, TappedRoutedEventArgs e)
{
string test = String.Empty;
string json = "a.json";
string content = String.Empty;
List<DayTask> mytasks = new List<DayTask>();
var deserializer = new DataContractJsonSerializer(typeof(List<DayTask>));
var stream = await ApplicationData.Current.LocalFolder.OpenStreamForReadAsync(json);
using (stream)
{
mytasks = (List<DayTask>)deserializer.ReadObject(stream);
}
foreach (var tasks in mytasks)
{
test += String.Format("Day:{0}:\n", tasks.Day);
foreach (var simpletask in tasks.Tasks)
{
test += String.Format("\tStart Time: {0}\n", simpletask.StartTime);
test += String.Format("\tEnd Time: {0}\n", simpletask.EndTime);
test += String.Format("\tDescription Time: {0}\n", simpletask.Description);
}
}
TextBlock.Text = test;
}
I deployed my app several times, and this is my result:
ReadData2() : 'System.IO.FileNotFoundException'
WriteData() -> ReadData1(): Data from a.json was shown
WriteData() -> ReadData2(): Data from creatList() was shown
WriteData() -> ReadData1(): Data from a.json was shown -> ReadData2(): Data from creatList() was shown
So that, i have some question:
Do i have 2 json files, one i created by adding into my project and the other one i created when ran WriteData() method ? What is their paths ?
If my file is data.json in DataSource folder, how can i write data to it ? I can read data from it using uri like GetSimpleTaskAsync() but i don't know how to write to it correctly. (I tried to convert object into string to write but can't read it again, i guess i wrote it in wrong way)
Sorry for my long post and my bad english :) Thank you very much

But i want to write data to the same file, so i added data in hardcore way:
Your are making confusion between ms-appx:/// and ms-appdata:/// folders (or ApplicationData.Current.LocalFolder )
The ms-appx folder is read-only. You can't write to it. (or you could edit your app code without passing through the certification process)
The file you wrote must be into the ms-appdata folder.

How to Save an Image from Nokia Imaging SDK

I am having issues with some sample code that I am using from one of Nokia's Imaging SDK samples they provide. Essentially I am trying to save an image to IsolatedStorage. The code that I am reusing has been used successfully elsewhere in the solution, but when I try to use it there are no errors but it does not proceed with the following statements. Essentially in the StorePhoto method, once IBuffer buffer = await App.PhotoModel.RenderFullBufferAsync(); is called no error occurs but no code below that which is actually performs the save to isolated storage operation is ran, so no image is ever saved.
SavePage.xaml.cs
private static string _photoModelPath = #"\Lockscreen\v1\PhotoModel";
private static string _photoModelBufferFilename = #"buffer.data";
public async static void StorePhoto()
{
string _photoModelPath = #"\Lockscreen\v1\LockScreen";
string _photoModelBufferFilename = #"buffer.data";
using (IsolatedStorageFile storage = IsolatedStorageFile.GetUserStoreForApplication())
{
if (!storage.DirectoryExists(_photoModelPath))
{
storage.CreateDirectory(_photoModelPath);
}
if (storage.FileExists(_photoModelPath + #"\" + _photoModelBufferFilename))
{
storage.DeleteFile(_photoModelPath + #"\" + _photoModelBufferFilename);
}
IBuffer buffer = await App.PhotoModel.RenderFullBufferAsync(); //code exiting method with no error
if (buffer != null)
{
IsolatedStorageFileStream originalFile = storage.CreateFile(_photoModelPath + #"\" + _photoModelBufferFilename);
Stream bufferStream = buffer.AsStream();
bufferStream.CopyTo(originalFile);
bufferStream.Flush();
bufferStream.Close();
bufferStream.Dispose();
originalFile.Flush();
originalFile.Close();
originalFile.Dispose();
}
}
}
MainPage.xaml.cs
private async void _saveItem_Click(object sender, EventArgs e)
{
Helpers.SaveHelper.StorePhoto(); //calling the StorePhoto method here
}
PhotoModel.cs (from Nokia Imaging SDK sample)
/// <summary>
/// Renders current image with applied filters to a buffer and returns it.
/// Meant to be used where the filtered image is for example going to be
/// saved to a file.
/// </summary>
/// <returns>Buffer containing the filtered image data</returns>
public async Task<IBuffer> RenderFullBufferAsync()
{
using (BufferImageSource source = new BufferImageSource(_buffer))
using (FilterEffect effect = new FilterEffect(source) { Filters = _components })
using (JpegRenderer renderer = new JpegRenderer(effect))
{
return await renderer.RenderAsync();
}
}

Turns out to solve this I had to put the code that saves the image into the same page that I was originally calling that method, and also it needs to be of type Task so that the async/await will work properly.