Develop Reference

Read .xz file from S3 in C#

I am trying to read file from S3 using below code:
var dir = new S3DirectoryInfo(_amazonS3Client, bucketName, folderName);
IS3FileSystemInfo[] files = dir.GetFileSystemInfos();
if(files.Length > 0)
{
_bucketKey = files[0].Name;
var request = new GetObjectRequest
{
BucketName = bucketName,
Key = $"{folderName}/{_bucketKey}"
};
using (GetObjectResponse response = await _amazonS3Client.GetObjectAsync(request))
using (Stream responseStream = response.ResponseStream)
using (var reader = new StreamReader(responseStream))
{
var responseBody = reader.ReadToEnd();
File.WriteAllText(newFilePath, responseBody);
}
}
It is working fine for a non-compressed file, however, I need some suggestions on how could I read the .xz file? My file is like DummyData_2020-07-21.csv.xz

I found a solution using SharpCompress and below code worked for me
var request = new GetObjectRequest
{
BucketName = bucketName,
Key = $"{folderName}/{bucketKey}"
};
using (var obj = _amazonS3Client.GetObject(request))
{
obj.WriteResponseStreamToFile($"{fileLocation}{bucketKey}");
}
using (Stream xz = new XZStream(File.OpenRead($"{fileLocation}{bucketKey}")))
using (TextReader reader = new StreamReader(xz))
{
var responseBody = reader.ReadToEnd();
File.WriteAllText(newFilePath, responseBody);
}

Authenticate login in website using certificate C#

I'm doing some research, and I want to logging into the ecac site using my company's certificate ... using Chrome I can access the system without any problem because chrome asks for the certificate to authenticate the login. I tried to implement a small test crawler, where I put my certificate in the request, but, the government website always returns error ... So I tried to use selenium with the chrome driver to login, but with that it is necessary that I select the certificate manually, and my idea is to do this automatically.
My test source is:
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;
namespace TestCrawler
{
class Program
{
static void Main(string[] args)
{
string host = #"https://cav.receita.fazenda.gov.br/autenticacao/login";
string certName = #"certificado.pfx";
string password = #"senha";
try
{
X509Certificate2 certificate = new X509Certificate2(certName, password);
ServicePointManager.CheckCertificateRevocationList = false;
ServicePointManager.ServerCertificateValidationCallback = (a, b, c, d) => true;
ServicePointManager.Expect100Continue = true;
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(host);
req.PreAuthenticate = true;
req.AllowAutoRedirect = true;
req.ClientCertificates.Add(certificate);
req.Method = "POST";
req.ContentType = "application/x-www-form-urlencoded";
string postData = "login-form-type=cert";
byte[] postBytes = Encoding.UTF8.GetBytes(postData);
req.ContentLength = postBytes.Length;
Stream postStream = req.GetRequestStream();
postStream.Write(postBytes, 0, postBytes.Length);
postStream.Flush();
postStream.Close();
WebResponse resp = req.GetResponse();
Stream stream = resp.GetResponseStream();
using (StreamReader reader = new StreamReader(stream))
{
string line = reader.ReadLine();
while (line != null)
{
Console.WriteLine(line);
line = reader.ReadLine();
}
}
stream.Close();
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
}
When analyzing the source of the ecac site, I have identified that in the login button by the certificate, a JS code is executed
onclick="javascript:document.loginCert.submit(); return false;
My another test:
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Security.Cryptography.X509Certificates;
namespace TestCrawler
{
class Program
{
private static string host = #"https://cav.receita.fazenda.gov.br/";
private static string certName = #"certificado.pfx";
private static string password = #"senha";
static async void Login()
{
X509Certificate2 certificate = new X509Certificate2(certName, password);
var handler = new WebRequestHandler();
handler.ClientCertificates.Add(certificate);
using (var client = new HttpClient(handler))
{
client.BaseAddress = new Uri(host);
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("text/html"));
// New code:
using (HttpResponseMessage response = await client.GetAsync("autenticacao/Login/Certificado"))
{
using (HttpContent content = response.Content)
{
string result = await content.ReadAsStringAsync();
}
}
}
}
static void Main(string[] args)
{
Login();
Console.ReadLine();
}
}
}

how to upload documents to sharepoint 2007 C#

I have no access to Sharepoint server, only like standard user from web page. I can upload manually there my documents. I tried to solve it via C# and I complet any code from examples from net. Our Sharepoint is 2007. My code run without any error. I put there control text to see if its proceed. All runs fine but nothing happens in Sharepoint page, no doc is uploaded. I have no idea why its do nothing :)
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Net;
namespace Sharepoint
{
class Program
{
public static void CopyStream(Stream read, Stream write)
{
int len; byte[] temp = new byte[1024];
while ((len = read.Read(temp, 0, temp.Length)) > 0)
{
write.Write(temp, 0, len);
/// Console.WriteLine("test");
}
}
static void Main(string[] args)
{
Uri destUri = new Uri("http://gaja/mBreSKCZ/mreports/sales/reportysales/Test_new.txt");
using (FileStream inStream = File.OpenRead(#"C:\Users\TK20382\Test_new.txt"))
{
WebRequest req = WebRequest.Create(destUri);
req.Method = "PUT";
req.Credentials = CredentialCache.DefaultCredentials; // assuming windows Auth
Console.WriteLine("test");
Console.ReadKey();
using (Stream outStream = req.GetRequestStream())
{
CopyStream(inStream, outStream);
}
}
}
}
}

You are missing HttpWebRequest.GetResponse Method which basically invokes PUT request. In addition if you are targeting .NET Framework >=2.0 version, then CopyStream method could be omitted and the line:
CopyStream(inStream, outStream);
replaced with:
inStream.CopyTo(outStream);
Modified version
public static string UploadFile(string targetUrl,ICredentials credentials, string sourcePath)
{
var request = WebRequest.Create(targetUrl);
request.Method = "PUT";
request.Credentials = credentials;
using (var fileStream = File.OpenRead(sourcePath))
using (var requestStream = request.GetRequestStream())
{
fileStream.CopyTo(requestStream);
}
using (var response = request.GetResponse())
using (var reader = new StreamReader(response.GetResponseStream()))
{
return reader.ReadToEnd();
}
}
Usage
UploadFile("https://contoso.intranet.com/documents/guide.docx", CredentialCache.DefaultCredentials, #"D:\guide.docx");
Alternatively WebClient.UploadFile Method could be utilized as shown below:
public static void UploadFile(string targeUrl, ICredentials credentials, string fileName)
{
using (var client = new WebClient())
{
client.Credentials = credentials;
client.UploadFile(targeUrl, "PUT", fileName);
}
}

How to compress HttpClient post and receive data in web api

I have the following web api client which sends data to server using json and gzip:
public void Remote_Push(BlockList Blocks)
{
// Pushes pending records to the remote server
using (var Client = new HttpClient())
{
Client.BaseAddress = new Uri(Context.ServerUrl);
Client.DefaultRequestHeaders.Accept.Clear();
Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var Content = JsonCompress(Blocks);
var T1 = Client.PostAsync("SyncPush/", Content); T1.Wait();
T1.Result.EnsureSuccess();
}
}
private static ByteArrayContent JsonCompress(object Data)
{
// Compress given data using gzip
var Bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(Data));
using (var Stream = new MemoryStream())
{
using (var Zipper = new GZipStream(Stream, CompressionMode.Compress, true)) Zipper.Write(Bytes, 0, Bytes.Length);
var Content = new ByteArrayContent(Stream.ToArray());
Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
Content.Headers.ContentEncoding.Add("gzip");
return Content;
}
}
On the server, I have created following action in web api controller:
[HttpPost]
public void SyncPush([FromBody]BlockList Blocks)
{
var Context = SyncCreateContext();
var Sync = new Processor(Context);
Sync.ServerPush(Blocks);
}
Previously, I have used PostAsJsonAsync on the client and it worked fine.
Now, I have switched to ByteArrayContent and gzip and no longer works, the Blocks is always null on the server.
What am I missing here, what is wrong or could be the problem?

Here is a sample console application to do what you are trying to do.
/*using System;
using System.IO;
using System.IO.Compression;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using Newtonsoft.Json;
using WebApi.Models;*/
class Program
{
static void Main(string[] args)
{
Console.WriteLine("Press any key to POST");
Console.ReadLine();
RemotePush(new BlockList());
Console.ReadLine();
}
public static async void RemotePush(BlockList blocks)
{
try
{
using (var client = new HttpClient())
{
try
{
Console.WriteLine("Please wait.");
client.BaseAddress = new Uri("http://localhost:52521/Home/");
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var content = JsonCompress(blocks);
var response = await client.PostAsync("SyncPush/", content);
using (var stream = await response.Content.ReadAsStreamAsync())
{
using (var streamReader = new StreamReader(stream))
{
Console.WriteLine(streamReader.ReadToEnd());
}
}
Console.WriteLine("Done.");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
private static MultipartFormDataContent JsonCompress(object data)
{
var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(data));
var stream = new MemoryStream();
using (var zipper = new GZipStream(stream, CompressionMode.Compress, true))
{
zipper.Write(bytes, 0, bytes.Length);
}
MultipartFormDataContent multipartContent = new MultipartFormDataContent();
multipartContent.Add(new StreamContent(stream), "gzipContent");
return multipartContent;
}
}
My controller is like this.
[System.Web.Mvc.HttpPost]
public JsonResult SyncPush(BlockList content)
{
try
{
if (content != null)
{
return Json("success", JsonRequestBehavior.AllowGet);
}
return Json("failed due to null", JsonRequestBehavior.AllowGet);
}
catch (Exception ex)
{
return Json("failed " + ex.Message, JsonRequestBehavior.AllowGet);
}
}

just FYI, since .NET Core is not covered and this question is still relevant working .NET Core code. I used brotli, since that is a widely accepted standard today.
using System.Text.Json.Serialization;
using System.Text.Json;
using System.Net.Http.Headers;
using System.IO.Compression;
public static class CompressedJsonHelper
{
private static readonly Lazy<JsonSerializerOptions>
Options = new(() =>
{
var opt = new JsonSerializerOptions { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
//opt.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase));
return opt;
});
public static HttpContent ToJson(this object data, bool noCompress = false)
{
if (noCompress)
{
ByteArrayContent byteContent = new (ToBytes(data));
byteContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
return byteContent;
}
MemoryStream memoryStream = new ();
BrotliStream compress = new (memoryStream, CompressionLevel.Optimal, true);
StreamContent streamContent = new (memoryStream);
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/json");
streamContent.Headers.ContentEncoding.Add("brotli");
JsonSerializer.Serialize(compress, data, Options.Value);
compress.Flush();
memoryStream.Position = 0;
return streamContent;
}
private static byte[] ToBytes(this object data) => JsonSerializer.SerializeToUtf8Bytes(data, Options.Value);
}
and the httpClient code:
HttpRequestMessage request = new(HttpMethod.Post, $"{yourbaseurl}/{path}")
{
Content = json.ToJson()
};
await _httpClient.SendAsync(request, ...) etc

Recursively deleting all files and folders from a given FTP path

I am trying to create a program that will recursively delete all files and folder from a given FTP path. My program is working well, if there are only a few files or directories in a given path. For some reason though, if there are several files in a directory, it just hangs on the ReadLine() line and eventually gets an IO error. Here is my code:
class Program
{
private const string FtpSite = "myftpsite.com";
private const string FtpUserName = "myusername";
private const string FtpPassword = "mypassword";
static void Main(string[] args)
{
DeleteFilesAndFolders("/");
}
private static void DeleteFilesAndFolders(string path)
{
if (path != null && (path.StartsWith(#"\\") || path.StartsWith("//")))
path = path.Remove(0, 1);
List<FileObject> files = DirectoryListing(path);
foreach (FileObject file in files.Where(file => !file.IsDirectory))
{
DeleteFile(path, file.FileName);
}
foreach (FileObject file in files.Where(file => file.IsDirectory))
{
DeleteFilesAndFolders(path + "/" + file.FileName);
DeleteFolder(path + "/" + file.FileName);
}
}
private static void DeleteFile(string path, string file)
{
var clsRequest = (FtpWebRequest)WebRequest.Create("ftp://" + FtpSite + path + "/" + file);
clsRequest.Credentials = new NetworkCredential(FtpUserName, FtpPassword);
clsRequest.Method = WebRequestMethods.Ftp.DeleteFile;
using (var response = (FtpWebResponse) clsRequest.GetResponse())
{
using (Stream datastream = response.GetResponseStream())
{
if (datastream == null)
return;
using (var sr = new StreamReader(datastream))
{
sr.ReadToEnd();
sr.Close();
}
datastream.Close();
response.Close();
}
}
}
private static void DeleteFolder(string path)
{
var clsRequest = (FtpWebRequest)WebRequest.Create("ftp://" + FtpSite + path);
clsRequest.Credentials = new NetworkCredential(FtpUserName, FtpPassword);
clsRequest.Method = WebRequestMethods.Ftp.RemoveDirectory;
using (var response = (FtpWebResponse)clsRequest.GetResponse())
{
using (Stream datastream = response.GetResponseStream())
{
if (datastream == null)
return;
using (var sr = new StreamReader(datastream))
{
sr.ReadToEnd();
sr.Close();
}
datastream.Close();
response.Close();
}
}
}
private static List<FileObject> DirectoryListing(string path)
{
var regex = new Regex(#"^([d-])([rwxt-]{3}){3}\s+\d{1,}\s+.*?(\d{1,})\s+(\w+\s+\d{1,2}\s+(?:\d{4})?)(\d{1,2}:\d{2})?\s+(.+?)\s?$",
RegexOptions.Compiled | RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
var request = (FtpWebRequest)WebRequest.Create("ftp://" + FtpSite + path);
request.Credentials = new NetworkCredential(FtpUserName, FtpPassword);
request.Method = WebRequestMethods.Ftp.ListDirectoryDetails;
var result = new List<FileObject>();
using (var response = (FtpWebResponse) request.GetResponse())
{
using (Stream responseStream = response.GetResponseStream())
{
if (responseStream == null)
return null;
using (var reader = new StreamReader(responseStream))
{
while (!reader.EndOfStream)
{
string r = reader.ReadLine();
if (string.IsNullOrWhiteSpace(r))
continue;
var reg = regex.Match(r);
var c = new FileObject
{
FileName = reg.Groups[6].Value,
IsDirectory = reg.Groups[1].Value.ToLower() == "d"
};
result.Add(c);
}
reader.Close();
}
response.Close();
}
}
return result;
}
}
And this is my simple container FileObject class:
internal class FileObject
{
public bool IsDirectory { get; set; }
public string FileName { get; set; }
}

My issue appeared to be fixed by using Active mode instead of Passive. I added this to my code:
request.UsePassive = false;
This appears to fix my issue. I tested it on a large batch of files with lots of subdirectories and sub-subdirectories and it seems to work flawless now.