直接将文件从FTP传输到Azure文件存储，而无需将其本地保留在内存或磁盘中 [英] Transfer files directly from FTP to Azure File Storage without keeping them locally in memory or disk

问题描述

我必须将文件从FTP传输到Azure文件存储.我的代码工作正常，但是我正在将这些文件传输到内存中，这不是最佳实践.因此，首先我将流读取到内存中的Byte数组.然后，将输出上传到Azure文件存储.

I have to transfer files from FTP to an Azure File Storage. My code works fine, but I'm transferring those files in memory which is not a best practice. So first I read the stream to an Byte array in memory. Then I upload the output to an Azure file storage.

现在，我知道最好异步进行此操作.但是我不知道这是否可行以及如何做到.

Now I know it's better to do this asynchronicaly. But I don't know if this is possible and how to do it.

我的代码:

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.WindowsAzure.Storage;
using System.Configuration;
using Microsoft.WindowsAzure.Storage.File;
using System.IO;
using Microsoft.Azure;
using System.Net;

namespace TransferFtpToAzure
{
    class Program
    {
        public static void Main(string[] args)
        {
            List<FileName> sourceFileList = new List<FileName>();
            List<FileName> targetFileList = new List<FileName>();

            string targetShareReference = ConfigurationManager.AppSettings["AzureShare"];
            string targetDirectoryReference = ConfigurationManager.AppSettings["Environment"] + "/" + Enums.AzureFolders.Mos + "/" + Enums.AzureFolders.In;
            string sourceURI = (ConfigurationManager.AppSettings["FtpConnectionString"] + ConfigurationManager.AppSettings["Environment"].ToUpper() +"/"+ Enums.FtpFolders.Mos + "/").Replace("\\","/");
            string sourceUser = ConfigurationManager.AppSettings["FtpServerUserName"];
            string sourcePass = ConfigurationManager.AppSettings["FtpServerPassword"];

            getFileLists(sourceURI, sourceUser, sourcePass, sourceFileList, targetShareReference, targetDirectoryReference, targetFileList);

            Console.WriteLine(sourceFileList.Count + " files found!");

            CheckLists(sourceFileList, targetFileList);
            targetFileList.Sort();

            Console.WriteLine(sourceFileList.Count + " unique files on sourceURI" + Environment.NewLine + "Attempting to move them.");

            foreach (var file in sourceFileList)
            {
                try
                {
                    CopyFile(file.fName, sourceURI, sourceUser, sourcePass, targetShareReference, targetDirectoryReference);
                }
                catch
                {
                    Console.WriteLine("There was move error with : " + file.fName);
                }
            }
        }

        public class FileName : IComparable<FileName>
        {
            public string fName { get; set; }
            public int CompareTo(FileName other)
            {
                return fName.CompareTo(other.fName);
            }
        }

        public static void CheckLists(List<FileName> sourceFileList, List<FileName> targetFileList)
        {
            for (int i = 0; i < sourceFileList.Count; i++)
            {
                if (targetFileList.BinarySearch(sourceFileList[i]) > 0)
                {
                    sourceFileList.RemoveAt(i);
                    i--;
                }
            }
        }

        public static void getFileLists(string sourceURI, string sourceUser, string sourcePass, List<FileName> sourceFileList, string targetShareReference, string targetDirectoryReference, List<FileName> targetFileList)
        {
            string line = "";
            /////////Source FileList
            FtpWebRequest sourceRequest;
            sourceRequest = (FtpWebRequest)WebRequest.Create(sourceURI);
            sourceRequest.Credentials = new NetworkCredential(sourceUser, sourcePass);
            sourceRequest.Method = WebRequestMethods.Ftp.ListDirectory;
            sourceRequest.UseBinary = true;
            sourceRequest.KeepAlive = false;
            sourceRequest.Timeout = -1;
            sourceRequest.UsePassive = true;
            FtpWebResponse sourceRespone = (FtpWebResponse)sourceRequest.GetResponse();
            //Creates a list(fileList) of the file names
            using (Stream responseStream = sourceRespone.GetResponseStream())
            {
                using (StreamReader reader = new StreamReader(responseStream))
                {
                    line = reader.ReadLine();
                    while (line != null)
                    {
                        var fileName = new FileName
                        {
                            fName = line
                        };
                        sourceFileList.Add(fileName);
                        line = reader.ReadLine();
                    }
                }
            }
            /////////////Target FileList
            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
            CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
            //var test = fileClient.ListShares();
            CloudFileShare fileShare = fileClient.GetShareReference(targetShareReference);
            if (fileShare.Exists())
            {
                CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference();
                if (rootDirectory.Exists())
                {
                    CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(targetDirectoryReference);
                    if (customDirectory.Exists())
                    {
                        var fileCollection = customDirectory.ListFilesAndDirectories().OfType<CloudFile>();
                        foreach (var item in fileCollection)
                        {
                            var fileName = new FileName
                            {
                                fName = item.Name
                            };
                            targetFileList.Add(fileName);
                        }
                    }
                }
            }
        }

        public static void CopyFile(string fileName, string sourceURI, string sourceUser, string sourcePass, string targetShareReference, string targetDirectoryReference)
        {
            try
            {
                FtpWebRequest request = (FtpWebRequest)WebRequest.Create(sourceURI + fileName);
                request.Method = WebRequestMethods.Ftp.DownloadFile;
                request.Credentials = new NetworkCredential(sourceUser, sourcePass);
                FtpWebResponse response = (FtpWebResponse)request.GetResponse();
                Stream responseStream = response.GetResponseStream();
                Upload(fileName, ToByteArray(responseStream), targetShareReference, targetDirectoryReference);
                responseStream.Close();
            }
            catch
            {
                Console.WriteLine("There was an error with :" + fileName);
            }
        }

        public static Byte[] ToByteArray(Stream stream)
        {
            MemoryStream ms = new MemoryStream();
            byte[] chunk = new byte[4096];
            int bytesRead;
            while ((bytesRead = stream.Read(chunk, 0, chunk.Length)) > 0)
            {
                ms.Write(chunk, 0, bytesRead);
            }

            return ms.ToArray();
        }

        public static bool Upload(string FileName, byte[] Image, string targetShareReference, string targetDirectoryReference)
        {
            try
            {
                CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
                CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
                //var test = fileClient.ListShares();
                CloudFileShare fileShare = fileClient.GetShareReference(targetShareReference);
                if (fileShare.Exists())
                {
                    CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference();
                    if (rootDirectory.Exists())
                    {
                        CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(targetDirectoryReference);
                        if (customDirectory.Exists())
                        {
                            var cloudFile = customDirectory.GetFileReference(FileName);
                            using (var stream = new MemoryStream(Image, writable: false))
                            {
                                cloudFile.UploadFromStream(stream);
                            }
                        }
                    }
                }
                return true;
            }
            catch
            {
                return false;
            }
        }
    }
}

推荐答案

如果我对您的理解正确，则希望避免在下载和上传之间将文件存储在内存中.

If I understand you correctly, you want to avoid storing the file in memory between the download and upload.

为此，请参见:
Azure函数可将文件从FTP复制到Blob存储.

For that see:
Azure function to copy files from FTP to blob storage.

这篇关于直接将文件从FTP传输到Azure文件存储，而无需将其本地保留在内存或磁盘中的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！