从FTP服务器下载大文件（〜150MB）挂起 [英] Downloading large files(~150MB) from FTP server hangs

问题描述

  using（System.IO.FileStream fileStream = System .IO.File.OpenWrite（filePath））
 {
 
 byte [] buffer = new byte [4096]; 
 
 int bytesRead = responseStream.Read（buffer，0，4096）; 
 while（bytesRead> 0）
 {
 fileStream.Write（buffer，0，bytesRead）; 
 bytesRead = responseStream.Read（buffer，0，4096）; 
 
 
 $ b  

创建responseStream： p>

  System.IO.Stream responseStream = GetFileAsStream（url，username，password，false）; 
 
 public static System.IO.Stream GetFileAsStream（string ftpUrl，string username，string password，bool usePassive）
 {
 System.Net.FtpWebRequest request =（System.Net。的FtpWebRequest）System.Net.WebRequest.Create（ftpUrl）; 
 request.KeepAlive = false; 
 request.ReadWriteTimeout = 120000; 
 request.Timeout = -1; 
 request.UsePassive = usePassive; 
 
 request.Credentials = new System.Net.NetworkCredential（username，password）; 
 
 request.Method = System.Net.WebRequestMethods.Ftp.DownloadFile; 
 
 System.IO.Stream fileResponseStream; 
 
 
 System.Net.FtpWebResponse fileResponse =（System.Net.FtpWebResponse）request.GetResponse（）; 
 
 
 
 fileResponseStream = fileResponse.GetResponseStream（）; 
 
返回fileResponseStream; 
} 
  

对于较小的文件可以正常工作，但是当文件较大时（例如150mb）该过程挂起。出于某种原因，程序不明白它已完成下载，并且仍然尝试读取更多字节。

我更喜欢不包含使用外部库的答案。谢谢

解决方案

我已经通过引入请求超时来解决我的问题 - 如果已达到，则使程序抛出WebException 。在这种情况下，程序将从其剩下的地方继续下载。这是我的代码：

这是一个方法，如果下载文件返回true，否则返回true否则

  Digitalez.DirectoryUtil.EnsureDirectoryExists（relativePath）; 
 
 string filePath = System.IO.Path.Combine（relativePath，fileInfo.Name）; 
 long length = Digitalez.FtpUtil.GetFileLength（fileInfo.FullPath，userName，password，usePassive）; 
 long offset = 0; 
 int retryCount = 10; 
 int？ readTimeout = 5 * 60 * 1000; //五分钟
 
 //如果文件存在，不要下载
 if（System.IO.File.Exists（filePath））
 {
 return假; 
} 
 
 while（retryCount> 0）
 {
 
 using（System.IO.Stream responseStream = Captator.Eifos.Net.FtpUtil。 GetFileAsStream（fileInfo.FullPath，userName，password，usePassive，offset，requestTimeout：readTimeout！= null？readTimeout.Value：System.Threading.Timeout.Infinite））
 {
 
 using（System .IO.FileStream fileStream = new System.IO.FileStream（filePath，System.IO.FileMode.Append））
 {
 byte [] buffer = new byte [4096]; 
 try 
 {
 int bytesRead = responseStream.Read（buffer，0，buffer.Length）; 
 
 while（bytesRead> 0）
 {
 fileStream.Write（buffer，0，bytesRead）; 
 
 bytesRead = responseStream.Read（buffer，0，buffer.Length）; 
} 
 
返回true; 
} 
 catch（System.Net.WebException）
 {
 //什么都不做 - 消耗此异常以强制重新读取文件的其余部分
} 
 
 
 if（System.IO.File.Exists（filePath））
 {
 offset = new System.IO.FileInfo（filePath）.Length; 
} 
 else 
 {
 offset = 0; 
} 
 
 retryCount--; 
 
 if（offset == length）
 {
 return true; 
} 
 
} 
} 
  

Digitalez。 FtpUtil：

  public static System.IO.Stream GetFileAsStream（string ftpUrl，string username，string password，bool usePassive，long offset， int requestTimeout）
 {
 System.Net.FtpWebRequest request =（System.Net.FtpWebRequest）System.Net.WebRequest.Create（ftpUrl）; 
 
 request.KeepAlive = false; 
 request.ReadWriteTimeout = requestTimeout; 
 request.Timeout = requestTimeout; 
 request.ContentOffset = offset; 
 request.UsePassive = usePassive; 
 request.UseBinary = true; 
 
 request.Credentials = new System.Net.NetworkCredential（username，password）; 
 
 request.Method = System.Net.WebRequestMethods.Ftp.DownloadFile; 
 
 System.IO.Stream fileResponseStream; 
 
 System.Net.FtpWebResponse fileResponse =（System.Net.FtpWebResponse）request.GetResponse（）; 
 
 fileResponseStream = fileResponse.GetResponseStream（）; 
 
返回fileResponseStream; 
 
 $ b $ public static long GetFileLength（string ftpUrl，string username，string password，bool usePassive）
 {
 System.Net.FtpWebRequest request =（System.Net .FtpWebRequest）System.Net.WebRequest.Create（ftpUrl）; 
 
 request.KeepAlive = false; 
 request.UsePassive = usePassive; 
 
 
 request.Credentials = new System.Net.NetworkCredential（username，password）; 
 request.Method = System.Net.WebRequestMethods.Ftp.GetFileSize; 
 
 System.Net.FtpWebResponse lengthResponse =（System.Net.FtpWebResponse）request.GetResponse（）; 
 long length = lengthResponse.ContentLength; 
 lengthResponse.Close（）; 
的退货长度; 
 
} 
  

我没有试过其他服务器，诀窍。

I am trying to download files from ftp server with this code:

using (System.IO.FileStream fileStream = System.IO.File.OpenWrite(filePath))
              {

                byte[] buffer = new byte[4096];

                int bytesRead = responseStream.Read(buffer, 0, 4096);
                while (bytesRead > 0)
                {
                  fileStream.Write(buffer, 0, bytesRead);                      
                  bytesRead = responseStream.Read(buffer, 0, 4096);
                }

              }

The creation of responseStream:

System.IO.Stream responseStream = GetFileAsStream(url, username, password, false);

public static System.IO.Stream GetFileAsStream(string ftpUrl, string username, string password, bool usePassive)
{
  System.Net.FtpWebRequest request = (System.Net.FtpWebRequest)System.Net.WebRequest.Create(ftpUrl);
  request.KeepAlive = false;
  request.ReadWriteTimeout = 120000;
  request.Timeout = -1;
  request.UsePassive = usePassive;

  request.Credentials = new System.Net.NetworkCredential(username, password);

  request.Method = System.Net.WebRequestMethods.Ftp.DownloadFile;

  System.IO.Stream fileResponseStream;


  System.Net.FtpWebResponse fileResponse = (System.Net.FtpWebResponse)request.GetResponse();



  fileResponseStream = fileResponse.GetResponseStream();

  return fileResponseStream;
}

It works fine with smaller files but when a file is bigger (e.g. 150mb) the process hangs. For some reason the program does not understand that it has completed the download and it still tries to read more bytes.

I prefer answers which do not include using external libraries. Thank you

解决方案

I have solved my problem by introducing a request timeout- which, if reached, makes the program to throw a WebException. In that case, the program resumes the download from the place it left of. Here's my code :

This is inside of a method which is returning true if the file is downloaded, false- otherwise

Digitalez.DirectoryUtil.EnsureDirectoryExists(relativePath);

        string filePath = System.IO.Path.Combine(relativePath, fileInfo.Name);
        long length = Digitalez.FtpUtil.GetFileLength(fileInfo.FullPath, userName, password, usePassive);
        long offset = 0;
        int retryCount = 10;
        int? readTimeout = 5 * 60 * 1000; //five minutes

        // if the file exists, do not download it
        if (System.IO.File.Exists(filePath))
        {
          return false;
        }

        while (retryCount > 0)
        {

          using (System.IO.Stream responseStream = Captator.Eifos.Net.FtpUtil.GetFileAsStream(fileInfo.FullPath, userName, password, usePassive, offset, requestTimeout: readTimeout != null ? readTimeout.Value : System.Threading.Timeout.Infinite))
          {

            using (System.IO.FileStream fileStream = new System.IO.FileStream(filePath, System.IO.FileMode.Append))
            {
              byte[] buffer = new byte[4096];
              try
              {
                int bytesRead = responseStream.Read(buffer, 0, buffer.Length);

                while (bytesRead > 0)
                {
                  fileStream.Write(buffer, 0, bytesRead);

                  bytesRead = responseStream.Read(buffer, 0, buffer.Length);
                }

                return true;
              }
              catch (System.Net.WebException)
              {
                // Do nothing - consume this exception to force a new read of the rest of the file
              }
            }

            if (System.IO.File.Exists(filePath))
            {
              offset = new System.IO.FileInfo(filePath).Length;
            }
            else
            {
              offset = 0;
            }

            retryCount--;

            if (offset == length)
            {
              return true;
            }

          }
        }

Digitalez.FtpUtil:

public static System.IO.Stream GetFileAsStream(string ftpUrl, string username, string password, bool usePassive, long offset, int requestTimeout)
{
  System.Net.FtpWebRequest request = (System.Net.FtpWebRequest)System.Net.WebRequest.Create(ftpUrl);

  request.KeepAlive = false;
  request.ReadWriteTimeout = requestTimeout;
  request.Timeout = requestTimeout;
  request.ContentOffset = offset;
  request.UsePassive = usePassive;
  request.UseBinary = true;

  request.Credentials = new System.Net.NetworkCredential(username, password);

  request.Method = System.Net.WebRequestMethods.Ftp.DownloadFile;

  System.IO.Stream fileResponseStream;

  System.Net.FtpWebResponse fileResponse = (System.Net.FtpWebResponse)request.GetResponse();

  fileResponseStream = fileResponse.GetResponseStream();

  return fileResponseStream;
}

public static long GetFileLength(string ftpUrl, string username, string password, bool usePassive)
{
  System.Net.FtpWebRequest request = (System.Net.FtpWebRequest)System.Net.WebRequest.Create(ftpUrl);

  request.KeepAlive = false;
  request.UsePassive = usePassive;


  request.Credentials = new System.Net.NetworkCredential(username, password);
  request.Method = System.Net.WebRequestMethods.Ftp.GetFileSize;

  System.Net.FtpWebResponse lengthResponse = (System.Net.FtpWebResponse)request.GetResponse();
  long length = lengthResponse.ContentLength;
  lengthResponse.Close();
  return length;

}

I haven't tried other servers but this certainly does the trick.

这篇关于从FTP服务器下载大文件（〜150MB）挂起的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！