flask - python上传大文件到服务器报错

问题描述

问 题

项目需要，在mac上将打包好的文件（四五百兆）自动上传到web后台，用了两种方式上传，都报了类似的错误，在windows和linux上测试不会报错，但是到了mac打包机上就会报错：
①第一种报错：

Traceback (most recent call last):
  File "my_upload.py", line 61, in <module>
    upload_pack(f)
  File "my_upload.py", line 54, in upload_pack
    r = requests.post(url, form, files=files)
  File "/Library/Python/2.7/site-packages/requests-2.9.1-py2.7.egg/requests/api.py", line 107, in post
    return request('post', url, data=data, json=json, **kwargs)
  File "/Library/Python/2.7/site-packages/requests-2.9.1-py2.7.egg/requests/api.py", line 53, in request
    return session.request(method=method, url=url, **kwargs)
  File "/Library/Python/2.7/site-packages/requests-2.9.1-py2.7.egg/requests/sessions.py", line 468, in request
    resp = self.send(prep, **send_kwargs)
  File "/Library/Python/2.7/site-packages/requests-2.9.1-py2.7.egg/requests/sessions.py", line 576, in send
    r = adapter.send(request, **kwargs)
  File "/Library/Python/2.7/site-packages/requests-2.9.1-py2.7.egg/requests/adapters.py", line 426, in send
    raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', error(32, 'Broken pipe'))

代码：

# -*- coding: UTF-8 -*-
import requests
import json
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
url = 'http://10.242.12.121:9096/auto_upload/'

# config
channel = unicode('test2').encode('utf-8')
form = {'platform': 'Android', 'channel': channel, 'publish_time': '2016-08-01 00:00', 'remarks': 'auto_test', 'code': 'HH'}
files = {'file': ('D3174449.apk', open('D3174449.apk', 'rb'), 'application/octet-stream')}    
print 'upload...'
s = requests.session()
r = s.post(url, form, files=files, timeout=500)
print r.text
# print unicode(json.loads(r.text)).encode('utf-8')
print 'upload_end'

第二种报错：urllib2.URLError: <urlopen error [Errno 32] Broken pipe>
代码：

#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
# buld post body data
import urllib2
import time

boundary = '----WebKitFormBoundary3hYfNZFUuz8yiMid'
data = []
data.append('--%s' % boundary)

data.append('Content-Disposition: form-data; name="%s"\r\n' % 'platform')
data.append('Android')
data.append('--%s' % boundary)

data.append('Content-Disposition: form-data; name="%s"\r\n' % 'channel')
data.append('360')
data.append('--%s' % boundary)

data.append('Content-Disposition: form-data; name="%s"\r\n' % 'publish_time')
data.append('2016-08-23 00:00')
data.append('--%s' % boundary)

data.append('Content-Disposition: form-data; name="%s"\r\n' % 'remarks')
data.append('hh2')
data.append('--%s' % boundary)

data.append('Content-Disposition: form-data; name="%s"\r\n' % 'code')
data.append('HH')
data.append('--%s' % boundary)

fr = open(u'./D3174449.apk', 'rb')
data.append('Content-Disposition: form-data; name="%s"; filename="D3174449.apk"' % 'file')
data.append('Content-Type: %s\r\n' % 'application/octet-stream')
data.append(fr.read())
fr.close()
data.append('--%s--\r\n' % boundary)

http_url = 'http://10.242.12.121:9096/auto_upload/'
http_body = '\r\n'.join(data)
print 'upload...'
try:
    # buld http request
    req = urllib2.Request(http_url, data=http_body)
    # header
    #req.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boundary)
    #req.add_header('User-Agent', 'Mozilla/5.0')
    #req.add_header('Referer', 'http://10.246.13.129:9099/auto_upload/')
    req.add_header('Accept', 'application/json, text/javascript, */*; q=0.01')
    req.add_header('Accept-Encoding', 'gzip, deflate')
    req.add_header('Accept-Language', 'zh-CN,zh;q=0.8')
    req.add_header('Connection', 'keep-alive')
    #req.add_header('Content-Length', '620')
    req.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boundary)
    req.add_header('Cookie', 'session=.eJyrVkosLclQsjLUUUrOT0lVsqpWUkhSslIKdgmsUqoFCRYVALkZGXkmBhYGSjpKqbmJmTkIEQeQAr281JLUxOJUveT8XKCStNKcnLzEXKBhSjGlFgaphjGlZilmlkq1AP-KIUg.Cp7ldw.cDtv8It9wXz12qb7JbI5gf08rew')
    req.add_header('Origin', 'http://10.246.13.129:9096')
    req.add_header('Referer', 'http://10.246.13.129:9096/auto_upload/')
    req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36')
    req.add_header('X-Requested-With', 'XMLHttpRequest')

    # post data to server
    resp = urllib2.urlopen(req, timeout=500)
    # get response
    qrcont = resp.read()
    print qrcont

except Exception, e:
    print e
print 'upload end'

搜索能力有限，去谷歌搜了下没有找到合适的解决办法，或者是理解不够：
https://github.com/kennethrei...

服务端代码：

def auto_upload():
    """
    上传文件
    """
    try:
        file = request.files['file']
    except:
        print 'file fail'
    print 'myautofile_debug', file
    platform = request.form.get('platform')
    channel = request.form.get('channel')
    publish_time = request.form.get('publish_time')
    remarks = request.form.get('remarks')
    code = request.form.get('code')
    #create channel if channel does not exist
    data = {'platform': platform, 'channel': channel, 'remarks': '', 'app_num': 0, 'code': code}
    result = app.config['mongo'].insert_channel_data(data)
    print 'result:', result
    #if result is not False:
        # 增加渠道目录
    platform_dir = os.path.join(app.config['UPLOAD_FOLDER'], code, platform)
    channel_dir = os.path.join(platform_dir, channel)
    print 'add_channel', platform_dir, channel_dir, os.path.exists(platform_dir), os.path.exists(channel_dir)
        #try:
    if not os.path.exists(platform_dir):
        os.mkdir(platform_dir)
    if not os.path.exists(channel_dir):
        os.mkdir(channel_dir)
        #except:
            #return Response(json.dumps('fail'), mimetype='application/json')

    # channel = channel.encode('utf-8')
    relative_path = code + os.sep + platform + os.sep + channel + os.sep
    #print 'upload_file', relative_path
    if file:
        # filename = secure_filename(file.filename)
        filename = file.filename
        filename = gen_file_name(filename, relative_path)
        mimetype = file.content_type
        #print 'auto_mimetype:', mimetype
        if not allowed_file(file.filename, platform):
            #print 'not allowed'
            result = uploadfile(name=filename, type=mimetype, size=0, not_allowed_msg="文件类型错误！")

        else:
            # save file to disk
            uploaded_file_path = os.path.join(app.config['UPLOAD_FOLDER'], relative_path + filename)
            file.save(uploaded_file_path)

            # get file size after saving
            size = os.path.getsize(uploaded_file_path)
            size_str = _formatFileSize(size)
            #print 'size', size, size_str

            # save data into database
            data = {}
            data['code'] = code
            data['platform'] = platform
            data['channel'] = channel
            data['filename'] = filename
            data['url'] = uploaded_file_path
            data['size'] = size_str
            data['publish_time'] = publish_time
            data['upload_time'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            data['remarks'] = remarks
            app.config['mongo'].insert_app_data(data)

            # return json for js call back
            result = uploadfile(name=filename, type=mimetype, size=size, relative_path = relative_path)
        return json.dumps({"files": [result.get_file()]})

解决方案

你这个一般都需要结合服务器端的软件代码、框架版本之类的来解决啊，不过既然你特意说了Windows、Linux和MAC的差异，我只知道一个比较重要的差异，就是MAC操作系统在对方shutdown的情况下不能保持半连接状态，一段时间后悔自动关闭，所以如果你的服务器端在还没有传输完文件的情况下就以某种方式SHUTDOWN了写的一端，就会在传输中出错，比如说先发送了返回值，再慢慢接收数据。没有别的信息的话只能先这么猜了。

这篇关于flask - python上传大文件到服务器报错的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！