Node.JS同时下载数百个文件 [英] Node.JS downloading hundreds of files simultaneously

问题描述

我正尝试同时下载超过100个文件.但是，当我执行下载功能时，我的Macbook在Windows中冻结(无法执行新任务)，在这两种情况下(空闲网络)也都没有下载(但没有冻结)并且没有下载进度.

I am trying to download more that 100 files at the same time. But when I execute the downloading function my macbook freezes(unable to execute new tasks) in windows also no download(but doesn't freeze) and no download progress in both case(idle network).

这是我的下载模块:

var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');

router.get('/', function (req, res, next) {
    links.find({dlStatus: false}, function (err, docs) {

        if (err) {
            console.log(err);
            res.end();
        } else if (!docs) {
            console.log('No incomplete downloads!');
            res.end();
        } else {
            for (var i = 0; i < docs.length; i++) {

                //todo scraping


                var video = youtubedl(docs[i].url, [], {cwd: __dirname});

                // Will be called when the download starts.
                video.on('info', function (info) {
                    console.log('Download started');
                    console.log(info);
                });

                video.pipe(fs.createWriteStream('./downloads/' + docs[i].id + '-' + i + '.mp4'));
                video.on('complete', function complete(info) {
                    links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
                        if (err)console.log(err);
                        else console.log('Download completed!')
                    });
                });
            }
        }
    });
});

module.exports = router;

现在任何人都可以在这里帮助我吗?我正在使用此模块下载文件.

Now can anyone please help me here? I am using this module for downloading files.

推荐答案

在这种情况下，解决方案正在使用 async .

The solution is using async in this case.

以这种方式尝试....使用async.each()

Try it this way....with async.each()

var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');
var async = require('async')

router.get('/', function (req, res, next) {
    links.find({dlStatus: false}, function (err, docs) {

        if (err) {
            console.log(err);
            res.end();
        } else if (!docs) {
            console.log('No incomplete downloads!');
            res.end();
        } else {
            async.each(docs,function(doc,cb){
              var video = youtubedl(doc.url, [], {cwd: __dirname});

              // Will be called when the download starts.
              video.on('info', function (info) {
                  console.log('Download started');
                  console.log(info);
              });

              video.pipe(fs.createWriteStream('./downloads/' + docs.id + '-' + i + '.mp4'));
              video.on('complete', function complete(info) {
                  links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
                      if (err){
                        console.log(err);
                        cb(err);
                      }
                      else {
                        console.log('Download completed!');
                        cb()
                      }
                  });
              });

            },function(err){
              if(err)
                return console.log(err);
              console.log("Every thing is done,Here!!");
            })
        }
    });
});

module.exports = router;

您也可以使用async.eachLimits()批量处理所有内容.

And you can process every thing in batch too using async.eachLimits().

这篇关于Node.JS同时下载数百个文件的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！