nodejs中使用async来对异步操作进行同步,避免多个异步同时启动引发连接错误

来源:互联网 发布:siam journal 知乎 编辑:程序博客网 时间:2024/05/29 09:16
</pre><pre name="code" class="python">/* 对于nodejs中由于几乎大部分的操作都是异步的,所以我们都是通过一层层的函数来实现。 而异步也会带来一些问题,那就是同时并发过度的request,就会导致链接失败(或则连接被关闭) 如何做到既能异步操作,又能控制并发数目,asyn库应该是可以满足需要 */const url = require('url');const request = require('request');const http = require('http');const fs = require('fs');const util = require('util');const async = require('async');function getUrl(id, arr) {    var arrs = [];    for (var i = 0; i < arr[1]; ++i) {        var n = i + '';        if (n !== '0') {            while (n.length < 3) {                n = 0 + n;            }        }        var r = util.format('http://img.zngirls.com/gallery/%d/%d/%s.jpg', id, arr[0], n);        arrs.push(r);    }    //console.log(arrs);    return arrs;}function direct_download(_url, filename, cb) {    var arrPath = filename.split('/');    if (arrPath.length > 1) {        var fullPath = '';        arrPath = arrPath.slice(0, -1);        do {            fullPath += arrPath.shift() + '/';            if (!fs.existsSync(fullPath)) {                fs.mkdir(fullPath);            }        } while (arrPath.length > 0);    }    var opt = {        proxy: 'http://proxy3.bj.petrochina:8080',        url: _url,        headers: {            referer: 'http://www.baidu.com',        }    }    request.get({        proxy: 'http://proxy3.bj.petrochina:8080',        url: _url,        headers: {            referer: 'http://www.baidu.com',        }    }).pipe(fs.createWriteStream(filename)).on('close', function () {        console.log("图片下载成功啦");        if (cb) {            cb(null, "successful !");            //callback貌似必须调用,第二个参数将传给下一个回调函数的result,result是一个数组        }    });    //download(_url, filename);    /*     request.get(opt, function(err,res, body){     if(err){     console.log(err);     console.log("有一张图片请求失败啦...");     }else{     fs.writeFile(filename, body, function(err){     if(err){     console.log(err);     console.log("有一张图片写入失败啦...");     }else{     console.log("图片下载成功啦");     if(cb){     cb(null,"successful !");     //callback貌似必须调用,第二个参数将传给下一个回调函数的result,result是一个数组     }     }     });     }     });     */}function download(ur, fileName) {    //实现下载(网络异步)和文件保存(IO异步操作)    var u = url.parse(ur);    var options = {        //代理服务器        host: 'proxy3.bj.petrochina',        port: 8080,        path: ur,        //增加请求头,绕过服务器检测        //headers : {Referer:'http://www.baidu.com',}        headers: {            Referer: 'http://www.baidu.com',            Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',            'Accept-Encoding': 'gzip, deflate, sdch',            'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',            Host: 't1.zngirls.com',            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',        }    };    http.get(options, function (res) {        //打开一个文件对象,在数据传输过程中,每次以块的形式        //写入到文件中        var fd = fs.openSync(fileName, 'w');        //监听数据传输        res.on('data', function (chunk) {            fs.writeSync(fd, chunk, 0, chunk.length);            //console.log(util.inspect(chunk,true));        });        //监听传输完成        res.on('end', function () {            fs.closeSync(fd);            console.log(`save to ${fileName}`);        });    });}function somany_request() {    var id = 18071;    //var gallery = [[18812, 49], [19695, 49], [19019, 43], [18214, 49], [16751, 54], [13207, 72], [13206, 68]];    var gallery = [[18812, 49], [19695, 49], [19019, 43], [18214, 49], [16751, 54], [13207, 72], [13206, 68]];    var pictures = []    gallery.forEach(function (i) {        var urls = getUrl(id, i);        urls.forEach(function (_url) {            var filename = _url.split('/').slice(-4).join('/');            pictures.push({url: _url, filename: filename});            //direct_download(_url, filename);        });    });    //开始启动异步下载    async.mapLimit(pictures, 10, function (pic, cb) {        console.log('已经有10张图片进入下载队列...');        direct_download(pic.url, pic.filename, cb);    }, function (err, result) {        if (err) {            console.log(err);        } else {            console.log('下载任务全部完成');        }    });}somany_request();//direct_download('http://img.zngirls.com/gallery/18071/18812/0.jpg', 'gallery/18071/18812/0.jpg');

0 0