基于nodeJS爬虫

最新开始学nodeJS（前端的高大上）

先说说中间需要用到的第三方依赖吧
直接贴代码：

const http = require('http')const fs = require('fs')const cheerio = require('cheerio')const iconv = require('iconv-lite')let request = require('request');

##FS NODEJS文件系统

fs.appendFile('message.txt', 'data to append', (err) => {if (err) throw err;console.log('The "data to append" was appended to file!');});
等等这里只列出一个，具体的请去参考官网API：http://nodejs.cn/api/fs.html#

#

##cheerio为服务器特别定制的，快速、灵活、实施的jQuery核心实现

var cheerio = require('cheerio'),    $ = cheerio.load('<h2>Hello world</h2>');let htmls = $('h2').html();console.log(htmls)//=> Hello there!

#

## iconv-lite 为防止当前访问网页文件乱码而出现的，通过buffer字符流来进行转化进行相应的转换字符编码

    iconv.decode(Buffer.Concat(html),'gb2312');

## 下面为源代码

//引入模块const http = require('http')const fs = require('fs')const cheerio = require('cheerio')const iconv = require('iconv-lite')let request = require('request');const zzxxxc = [    'http://www.88dushu.com/xiaoshuo/69/69252/'];// http://www.biquge.cm/8/8726/let bookLists = [];let bookName = '';let m = 0;hrefs();/** * 初始化存储章节 */function hrefs() {    if (m < zzxxxc.length) {        http.get(zzxxxc[m], res => {            const html = []            res.on('data', (chunk) => {                html.push(chunk)            })            res.on('end', () => {                const html1 = iconv.decode(Buffer.concat(html), 'gb2312');                const $ = cheerio.load(html1);                var ddsli = {};                let result = $('.mulu').find('li');                bookName = $('.rt h1').text();                let key = ''; // 用于存储书名                result.each((i, elem) => {                    let $hrefTag = $(elem).find('a') || false;                    if ($hrefTag) {                        $hrefTag.each((i, elem) => {                            let bookCharInfo = {};                            if ($(elem).text().trim() !== '插图') {                                bookCharInfo.charName = $(elem).text().trim();                                bookCharInfo.charUrl = zzxxxc[m] + $(elem).attr('href');                                bookLists.push(bookCharInfo);                            }                        });                    }                });                m++;                txtWord()            });        })    } else {        return '';    }}let i = 0;/** * 循环查询页面存储 */function txtWord() {    let hrefUrls = bookLists[i].charUrl;    request({        url: hrefUrls,        headers: {            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36',        },        encoding: null    }, function(error, response, body) {        if (response && response.statusCode == 200) {            // 获取数据完毕后，使用iconv-lite转码，decedo中为Buffer对象，Buffer.concat为数组            const html1 = iconv.decode(body, 'gb2312');            const $ = cheerio.load(html1);            var ddsli = {}; // 处理数据            var bokname = $('.novel h1').text() + '.'.replace(/./g, '\n');            var contentTxt = $('.yd_text2').text().trim().replace(/\s/g, "").replace(/。/g, '。\n') + '.'.replace(/./g, '\r\n');            fs.appendFile('./' + bookName + '.txt', (bokname + contentTxt), function(err) {                if (err) {} else {                    if (i < bookLists.length - 1) {                        i++;                        console.log('开始下载==>' + bokname.replace(/[\r\n]/g, ""));                        txtWord();                    } else {                        console.log('下载完成,进入下一本');                        hrefs();                    }                }            });        } else {            txtWord();        }    });}/** * 剔除空格 * @param {*字符串} str  */function trim(str) {    return str.replace(/(^\s*)|(\s*$)/g, '').replace(/ /g, '')}

如有转载：请提供出处谢谢