使用sync-request和cheerio写爬虫

npm install sync-request

var request = require('sync-request');
var html = request(method, url, options).getBody().toString();

npm install cheerio

var cheerio = require('cheerio');
var $ = cheerio.load(html);

var cheerio = require('cheerio');
var request = require('sync-request');
var fs = require('fs');

var ofile = fs.openSync('out.html', 'w');

var url = 'http://www.runoob.com/html/html-tutorial.html';
var html = request('GET', url).getBody().toString();

var toc = getToc(html);
for(var i in toc) {
    var url = toc[i];
    console.log('page: ' + url);
    html = request('GET', url).getBody().toString();
    var content = getContent(html);
    fs.writeSync(ofile, content, null, 'utf-8');
}

fs.closeSync(ofile);
console.log('Done..');


function getToc(html)  {

    var $ = cheerio.load(html);

    var $list = $('#leftcolumn').find('a');
    var res = [];
    for(var i = 0; i < $list.length; i++)
    {
        var url = $list.eq(i).attr('href');
        res.push('http://www.runoob.com/' + url);
    }
    return res;
}


function getContent(html) {
    var $ = cheerio.load(html);
    var content = $('#content').html();
    return content;
}