-
Notifications
You must be signed in to change notification settings - Fork 0
/
zolXiaoHua.js
35 lines (31 loc) · 865 Bytes
/
zolXiaoHua.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
var Crawler = require("crawler");
var fs = require('fs');
var file = "zolConfig.json"
var config = JSON.parse(fs.readFileSync(file));
console.log(config)
/*
var ifrom = process.argv[2];//5400
var ito = process.argv[3] ;//5500;
*/
var ifrom = config.iStart
var ito = config.iEnd
var fileStream = ifrom+'-'+ito+'.txt';
var c = new Crawler({
//rateLimit: 1000, // `maxConnections` will be forced to 1
callback: function(err, res, done){
var article = res.$(".article-text").text();
fs.appendFile(fileStream, article, function (err) {
if(err){
return console.log(err);
}else {
console.log(article);
}
});
done();
}
});
var arrUrl = [];
for (i=ifrom; i<ito; i++){
arrUrl.push('http://xiaohua.zol.com.cn/detail5/'+i+'.html');
}
c.queue(arrUrl);//between two tasks, minimum time gap is 1000 (ms)