Nodejs全站开发学习系列 & 深入浅出Node学习笔记 & Spider抓取

https://course.tianmaying.com/node

这个系列的文章看起来很不错，值得学习一下。

/Users/baidu/Documents/Data/Interview/Web-Server开发/深入浅出Node.js-f46c.pdf

深入浅出Node笔记：

// math.js

exports.add = function () {

var sum = 0, i = 0,

args = arguments,

l = args.length; while (i < l) {

  sum += args[i++]; }

return sum; };

// program.js

var math = require('math'); exports.increment = function (val) {

return math.add(val, 1); };

写一个例子：

$ cat module_path.js

console.log(module.paths);

运行：

$ node module_path.js

[ '/Users/baidu/Documents/Data/Work/Code/Self/nodejs/node_modules',

  '/Users/baidu/Documents/Data/Work/Code/Self/node_modules',

  '/Users/baidu/Documents/Data/Work/Code/node_modules',

  '/Users/baidu/Documents/Data/Work/node_modules',

  '/Users/baidu/Documents/Data/node_modules',

  '/Users/baidu/Documents/node_modules',

  '/Users/baidu/node_modules',

  '/Users/node_modules',

  '/node_modules' ]

看到P66，有点枯燥。

http://blog.csdn.net/g9yuayon/article/details/1568980

明眼老大们自然可以看出这是函数编程的风格。其实JavaScript本就是采用C语言句法的简化版LISP，异常灵活。

用下面的例子看看：

http://blog.csdn.net/u012273376/article/details/52736906

利用nodejs做爬虫

上文中分析了设计和实现的过程，最后是利用从一个json文件里获取出数据，并存储来实现的。

http://www.bilibili.com/index/index-icon.json

内容类似：

{"fix":[{"id":"568","type":"fix","title":"\u4e1b\u6797","deltime":"0","posttime":"1475232463","edittime":"1475232495","sttime":"1475145660","endtime":"0","state":"1","icon":"http:\/\/i0.hdslb.com\/group1\/M00\/B8\/16\/oYYBAFfuQueAcLCTAAAlMkM6OCk652.gif","weight":"2","links":["http:\/\/search.bilibili.com\/all?keyword=%E4%B8%9B%E6%9E%97"]},{"id":"567","type":"fix","title":"\u975e\u6d32\u6b22\u8fce\u4f60","deltime":"0","posttime":"1475232419","edittime":"1475232446","sttime":"1475145660","endtime":"0","state":"1","icon":"http:\/\/i0.hdslb.com\/group1\/M00\/B8\/16\/oYYBAFfuQa2AeYm6AACICFF4-Wk024.gif","weight":"2","links":["http:\/\/search.bilibili.com\/all?keyword=%E9%9D%9E%E6%B4%B2%E6%AC%A2%E8%BF%8E%E4%BD%A0"]},{"id":"566","type":"fix","title":"\u806a\u54e5","deltime":"0","posttime":"1475232380","edittime":"1475232380","sttime":"1475145660","endtime":"0","state":"1","icon":"http:\/\/i0.hdslb.com\/group1\/M00\/B8\/16\/oYYBAFfuQa6ABU5vAAAQAHdNgM4913.gif","weight":"2","links":["http:\/\/search.bilibili.com\/all?keyword=%E8%81%AA%E5%93%A5"]},{"id":"565","type":"fix","title":"\u975e\u6d32\u4eba (..\u2022\u02d8_\u02d8\u2022..)","deltime":"0","posttime":"1475232306","edittime":"1475232339","sttime":"1475145660","endtime":"0","state":"1","icon":"http:\/\/i0.hdslb.com\/group1\/M00\/B8\/16\/oYYBAFfuQauAQVjgAAAZKlLn9aM964.gif","weight":"1","links":["http:\/\/search.bilibili.com\/all?keyword=%E9%9D%9E%E6%B4%B2"]},{"id":"564","type":"fix","title":"SSR","deltime":"0","posttime":"1475232266","edittime":"1475232273","sttime":"1475145660","endtime":"0","state":"1","icon":"http:\/\/i0.hdslb.com\/group1\/M00\/B8\/16\/oYYBAFfuQaiAHLMeAABMJhP3294905.gif","weight":"3","links":["http:\/\/search.bilibili.com\/all?keyword=SSR"]},{"id":"563","type":"fix","title":"\u9759\u7535","deltime":"0","posttime":"1475232209","edittime":"1475232209","sttime":"1475145660","endtime":"0","state":"1","icon":"http:\/\/i0.hdslb.com\/group1\/M00\/B8\/16\/oYYBAFfuQaaAVYlOAAAPBHWFJBA697.gif","weight":"1","links":["http:\/\/search.bilibili.com\/all?keyword=%E9%9D%99%E7%94%B5"]},{"id":"562","type":"fix","title":"\u8214\u51b0\u68d2","deltime":"0","posttime":"1474973710","edittime":"1474973710","sttime":"1474887000","endtime":"0","state":"1","icon":"http:\/\/i0.hdslb.com\/group1\/M00\/B8\/10\/oYYBAFfqTrOAOUUFAAASBQb9Glo754.gif","weight":"1","links":["http:\/\/search.bilibili.com\/all?keyword=%E8%88%94%E5%86%B0"]},

......

在这个目录 /Users/baidu/Documents/Data/Work/Code/Self/nodejs/bilibili_spider 写了代码 index.js

const fs = require('fs')

const request = require('request')

function getJsonFile(jurl) {

  request({

    url: jurl,

    gzip: true

  }, function(err, res, body) {

    if (!err && res.statusCode == 200) {

      console.log('===获取Json成功===');

      let result = JSON.parse(body);

      for (let i=0; i<result.fix.length; i++) {

        saveGif(result.fix[i].icon, result.fix[i].title);

      }

    }

    else {

      console.log('===Error info===', err, 'Code:'+res.statusCode);

      return false;

    }

  }

  );

}

function saveGif(url, title) {

console.log('存储图片=>' + title);

request(url).pipe(fs.createWriteStream('./gif/'+title+'.gif'));

console.log('图片' + title + '存储完成');

}

其中有几点需要说明和了解：

1. (() => {

  const jsonUrl = 'http://www.bilibili.com/index/index-icon.json';

  getJsonFile(jsonUrl);

})();

这是一个匿名函数。类似 ()()； 这样的就是匿名函数。为什么要加两个括号呢，因为第二个括号是用来调用的。

2. request是需要用npm来安装的

3. request(url).pipe(fs.createWriteStream('./gif/'+title+'.gif'));

这个用法要了解。

4. 如果目录(gif)不存在，文件是无法创建成功的。命令行看到打印出了结果，是因为JS是异步执行的。

第一次运行时候报错：

SyntaxError: Block-scoped declarations (let, const, function, class) not yet supported outside strict mode

上网搜了之后，在第一行增加了严格模式的声明：

"use strict";

然后运行，报了另外的错：

Error: options.uri is a required argument

上网查了半天，没想到在匿名函数前面加个log居然就好了。。。代码片段如下：

console.log("hi0");

(() => {

  //console.log("hi1");

  const jsonUrl = 'http://www.bilibili.com/index/index-icon.json';

  //console.log("hi2");

  getJsonFile(jsonUrl);

  //console.log("hi3");

})();

function getJsonFile(jurl) {

  //jurl = url.parse(jurl);

  request({

    url: jurl,

    gzip: true

  }, function(err, res, body) {
......

运行结果如下：

$ node index.js

hi0

===获取Json成功===

存储图片=>丛林

图片丛林存储完成

存储图片=>非洲欢迎你

图片非洲欢迎你存储完成

存储图片=>聪哥

图片聪哥存储完成

存储图片=>非洲人 (..•˘_˘•..)

图片非洲人 (..•˘_˘•..)存储完成

文件内容如下：

莫非是由于异步调用的原因？

需要增加一个console.log，增加对异步调用的延迟？

未解。。。

后来又上网查了查，增加url模块的校验，感觉跟之前的也没差别：

"use strict";

const fs = require('fs')

const request = require('request')

const url = require('url');

//console.log("hi0");

(() => {

  //console.log("hi1");

  const jsonUrl = 'http://www.bilibili.com/index/index-icon.json';

  //console.log("hi2");

  getJsonFile(jsonUrl);

  //console.log("hi3");

})();

function getJsonFile(jurl) {

  jurl = url.parse(jurl);

  request({

    url: jurl,

    gzip: true

  }, function(err, res, body) {

    if (!err && res.statusCode == 200) {

      console.log('===获取Json成功===');

      let result = JSON.parse(body);

      for (let i=0; i<result.fix.length; i++) {

        saveGif(result.fix[i].icon, result.fix[i].title);

      }

    }

    else {

      console.log('===Error info===', err, 'Code:'+res.statusCode);

      return false;

    }

  }

  );

}

......

可以看到，增加url.parse，去掉了console.log，调用也能成功了。实在是不知所以然。

然后我又仿照下面这个程序：

http://cnodejs.org/topic/54bdaac4514ea9146862abee

写了一个抓取lofter上图片的程序：

var fs = require('fs');

var request = require('request');

var cheerio = require('cheerio');

var mkdirp = require('mkdirp');

var url = 'http://loftermeirenzhi.lofter.com/tag/人像?page=';

var dir = './images';

mkdirp(dir, function(err) {

  if (err) {

    console.log(err);

  }

});

var getImages = function(indexes) {

    for (var i=1; i<=indexes; i++) {

      var newUrl = url + i;

      request(newUrl, function(error, response, body) {

        if (!error && response.statusCode == 200) {

          console.log(body);

          var $ = cheerio.load(body);

          // selector

          $('.img img').each(function() {

            var src = $(this).attr('src');

            console.log('正在下载' + src);

            download(src, dir);

          });

        }

      });

    }

}

var download = function(url, dir) {

    var fileName = Math.floor(Math.random()*100000) + url.substr(-4, 4);

    request.head(url, function(err, res, body) {

      request(url).pipe(fs.createWriteStream(dir+'/'+fileName+'.jpg'));

    });

};

getImages(1);

上面这一句我不是非常明白，感觉应该是先class再标签

$('.img img')

实际去抓的时候，发现只能抓下一个头图。而页面中实际的图是没有能够抓下来的。

巴特西

Nodejs全站开发学习系列 & 深入浅出Node学习笔记 & Spider抓取

最新文章

热门文章