Scrapy是一个为了爬取网站数据，提取结构性数据而编写的应用框架。可以应用在包括数据挖掘， 信息处理或存储历史数据等一系列的程序中。其最初是为了页面抓取(更确切来说,网络抓取)所设计的， 也可以应用在获取API所返回的数据(比如Web Services)或者通用的网络爬虫。

yum install python-devel

yum install libffi-devel

yum install openssl-devel

pip install pyopenssl

yum install python-lxml

yum install libxml2-devel

yum install libxslt-devel

pip install service-identity

pip install scrapy

pip install scrapy -U

scrapy bench

import scrapy

class StackOverflowSpider(scrapy.Spider):

    name = 'stackoverflow'

    start_urls = ['http://stackoverflow.com/questions?sort=votes']

    def parse(self, response):

        for href in response.css('.question-summary h3 a::attr(href)'):

            full_url = response.urljoin(href.extract())

            yield scrapy.Request(full_url, callback=self.parse_question)

    def parse_question(self, response):

        yield {

            'title': response.css('h1 a::text').extract()[0],

            'votes': response.css('.question .vote-count-post::text').extract()[0],

            'body': response.css('.question .post-text').extract()[0],

            'tags': response.css('.question .post-tag::text').extract(),

            'link': response.url,

        }

scrapy runspider stackoverflow_spider.py -o top-stackoverflow-questions.json

[{

    "body": "... LONG HTML HERE ...",

    "link": "http://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-an-unsorted-array",

    "tags": ["java", "c++", "performance", "optimization"],

    "title": "Why is processing a sorted array faster than an unsorted array?",

    "votes": ""

},

{

    "body": "... LONG HTML HERE ...",

    "link": "http://stackoverflow.com/questions/1260748/how-do-i-remove-a-git-submodule",

    "tags": ["git", "git-submodules"],

    "title": "How do I remove a Git submodule?",

    "votes": ""

},

...]