pip install Scrapy

pip uninstall Scrapy

scrapy startproject crawlquote#crawlquote这是我起的项目名

scrapy genspider quotes quotes.toscrape.com

scrapy crawl quotes

import scrapy

class CrawlquoteItem(scrapy.Item):

    # define the fields for your item here like:

    # name = scrapy.Field()

    text = scrapy.Field()

    author = scrapy.Field()

    tags = scrapy.Field()

# -*- coding: utf- -*-

import scrapy

from  crawlquote.items import CrawlquoteItem

class QuotesSpider(scrapy.Spider):

    name = 'quotes'

    allowed_domains = ['quotes.toscrape.com']

    start_urls = ['http://quotes.toscrape.com/']

    def parse(self, response):

        quotes = response.css('.quote')

        for quote in quotes:

            item = CrawlquoteItem()

            text = quote.css('.text::text').extract_first()  # 获取一个

            author = quote.css('.author::text').extract_first()

            tags = quote.css('.tags .tag::text').extract()

            item['text'] = text

            item['author'] = author

            item['tags'] = tags

            yield item  # 将网页中的内容重新生成一个item以便于后面的认识

        next = response.css('.pager .next a::attr(href)').extract_first()

        url = response.urljoin(next)  # urljoin翻页

        yield scrapy.Request(url=url, callback=self.parse)  # 递归调用

# -*- coding: utf- -*-

# Define your item pipelines here

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymongo

from scrapy.exceptions import DropItem

class TextPipeline(object):

    def __init__(self):

        self.limit = 

    def process_item(self, item, spider):  # 对重新生成的item进行再制作

        if item['text']:

            if len(item['text']) > self.limit:

                item['text'] = item['text'][:self.limit].rstrip() + '...'

            return item

        else:

            return DropItem('Missing Text')

class MongoPipeline(object):  # 与数据库有关的操作

    def __init__(self, mongo_uri, mongo_db):  # （） MongoPipeline构造函数

        self.mongo_uri = mongo_uri

        self.mongo_db = mongo_db

    @classmethod

    def from_crawler(cls, crawler):  # （）读取settings里面的值，类方法

        return cls(

            mongo_uri=crawler.settings.get('MONGO_URI'),

            mongo_db=crawler.settings.get('MONGO_DB')

        )

    def open_spider(self, spider):  # （）爬虫启动时需要的操作

        self.client = pymongo.MongoClient(self.mongo_uri)

        self.db = self.client[self.mongo_db]

    def process_item(self, item, spider):  # 保存到mongodb数据库

        name = item.__class__.__name__

        self.db[name].insert(dict(item))

        return item

    def close_spider(self, spider):  # 关闭mongodb

        self.client.close()

BOT_NAME = 'crawlquote'

SPIDER_MODULES = ['crawlquote.spiders']

NEWSPIDER_MODULE = 'crawlquote.spiders'

#数据库链接

MONGO_URI = 'localhost'

MONGO_DB = 'crawlquote'

#项目管道开启

ITEM_PIPELINES = {

    'crawlquote.pipelines.TextPipeline': ,

    'crawlquote.pipelines.MongoPipeline': ,

}