scrapy startproject myProject

import scrapy

class MyprojectItem(scrapy.Item):

    number = scrapy.Field() #帖子编号

    title = scrapy.Field() #帖子标题

    content = scrapy.Field() #帖子内容

    status = scrapy.Field() #处理状态

scrapy genspider sun wz.sun0769.com

import scrapy

from scrapy.linkextractors import LinkExtractor

from scrapy.spiders import CrawlSpider, Rule

from myProject.items import MyprojectItem

class SunSpider(CrawlSpider):

    name = 'sun'

    allowed_domains = ['wz.sun0769.com']

    start_urls = ['http://wz.sun0769.com/political/index/politicsNewest?id=1&page=1']

    rules = (

        Rule(LinkExtractor(allow=r'id=\d+&page=\d+')),#每一页的匹配规则，默认follow=True跟进

        Rule(LinkExtractor(allow=r'politics/index\?id=\d+'), callback='parse_item'),#每个帖子的匹配规则，默认不跟进

    )

    def parse_item(self, response):

        item = MyprojectItem()

        title = response.xpath('//div[@class="mr-three"]/p[@class="focus-details"]/text()').extract()[0] #投诉标题

        status = response.xpath('//div[@class="focus-date clear focus-date-list"]/span[3]/text()').extract()[0].split()[1] #处理状态

        number = response.xpath('//div[@class="focus-date clear focus-date-list"]/span[4]/text()').extract()[0].split('：')[-1] #投诉编号

        content = response.xpath('//div[@class="details-box"]/pre/text()').extract()[0] #投诉内容

        item['title'] = title

        item['status'] = status

        item['number'] = number

        item['content'] = content

        yield item

import random

import json

import redis

from myProject.settings import USER_AGENTS

class MyprojectDownloaderMiddleware:

    def __init__(self):

        super().__init__()

        self.r = redis.StrictRedis()#创建redis数据库连接客户端，用于取里面存储的代理Ip和port

    def process_request(self, request, spider):

        proxy_list = json.loads(self.r.get('proxy_list').decode())

        proxy = random.choice(proxy_list)

        request.headers['User-Agent'] = random.choice(USER_AGENTS) #设置user-agent

        request.meta['proxy'] ='http://'+proxy['ip']+':'+str(proxy['port']) #设置代理ip和port

        return None

import pymongo

class MyprojectPipeline:

    def open_spider(self,spider):

        self.client = pymongo.MongoClient('mongodb://localhost:27017')#创建mongodb数据库连接客户端

        self.db = self.client.test #选择test数据库

        self.collection = self.db.sunInfo #选择sunInfo集合存储数据

    def process_item(self, item, spider):

        self.collection.insert_one(dict(item))

        return item

    def close_spider(self,spider):

        self.client.close()

BOT_NAME = 'myProject'

SPIDER_MODULES = ['myProject.spiders']

NEWSPIDER_MODULE = 'myProject.spiders'

ROBOTSTXT_OBEY = False

COOKIES_ENABLED = False

DOWNLOAD_DELAY = 1.0#设置延迟，减轻服务器压力，从而降低被查封的风险

#设置下载中间件

DOWNLOADER_MIDDLEWARES = {

   'myProject.middlewares.MyprojectDownloaderMiddleware': 543,

}

#设置item管道

ITEM_PIPELINES = {

   'myProject.pipelines.MyprojectPipeline': 300,

}

#可供选择的User-Agent，可通过网上查找更多，随机切换，起到反爬虫作用

USER_AGENTS = [

    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",

    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",

    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",

    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",

    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",

    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",

    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",

    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5"

    ]

scrapy crawl sun