xicidaili.py

# -*- coding: utf-8 -*-

import scrapy

# 继承scrapy,Spider类

class XicidailiSpider(scrapy.Spider):

    name = 'xicidaili'

    allowed_domains = ['xicidaili.com']

    start_urls = ['https://www.xicidaili.com/nn/',

                  "https://www.xicidaili.com/nt/",

                  "https://www.xicidaili.com/wn/,"

                  "https://www.xicidaili.com/wt/"]

    # 解析响应数据，提取数据和网址等。

    def parse(self, response):

        selectors = response.xpath('//tr')

        for selector in selectors:

            ip = selector.xpath("./td[2]/text()").get()

            port = selector.xpath("./td[3]/text()").get()     #.代表当前节点下

            country = selector.xpath("./td[4]/a/text()").get()   # get()和extract_first() 功能相同，getall()获取多个

            # print(ip,port,country)

            Items={

                "ip":ip,

                "port":port,

                "country":country

            }

            yield  Items

        """

        # 翻页操作

        # 获取下一页的标签

        next_page = response.xpath("//a[@class='next_page']/@href").get()

        # 判断next_page是否有值，也就是是否到了最后一页

        if next_page:

            # 拼接网页url---response.urljoin

            next_url = response.urljoin(next_page)

            # 判断最后一页是否

            yield  scrapy.Request(next_url,callback=self.parse)   # 回调函数不要加括号

    """

# -*- coding: utf-8 -*-

# settings.py设置

# Scrapy settings for xicidailiSpider project

#

# For simplicity, this file contains only settings considered important or

# commonly used. You can find more settings consulting the documentation:

#

#     https://doc.scrapy.org/en/latest/topics/settings.html

#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html

#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'xicidailiSpider'

SPIDER_MODULES = ['xicidailiSpider.spiders']

NEWSPIDER_MODULE = 'xicidailiSpider.spiders'

# 设置到处文件的字符编码

FEED_EXPORT_ENCODING ="UTF8"

# Crawl responsibly by identifying yourself (and your website) on the user-agent

#USER_AGENT = 'xicidailiSpider (+http://www.yourdomain.com)'

# Obey robots.txt rules

# 是否准售robots.txt协议，不遵守

ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)

#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)

# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay

# See also autothrottle settings and docs

#DOWNLOAD_DELAY = 3

# The download delay setting will honor only one of:

#CONCURRENT_REQUESTS_PER_DOMAIN = 16

#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)

#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)

#TELNETCONSOLE_ENABLED = False

# Override the default request headers:

DEFAULT_REQUEST_HEADERS = {

   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

   'Accept-Language': 'en',

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) \

    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'

}

# Enable or disable spider middlewares

# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html

#SPIDER_MIDDLEWARES = {

#    'xicidailiSpider.middlewares.XicidailispiderSpiderMiddleware': 543,

#}

# Enable or disable downloader middlewares

# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html

#DOWNLOADER_MIDDLEWARES = {

#    'xicidailiSpider.middlewares.XicidailispiderDownloaderMiddleware': 543,

#}

# Enable or disable extensions

# See https://doc.scrapy.org/en/latest/topics/extensions.html

#EXTENSIONS = {

#    'scrapy.extensions.telnet.TelnetConsole': None,

#}

# Configure item pipelines

# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html

#ITEM_PIPELINES = {

#    'xicidailiSpider.pipelines.XicidailispiderPipeline': 300,

#}

# Enable and configure the AutoThrottle extension (disabled by default)

# See https://doc.scrapy.org/en/latest/topics/autothrottle.html

#AUTOTHROTTLE_ENABLED = True

# The initial download delay

#AUTOTHROTTLE_START_DELAY = 5

# The maximum download delay to be set in case of high latencies

#AUTOTHROTTLE_MAX_DELAY = 60

# The average number of requests Scrapy should be sending in parallel to

# each remote server

#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0

# Enable showing throttling stats for every response received:

#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)

# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings

#HTTPCACHE_ENABLED = True

#HTTPCACHE_EXPIRATION_SECS = 0

#HTTPCACHE_DIR = 'httpcache'

#HTTPCACHE_IGNORE_HTTP_CODES = []

#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'