scrapy startproject pfive

scrapy genspider pfive_spider www.17500.cn

start_urls = ['http://www.17500.cn/p5/all.php']

class PfiveItem(scrapy.Item):

    #开奖期号

    awardID = scrapy.Field()

    #开奖日期

    awardDate = scrapy.Field()

    #开奖号码

    awardNum = scrapy.Field()

class PfiveSpiderSpider(scrapy.Spider):

    name = 'pfive_spider'

    allowed_domains = ['www.17500.cn']

    start_urls = ['http://www.17500.cn/p5/all.php']

    def parse(self, response):

        list = response.xpath("//table/tbody/tr/td/table/tbody/tr[3]/td[@class='normal']/table/tbody/tr[@bgcolor='#ffffff']")

        for l in list:

            pfiveItem = PfiveItem()

            pfiveItem['awardID'] = l.xpath('./td[1]/text()').extract_first()

            pfiveItem['awardDate'] = l.xpath('./td[2]/text()').extract_first()

            pfiveItem['awardNum'] = l.xpath('./td[3]/text()').extract_first()

            yield pfiveItem

# Obey robots.txt rules

ROBOTSTXT_OBEY = False

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'

from scrapy import cmdline

cmdline.execute('scrapy crawl pfive_spider'.split())

class JavaScriptMiddleware(object):

    def process_request(self, request, spider):

        if spider.name == "pfive_spider":

            driver = webdriver.Chrome("G:\\Crawler\chromedriver.exe") #指定使用的浏览器

            driver.get(request.url)

            time.sleep(1)

            js = "var q=document.documentElement.scrollTop=10000" #模拟浏览页面

            driver.execute_script(js) #可执行js，模仿用户操作。此处为将页面拉至最底端。

            time.sleep(3)

            body = driver.page_source

            print ("访问"+request.url)

            return HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)

        else:

            return None