用scrapy爬取京东商城的商品信息

软件环境：

 gevent (1.2.2)

 greenlet (0.4.12)

 lxml (4.1.1)

 pymongo (3.6.0)

 pyOpenSSL (17.5.0)

 requests (2.18.4)

 Scrapy (1.5.0)

 SQLAlchemy (1.2.0)

 Twisted (17.9.0)

 wheel (0.30.0)

1.创建爬虫项目

2创建京东网站爬虫. 进入爬虫项目目录，执行命令：

scrapy genspider jd www.jd.com

会在spiders目录下会创建和你起的名字一样的py文件：jd.py，这个文件就是用来写你爬虫的请求和响应逻辑的

3. jd.py文件配置

分析的amazon网站的url规则：

https://search.jd.com/Search?

以防关键字是中文，所以要做urlencode

1.首先写一个start_request函数，用来发送第一次请求，并把请求结果发给回调函数parse_index，同时把reponse返回值传递给回调函数,response类型<class 'scrapy.http.response.html.HtmlResponse'>

     def start_requests(self):

         # https://www.amazon.cn/s/ref=nb_sb_ss_i_1_6?field-keywords=macbook+pro

         # 拼接处符合条件的URL地址

         # 并通过scrapy.Requst封装请求，并调用回调函数parse_index处理,同时会把response传递给回调函数

         url = 'https://search.jd.com/Search?'

         # 拼接的时候field-keywords后面是不加等号的

         url += urlencode({"keyword": self.keyword, "enc": "utf-8"})

         yield scrapy.Request(url,

                              callback=self.parse_index,

                              )

2.parse_index从reponse中获取所有的产品详情页url地址，并遍历所有的url地址发送request请求，同时调用回调函数parse_detail去处理结果

 def parse_detail(self, response):

     """

     接收parse_index的回调，并接收response返回值，并解析response

     :param response:

     :return:

     """

     jd_url = response.url

     sku = jd_url.split('/')[-1].strip(".html")

     # price信息是通过jsonp获取，可以通过开发者工具中的script找到它的请求地址

     price_url = "https://p.3.cn/prices/mgets?skuIds=J_" + sku

     response_price = requests.get(price_url)

     # extraParam={"originid":"1"}  skuIds=J_3726834

     # 这里是物流信息的请求地址，也是通过jsonp发送的，但目前没有找到它的参数怎么获取的，这个是一个固定的参数，如果有哪位大佬知道，好望指教

     express_url = "https://c0.3.cn/stock?skuId=3726834&area=1_72_4137_0&cat=9987,653,655&extraParam={%22originid%22:%221%22}"

     response_express = requests.get(express_url)

     response_express = json.loads(response_express.text)['stock']['serviceInfo'].split('>')[1].split('<')[0]

     title = response.xpath('//*[@class="sku-name"]/text()').extract_first().strip()

     price = json.loads(response_price.text)[0]['p']

     delivery_method = response_express

     # # 把需要的数据保存到Item中，用来会后续储存做准备

     item = AmazonItem()

     item['title'] = title

     item['price'] = price

     item['delivery_method'] = delivery_method

     # 最后返回item，如果返回的数据类型是item，engine会检测到并把返回值发给pipelines处理

     return item

4. item.py配置

 import scrapy

 class JdItem(scrapy.Item):

     # define the fields for your item here like:

     # name = scrapy.Field()

     # amazome Item

     title = scrapy.Field()

     price = scrapy.Field()

     delivery_method = scrapy.Field()

5. pipelines.py配置

 from pymongo import MongoClient

 class MongoPipeline(object):

     """

     用来保存数据到MongoDB的pipeline

     """

     def __init__(self, db, collection, host, port, user, pwd):

         """

         连接数据库

         :param db: databaes name

         :param collection: table name

         :param host: the ip for server

         :param port: thr port for server

         :param user: the username for login

         :param pwd: the password for login

         """

         self.db = db

         self.collection = collection

         self.host = host

         self.port = port

         self.user = user

         self.pwd = pwd

     @classmethod

     def from_crawler(cls, crawler):

         """

         this classmethod is used for to get the configuration from settings

         :param crwaler:

         :return:

         """

         db = crawler.settings.get('DB')

         collection = crawler.settings.get('COLLECTION')

         host = crawler.settings.get('HOST')

         port = crawler.settings.get('PORT')

         user = crawler.settings.get('USER')

         pwd = crawler.settings.get('PWD')

         return cls(db, collection, host, port, user, pwd)

     def open_spider(self, spider):

         """

         run once time when the spider is starting

         :param spider:

         :return:

         """

         # 连接数据库

         self.client = MongoClient("mongodb://%s:%s@%s:%s" % (

             self.user,

             self.pwd,

             self.host,

             self.port

         ))

     def process_item(self, item, spider):

         """

         storage the data into database

         :param item:

         :param spider:

         :return:

         """
　　　　　　# 获取item数据，并转换成字典格式

         d = dict(item)
　　　　　　 # 有空值得不保存

         if all(d.values()):
　　　　　　　　　　# 保存到mongodb中

             self.client[self.db][self.collection].save(d)

         return item

         # 表示将item丢弃，不会被后续pipeline处理

         # raise DropItem()

6. 配置文件

 # database server

 DB = "jd"

 COLLECTION = "goods"

 HOST = "127.0.0.1"

 PORT = 27017

 USER = "root"

 PWD = ""

 ITEM_PIPELINES = {

    'MyScrapy.pipelines.MongoPipeline': 300,

 }

巴特西