#!/usr/bin/env python

#coding:utf-

import urllib2,sys,re,os,string

reload(sys);

sys.setdefaultencoding('utf8');

#url="http://www.dianping.com/search/category/1/20/g122"

def httpCrawler(url):

    #first page

    content = httpRequest(url)

    #other page

    #for pageNo in range(,):

    #    content = httpRequest(url)

    shops=parseHtml(content)

    getAllPages(shops)

    unpackOneShop()

    #saveData(shops)

def httpRequest(url):

    try:

        html = None

        req_header = {

            'User-Agent':'Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0'

            #'Accept':'text/html;q=0.9,*/*;q=0.8',

            #'Accept-Language':'en-US,en;q=0.5',

            #'Accept-Encoding':'gzip',

            #'Host':'j3.s2.dpfile.com',

            #'Connection':'keep-alive',

            #'Referer':'http://www.baidu.com'

        }

        req_timeout =

        req = urllib2.Request(url,None,req_header)

        resp = urllib2.urlopen(req,None,req_timeout)

        html = resp.read().decode('utf-8')

        #print html

    finally:

        if resp:

            resp.close()

    return html

def parseHtml(html):

    content = None

    #shops = re.findall(r'<li class="" >(.+?)<\li>',html,re.S)

    shops = re.findall(r'<div class="shop-list J_shop-list shop-all-list" id="shop-all-list">\n<ul>.*?</ul>\n</div>',html,re.S)

    return shops

def getAllPages(shops):

    #total  pages

    getEachShop(shops)

    print "################one page done."

global t_OneShop

t_OneShop=['']**

def getEachShop(shops):

    global t_OneShop

    t_OneShop=['']**

    t_start=

    shops_string=''.join(shops)

    i=

    t_start = shops_string.find(r'<li class="" >')

    all_end = shops_string.rfind('</li>')

    while i<= and t_start and all_end:

        t_start = shops_string.find(r'<li class="" >',t_start)

        t_end = shops_string.find('</li>',t_start,all_end)

        #print "t_start:",t_start

        #print "t_end:",t_end

        t_OneShop[i] = shops_string[t_start:t_end]

        #print t_OneShop[i]

        t_start=t_end

        i=i+

def unpackOneShop():

    global t_OneShop

    f = open('./zhubao/shops.csv', 'w')

    f.write('\xEF\xBB\xBF')

    f.write('名称,地址,人均消费,,,')

    f.write('\r\n')

    f.close()

    for i in range(,):

        #print t_OneShop[i]

        f = open('./zhubao/shops.csv', 'ab+')

        ShopName = re.findall(r'<h4>(.*?)</h4>',t_OneShop[i])

    #ShopDistrict =

        address = re.findall(r'<span class="addr">(.*?)</span>',t_OneShop[i])

        mean_price = re.findall(r'mean-price" target="_blank" >(.*?)</span>',t_OneShop[i],re.S)

        averageComsumption = re.findall(r'<b>(.*?)</b>',''.join(mean_price),re.S)

        print 'mean_price:',mean_price

        print 'average::',averageComsumption

        ShopName.extend(address)

        ShopName.extend(averageComsumption)

        print (','.join(ShopName)).replace('\n',''),'\r\n'

        f.write((''.join(','.join(ShopName)).replace('\n','')))

        f.write('\r\n')

        f.close()

    #iprovince =

    #city =

    #adminDistrict =

def saveData(data):

    if not os.path.exists('./zhubao'):

        os.mkdir(r'./zhubao')

    f = open('./zhubao/zhubao_shops.csv', 'wb')

    f.write(data)

    f.close()

if __name__ == '__main__':

    url="http://www.dianping.com/search/category/1/20/g122"

    httpCrawler(url)

'''

python2. 没有urllib.request

多线程

gevent

爬虫系统基本的结构：

.网络请求；

最简单的工具就是urllib、urllib2。这两个工具可以实现基本的下载功能，如果进阶想要异步可以使用多线程，如果想效率更高采用非阻塞方案tornado和curl可以实现非阻塞的下载。

.抓取结构化数据；

要想在页面中找到新链接需要对页面解析和对url排重，正则和DOM都可以实现这个功能，看自己熟悉哪一种。

正则感觉速度较快一些，DOM相对较慢并且复杂一点，如果只是为了要url正则可以解决，如果还想要页面中其他的结构或者内容DOM比较方便。

url的排重两小可以用memcache或者redis，量大就要用到bloomfilter。

.数据存储；

抓的少怎么存都行，抓的多并且要方便读取那就要好好设计了，用哈希分布存储在RDBMS上或者直接存在HBase上都要看你的数据量和具体需求。

'''