# -*- coding: utf-8 -*-

 """

 Created on Wed Jul 10 14:40:41 2019

 @author: Administrator

 """

 import pandas as pd

 import requests

 from bs4 import BeautifulSoup

 import time 

 url1 = 'http://www.mca.gov.cn/article/sj/xzqh//1980/'

 headers = {'content-type': 'application/json',

                'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0'}

 # 1. 获取所有链接========================================================================

 def f1(url1):

     '2018-1980年中华人民共和国行政区划代码 的所有链接'

     #requests发出请求，设置url，header参数

     response = requests.get(url1, headers=headers, timeout=200, verify=False)

     soup = BeautifulSoup(response.text,'lxml') #将网页源码返回为BeautifulSoup类型

     _tmp1 = soup.select('td.arlisttd')

     end_1 = []

     for i in _tmp1:

         _a = i.select('a')[0].get('href')

         _b = i.select('a')[0].get('title')[:4]

         end_1.append(['http://www.mca.gov.cn'+_a,_b])

     return end_1

 end_2=[]

 for i in ['','?2','?3']:

     end_2 = end_2+f1(url1+i)

 def f2(url1='http://www.mca.gov.cn/article/sj/xzqh/2019/'):

     '2019年中华人民共和国行政区划代码'

     response = requests.get(url1, headers=headers, timeout=200, verify=False)

     soup = BeautifulSoup(response.text,'lxml')

     _tmp1 = soup.select('td.arlisttd')

     end_1 = []

     for i in _tmp1:

         _a = i.select('a')[0].get('href')

         _b = i.select('a')[0].get('title')[:7]

         end_1.append(['http://www.mca.gov.cn'+_a,_b])

     return end_1

 end_2 = end_2+f2()

 # 2. 获取数据========================================================================

 def f3(url1='http://www.mca.gov.cn/article/sj/xzqh/1980/201903/20190300014989.shtml'):

     #url1='http://www.mca.gov.cn/article/sj/xzqh/1980/201507/20150715854922.shtml'

     #url1='http://www.mca.gov.cn/article/sj/xzqh/1980/201507/20150715854918.shtml'

     #

     response = requests.get(url1, headers=headers, timeout=200, verify=False)

     soup = BeautifulSoup(response.text,'lxml')

     _txt = soup.select('script')[4].get_text().strip().replace('window.location.href="','').strip('";')

     if _txt[-4:]=='html':

         print('script!')

         url2 = _txt

     else:

         _tmp1 = soup.select('div.artext > div > p > a')

         if len(_tmp1)==0:

             _tmp1 = soup.select('div#zoom > a')

         url2 = _tmp1[0].get('href')

     print(url2)

     #return url2

     #url2='http://www.mca.gov.cn/article/sj/tjbz/a/201713/201708220856.html'

     time.sleep(0.5)

     response = requests.get(url2, headers=headers, timeout=200, verify=False)

     #将网页源码返回为BeautifulSoup类型

     soup = BeautifulSoup(response.text,'lxml')

     _tmp1 = soup.select('table > tr[height="19"]')

     end_1 = []

     if len(_tmp1)>5:

         for i in _tmp1:

             _a = i.select('td')[1].get_text().strip()

             if len(_a)>15: #部分数据页面，最后一行是备注。

                 continue

             else:

                 _b = i.select('td')[2].get_text().strip()

                 end_1.append([_a,_b])

     else:

         _tmp1 = soup.select('table > tr[height="20"]')

         for i in _tmp1:

             _a = i.select('td')[0].get_text().strip()

             if len(_a)>15 or _a=='行政区划代码': #部分数据页面，最后一行是备注。

                 continue

             else:

                 _b = i.select('td')[1].get_text().strip()

                 end_1.append([_a,_b])

     return end_1

 #循环对每个链接 获取数据

 end_3=[];#end_4=[]

 for j in range(len(end_2)):

     item = end_2[j]

     if ''  in item[1] or ''  in item[1]:

         print(j,item[0],item[1])

         tmp2 = f3(item[0])

         print('.')

         end_3.extend([[item[1]]+i for i in tmp2])

         #end_4.append(tmp2)

         time.sleep(0.1)

 df_result = pd.DataFrame(end_3)

 #pd.DataFrame(end_4).to_excel('所有连接.xlsx',index=False)

 df_result.to_excel('地区编码.xlsx',index=False)

 '''

 #\3 2019年5月份县以上行政区划代码_3852 > table > tbody > tr:nth-child(4)

 #list_content > div.list_right > div > ul > table > tbody > tr:nth-child(1) > td.arlisttd > a

 '''