# 头条街拍图片爬取

 1 import re

 import requests

 from urllib import request

 import json

 import os

 i = 0

 headers = {

     'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'

 }

 while True:

     pag_all_url = 'https://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab'

     i += 20

     full_pag_url = pag_all_url.format(i)

     # print(full_pag_url) # 街拍的首页 根据不同的i来请求ajax,从而获得所有的街拍网址 像这样https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D

     pag_html = requests.get(full_pag_url,headers = headers).text

     pag_html_str = str(json.loads(pag_html))

     # print(pag_html_str) #把网页转化为字符串 进行正则匹配

     img_pag_id = re.findall(r'\'item_source_url\': \'\/group\/(\d*)\/\',',pag_html_str)

     # print(img_pag_id) #获得每个街拍的url like this--->https://www.toutiao.com/a6590127156037157379/

     for l in img_pag_id:    #图片下载

         img_all_url = 'https://www.toutiao.com/a{}'

         full_url = img_all_url.format(l)

         # print(full_url)#图片的url  print(full_pag_url)#图片所在的url

         html = requests.get(full_url,headers=headers).text

         pattern = r'gallery: JSON\.parse\((.*)\),'

         ans1 = re.search(pattern,html)

         try:

             ans1_str = json.loads(ans1[1])

             ans1_dic = json.loads(ans1_str)

             # print(ans1_dic)

             # if not os.path.exists('1'):

             #     os.mkdir('1')

             for q in ans1_dic['sub_images']:

                 img_url = q['url']

                 print(img_url)

                 filename = '1/' + img_url.split('/')[-1] + '.jpg'

                 request.urlretrieve(img_url, filename)

         except:continue