create table grilsbase

(

id int primary key auto_increment,

name varchar(50),height varchar(50),bwh varchar(50),title varchar(100),img_upload varchar(100),pc_img_upload varchar(100),

resource_id varchar(50),totals varchar(50),recommend_id varchar(50),date varchar(50),headimg_upload varchar(50),

show_datetime varchar(50),client_show_datetime varchar(50),video_duration varchar(50),free_select varchar(50),

trial_time varchar(50),viewtimes varchar(50),coop_customselect_654 varchar(50),coop_id varchar(50),tag_class varchar(50),

tag_name varchar(50),playerid varchar(50),block_detailid varchar(50),type varchar(50),istop varchar(50)

)

 #coding:utf-8

 import  requests

 import os

 import json

 import re

 import MySQLdb

 import threading

 #获取数据url

 gilsUrl='http://act.vip.xunlei.com/ugirls/js/ugirlsdata.js'

 gilsDetailUrl='http://meitu.xunlei.com/detail.html'

 gilsImgUrl='http://data.meitu.xunlei.com/data/image/%s/%s'

 executor = threading.BoundedSemaphore(10)

 regex=re.compile('\/([^\/]*?\.jpg)$')

 regexhead=re.compile('\/([^\/]*?)\.jpg$')

 class MySQL:

     def __init__(self,host,user,pwd,db):

         self.host=host

         self.user=user

         self.db=db

         self.pwd=pwd

     def GetConnect(self):

         if not self.db:

             raise(NameError,'没有目标数据库')

         self.connect=MySQLdb.connect(host=self.host,user=self.user,password=self.pwd,database=self.db,port=3306,charset='utf8')

         cur=self.connect.cursor()

         if not cur:

             raise(NameError,'数据库访问失败')

         else:

             return cur

     def ExecSql(self,sql):

          cur=self.GetConnect()

          cur.execute(sql)

          self.connect.commit()

          self.connect.close()

     def ExecQuery(self,sql):

         cur=self.GetConnect()

         cur.execute(sql)

         resList = cur.fetchall()

         self.connect.close()

         return resList   

 def getGirlsData():

     regex=re.compile("var ugirlsData=(.+)")

     r=requests.get(gilsUrl)

     jsond=regex.findall(r.text)

     with open('ugirlsdata.json','w+',encoding='utf-8') as f:

          f.write(jsond[0])

     #print('写入json成功')

     return json.loads(jsond[0])

 def getImgName(imgurl):

     if(imgurl==''):

         return ''

     m=regex.findall(imgurl)

     if m is None:

         return ''

     else:

         return  m[0] if len(m)>0 else ''

 def getImgNameHead(imgurl):

     if(imgurl==''):

         return ''

     m=regexhead.findall(imgurl)

     if m is None:

         return ''

     else:

         return  m[0] if len(m)>0 else ''

 def WriteDB(jsdata):

     ms = MySQL(host="192.168.0.108", user="lin", pwd="", db="grils")

     for data in jsdata:

         sql="insert into grilsbase(\

         name,height,bwh,title,img_upload,pc_img_upload,resource_id,totals,recommend_id,\

         date,headimg_upload,show_datetime,client_show_datetime,video_duration,free_select,trial_time,\

         viewtimes,coop_customselect_654,coop_id,tag_class,tag_name,playerid,block_detailid,type,istop)\

         values('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s','%s')" % \

         (data['name'],data['height'],data['bwh'],data['title'],getImgName(data.get('img_upload','')),data['pc_img_upload'],data['resource_id'],data["totals"],data["recommend_id"], \

         data['date'],getImgName(data.get("headimg_upload",'')),data["show_datetime"],data["client_show_datetime"],data["video_duration"],data["free_select"],data["trial_time"], \

         data['viewtimes'],data['coop_customselect_654'],data['coop_id'],data.get('tag_class',''),data.get('tag_name',''),data.get('playerid',''),data['block_detailid'],data['type'],data['istop'])

         #print(sql)

         ms.ExecSql(sql)

         print('完成'+data['name']+'数据更新...')

         DownImg(data['name'],data["totals"],data['resource_id'],data["headimg_upload"],data["img_upload"])

 def DownImg(name,totals,resource_id,headimg_upload,img_upload):

     path=creatFile(resource_id)

     if headimg_upload.strip()!='':

         #os.remove('./pic/'+resource_id+'/'+getImgName(headimg_upload)+'.jpg')

         DownImgRun(headimg_upload,path,getImgNameHead(headimg_upload))

     if img_upload.strip()!='':

         #os.remove('./pic/'+resource_id+'/'+getImgName(img_upload)+'.jpg')

         DownImgRun(img_upload,path,getImgNameHead(img_upload))

     #print('正在下载'+name+'图片')

     for i in range(1,int(totals)+1):

         url=gilsImgUrl%(resource_id,str(i)+'.jpg')

         DownImgRun(url,path,i)

         #t=threading.Thread(target=DownImgRun,args={url,path,i})

         #t.start()

         #t.join()

 def DownImgRun(url,path,i):

     #print(url)

     r=requests.get(url)

     if(r.status_code==200):

         with open(path+'/'+str(i)+'.jpg','wb') as fimg:

             fimg.write(r.content)

 def creatFile(dirname):

     path='./pic/'+dirname

     if os.path.exists(path):

         return path

     else:

         os.makedirs(path)

         return  path

 if __name__ == '__main__':

     gri=getGirlsData()

     WriteDB(gri)