CSIC_716_20191028【爬小破站】

1、爬取小破站的弹幕

2、展示爬取内容

打开网页，用教的方法找到cid 和header

import requests

from bs4 import BeautufulSoup

import pandas as pd

import re

header={

'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'

}#小破站的header可以通过谷歌浏览器的network中的request header获取

url=' http://comment.破站.com/125507930.xml' #以爬取小破站cid=125507930为例

response=request.get(headers=header,url=url) #获取响应

response.encoding=response.apparent_encoding

data=response.text

soup=BeautifulSoup(data,'lxml' )#解析

dlist=soup.find_all('d') #将内容按照规律向需要获取的内容筛选靠拢,存入dlist

d_list=[ ]

for i in d_list

danmu{ }

danmu['弹幕']=i.text

d_list.append(danmu) #将取到的值放到d_list中

df=pd.dateframe(d_list) #df应该还是列表，pd是pandas

f.open('sign.txt','encoding='utf-8')

for p in danmu['danmu'].value:

pat=re.compile(r'[一-龥]+')

filter_data=re.findall(pattern=pat,string=p )#筛选

f.write( " ".join(filter_data)) #保存数据进入sign文件

f.close()

------------------------------------------------------------------------------------------------------------------------------------------------------

利用上面获得的文件 sign.txt进行展示

import jieba

from imageio import imread

from wordcloud import WordCloud

f.open('sign.txt ',encoding='utf-8' )

data=f.read()

result=" ".join(jieba.lcut(data))

f.close( )

mask_color=imread('XXXX.jpg')

wc=WordCloud(

font_path='font的路径',

mask=mask_color,

width=xx,

height=xx,

)

wc.generate(result)

wc.to_file('xxxx.png')

plt.imshow( wc)

plt.show()

----------------------------------------------------以下为正式代码将两者合并--------------------------------------------------------------------------------------------------

如果不生成中间文件，爬完网页直接生成图片，代码合并，如下所示

 #coding:utf-8

 import requests

 import jieba

 import pandas as pd

 import re

 import matplotlib.pyplot as plt

 from bs4 import BeautifulSoup

 from imageio import imread

 from wordcloud import WordCloud

 header={

 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.70 Safari/537.36'

 }

 url='http://comment.bilibili.com/122593266.xml'  #换一下数字的cid即可

 response = requests.get(url=url,headers=header)

 response.encoding=response.apparent_encoding

 data=response.text

 suop=BeautifulSoup(data,'lxml')

 list_filter=suop.find_all('d')

 plist=[]

 for i in list_filter:

     danmu={}

     danmu['弹幕']= i.text

     plist.append(danmu)

 df=pd.DataFrame(plist)

 reslist=[]

 for p in df['弹幕'].values:

     txtfilter= re.compile(r'[一-龥]+')

     res=re.findall(pattern=txtfilter,string=p)

     result = " ".join(res)

     reslist.append(result)

 result= " ".join(reslist)

 finalResult=" ".join(jieba.lcut(result))

 mask_color=imread('五角星.jpg')  #图片可以随意替换，放在project中即可

 wc=WordCloud(

     font_path=r'C:\Windows\Fonts\STLITI.TTF',

     mask=mask_color,

     width=1920,

     height=1080,

     background_color='white'

 )

 wc.generate(finalResult)

 wc.to_file('hunt.png')

 plt.imshow(wc)

 plt.show( )

巴特西

CSIC_716_20191028【爬小破站】

最新文章

热门文章