Python_爬虫_urllib解析库

简介：提取网页保存到txt文件中 + 解析txt文件内容，取出内容

from urllib import request

import re.json

url="http://www.163.com"

response = request.urlopen(url).read().decode("gbk")

# 写入一个txt文件

with open("163/163.txt","w",encoding="gbk")as f:

    f.write(str(response))

# 读取,用json序列化后遍历出来，取出键里面的值

with open("163/163.txt",'r+',encoding="utf-8")as f:

    content=f.read()

c1=content.replace(" ",'').replace("\n",'') #去掉空格、换行

c2="["+c1+"]"

cc=json.loads.(c2)  #序列化

for i in cc:

    print(x['title'])   #取出每个项里面“title”键的值

    print(x['docurl'])

简介：提取美团链接保存在一个txt文件中

#coding=utf-8

from urllib import request

from bs4 import BeautifulSoup

req=request.urlopen("http://hotel.meituan.com/xian/")

content=req.read().decode("utf8")

bsObj=BeautifulSoup(content,"html.parser")

pcontent=bsObj.findAll("a",{"class":"poi-title"})

i=1

with open("meituan/url.txt","a+",encoding="utf8") as f:

        for x in pcontent:

                f.write(x['href']+"\n")    #取出 标签 里面的信息

                f.write(x.get_text())    #取出正文

                print("第"+'int(%s)'%(i)+"条url")

                i+=1

范例：功能实现后提高代码质量

#coding=utf-8

#获取当前地址下的所有酒店url地址

from urllib import request,error

from bs4 import BeautifulSoup

import json

for page in range(3):

        url="https://ihotel.meituan.com/hbsearch/HotelSearch?utm_medium=pc&version_name=999.9&cateId=20&attr_28=129&uuid=12B729E22135402D5CBC1432A179A735CF81DF50626153919EC2C66D46DCB233%401517811001478&cityId=42&offset="+str(page*20)+"&limit=20&startDay=20180205&endDay=20180205&q=&sort=defaults"

        try:

                req=request.urlopen(url)

                content=req.read().decode("utf8")

##                bsObj=BeautifulSoup(content,"html.parser")

##                pcontent=bsObj.findAll("a",{"class":"poi-title"})

                content_dict=json.loads(content)

                with open("meituan/url.txt","a+",encoding="utf8") as f:

                        for x in content_dict['data']['searchresult']:

                                print(x['poiid'])

                                hotel_url="http://hotel.meituan.com/%s/"%x['poiid']

                                f.write(hotel_url+"\n")

        except error.URLError as e:

                print(e.reason)

urllib添加代理IP

# -*- coding: UTF-8 -*-

from urllib import request

if __name__ == "__main__":

    #访问网址

    url = 'http://2017.ip138.com/ic.asp'

    #url = 'http://www.whatismyip.com.tw'

    #这是代理IP

    proxy = {'http':'113.124.226.174:808'}

    #创建ProxyHandler

    proxy_support = request.ProxyHandler(proxy)

    #创建Opener

    opener = request.build_opener(proxy_support)

    #添加UserAngent

    opener.addheaders = [

        ('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36'),

        ('Host','www.whatismyip.com.tw')    #这个网站能检测IP地址，所以用这个作为示例

    ]

    #安装OPener

    request.install_opener(opener)

    #使用自己安装好的Opener

    response = request.urlopen(url)

    #读取相应信息并解码

    html = response.read().decode("gbk")

    #打印信息

    print(html)

巴特西

Python_爬虫_urllib解析库

简介：提取网页保存到txt文件中 + 解析txt文件内容，取出内容

简介：提取美团链接保存在一个txt文件中

范例：功能实现后提高代码质量

urllib添加代理IP

最新文章

热门文章