#coding:utf-8

import requests, json, time, re, os, sys, time

import urllib2

#设置为utf-8模式

reload(sys)

sys.setdefaultencoding( "utf-8" )

def getHtml(url):

    response = urllib2.urlopen(url)

    html = response.read()

    #可以根据编码格式进行编码

    #html = unicode(html,'utf-8')

    return html

url = 'https://www.cnblogs.com/'

print getHtml(url)

def getHtml(url):

    #使用将urllib2.Request()实例化,需要访问的URL地址则作为Request实例的参数

    request = urllib2.Request(url)

    #Request对象作为urlopen()方法的参数,发送给服务器并接收响应的类文件对象

    response = urllib2.urlopen(request)

    #类文件对象支持文件对象操作方法

    #如read()方法读取返回文件对象的全部内容并将其转换成字符串格式并赋值给html

    html = response.read()

    #可以根据编码格式进行编码

    #html = unicode(html,'utf-8')

    return html 

url = 'https://www.cnblogs.com/'

print getHtml(url)

def getHtml(url):

    #构造ua

    ua_header = {"User-Agent":"Mozzila/5.0(compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}

    #url连同headers一起构造Request请求,这个请求将附带IE9.0浏览器的User-Agent

    request = urllib2.Request(url,headers=ua_header)

    #设置超时时间

    response = urllib2.urlopen(request,timeout=60)

    html = response.read()

    return html

url = 'https://www.cnblogs.com/'

print getHtml(url)

def getHtml(url):

    ua = {"User-Agent":"Mozzila/5.0(compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0;"}

    request = urllib2.Request(url)

    #也可以通过Request.add_header()添加/修改一个特定的header

    request.add_header("Connection","keep-alive")

    response = urllib2.urlopen(request)

    html = response.read()

    #查看响应码

    print '相应码为:',response.code

    #也可以通过Request.get_header()查看header信息

    print "Connection:",request.get_header("Connection")

    #或者

    print request.get_header(header_name = "Connection")

    #print html

    return html

#coding:utf-8

import requests, json, time, re, os, sys, time

import urllib2

import random

#设置为utf-8模式

reload(sys)

sys.setdefaultencoding( "utf-8" )

def getHtml(url):

    #定义ua池,每次随机取出一个值

    ua_list = ["Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv2.0.1) Gecko/20100101 Firefox/4.0.1","Mozilla/5.0 (Windows NT 6.1; rv2.0.1) Gecko/20100101 Firefox/4.0.1","Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11","Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11"]

    user_agent = random.choice(ua_list)

    #print user_agent

    request = urllib2.Request(url)

    request.add_header("Connection","keep-alive")

    request.add_header("User-Agent",user_agent)

    response = urllib2.urlopen(request,data=None,timeout=60)

    html = response.read()

    #print '响应码为:',response.code

    #print 'URL:',response.geturl()

    #print 'Info:',response.info()