Python3中urllib详细使用方法

urllib是python的一个获取url(Uniform Resource Locators,统一资源定址器)了，我们可以利用它来抓取远程的数据进行保存哦，下面整理了一些关于urllib使用中的一些关于header,代理,超时,认证,异常处理处理方法，下面一起来看看。

python3 抓取网页资源的 N 种方法：

1、最简单

[python] view plain copy

print?

1. import urllib.request  
2. response = urllib.request.urlopen(’http://python.org/’)  
3. html = response.read()  

import urllib.requestresponse = urllib.request.urlopen('http://python.org/')html = response.read()

2、使用 Request

[python] view plain copy

print?

1. import urllib.request  
2. req = urllib.request.Request(’http://python.org/’)  
3. response = urllib.request.urlopen(req)  
4. the_page = response.read()  

import urllib.requestreq = urllib.request.Request('http://python.org/')response = urllib.request.urlopen(req)the_page = response.read()

3、发送数据

[python] view plain copy

print?

1. #! /usr/bin/env python3  
2. import urllib.parse  
3. import urllib.request  
4. url = ’http://localhost/login.php’  
5. user_agent = ’Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)’  
6. values = {  
7. ’act’ : ‘login’,  
8. ’login[email]’ : ‘[e-mail]’,  
9. ’login[password]’ : ‘123456’  
10. }  
11. data = urllib.parse.urlencode(values)  
12. req = urllib.request.Request(url, data)  
13. req.add_header(’Referer’, ‘http://www.python.org/’)  
14. response = urllib.request.urlopen(req)  
15. the_page = response.read()  
16. print(the_page.decode(“utf8”))  

#! /usr/bin/env python3import urllib.parseimport urllib.requesturl = 'http://localhost/login.php'user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'values = {'act' : 'login','login[email]' : '[e-mail]','login[password]' : '123456'}data = urllib.parse.urlencode(values)req = urllib.request.Request(url, data)req.add_header('Referer', 'http://www.python.org/')response = urllib.request.urlopen(req)the_page = response.read()print(the_page.decode("utf8"))

4、发送数据和header

[python] view plain copy

print?

1. #! /usr/bin/env python3  
2. import urllib.parse  
3. import urllib.request  
4. url = ’http://localhost/login.php’  
5. user_agent = ’Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)’  
6. values = {  
7. ’act’ : ‘login’,  
8. ’login[email]’ : ‘[e-mail]’,  
9. ’login[password]’ : ‘123456’  
10. }  
11. headers = { ’User-Agent’ : user_agent }  
12. data = urllib.parse.urlencode(values)  
13. req = urllib.request.Request(url, data, headers)  
14. response = urllib.request.urlopen(req)  
15. the_page = response.read()  
16. print(the_page.decode(“utf8”))  

#! /usr/bin/env python3import urllib.parseimport urllib.requesturl = 'http://localhost/login.php'user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'values = {'act' : 'login','login[email]' : '[e-mail]','login[password]' : '123456'}headers = { 'User-Agent' : user_agent }data = urllib.parse.urlencode(values)req = urllib.request.Request(url, data, headers)response = urllib.request.urlopen(req)the_page = response.read()print(the_page.decode("utf8"))

5、http 错误

[python] view plain copy

print?

1. #! /usr/bin/env python3  
2. import urllib.request  
3. req = urllib.request.Request(’http://www.111cn.net ’)  
4. try:  
5. urllib.request.urlopen(req)  
6. except urllib.error.HTTPError as e:  
7. print(e.code)  
8. print(e.read().decode(“utf8”))  

#! /usr/bin/env python3import urllib.requestreq = urllib.request.Request('http://www.111cn.net ')try:urllib.request.urlopen(req)except urllib.error.HTTPError as e:print(e.code)print(e.read().decode("utf8"))

6、异常处理1

[python] view plain copy

print?

1. #! /usr/bin/env python3  
2. from urllib.request import Request, urlopen  
3. from urllib.error import URLError, HTTPError  
4. req = Request(”http://www.111cn.net /”)  
5. try:  
6. response = urlopen(req)  
7. except HTTPError as e:  
8. print(‘The server couldn’t fulfill the request.’)  
9. print(‘Error code: ’, e.code)  
10. except URLError as e:  
11. print(‘We failed to reach a server.’)  
12. print(‘Reason: ’, e.reason)  
13. else:  
14. print(“good!”)  
15. print(response.read().decode(“utf8”))  

#! /usr/bin/env python3from urllib.request import Request, urlopenfrom urllib.error import URLError, HTTPErrorreq = Request("http://www.111cn.net /")try:response = urlopen(req)except HTTPError as e:print('The server couldn't fulfill the request.')print('Error code: ', e.code)except URLError as e:print('We failed to reach a server.')print('Reason: ', e.reason)else:print("good!")print(response.read().decode("utf8"))

7、异常处理2

[python] view plain copy

print?

1. #! /usr/bin/env python3  
2. from urllib.request import Request, urlopen  
3. from urllib.error import  URLError  
4. req = Request(”http://www.111cn.net /”)  
5. try:  
6. response = urlopen(req)  
7. except URLError as e:  
8. if hasattr(e, ‘reason’):  
9. print(‘We failed to reach a server.’)  
10. print(‘Reason: ’, e.reason)  
11. elif hasattr(e, ‘code’):  
12. print(‘The server couldn’t fulfill the request.’)  
13. print(‘Error code: ’, e.code)  
14. else:  
15. print(“good!”)  
16. print(response.read().decode(“utf8”))  

#! /usr/bin/env python3from urllib.request import Request, urlopenfrom urllib.error import  URLErrorreq = Request("http://www.111cn.net /")try:response = urlopen(req)except URLError as e:if hasattr(e, 'reason'):print('We failed to reach a server.')print('Reason: ', e.reason)elif hasattr(e, 'code'):print('The server couldn't fulfill the request.')print('Error code: ', e.code)else:print("good!")print(response.read().decode("utf8"))

8、HTTP 认证

[python] view plain copy

print?

1. #! /usr/bin/env python3  
2. import urllib.request  
3. # create a password manager  
4. password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()  
5. # Add the username and password.  
6. # If we knew the realm, we could use it instead of None.  
7. top_level_url = ”https://www.111cn.net /”  
8. password_mgr.add_password(None, top_level_url, ‘rekfan’, ‘xxxxxx’)  
9. handler = urllib.request.HTTPBasicAuthHandler(password_mgr)  
10. # create “opener” (OpenerDirector instance)  
11. opener = urllib.request.build_opener(handler)  
12. # use the opener to fetch a URL  
13. a_url = ”https://www.111cn.net /”  
14. x = opener.open(a_url)  
15. print(x.read())  
16. # Install the opener.  
17. # Now all calls to urllib.request.urlopen use our opener.  
18. urllib.request.install_opener(opener)  
19. a = urllib.request.urlopen(a_url).read().decode(’utf8’)  
20. print(a)  

#! /usr/bin/env python3import urllib.request

# create a password managerpassword_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()# Add the username and password.# If we knew the realm, we could use it instead of None.top_level_url = "https://www.111cn.net /"password_mgr.add_password(None, top_level_url, 'rekfan', 'xxxxxx')handler = urllib.request.HTTPBasicAuthHandler(password_mgr)# create "opener" (OpenerDirector instance)opener = urllib.request.build_opener(handler)# use the opener to fetch a URLa_url = "https://www.111cn.net /"x = opener.open(a_url)print(x.read())# Install the opener.# Now all calls to urllib.request.urlopen use our opener.urllib.request.install_opener(opener)a = urllib.request.urlopen(a_url).read().decode('utf8')print(a)9、使用代理

[python] view plain copy

print?

1. #! /usr/bin/env python3  
2. import urllib.request  
3. proxy_support = urllib.request.ProxyHandler({’sock5’: ‘localhost:1080’})  
4. opener = urllib.request.build_opener(proxy_support)  
5. urllib.request.install_opener(opener)  
6. a = urllib.request.urlopen(”http://www.111cn.net ”).read().decode(“utf8”)  
7. print(a)  

#! /usr/bin/env python3import urllib.requestproxy_support = urllib.request.ProxyHandler({'sock5': 'localhost:1080'})opener = urllib.request.build_opener(proxy_support)urllib.request.install_opener(opener)a = urllib.request.urlopen("http://www.111cn.net ").read().decode("utf8")print(a)

10、超时

[python] view plain copy

print?

1. #! /usr/bin/env python3  
2. import socket  
3. import urllib.request  
4. # timeout in seconds  
5. timeout = 2  
6. socket.setdefaulttimeout(timeout)  
7. # this call to urllib.request.urlopen now uses the default timeout  
8. # we have set in the socket module  
9. req = urllib.request.Request(’http://www.111cn.net /’)  
10. a = urllib.request.urlopen(req).read()  
11. print(a)  

#! /usr/bin/env python3import socketimport urllib.request

# timeout in secondstimeout = 2socket.setdefaulttimeout(timeout)# this call to urllib.request.urlopen now uses the default timeout# we have set in the socket modulereq = urllib.request.Request('http://www.111cn.net /')a = urllib.request.urlopen(req).read()print(a)