python urllib* 获取网页信息

来源：互联网发布：json转字符串编辑：程序博客网时间：2024/05/22 03:27
直接上代码吧，从简单到复杂，同时涉及到cookie的使用
import urllib import urllib2import cookielibimport json########################simple urllib2##############################################################url="http://www.baidu.com/s"request=urllib2.Request(url)print urllib2.urlopen(request).geturl()######################with params########################################url='http://www.baidu.com/s'data={"wd":"hello",}print urllib.urlencode(data)request=urllib2.Request(url,urllib.urlencode(data))print urllib2.urlopen(request).geturl()#####################with headers#######################################headers={"User-agent":"Mozilla/4.0(compatible); MSIE 6.0; Windows NT 5.1"}url='http://www.baidu.com/s'request=urllib2.Request(url,urllib.urlencode(data),headers)print urllib2.urlopen(request).geturl()    #############################################method one##################################################url="https://openapi.hellocdn.com/api/rest/login"headers={"User-agent":"Mozilla/4.0(compatible); MSIE 6.0; Windows NT 5.1"}login_info={"user":"someone@some.com","pass":"someonespassword","output":"json"}login_info=urllib.urlencode(login_info)cj=cookielib.CookieJar()opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))request=urllib2.Request(url,login_info,headers)response=opener.open(request)token=json.loads(response.read())["loginResponse"]["session"]["sessionToken"]url="https://openapi.hellocdn.com/stat/rest/traffic/responseCode/edge"login_info={"sessionToken":token,"apiKey":"test","fromDate":"20140701","toDate":"20140702","timeInterval":"0","output":"json"}headers={"User-agent":"Mozilla/4.0(compatible); MSIE 6.0; Windows NT 5.1"}login_info=urllib.urlencode(login_info)request=urllib2.Request(url,login_info,headers)response=opener.open(request)print "response",responseprint "url",response.geturl()headers=response.info()print "headers",headersfor (k,v) in headers.items():    print k,"#"*5,v#############################################method two##################################################from cookielib import LWPCookieJarimport requestsurl="https://openapi.hellocdn.com/api/rest/login"headers={"User-agent":"Mozilla/4.0(compatible); MSIE 6.0; Windows NT 5.1"}login_info={"user":"someone@some.com","pass":"someonespassword","output":"json"}jar = LWPCookieJar('cookies.txt')r = requests.get(url, cookies=jar,params=login_info,headers=headers)token=r.json()["loginResponse"]["session"]["sessionToken"]jar.save()url="https://openapi.hellocdn.com/stat/rest/traffic/responseCode/edge"login_info={"sessionToken":token,"apiKey":"test","fromDate":"20140701","toDate":"20140702","timeInterval":"0","output":"json"}jar = LWPCookieJar('cookies.txt')jar.load()r = requests.get(url, cookies=jar,params=login_info,headers=headers)
0 0