python的urllib2和urllib爬虫及区别

来源：互联网发布：小蜜蜂软件官方网编辑：程序博客网时间：2024/06/05 14:19

urllib和urllib2的区别，urllib只能根据url爬取，不能设置头信息，但可以对url进行编码；urllib2可以设置request请求头信息

# -*- coding: utf-8 -*-import urllibf = urllib.urlopen('http://www.cnblogs.com/')result = f.read()   #读取html页面的第一行print result

# -*- coding: utf-8 -*-import urllib2import sysdef clawer(url):    send_headers={                  'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',                  'Connection':'keep-alive',                  'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0'                  }    req = urllib2.Request(url,headers=send_headers)    r = urllib2.urlopen(req)    html = r.read()        #返回网页内容    receive_header = r.info()     #返回的报头信息    print sys.getfilesystemencoding()    #html = html.decode('utf-8','replace').encode(sys.getfilesystemencoding()) #转码:避免输出出现乱码    #print receive_header    print html.decode('gbk')url= "http://www.jb51.net"url = "http://uvs.youdao.com/search?site=photogalaxy.163.com&sort=time&channelid=&q=%B7%C9%BB%FA&length=10&start=0" #网页地址clawer(url)

import urllibimport urllib2values = {"username":"1016903103@qq.com","password":"XXXX"}data = urllib.urlencode(values)url = "https://passport.csdn.net/account/login?from=http://my.csdn.net/my/mycsdn"request = urllib2.Request(url,data)response = urllib2.urlopen(request)print response.read()

0 0