用python实现一个阅读糗事百科热门的爬虫应用

来源：互联网发布：mac直接进入win10 编辑：程序博客网时间：2024/06/04 17:56

代码

#coding=utf-8import urllibimport urllib2import re#糗事百科爬虫类class QSBK:#初始化方法，定义一些变量def __init__(self):self.enable = True#初始化页码self.pageIndex = 1#初始化headersself.user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'self.headers = {'User-Agent':self.user_agent}#请求html页面方法def getHtml(self,pageIndex):try:#利用urllib2的相关功能爬取html内容url = "http://www.qiushibaike.com/hot/page/"+str(pageIndex)request = urllib2.Request(url,headers = self.headers)response = urllib2.urlopen(request)return response.read().decode('utf-8')except urllib2.URLError,e:if hasattr(e,reason):print '请求失败，原因：'+e.reasonif hasattr(e,code):print '请求失败，错误代码：'+e.codereturn None#利用正则表达式匹配出段子及相关内容def getDuanzi(self,pageIndex):contents = self.getHtml(pageIndex)if not contents:print '内容请求失败'return Nonepattern = re.compile('<div.*?class="author.*?>.*?<a.*?</a>.*?<a.*?>.*?<h2>(.*?)</h2>.*?</a>.*?<div.*?class'+                     '="content".*?>(.*?)</div>(.*?)<div class="stats.*?class="number">(.*?)</i>',re.S)items = re.findall(pattern,contents)return items#实现阅读段子的相关功能def getOneStory(self):items = self.getDuanzi(self.pageIndex)for item in items:#实现回车一下出来一个段子，输入q退出input = raw_input()if input == 'q':print '结束阅读'self.enable = FalsereturnhaveImg = re.search('img',item[2])#剔除有图片的段子if not haveImg:haveTag = re.search('<br/>',item[1])#把段子中的换行标签剔除if haveTag:noTag = re.sub('<br/>','',item[1])print u"发布人：%s\t点赞数:%d\n段子:%s"%(item[0],int(item[3]),noTag)else:print u"发布人：%s\t点赞数:%d\n段子:%s"%(item[0],int(item[3]),item[1])#开始方法def start(self):#从控制台取得用户要读的页数def givePage():input = raw_input('你想看几页段子？')try:isinstance(int(input),int)except:print "请输入数字！" givePage()return int(input)pages = givePage()while (self.enable == True)&(pages>=self.pageIndex):self.getOneStory()self.pageIndex += 1spider = QSBK()spider.start()

主要实现了按一下回车出来一个段子的功能

用了python的urllib2和re模块

0 0