python最简单的爬虫
来源:互联网 发布:小企业会计记账软件 编辑:程序博客网 时间:2024/06/05 20:01
主要5部分:主函数,url管理器,网页下载器,网页输出器,网页解析器
(用了set,但是下面的代码并没有突出set的优势。后续可改进)
主要入口函数, spider_main:
import url_manager, html_downloader, html_outputer, html_parserclass SpiderMain(object): def __init__(self): self.urls = url_manager.UrlManager() self.downloader = html_downloader.HtmlDownloader() self.parser = html_parser.HtmlParser() self.outputer = html_outputer.HtmlOutputer() def craw(self, root_url): count = 1 self.urls.add_new_url(root_url)#根节点入队 while self.urls.has_new_url():#类似广度优先搜索 try: new_url = self.urls.get_new_url()#从队列中得到一个新的url print 'craw %d : %s' % (count, new_url)#哪个url正在被爬取 html_cont = self.downloader.download(new_url)#把这个网页爬下来 new_urls, new_data = self.parser.parse(new_url, html_cont)#解析内部的url和数据 self.urls.add_new_urls(new_urls)#将又爬到的url添加到队列中 self.outputer.collect_data(new_data)#收集数据 self.outputer.output_html()#打印数据 if count == 10: break count += 1 except Exception as e: print e print 'craw failed'if __name__ == "__main__": root_url = "http://baike.baidu.com/view/21087.htm" #根节点 obj_spider = SpiderMain() obj_spider.craw(root_url)
url管理器:
class UrlManager(object): def __init__(self): self.new_urls = set() self.old_urls = set() def get_new_url(self): new_url = self.new_urls.pop() self.old_urls.add(new_url) return new_url def add_new_url(self, url):#添加新的url到队列中 if url is None: return if url not in self.new_urls and url not in self.old_urls: self.new_urls.add(url) def add_new_urls(self, urls):#批量添加 if urls is None or len(urls) == 0:#没爬到或者爬到了空串 return for url in urls: self.add_new_url(url) def has_new_url(self):#判断队列是否为空 return len(self.new_urls) != 0
网页下载器:
import urllib2class HtmlDownloader(object): def download(self, url): if url is None: return None response = urllib2.urlopen(url)#打开url,并获取返回值 if response.getcode() != 200: return None return response.read()#返回网页的全部内容
网页解析器:
import reimport urlparsefrom bs4 import BeautifulSoupclass HtmlParser(object): def _get_new_urls(self, page_url, soup):#从一堆文字中获取url(正则匹配) new_urls = set() links = soup.find_all('a', href = re.compile(r"/view/\d+\.htm"))#获取a标签中,href为指定格式的标签全部内容 for link in links: new_url = link['href']#取其中的href new_full_url = urlparse.urljoin(page_url, new_url)#拼接成完整链接 new_urls.add(new_full_url) return new_urls def _get_new_data(self, page_url, soup):#从一堆文字中获取数据 res_data = {}#键值对组合,title键值对,summary键值对 #url res_data['url'] = page_url #<dd class="lemmaWgt-lemmaTitle-title"> title_node = soup.find('dd', class_ = "lemmaWgt-lemmaTitle-title").find('h1') res_data['title'] = title_node.get_text() #print res_data['title'] #<div class="lemma-summary" label-module="lemmaSummary"> summary_node = soup.find('div', class_ = "lemma-summary") res_data['summary'] = summary_node.get_text() #print res_data['summary'] return res_data def parse(self, page_url, html_cont): if page_url is None or html_cont is None: return soup = BeautifulSoup(html_cont, 'html.parser', from_encoding = 'utf-8') new_urls = self._get_new_urls(page_url, soup) new_data = self._get_new_data(page_url, soup) return new_urls, new_data
网页输出器:
class HtmlOutputer(object): def __init__(self): self.datas = [] def collect_data(self, data):#将数据(键值对们)组成列表 if data is None: return self.datas.append(data) def output_html(self): fout = open("F:\\output.html", 'w') fout.write("<html>") fout.write("<body>") fout.write("<table>") for data in self.datas: print data['title'] fout.write("<tr>") fout.write("<td>%s</td>" % data['url']) fout.write("<td>%s</td>" % data['title'].encode('utf-8')) fout.write("<td>%s</td>" % data['summary'].encode('utf-8')) fout.write("</tr>") fout.write("</table>") fout.write("</body>") fout.write("</html>") fout.close()
0 0
- 最简单的python爬虫
- python最简单的爬虫
- 最简单的python 爬虫
- python 3 最简单的小爬虫
- Python 3.0最简单的爬虫
- 最简单的Python网页爬虫
- Python 3.0最简单的爬虫
- python最简单的小爬虫
- python爬虫——写出最简单的网页爬虫
- Python简单的爬虫
- 简单的python爬虫
- 简单的Python 爬虫
- Python网络爬虫4 ---- Linux下编写最简单的scrapy网络爬虫项目
- python 爬虫试手,好简单的爬虫
- PYTHON 爬虫简单的认识
- python简单的爬虫代码
- [python]简单的网络爬虫
- Python实现简单的爬虫
- leetcode 485 python
- 19、C++成绩统计
- 类与类之间的几种关系
- MapReduce的Shuffle过程介绍
- angularjs2--tab页调用父页面的方法
- python最简单的爬虫
- 插入排序
- [BZOJ1998][Hnoi2010]Fsk物品调度(置换群+并查集)
- Node.js request实现技术灌水
- 链表判环练习
- 全排列计算(康托展开)
- JSON简单介绍
- 喝汽水问题
- android recent key长按事件弹起触发最近列表故障分析