爬取双色球的开奖记录

需要twisted, pymongo,BeautifulSoup.

 

#!/usr/bin/env python# encoding: utf-8"""lott.pyCreated by Daniel Yang on 2011-01-23.Copyright (c) 2011 Yang. All rights reserved."""import sysimport osfrom twisted.internet import reactor, deferfrom twisted.web.client import getPagefrom BeautifulSoup import BeautifulSoupfrom pymongo import Connectionconnection = Connection()db = connection.lotttable = db.double_colorBASE_URL = "http://kaijiang.zhcw.com/zhcw/html/ssq/list_%s.html"def handleError(error):print 'got errpr', errordef processPage(pageContent):print 'got page', len(pageContent)soup = BeautifulSoup(pageContent)trs = soup.findAll('tr') for tr in trs:tds = tr.findAll('td')if len(tds) == 7:id = table.insert({'date':tds[0].text, 'issue': tds[1].text, 'lott':tds[2].text,'amount':tds[3].text, 'first':tds[4].text, 'second':tds[5].text}, safe=True);print 'inserted ', idfor i in range(1, 59):url = BASE_URL % ipageFetchedDeferred = getPage(url)pageFetchedDeferred.addCallback(processPage)pageFetchedDeferred.addErrback(handleError)reactor.run()