批量提取 sitemap.xml 中的链接生成 sitemap.txt (Python脚本)

题目读起来很绕，是这样的，我的小站用了dedecms的一个插件，它能生成若干个 google sitemap 的 .xml 文件，然而却没有对应的利于百度收录的插件可使用。在网上查了一下，说是要提取这些 .xml 文件中的链接出来，生成一个对应的 sitemap.txt 才行，然后以如下格式放到 robot.txt 中

Sitemap: http://your site/sitemap.xml. http://your site/sitemap.txt

可是一个一个手动提取其中的链接太麻烦了，我用了20分钟写了一个 python 脚本，方便这个操作，当然程序还很简陋，但也足够我用了。

现在的功能是：从网站下载 sitemap.xml 文件，提取其中的链接生成 sitemap.txt 然后再传回网站，最后生成一个 robot.txt 作为参考。因为文件较多，我用了一个线程池,加快处理速度。

这个小脚本是在 linux 下写的，测试运行良好，我没有在 win32 平台下测试，但也应该能够工作。

#!/usr/bin/env python#coding=utf-8############################################################### dositemap.py By http://www.python8.org## author: [e-mail]##############################################################import reimport globimport osimport urllibimport urlparsefrom ftplib import FTPfrom common import *p = re.compile(r"<loc>(.*)</loc>",re.I)sitemapname = 'googlesitemap'ftpdict = {'server':'218.95.37.111','port':21,'user':'testuser','pwd':'littlesecret'}def do_robot(filename,baseurl = 'http://www.python8.org/'): f = open('robot.txt','a') basefilename = os.path.splitext(os.path.split(filename)[1])[0] webfilename0 = urlparse.urljoin(baseurl,basefilename+'.xml') webfilename1 = urlparse.urljoin(baseurl,basefilename+'.txt') f.write(webfilename0 + '. ' + webfilename1 + os.linesep) f.close()def uploadsitemaptxt(filename): ftp = FTP() ftp.connect('%(server)s' % ftpdict, '%(port)d' % ftpdict) ftp.login('%(user)s' % ftpdict, '%(pwd)s' % ftpdict) ftp.cwd('wwwroot/python8_org') bufsize = 1024 filehandle = open(filename,'rb') ftp.storbinary('STOR %s' % filename,filehandle,bufsize) filehandle.close() ftp.quit() def dositemap(filename,baseurl='http://www.python8.org/'): webfile = urlparse.urljoin(baseurl,filename) urllib.urlretrieve(webfile,filename) if os.path.exists(filename): sitemap2txt(filename) def sitemap2txt(filename): data = file(filename,'r').read() urlList = p.findall(data) newfilename = os.path.splitext(os.path.split(filename)[1])[0] + '.txt' urlList = (eachurl + os.linesep for eachurl in urlList if urlList) file(newfilename,'w').writelines(urlList) if os.path.exists(newfilename): uploadsitemaptxt(newfilename) print filename.ljust(20),' ==> ',newfilename.ljust(20),'OK!' def cleaner(): from glob import glob [os.remove(eachfile) for eachfile in glob('g*.xml')] [os.remove(eachfile) for eachfile in glob('g*.txt')] print 'all temp file cleared' def main(): print 'Initializing, please wait...' if os.path.exists('robot.txt'): os.remove('robot.txt') wm = WorkerManager(10) #开启10个线程同时处理，空间商很生气 :D for each in xrange(1,101,1):#我有100个 .xml 脚本，文件名分别是 googlesitemap_1.xml 到 googlesitemap_101.xml eachfile = sitemapname + '_%s.xml' % each do_robot(eachfile) wm.add_job(dositemap,eachfile) wm.start() wm.wait_for_complete() cleaner() #清理文件 if __name__=='__main__': main()

脚本中引用的 common 模块中存放了一个线程池（抄来的 -O-），代码如下：

import Queue, threadingimport threading# working thread class Worker(threading.Thread): worker_count = 0 def __init__( self, workQueue, resultQueue, timeout = 0, **kwds): threading.Thread.__init__( self, **kwds ) self.id = Worker.worker_count Worker.worker_count += 1 self.setDaemon( True ) self.workQueue = workQueue self.resultQueue = resultQueue self.timeout = timeout self.thread_evt = threading.Event() def stop(self): self.thread_evt.set() def run( self ): ''' the get-some-work, do-some-work main loop of worker threads ''' while True: try: callable, args, kwds = self.workQueue.get(timeout=self.timeout) res = callable(*args, **kwds) #print "worker[%2d]: %s" % (self.id, str(res) ) self.resultQueue.put( res ) # except Queue.Empty: # break except: break #print 'worker[%2d]' % self.id, sys.exc_info()[:2] class WorkerManager: def __init__( self, num_of_workers=10, timeout = 1): self.workQueue = Queue.Queue() self.resultQueue = Queue.Queue() self.workers = [] self.timeout = timeout self._recruitThreads( num_of_workers ) def _recruitThreads( self, num_of_workers ): for i in range( num_of_workers ): worker = Worker( self.workQueue, self.resultQueue, self.timeout ) self.workers.append(worker) def stop(self): for w in self.workers: w.stop() def start(self): for w in self.workers: w.start() def wait_for_complete( self): # ...then, wait for each of them to terminate: while len(self.workers): worker = self.workers.pop() worker.join( ) if worker.isAlive() and not self.workQueue.empty(): self.workers.append( worker ) # print "All jobs completed." def add_job( self, callable, *args, **kwds ): self.workQueue.put( (callable, args, kwds) ) def get_result( self, *args, **kwds ): return self.resultQueue.get( *args, **kwds )