python 爬取 apache 遍历目录 文件

apache这样配置的时候可以目录遍历：

<Directory /var/www/>
Options Indexes FollowSymLinks
#Options FollowSymLinks
AllowOverride NoneRequire all 

grantedDirectoryIndex index.html index.php 

</Directory>

大概长这样：

python代码

#coding=utf-8import requestsimport urllibfrom bs4 import BeautifulSoupimport os'''scrapy a website using apache default folders'''URL = 'http://198.255.15.22/'#os.makedirs(r'./'+dirName)def download(URL,URI):#print '->>>>>>>>>>>>>>>>>>>>>>>>>>'+URL,'--',URIr = requests.get(URL+URI)soup = BeautifulSoup(r.text)links =  soup.find_all('a')for i in links:if i.text == '[To Parent Directory]':continuename =  i['href']print nameif name == '/':continue#directoryif name[-1] == '/'  : download(URL, name)print nameif not os.path.exists(r'.' + name):os.makedirs(r'.' + name)#file else:try:filename = name.split('/')[-1]#print 'filename  ',filenameif not os.path.exists('./' + URI + filename):urllib.urlretrieve(URL + name, './' + URI + filename)print 'download->',URL + nameelse:print 'exists', URL + nameexcept:print URL + name, 'failed!'##def downloadif __name__ == '__main__':URL = ['http://*.*.*.*/', 'http://*.*.*.*/']for url in URL:download(url,'/')