爬虫豆瓣完整版

from bs4 import BeautifulSoupimport requests, time, pymongofrom multiprocessing import Pooluser_agent = 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36'headers ={'User-Agent':user_agent}proxies = {'http':'60.178.173.125'}#创建数据库client = pymongo.MongoClient('localhost', 27017)    #激活客户端Douban = client['douban']   #创建数据库名称url_list = Douban['url_list']   #创建表用于存放每个电影的链接item_list = Douban['item_list']     #创建表用于放每个电影的详细信息#获取250个电影的列表页，一页有25个电影，所以是10页start_url = ['https://movie.douban.com/top250?start={}&filter='.format(str(i)) for i in range(0, 250, 25)]url0 = 'https://movie.douban.com/top250?start=0&filter='#获取某一个电影详情信息url_one = 'https://movie.douban.com/subject/6146955/'url_two = 'https://movie.douban.com/subject/1291546/'url_404 = 'https://movie.douban.com/subject/5912992/'link_error = []     #用于存放def get_movie_info(url):    wb_data = requests.get(url,headers = headers)    soup = BeautifulSoup(wb_data.text, 'lxml')    time.sleep(4)    if (soup.find('title').text == "页面不存在"):    #如果详情页不存在就跳过（其实应该在导航页爬取，这样排名会完整，只不过缺失一些信息        link_error.append(url)        pass    else:        genres = soup.find_all(property = "v:genre")    #类型（类型会有多个,是一个列表）        runtime = soup.find(property = "v:runtime")     #时长        ytime = soup.find(property = "v:initialReleaseDate")        #年份（只要年份不要月）        movie = {            'number': soup.select('div.top250 > span.top250-no')[0].string,            'title': soup.find(property="v:itemreviewed").string,  # 定位电影名            'score': soup.find(property = "v:average").string,    #定位电影评分            'comments_count': soup.find(property = "v:votes").string,    #定位评论数量            'genre': list(i.string for i in genres),            'runtime': soup.find(property = "v:runtime").string if soup.find_all('span','v:runtime') else None,            'year':ytime.string[0:4],            'url': url        }        #有些runtime没有属性，特殊处理        if (movie['runtime'] == None):            # item_list.insert_one(movie)      #插入数据库中            s = soup.find_all(id = 'info')[0].get_text(strip = True)            # print(s.find("分"))            k = s.find("分")            j = s.find("片长")            # print(s[j+3:k+1])            # print(movie['runtime'])            movie['runtime'] = s[j+3:k+1]        print(movie)        item_list.insert_one(movie)      #插入数据库中# get_movie_info(url_one)#获取每一页的电影链接def get_movie_link(url):    wb_data = requests.get(url,headers=headers)    soup = BeautifulSoup(wb_data.text, 'lxml')    links = soup.select('div > div.info > div.hd > a ')    for i in links:        item_link = i.get('href')        url_list.insert_one({'url':item_link})      #将电影链接放入数据库中        print(item_link)#将所有页里面的链接放入links4# for i in start_url:#     get_movie_link(i)##将数据库中存取的电影链接提取给get_movie_info,进行详情页的爬取# for i in url_list.find(no_cursor_timeout  = True):      #默认MongoDB过十分钟还没有处理完毕会异常，加参数永不停，要手动关闭#     # print(i['url'])#     item_url = i['url']#     get_movie_info(item_url)#断点设计db_urls = [item['url'] for item in url_list.find()]     #找到总共的urlindex_urls = [item['url'] for item in item_list.find()]     #已经抓取的urlx = set(db_urls)y = set(index_urls)rest_of_urls = x-y# for i in rest_of_urls:#     get_movie_info(i)print(rest_of_urls)print(len(link_error))# #数据库操作# # print(url_list.find()['url'])     #总数# print(url_list.count())