selenium+python+BeautifulSoup爬取知乎文章信息

本文通过selenium+python+BeautifulSoup来爬取知乎文章信息。

#知乎推荐文章爬取#2017/8/6# -*- encoding = utf-8 -*-from selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.common.exceptions import TimeoutExceptionfrom selenium.common.exceptions import NoSuchElementExceptionfrom bs4 import BeautifulSoupimport csvimport osimport timeimport redriver = webdriver.Chrome()#登录知乎def putcookies(account,password):    try:        driver.get('https://www.zhihu.com/#signin')        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.index-tab-navs > div > a.active")))        botton = driver.find_element_by_css_selector('body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.view.view-signin > div.qrcode-signin-container > div.qrcode-signin-step1 > div.qrcode-signin-cut-button > span')        botton.click()        form = driver.find_element_by_css_selector('body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.view.view-signin > form > div.group-inputs > div.account.input-wrapper > input[type="text"]')        pas = driver.find_element_by_css_selector('body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.view.view-signin > form > div.group-inputs > div.verification.input-wrapper > input[type="password"]')        sub = driver.find_element_by_css_selector('body > div.index-main > div > div.desk-front.sign-flow.clearfix.sign-flow-simple > div.view.view-signin > form > div.button-wrapper.command > button')        form.send_keys(account)        pas.send_keys(password)        sub.click()        try:            print('请手动输入验证码')            driver.implicitly_wait(10)            driver.find_element_by_css_selector('#root > div > div:nth-child(2) > header > div > div.SearchBar > button')        except NoSuchElementException:              sub.click()    except:        putcookies(account,password)#滑动页面def change_page(num):    WebDriverWait(driver,10).until(EC.presence_of_element_located((By.CSS_SELECTOR,'#root > div > div:nth-child(2) > header > div > div.SearchBar > button')))    for i in range(num):        driver.execute_script('window.scrollTo(0, document.body.scrollHeight)')          time.sleep(3)#解析页面def findinf(html):    soup = BeautifulSoup(html,'lxml')    r = re.compile('(\d+)')    links = soup.find_all('div',class_='Card TopstoryItem')    for link in links:        try:            maininf = link.find(class_='Feed-meta-item').get_text()[-3:]#话题            writer = link.find(class_='AuthorInfo-head').get_text()#作者        except:            continue        try:            intd = link.find('div',class_='RichText AuthorInfo-badgeText').string#作者个人介绍        except:            intd = ''        title = link.find('h2',class_='ContentItem-title').get_text()#标题        href = 'https://www.zhihu.com' + link.find('h2',class_='ContentItem-title').a['href']#文章链接        try:            support = link.find(class_='Button VoteButton VoteButton--up').get_text()#点赞        except:            support = link.find(class_='Button LikeButton ContentItem-action').get_text()#点赞        try:            talking = r.match(link.find('button',class_='Button ContentItem-action Button--plain').get_text()[:-3]).group()#评论数        except:            talking = ''        content = link.find('span',class_='RichText CopyrightRichText-richText').get_text()#摘要        yield  {                'maininf': maininf,                'writer':writer,                'intd':intd,                'title':title,                'support':support,                'talking':talking,                'content':content,                'href':href,            }#创建一个文件夹def make(path):    if not os.path.exists(path):        os.makedirs(path)#保存数据def save_to_csv(inf,path):    with open(path + '知乎文章信息概要采集.csv','a') as f:        writer = csv.writer(f)        writer.writerow(['标题','作者','话题','作者个人介绍','点赞数','评论数','文章链接','摘要'])        try:            for i in inf:                writer.writerow([i['title'],i['writer'],i['maininf'],i['intd'],i['support'],i['talking'],i['href'],i['content']])        except:            pass#主函数def main(account,password,num):    path = 'D:/数据/知乎文章/'    putcookies(account,password)    change_page(num)       inf = findinf(driver.page_source)    make(path)    print('---'*43)    print('{:^60}'.format('知乎文章概要'))    print("***"*43)    for i in findinf(driver.page_source):        print('标题:{:<10s}'.format(i['title']))        print('作者:{:>3s}'.format(i['writer']),end = ' '*5)        print("话题:{:>3s}".format(i['maininf']))        print('作者个人介绍:')        print('{:<5s}'.format(i['intd']))        print('点赞数:{:<2s}'.format(i['support']),end = ' '*5)        print("评论数:{:3s}".format(i['talking']))        print("文章链接:" + i['href'])        print("摘要：")        print('{:<5s}'.format(i['content']))        print('---' *43)    save_to_csv(inf,path)#执行程序if __name__ == '__main__':    num = int(input('请输入要爬取的页面数：'))    account = input("请输入知乎账号：")    password = input("请输入知乎密码：")    time_start = time.time()    main(account,password,num)    print("^^^"*43)    print("共耗时{}秒".format(time.time()-time_start))    driver.quit()