pyhton爬虫（12）——抓取携程酒店评论数据

本文主要目标是抓取携程酒店基本信息和用户评论数据。具体来说，酒店基本信息包括：酒店名、酒店星级、酒店最低房价、用户推荐比、酒店总评分等；用户评论数据包括：用户评论时间，用户评分，评论内容等。

实现代码如下所示：

# -*- coding: utf-8 -*-"""Created on Mon Aug  7 21:05:03 2017@author: Administrator"""import urllib.requestfrom bs4 import BeautifulSoupimport pymysql.cursors'''目标:获取酒店名称和酒店星级'''url1 = 'http://hotels.ctrip.com/hotel/435383.html'html1 = urllib.request.urlopen(url1).read().decode('utf-8')#1.获取酒店名称信息soup1 = BeautifulSoup(html1,'lxml')result1 = soup1.find_all(attrs={"itemprop":"name"})hotelName = result1[0].stringprint("酒店名称为:{}".format(hotelName))#2.获取酒店星级信息soup12 = BeautifulSoup(html1,'lxml')result12 = soup12.find_all(attrs={"class":"grade"})#print(result1)result12 =str(result12)soup13 = BeautifulSoup(result12,'lxml')result13 = soup13.find_all('span')hotelStar = result13[0]['title']print("酒店星级为:{}".format(hotelStar))'''目标:获取酒店最低房价和评论总数'''url2 = "http://m.ctrip.com/html5/hotel/HotelDetail/435383.html"html2 = urllib.request.urlopen(url2).read().decode('utf-8')#获取酒店最低价soup2 = BeautifulSoup(html2,'lxml')result2 = soup2.find_all(attrs={"class":"js-cas-p"})lowPrice = result2[0]['data-cas-p']print("酒店最低房价为:{}".format(lowPrice))#评论总数result21 = soup2.find_all(attrs={"class":"dt-color12 dt-fn15"})commentCounts =  result21[0].stringprint("评论总数为:{}".format(commentCounts))'''目标:获取酒店卫生评分、环境评分、服务评分、设施评分、用户推荐比、用户评分、评价内容'''url3 = 'http://m.ctrip.com/html5/hotel/HotelDetail/dianping/435383.html'html3 = urllib.request.urlopen(url3).read().decode('utf-8')soup3 = BeautifulSoup(html3,'lxml')#获取酒店各项评分数据result32 = soup3.find_all(attrs={"class":"ve-txt"})result32 = str(result32)soup32 = BeautifulSoup(result32,'lxml')result33 = soup32.find_all('em')userRecommendRate = result33[0].stringhRating = result33[1].stringeRating = result33[2].stringsRating = result33[3].stringiRating = result33[4].stringprint("用户推荐为:{}".format(userRecommendRate))print("卫生评分为:{}分".format(hRating))print("环境评分为:{}分".format(eRating))print("服务评分为:{}分".format(sRating))print("设施评分为:{}分".format(iRating))#提取用户评论数据result34 = soup3.find_all(attrs={"class":"hotel-cell-num"})result34 = str(result34[1])soup33 = BeautifulSoup(result34,'lxml')result35 = soup33.find_all('p')for i in range(0,10):    #userName = result35[i].get_text()    result36 = soup3.find_all(attrs={"itemprop":"datePublished"})    datePublished = result36[i].string    print("评论发表于:{}".format(datePublished))    result37 = soup3.find_all(attrs={"itemprop":"ratingValue"})    userRating = result37[i].string    print("评分为:{}".format(userRating))    result38 = soup3.find_all(attrs={"class":"tree-ellips-line6 js_arr"})    commentText = result38[i].get_text()    print("评论内容为:{}".format(commentText))    '''        数据库操作    '''    #获取数据库链接    connection  = pymysql.connect(host = 'localhost',                              user = 'root',                              password = '123456',                              db = 'xiecheng',                              charset = 'utf8mb4')    try:        #获取会话指针        with connection.cursor() as cursor:            #创建sql语句            sql = "insert into `macro-polo` (`hotelName`,`hotelStar`,`lowPrice`,`commentCounts`,`userRecommendRate`,`hRating`,`eRating`,`sRating`,`iRating`,`datePublished`,`userRating`,`commentText`) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"            #执行sql语句            cursor.execute(sql,(hotelName,hotelStar,lowPrice,commentCounts,userRecommendRate,hRating,eRating,sRating,iRating,datePublished,userRating,commentText))            #提交数据库            connection.commit()    finally:        connection.close()

运行结果如下下图所示：

MySQL数据库中的数据存储如下图所示：