python网络爬虫（二）

9.抓取新闻评论数

import requestsimport reimport json#设置评论URLcommentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&newsid=comos-fynfvar5623201&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20'def getCommentCount(newsURL):    m = re.search('doc-i(.*).shtml',newsURL)    newsID = m.group(1)#提取新闻表示号    comments = requests.get(commentURL.format(newsID))#将newsID填进commentURL的{}中去    jd = json.loads(comments.text.strip('var data='))    print(jd['result']['count']['total'])news = 'http://news.sina.com.cn/c/zs/2017-11-01/doc-ifynfvar5623201.shtml'getCommentCount(news)

运行结果：

10.获取新闻标识符

有两种方法，如代码所示：

import requests# from bs4 import BeautifulSoup# res = requests.get('http://news.sina.com.cn/china/')# res = requests.get('http://news.sina.com.cn/c/zs/2017-11-01/doc-ifynfvar5623201.shtml')# res = requests.get('http://comment5.news.sina.com.cn/page/info?version=1&format=js&\# channel=gn&newsid=comos-fynfvar5623201&group=&compress=0&ie=utf-8&oe=utf-8&page=1&\# page_size=20')# res.encoding = 'utf-8'# print(res.text)# soup = BeautifulSoup(res.text,'html.parser')# print(soup.text)# import json# jd = json.loads(res.text.strip('var data='))#剔除多余字符串，并将数据加载至变量jd中# print(jd['result']['count']['total'])# jd['result']['count']['total']# soup.select('#commentCount1')#方法1，使用字符串切片方法获取news = 'http://news.sina.com.cn/c/zs/2017-11-01/doc-ifynfvar5623201.shtml'news_id = news.split('/')[-1].rstrip('.shtml').lstrip('doc-i')#将地址以/分片获取最后一个分片，移除.shtml后缀，剔除doc-i前缀，最终获取到新闻标识号print(news_id)#方法2，使用正则表达式方法获取新闻表示号import rem = re.search('doc-i(.*).shtml',news)#doc-i(.*).shtml匹配以doc-i为前缀，以.shtml为后缀的字符串print(m.group(0))#m.group(0)为搜寻的满足匹配模式的字符串print(m.group(1))#m.group(1)可获取到匹配模式（）中的内容

运行结果：

建立评论数抽取函数

import requestsimport reimport json#设置评论URLcommentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&newsid=comos-fynfvar5623201&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20'def getCommentCount(newsURL):    m = re.search('doc-i(.*).shtml',newsURL)    newsID = m.group(1)#提取新闻表示号    comments = requests.get(commentURL.format(newsID))#将newsID填进commentURL的{}中去    jd = json.loads(comments.text.strip('var data='))    print(jd['result']['count']['total'])news = 'http://news.sina.com.cn/c/zs/2017-11-01/doc-ifynfvar5623201.shtml'getCommentCount(news)

运行结果：

完成内文信息抽取函数

#将新闻相关信息封装为字典回传import requestsfrom bs4 import BeautifulSoupimport reimport jsonfrom datetime import datetime#新闻评论链接commentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&newsid=comos-fynfvar5623201&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20'#新闻地址newsURL = 'http://news.sina.com.cn/c/zs/2017-11-01/doc-ifynfvar5623201.shtml'#获取新闻评论数def getCommentCount(newsURL):    m = re.search('doc-i(.*).shtml',newsURL)    newsID = m.group(1)#提取新闻表示号    comments = requests.get(commentURL.format(newsID))#将newsID填进commentURL的{}中去    jd = json.loads(comments.text.strip('var data='))    return jd['result']['count']['total']#获取新闻详细内容def getNewsDetail(newsURL):    result = {}#将数据封装在字典中返回    res = requests.get(newsURL)#获取响应内容    res.encoding = 'utf-8'    soup = BeautifulSoup(res.text,'html.parser')#将数据封装为DOM结构    result['title'] = soup.select('#artibodyTitle')[0].text#获取文章标题    result['newsSource'] = soup.select('.time-source span a')[0].text#获取新闻来源    timeSource = soup.select('.time-source')[0].contents[0].strip()#获取新闻时间字符串    result['dt'] = datetime.strptime(timeSource,'%Y年%m月%d日%H:%M')#将字符串转换为时间格式    result['article'] = ' '.join([p.text.strip() for p in soup.select("#artibody p")[:-1]])#获取文章内容    result['editor'] = soup.select('.article-editor')[0].text.strip("责任编辑：")#获取文章责任编辑    result['comments'] = getCommentCount(newsURL)#调用函数获取文章评论数    return resultgetNewsDetail(newsURL)   

运行结果：

查找分页链接

import requestsimport jsonres = requests.get('http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page=4&callback=newsloadercallback&_=1509521588694')jd = json.loads(res.text.lstrip(' newsloadercallback(').rstrip(');'))for ent in jd['result']['data']:    print(ent['url'])

运行结果：

建立分析清单链接函数

def parseListLinks(url):    newsdetails = []    res = requests.get(url)    jd = json.loads(res.text.lstrip(' newsloadercallback(').rstrip(');'))    for ent in jd['result']['data']:        newsdetails.append(getNewsDetail(ent['url']))    return newsdetailsurl = 'http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page=4&callback=newsloadercallback&_=1509521588694'print(parseListLinks(url))

运行结果：

使用for循环产生多页链接

url = 'http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page={}&callback=newsloadercallback&_=1509521588694'for i in range(1,10):    print(url.format(i))

运行结果：

批次抓取每页新闻信息

url = 'http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page={}&callback=newsloadercallback&_=1509521588694'news_total = []for i in range(1,3):    newsurl = url.format(i)    newsary = parseListLinks(newsurl)    news_total.extend(newsary)length = len(news_total)print(length)

运行结果（打印出文章数）：

备注：
注意python中append和extend的区别：
list.append(object) 向列表中添加一个对象object
list.extend(sequence) 把一个序列seq的内容添加到列表中

使用append的时候，是将new_media看作一个对象，整体打包添加到music_media对象中。

使用extend的时候，是将new_media看作一个序列，将这个序列和music_media序列合并，并放在其后面。