学习笔记—Python网络爬虫(二)
来源:互联网 发布:centos6 配置yum 编辑:程序博客网 时间:2024/04/29 14:40
一、获取指定页码的所有新闻
import requestsimport json
import re
from bs4 import BeautifulSoup
from datetime import datetime
commentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&\
newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&\
page_size=20&jsvar=loader_1509032030204_38862404'
def getCommentCounts(newsurl):
m = re.search('doc-i(.*).shtml', newsurl)
newsid = m.group(1)#获取"()"里面的内容
comments = requests.get(commentURL.format(newsid))
jd = json.loads(comments.text.strip('var loader_1509032030204_38862404='))
return jd['result']['count']['total']
def getNewsDetail(newsurl):
result = {}
res = requests.get(newsurl)
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')
result['title'] = soup.select('#artibodyTitle')[0].text
result['newssource'] = soup.select('.time-source span a')[0].text
timesource = soup.select('.time-source')[0].contents[0].strip()
result['dt'] = datetime.strptime(timesource, '%Y年%m月%d日%H:%M')
result['article'] = ' '.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
result['editor'] = soup.select('.article-editor')[0].text.lstrip('责任编辑:')
result['comments'] = getCommentCounts(newsurl)
return result
def parseListLink(url):
newsdetails = []
res = requests.get(url)
jd = json.loads(res.text.limport requests
import json
import re
from bs4 import BeautifulSoup
from datetime import datetime
commentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&\
newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&\
page_size=20&jsvar=loader_1509032030204_38862404'
def getCommentCounts(newsurl):
m = re.search('doc-i(.*).shtml', newsurl)
newsid = m.group(1)#获取"()"里面的内容
comments = requests.get(commentURL.format(newsid))
jd = json.loads(comments.text.strip('var loader_1509032030204_38862404='))
return jd['result']['count']['total']
def getNewsDetail(newsurl):
result = {}
res = requests.get(newsurl)
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')
result['title'] = soup.select('#artibodyTitle')[0].text
result['newssource'] = soup.select('.time-source span a')[0].text
timesource = soup.select('.time-source')[0].contents[0].strip()
result['dt'] = datetime.strptime(timesource, '%Y年%m月%d日%H:%M')
result['article'] = ' '.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
result['editor'] = soup.select('.article-editor')[0].text.lstrip('责任编辑:')
result['comments'] = getCommentCounts(newsurl)
return result
def parseListLink(url):
newsdetails = []
res = requests.get(url)
jd = json.loads(res.text.lstrip(' newsloadercallback(').rstrip(');'))
for ent in jd['result']['data']:
newsdetails.append(getNewsDetail(ent['url']))
return newsdetails
url = 'http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page=1&callback=newsloadercallback&_=1509170858142'
parseListLink(url)strip(' newsloadercallback(').rstrip(');'))
for ent in jd['result']['data']:
newsdetails.append(getNewsDetail(ent['url']))
return newsdetails
url = 'http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page=1&callback=newsloadercallback&_=1509170858142'
parseListLink(url)
二、获取多页新闻内文
#抓取多页新闻内文
url = 'http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gnxw&cat_2==gdxw1||=gatxw||=zs-pl||=mtjj&level==1||=2&show_ext=1&show_all=1&show_num=22&tag=1&format=json&page={}&callback=newsloadercallback&_=1509170858142'
news_total = []
for i in range(1,3):
newsurl = url.format(i)
newsary = parseListLink(newsurl)
news_total.extend(newsary)
三、将数据保存至Excel表格中
import pandas
df = pandas.DataFrame(news_total)
四、将数据保存至news.sqlite资料库中
#将数据保存至news.sqlite资料库中
import sqlite3
with sqlite3.connect('news.sqlite') as db:
df.to_sql('news', con = db)
#将数据从news.sqlite资料库中取出
with sqlite3.connect('news.sqlite') as db:
df2 = pandas.read_sql_query('SELECT * FROM news', con = db)
阅读全文
0 0
- 学习笔记—Python网络爬虫(二)
- Python学习笔记(二)网络刷博器爬虫
- 学习python写网络爬虫(二)
- 网络爬虫学习笔记(二)
- Python网络爬虫学习笔记(一)
- 学习笔记—Python网络爬虫(一)
- python网络爬虫学习笔记
- Python--网络爬虫学习笔记
- python网络爬虫学习笔记
- python网络爬虫(二)
- python网络爬虫(二)
- Python网络爬虫学习scrapy(二)
- 网络爬虫学习(二)
- python 爬虫笔记(二)
- Python网络爬虫学习笔记(第一周)
- 精通python网络爬虫学习笔记(1)
- 用python 写网络爬虫 学习笔记
- Python网络爬虫学习笔记(定向)
- MemoryStream导出 Excel文件
- tomcat8.0 启动报错 java.io.StreamCorruptedException: invalid type code: 3F
- mpchart 如何对y轴取整及对y轴上对应显示的数据取整数
- 利用nginx使域名解析到tomcat相应的目录下
- Unity 3D 对手机屏幕触摸的控制脚本
- 学习笔记—Python网络爬虫(二)
- CSIRO新研究成果:电动汽车电池长寿的福音
- 王思聪抛英雄互娱股份只是玩票 还有惊喜?
- gcc g++ 将所有警告转换为错误
- toJSONString(obj)对象属性值相同显示fail改用toJSONString(obj,SerializerFeature.DisableCircularReferenceDetect)
- 明星衣橱“发难”媒体记者,2016年是创业公司讣告之年吗?
- Android实践之ScrollView中滑动冲突处理,mapView百度地图滑动冲突
- 如何修改PHP的memory_limit限制
- 哎,为什么,我没有回复评论,现在想回复评论,我的csdn积分也没有了。。