学习笔记—Python网络爬虫（一）

来源：互联网发布：淘宝返利网排行榜编辑：程序博客网时间：2024/04/27 12:08

一、网络资源（URLs）撷取套件。

可以使用REST操作（POST, PUT, GET, DELETE）存取网络资源。以新浪新闻网站为例：

import requests
res = requests.get('http://news.sina.com.cn/china/')
res.encoding = 'utf-8'
print(res.text)

二、BeautifulSoup4

2.1 将保存的网络资源转化为DOM Tree，可提取每个节点文字信息。

from bs4 import BeautifulSoup
html_sample = ' \
<html> \
<body> \
<h1 id='title'>Hello World</h1> \
<a href="#" class="link">This is link1</a> \
<a href="# link2" class="link">This is link2</a> \
</body> \
</html>'

soup = BeautifulSoup(html_sample, 'html.parser')
print(soup.text)

2.2 找出所有含有特定标签的HTML元素

2.2.1 使用select找出含有h1标签的元素：

soup = BeautifulSoup(html_sample)

header = soup.select('h1') #返回的是一个list

print(header)

2.2.2 取得含有特定CSS属性的元素

1）使用select找出所有id为title的元素（id前需加#）

alink = soup.select('#title')

2）使用select找出所有class为link 的元素（class前需加.）

soup = BeautifulSoup(html_sample)

for link in soup.select('.link'):

print(link)

2.2.3 取得所有a标签内的链接

from bs4 import BeautifulSoup
a = '<a href="#" qoo=123 abc=456>I am ZZC</a>'
soup2 = BeautifulSoup(a, 'html.parser')
print(soup2.select('a')[0].text)

三、实例一：获取新浪新闻“时间”、“事件”、“链接”信息

import requests
from bs4 import BeautifulSoup
res = requests.get('http://news.sina.com.cn/china/')
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')

for news in soup.select('.news-item'):
# print(news)
if len(news.select('h2'))>0:
h2 = news.select('h2')[0].text
a = news.select('a')[0]['href']
time = news.select('.time')[0].text
print(time,h2,a)

四、实例二：整理新闻内文

import requests
from bs4 import BeautifulSoup
from datetime import datetime
res = requests.get('http://news.sina.com.cn/o/2017-10-26/doc-ifynfrfm9351229.shtml')
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')

#获取文章标题
title = soup.select('#artibodyTitle')[0].text
# print(title)
#获取文章时间
time = soup.select('.time-source')[0].contents[0].strip()
dt = datetime.strptime(time, '%Y年%m月%d日%H:%M')
dt.strftime('%Y/%m/%d')
soup.select('.time-source span a')[0].text
#整理新闻内文
article = []
for p in soup.select('#artibody p')[:-1]:
article.append(p.text.strip())
' '.join(article)
#获取编辑信息
soup.select('.article-editor')[0].text.lstrip('责任编辑：')
#获取评论信息
res2 = requests.get('http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&newsid=comos-fynfrfm9351229&group=&compress=0&ie=utf-8&oe=utf-8&page=1&page_size=20&jsvar=loader_1509032030204_38862404')
# res2.text
import json
jd = json.loads(res2.text.strip('var loader_1509032030204_38862404='))
jd['result']['count']['total']
#获取新闻编号(id)
newsurl = 'http://news.sina.com.cn/o/2017-10-26/doc-ifynfrfm9351229.shtml'
# newsid = newsurl.split('/')[-1].rstrip('.shtml').lstrip('doc-i')
# newsid
import re
m = re.search('doc-i(.*).shtml', newsurl)
# newsall = m.group(0)#获取所有match到的
newsid = m.group(1)#获取"()"里面的内容

上述代码可整理成下面的函数式：

import requests
import json
import re
from bs4 import BeautifulSoup
from datetime import datetime

news = 'http://news.sina.com.cn/o/2017-10-26/doc-ifynfrfm9351229.shtml'
commentURL = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&\
newsid=comos-{}&group=&compress=0&ie=utf-8&oe=utf-8&page=1&\
page_size=20&jsvar=loader_1509032030204_38862404'

def getCommentCounts(newsurl):
m = re.search('doc-i(.*).shtml', newsurl)
newsid = m.group(1)#获取"()"里面的内容
comments = requests.get(commentURL.format(newsid))
jd = json.loads(comments.text.strip('var loader_1509032030204_38862404='))
return jd['result']['count']['total']

def getNewsDetail(newsurl):
result = {}
res = requests.get(newsurl)
res.encoding = 'utf-8'
soup = BeautifulSoup(res.text, 'html.parser')
result['title'] = soup.select('#artibodyTitle')[0].text
result['newssource'] = soup.select('.time-source span a')[0].text
timesource = soup.select('.time-source')[0].contents[0].strip()
result['dt'] = datetime.strptime(timesource, '%Y年%m月%d日%H:%M')
result['article'] = ' '.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
result['editor'] = soup.select('.article-editor')[0].text.lstrip('责任编辑：')
result['comments'] = getCommentCounts(newsurl)
return result

getNewsDetail(news)

阅读全文

0 0