今日头条街拍爬取

本次练习使用到的知识点有

* Requests 库的使用

* BeautifulShop 库的使用

* 正则表达式的使用

* pymongo 库的使用

1、项目流程分析

2、中心调度

#中心调度

defmain(offset):

#获取列表页

index_data = get_page_index(offset,KEYWORDS)

ifindex_dataisNone:

print("offset:"+offset+"异常")

return

#解析列表页获取所有详情页的url

forurlinparse_page_index(index_data):

#获取详情页

detail_data = get_page_detail(url)

ifdetail_dataisNone:

print('url:%s异常'.format(url))

pass

#解析详情页

data = parse_page_detail(detail_data, url)

ifdataisNone:

continue

save_to_mongo(data)

3、请求和解析列表页

#请求获取列表页的响应数据

defget_page_index(offset,keywords):

params = {

'offset':offset,

'format':'json',

'keyword':KEYWORDS,

'cur_tab':3,

'autoload':'true',

'count':20

}

try:

response = requests.get('http://www.toutiao.com/search_content/',params=params)

ifresponse.status_code==200:

returnresponse.text

returnNone

exceptRequestExceptionase:

returnNone

#解析列表页

defparse_page_index(text):

try:

data = json.loads(text)

ifdataand'data'indata.keys():

foritemindata.get('data'):

yielditem.get('article_url')

exceptJSONDecodeErrorase:

print('解析异常')

return[]

4、请求和解析详情页

#解析详情页面

defparse_page_detail(html, url):

soup = BeautifulSoup(html,'lxml')

#获取页面的标题

title = soup.title.string

image_pattern = re.compile('var gallery = (.*?);',re.S)

result = image_pattern.search(html)

ifresult:

try:

data = json.loads(result.group(1))

ifdataand'sub_images'indata.keys():

#获取所有的image的url

images = [item.get('url')foritemindata.get('sub_images')]

forimageinimages:

#下载图片

download_image(image)

return{'title':title,'url':url,'images':images}

exceptJSONDecodeErrorase:

returnNone

returnNone

5、下载图片和保存至Mongodb

#获取图片的二进制流

defdownload_image(url):

try:

print('图片'+url+'正在下载')

response = requests.get(url)

ifresponse.status_code ==200:

#保存图片

save_image(response.content)

exceptRequestExceptionase:

print('异常image:'+url)

pass

#保存二进制流至文件

defsave_image(content):

file_path ='{0}/images/{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')

ifnotos.path.exists(file_path):

withopen(file_path,'wb+')asfile:

file.write(content)

file.close()

defsave_to_mongo(data):

ifdb[MONGO_TABLE].insert(data):

print('成功保存'+data['title'])

returnTrue

returnFalse

6、完整代码

#!/usr/bin/python

# -*- coding: utf-8 -*-

importos

importre

importrequests

importpymongo

importjson

fromhashlibimportmd5

frombs4importBeautifulSoup

fromsettingimport*

fromrequests.exceptionsimportRequestException

fromjson.decoderimportJSONDecodeError

frommultiprocessingimportPool

client = pymongo.MongoClient(MONGO_URL)

db = client[MONGO_DB]

#请求获取列表页的响应数据

defget_page_index(offset,keywords):

params = {

'offset':offset,

'format':'json',

'keyword':KEYWORDS,

'cur_tab':3,

'autoload':'true',

'count':20

}

try:

response = requests.get('http://www.toutiao.com/search_content/',params=params)

ifresponse.status_code==200:

returnresponse.text

returnNone

exceptRequestExceptionase:

returnNone

#解析列表页

defparse_page_index(text):

try:

data = json.loads(text)

ifdataand'data'indata.keys():

foritemindata.get('data'):

yielditem.get('article_url')

exceptJSONDecodeErrorase:

print('解析异常')

return[]

#请求获取详情页面的响应数据

defget_page_detail(url):

response = requests.get(url)

try:

ifresponse.status_code==200:

returnresponse.text

returnNone

exceptRequestExceptionase:

returnNone

#解析详情页面

defparse_page_detail(html, url):

soup = BeautifulSoup(html,'lxml')

#获取页面的标题

title = soup.title.string

image_pattern = re.compile('var gallery = (.*?);',re.S)

result = image_pattern.search(html)

ifresult:

try:

data = json.loads(result.group(1))

ifdataand'sub_images'indata.keys():

#获取所有的image的url

images = [item.get('url')foritemindata.get('sub_images')]

forimageinimages:

#下载图片

download_image(image)

return{'title':title,'url':url,'images':images}

exceptJSONDecodeErrorase:

returnNone

returnNone

#获取图片的二进制流

defdownload_image(url):

try:

print('图片'+url+'正在下载')

response = requests.get(url)

ifresponse.status_code ==200:

#保存图片

save_image(response.content)

exceptRequestExceptionase:

print('异常image:'+url)

pass

#保存二进制流至文件

defsave_image(content):

file_path ='{0}/images/{1}.{2}'.format(os.getcwd(), md5(content).hexdigest(), 'jpg')

ifnotos.path.exists(file_path):

withopen(file_path,'wb+')asfile:

file.write(content)

file.close()

defsave_to_mongo(data):

ifdb[MONGO_TABLE].insert(data):

print('成功保存'+data['title'])

returnTrue

returnFalse

#中心调度

defmain(offset):

#获取列表页

index_data = get_page_index(offset,KEYWORDS)

ifindex_dataisNone:

print("offset:"+offset+"异常")

return

#解析列表页获取所有详情页的url

forurlinparse_page_index(index_data):

#获取详情页

detail_data = get_page_detail(url)

ifdetail_dataisNone:

print('url:%s异常'.format(url))

pass

#解析详情页

data = parse_page_detail(detail_data, url)

ifdataisNone:

continue

save_to_mongo(data)

if__name__=='__main__':

groups = [x*20forxinrange(GROUP_START,GROUP_END+1)]

pool = Pool()

pool.map(main, groups)

7、运行结果