爬虫实战1--运用selenium框架单线程爬取妹子图片

妹子虽好，可惜触摸不到。

其他不说，直接上代码：

#-*- coding:utf-8 -*-
import selenium
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import ui
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import urllib
import os
import re
import math
import threading




# 一直等待某元素可见，默认超时10秒
def is_visible(driver,locator, timeout=10):
    try:
        ui.WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.XPATH, locator)))
        return True
    except selenium.common.exceptions.TimeoutException as e:
        return False
# 一直等待某个元素消失，默认超时10秒
def is_not_visible(driver,locator, timeout=10):
    try:
        ui.WebDriverWait(driver, timeout).until_not(EC.visibility_of_element_located((By.XPATH, locator)))
        return True
    except selenium.common.exceptions.TimeoutException as e:
        return False




#获得该页的所有相册链接和描述信息
def get_all_links_per_page(browser):
     album_links_list = browser.find_elements_by_xpath("//ul/li/span[1]/a")
     info = []
     for x in album_links_list:
         link = x.get_attribute('href')
         desc = x.text
         dic = {}
         dic['link'] = link
         dic['desc'] = desc
         info.append(dic)
         print("链接:%s,描述:%s" %(link,desc))
     return info


#下载一个相册
def download_per_album(browser,url):
browser.get(url['link'])
#获取相册描述信息
urls = url['desc']
#描述信息中可能含有一些符号，比如/ ? !等，后面我们要以相册的描述信息作为目录名
#创建目录，所以要把这些特殊符号去除，这里我一律用'_'替换
urls = re.sub(r'[^\w]','_',urls)

print(urls)

#图片的存放目录
img_path = 'd:/'+ urls + '/'

#判断目录是否存在
if not os.path.exists(img_path):
os.mkdir('d:/'+ urls)#不存在则创建相册目录

#查看该相册有多少张图片
total_page = int(browser.find_element_by_xpath("/html/body/div[2]/div[1]/div[4]/a[5]/span").text)
print("total_pic:%d\n" %(total_page))
img = browser.find_element_by_xpath("/html/body/div[2]/div[1]/div[3]/p/a/img")

#获取图片链接
img_link = img.get_attribute('src')

#添加headers，否则会被妹子图识别为爬虫，下的图都是一张一样的防盗链图
opener=urllib.request.build_opener()
opener.addheaders=[('Cookie','Hm_lvt_dbc355aef238b6c32b43eacbbf161c3c=1513017534,1513788797'), ('Referer','http://www.mzitu.com/'),('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36')]
urllib.request.install_opener(opener)


print("正在下载图片:%s\n" %(img_link))

#下载该相册第一张图片
urllib.request.urlretrieve(img_link, (img_path + '0.jpg'))

#下载该相册剩余所有图片
for i in range(1,total_page):
next_btn = browser.find_element_by_xpath("/html/body/div[2]/div[1]/div[4]/a[6]")#查找下一页按钮
next_btn.click()#点击下一页

#检查图片元素是否存在，默认超时10秒
is_visible(browser,'/html/body/div[2]/div[1]/div[3]/p/a/img')
img = browser.find_element_by_xpath("/html/body/div[2]/div[1]/div[3]/p/a/img")
img_link = img.get_attribute('src')
print("正在下载图片:%s\n" %(img_link))
img_name = str(i)+'.jpg'#图片文件名，采用：数字.jpg的形式
urllib.request.urlretrieve(img_link, (img_path + img_name))
return


if __name__ == '__main__':
    try:
        browser = webdriver.Chrome()
        url = "http://www.mzitu.com/"
        browser.get(url)
        
        #获得总页数
        total_page = int(browser.find_element_by_xpath("/html/body/div[2]/div[1]/div[2]/nav/div/a[4]").text)
        print("total_page:%d" %(total_page))




        #获得一页上所有的相册链接
        info_list = get_all_links_per_page(browser)
        for url in info_list:
            print("开始下载相册：%s\n" %(url['desc']))
            download_per_album(browser,url)




    except Exception as e:
        print("异常信息：")
        print(repr(e));