python +selenium+phantomjs 登录爬取新浪微博动态js页面

登录新浪微博

最近新浪微博好烦，都取消不了验证码这个难搞得东西，而且跳来跳去，一改版以前的代码就都不能用了。目前整理的资料有三种方法：

 1. 设Cookie：简单粗暴，免去了模拟登录的好多麻烦，只是要定期更新 2. 模拟登录：验证码是个大麻烦，有把验证码图片截下来，本地识别控制台输入验证 3. 扫码登录 ：用app的扫码登录

其实除了设cookie ，在有验证码下，后两种方法差不多，都要人工干预。设cookie就不再说了。下面给出后两种方法的代码，代码来自GitHub - xchaoinfo/fuck-login: 模拟登录一些知名的网站，为了方便爬取需要登录的网站

模拟登录

#!/usr/bin/env python3# -*- coding: utf-8 -*-'''Required- requests (必须)- rsa (必须)- pillow (可选)Info- author : "xchaoinfo"- email  : "[e-mail]"- date   : "2016.3.7"'''import timeimport base64import rsaimport binasciiimport requestsimport reimport randomtry:    from PIL import Imageexcept:    passtry:    from urllib.parse import quote_plusexcept:    from urllib import quote_plus'''如果没有开启登录保护，不用输入验证码就可以登录如果开启登录保护，需要输入验证码'''# 构造 Request headersagent = 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0'headers = {    'User-Agent': agent}session = requests.session()# 访问 初始页面带上 cookieindex_url = "http://weibo.com/login.php"try:    session.get(index_url, headers=headers, timeout=2)except:    session.get(index_url, headers=headers)try:    input = raw_inputexcept:    passdef get_su(username):    """    对 email 地址和手机号码 先 javascript 中 encodeURIComponent    对应 Python 3 中的是 urllib.parse.quote_plus    然后在 base64 加密后decode    """    username_quote = quote_plus(username)    username_base64 = base64.b64encode(username_quote.encode("utf-8"))    return username_base64.decode("utf-8")# 预登陆获得 servertime, nonce, pubkey, rsakvdef get_server_data(su):    pre_url = "http://login.sina.com.cn/sso/prelogin.php?entry=weibo&callback=sinaSSOController.preloginCallBack&su="    pre_url = pre_url + su + "&rsakt=mod&checkpin=1&client=ssologin.js(v1.4.18)&_="    pre_url = pre_url + str(int(time.time() * 1000))    pre_data_res = session.get(pre_url, headers=headers)    sever_data = eval(pre_data_res.content.decode("utf-8").replace("sinaSSOController.preloginCallBack", ''))    return sever_data# print(sever_data)def get_password(password, servertime, nonce, pubkey):    rsaPublickey = int(pubkey, 16)    key = rsa.PublicKey(rsaPublickey, 65537)  # 创建公钥    message = str(servertime) + '\t' + str(nonce) + '\n' + str(password)  # 拼接明文js加密文件中得到    message = message.encode("utf-8")    passwd = rsa.encrypt(message, key)  # 加密    passwd = binascii.b2a_hex(passwd)  # 将加密信息转换为16进制。    return passwddef get_cha(pcid):    cha_url = "http://login.sina.com.cn/cgi/pin.php?r="    cha_url = cha_url + str(int(random.random() * 100000000)) + "&s=0&p="    cha_url = cha_url + pcid    cha_page = session.get(cha_url, headers=headers)    with open("cha.jpg", 'wb') as f:        f.write(cha_page.content)        f.close()    try:        im = Image.open("cha.jpg")        im.show()        im.close()    except:        print(u"请到当前目录下，找到验证码后输入")def login(username, password):    # su 是加密后的用户名    su = get_su(username)    sever_data = get_server_data(su)    servertime = sever_data["servertime"]    nonce = sever_data['nonce']    rsakv = sever_data["rsakv"]    pubkey = sever_data["pubkey"]    showpin = sever_data["showpin"]    password_secret = get_password(password, servertime, nonce, pubkey)    postdata = {        'entry': 'weibo',        'gateway': '1',        'from': '',        'savestate': '7',        'useticket': '1',        'pagerefer': "http://login.sina.com.cn/sso/logout.php?entry=miniblog&r=http%3A%2F%2Fweibo.com%2Flogout.php%3Fbackurl",        'vsnf': '1',        'su': su,        'service': 'miniblog',        'servertime': servertime,        'nonce': nonce,        'pwencode': 'rsa2',        'rsakv': rsakv,        'sp': password_secret,        'sr': '1366*768',        'encoding': 'UTF-8',        'prelt': '115',        'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',        'returntype': 'META'        }    login_url = 'http://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.18)'    if showpin == 0:        login_page = session.post(login_url, data=postdata, headers=headers)    else:        pcid = sever_data["pcid"]        get_cha(pcid)        postdata['door'] = input(u"请输入验证码")        login_page = session.post(login_url, data=postdata, headers=headers)    login_loop = (login_page.content.decode("GBK"))    # print(login_loop)    pa = r'location\.replace\([\'"](.*?)[\'"]\)'    loop_url = re.findall(pa, login_loop)[0]    # print(loop_url)    # 此出还可以加上一个是否登录成功的判断，下次改进的时候写上    login_index = session.get(loop_url, headers=headers)    uuid = login_index.text    uuid_pa = r'"uniqueid":"(.*?)"'    print uuid    uuid_res = re.findall(uuid_pa, uuid, re.S)[0]    web_weibo_url = "http://weibo.com/%s/profile?topnav=1&wvr=6&is_all=1" % uuid_res    weibo_page = session.get(web_weibo_url, headers=headers)    weibo_pa = r'<title>(.*?)</title>'    # print(weibo_page.content.decode("utf-8"))    userID = re.findall(weibo_pa, weibo_page.content.decode("utf-8", 'ignore'), re.S)[0]    print(u"欢迎你 %s, 你在正在使用 xchaoinfo 写的模拟登录微博" % userID)if __name__ == "__main__":    # username = input(u'用户名：')    # password = input(u'密码：')    username = "*****"    password = "*****"    login(username, password)

扫码

#!/usr/bin/env python# -*- coding: utf-8 -*-'''在python2.7中测试通过Required- requests (必须)- pillow (可选)Info- author : "fangc"- email  : "[e-mail]"- date   : "2016.3.16"'''import requestsimport cookielibimport timeimport reimport systry:    from PIL import Imageexcept:    passimport threading# 构造 Request headersagent = 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0'headers = {    'User-Agent': agent}session = requests.session()session.cookies = cookielib.LWPCookieJar('weibo_cookies.txt')# 访问 初始页面带上 cookieindex_url = "http://weibo.com/"try:    session.get(index_url, headers=headers, timeout=2)except:    session.get(index_url, headers=headers)def open_img(image_name):    """打开图片    :param image_name: 图片的路径    :return:    """    im = Image.open(image_name)    im.show()    im.close()def login():    """登录主函数    :return:    """    image_name, qrcode_qrid = get_qrcode()    try:        # TODO(@fangc):用此方法打开图片不会退出，可以直接命令行用open打开，粗暴简单.        thread = threading.Thread(target=open_img, name="open", args=(image_name,))        thread.start()        print(u"请用手机微博扫描二维码"              u"微博二维码扫描在主页右上角!")    except:        print(u"请到当前目录下，打开二维码后用手机微博扫描二维码"              u"微博二维码扫描在主页右上角!")    # 下面判断是否已经扫描了二维码    statu = 0    while not statu:        qrcode_check_page = scan_qrcode(qrcode_qrid, str(long(time.time() * 10000)))        if "50114002" in qrcode_check_page:            statu = 1            print(u"---成功扫描，请在手机点击确认以登录---")        time.sleep(2)    # 下面判断是否已经点击登录,并获取alt的内容    while statu:        qrcode_click_page = scan_qrcode(qrcode_qrid, str(long(time.time() * 100000)))        if "succ" in qrcode_click_page:            # 登录成功后显示的是如下内容,需要获取到alt的内容            # {"retcode":20000000,"msg":"succ","data":{"alt":"ALT-MTgxODQ3MTYyMQ==-sdfsfsdfsdfsfsdf-39A12129240435A0D"}}            statu = 0            alt = re.search(r'"alt":"(?P<alt>[\w\-\=]*)"', qrcode_click_page).group("alt")            print(u"---登录成功---")        time.sleep(2)    # 下面是登录请求获取登录的跨域请求    params = {        "entry": "weibo",        "returntype": "TEXT",        "crossdomain": 1,        "cdult": 3,        "domain": "weibo.com",        "alt": alt,        "savestate": 30,        "callback": "STK_" + str(long(time.time() * 100000))    }    login_url_list = "http://login.sina.com.cn/sso/login.php"    login_list_page = session.get(login_url_list, params=params, headers=headers)    # 返回的数据如下所示，需要提取出4个url    # STK_145809336258600({"retcode":"0","uid":"1111111","nick":"*****@sina.cn","crossDomainUrlList":    # ["http:***************","http:\/\***************","http:\/\/***************","http:\/\/***************"]});    url_list = [i.replace("\/", "/") for i in login_list_page.content.split('"') if "http" in i]    for i in url_list:        session.get(i, headers=headers)        time.sleep(0.5)    session.cookies.save(ignore_discard=True, ignore_expires=True)    print(u"欢迎你, 你在正在使用 fangc 写的模拟登录微博")    is_login()def get_qrcode():    """获取二维码图片以及二维码编号    :return: qrcode_image, qrcode_qrid    """    qrcode_before = "http://login.sina.com.cn/sso/qrcode/image?entry=weibo&size=180&callback=STK_" + str(        long(time.time() * 10000))    qrcode_before_page = session.get(qrcode_before, headers=headers)    if qrcode_before_page.status_code != 200:        sys.exit(u"可能微博改了接口!请联系作者修改")    qrcode_before_data = qrcode_before_page.content    qrcode_image = re.search(r'"image":"(?P<image>.*?)"', qrcode_before_data).group("image").replace("\/", "/")    qrcode_qrid = re.search(r'"qrid":"(?P<qrid>[\w\-]*)"', qrcode_before_data).group("qrid")    cha_page = session.get(qrcode_image, headers=headers)    image_name = u"cha." + cha_page.headers['content-type'].split("/")[1]    with open(image_name, 'wb') as f:        f.write(cha_page.content)        f.close()    return image_name, qrcode_qriddef scan_qrcode(qrcode_qrid, _time):    """判断是否扫码等需要    :param qrcode_qrid:    :return: html    """    params = {        "entry": "weibo",        "qrid": qrcode_qrid,        "callback": "STK_" + _time    }    qrcode_check = "http://login.sina.com.cn/sso/qrcode/check"    return session.get(qrcode_check, params=params, headers=headers).contentdef is_login():    """判断是否登录成功    :return: 登录成功返回True，失败返回False    """    try:        session.cookies.load(ignore_discard=True, ignore_expires=True)    except:        print(u"没有检测到cookie文件")        return False    url = "http://weibo.com/"    my_page = session.get(url, headers=headers)    if "我的首页" in my_page.content:        return True    else:        return Falseif __name__ == '__main__':    login()    url = "http://weibo.com/zhangfeng0729?from=friends_dynamic&refer_flag=0000011002_&is_hot=1"    page = session.get(url, headers=headers)    # print session.cookies    print page.content

完成了上述工作只是能访问页面而已，但是获取的页面是js动态生成的，什么都搞不到！好伤！
比如 http://weibo.com/zhangfeng0729?from=friends_dynamic&refer_flag=0000011002_&is_hot=1
请求会得到

什么鬼，关键部分什么都没有，再看看浏览器解析的

PhantomJS

基础的爬虫只能爬取单纯的html代码，但是有很多页面是js渲染的，html源码里面神马都没有。这是就需要借助于像浏览器一样渲染js页面的工具了。

PhantomJS是一个无界面的,可脚本编程的WebKit浏览器引擎。它原生支持多种web 标准：DOM 操作，CSS选择器，JSON，Canvas 以及SVG。

下载地址：http://phantomjs.org/download.html

该部分参考
1. 网易新闻评论爬虫python+selenium+PhantomJS
2. Setting timeout on selenium webdriver.PhantomJS

默认方式会加载整个页面，包括css ,js， 图片， 链接等， 上面参考2 设置等待时间，不加载图片，启用缓存等，会让加载速度加快

不说了，直接上代码

# coding=utf-8import requestsfrom selenium import webdriverimport timeurl = "http://weibo.com/zhangfeng0729?from=friends_dynamic&refer_flag=0000011002_&is_hot=1"# webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.Cookie'] = 'SINAGLOBAL=3955422793326.2764.1451802953297; wb_publish_vip_1888964862=2; YF-Page-G0=9a31b867b34a0b4839fa27a4ab6ec79f; _s_tentry=123.sogou.com; Apache=3233757158741.355.1460597853009; ULV=1460597853022:23:7:4:3233757158741.355.1460597853009:1460533608651; YF-V5-G0=24e0459613d3bbdec61239bc81c89e13; YF-Ugrow-G0=3a02f95fa8b3c9dc73c74bc9f2ca4fc6; login_sid_t=9c9ff740ffffbdf23415b871aa319ec0; TC-Ugrow-G0=e66b2e50a7e7f417f6cc12eec600f517; TC-V5-G0=7e5b74ea4beaaa98b5f592db11c2eeb9; myuid=5898063885; TC-Page-G0=1e758cd0025b6b0d876f76c087f85f2c; wvr=6; SSOLoginState=1460604791; un=[e-mail]; user_active=201604141220; user_unver=b37741d33507619aa07b6d512be0dd67; SUS=SID-5898063885-1460616073-XD-a0vlz-c7f218016d290c52b4297371d3942ce7; SUE=es%3D5ec8e6a39470b9d75e4cf930977a7449%26ev%3Dv1%26es2%3D2b500477fcc5dc1560b4aee4d5ca9be5%26rs0%3Daeb6LPcKR9mfwMB31yJquO6NSXgRd9BpSLAcKEdCQjtUOM7pIzDblRVIOOItI22Uc6pLjTxUc6k7s0zIgIxar9JlVdPjOiHEGijE2ucUP6GjFv%252BJWntWR7szF3qcWknPBJzemeHiThBGRF0bAcNFHcH%252BY9VfrTaIVjXCUsO93B8%253D%26rv%3D0; SUP=cv%3D1%26bt%3D1460616073%26et%3D1460702473%26d%3Dc909%26i%3D2ce7%26us%3D1%26vf%3D0%26vt%3D0%26ac%3D27%26st%3D0%26uid%3D5898063885%26name%3D1654916845%2540qq.com%26nick%3Dforfun2016%26fmp%3D%26lcp%3D; SUB=_2A256C0vZDeTxGeNG4loR9i3EwzmIHXVZYToRrDV8PUJbvNAPLXX2kW9LHet_3CJqcXTqa57ddP1X0MxCpd6bSA..; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WhFdfTB9.lOPTcSpGQskYri5JpX5K-t; SUHB=0z1QUpglvHBm_i; ALF=1492143611; UOR=far.tooold.cn,widget.weibo.com,login.sina.com.cn; WBtopGlobal_register_version=ab9111fb56d70a2b'#这个地方可以设置 header, 代理等一些参数cap = webdriver.DesiredCapabilities.PHANTOMJScap["phantomjs.page.settings.resourceTimeout"] = 1000cap["phantomjs.page.settings.loadImages"] = Falsecap["phantomjs.page.settings.disk-cache"] = Truecap["phantomjs.page.customHeaders.Cookie"] = 'SINAGLOBAL=3955422793326.2764.1451802953297; ' #我删掉了一大部分# browser = webdriver.PhantomJS(desired_capabilities=cap)driver = webdriver.PhantomJS(executable_path="G:\\code\\python\\phantomjs-2.1.1\\phantomjs-2.1.1-windows\\bin\\phantomjs.exe", desired_capabilities=cap)driver.set_window_size(1120, 1000000)driver.get(url)# if diver.find_element_by_link_text("下一页"):#    pass# else:#     time.sleep(1)contents = driver.find_elements_by_class_name("WB_cardwrap")for content in contents:    print content.text

输出结果为

691关注37193粉丝3305微博添加粉丝群微关系共同关注(1)微博客服查看更多 a文章 [9]十大银行员工薪酬调查：平均年薪最低22万也算寒冬？据说看过这篇文章的同学都考上了银行查看更多 a相册查看更多a银行校园招聘张峰的粉丝也关注银行招聘网官博银行招聘网(yinhangzhaopin.com)官方微博+关注银行招聘资讯...微信公众账号：yinhangqiuzhi 汇集银行招聘信息，全方位解读银行招聘笔试面试历年真题、考查知识点、复习重点，为你敲开银行大门。+关注应届生求职网应届生求职网YingJieSheng.COM官方微博，网址：http://www.yingjiesheng.com+关注更多 a微盘文件分享2014年银行校园招聘答疑分享者：银行校园招聘张峰浏览量:715分享银行面试到底是什么？分享者：银行校园招聘张峰浏览量:1201分享2013年中国农业银行校园招聘备考专用分享者：银行校园招聘张峰浏览量:6274分享更多 a赞##后面的省略...

解析是解析出来了，但是还是存在一个大问题

新浪微博在浏览器中也是，他不会一次性给你加载完一页，等到你将页面滑到第才给你加载完，好烦啊！这样模拟点下一页也搞不成！！

废了好长时间，还没找到解决方法，麻烦知道的给我留言通知声，万分感谢！！！

先这样吧，之后再慢慢了解phantomjs 的详情，搞一些高级的用法