处理JS分页加载的网页_recv

# # shark-pager-next# # h3 class:ellipsis# # span class:dy-num fr# # shark-pager-disable-nextfrom selenium import webdriverfrom bs4  import BeautifulSoup as bsimport unittestclass douyu(unittest.TestCase):#初始化方法def setUp(self):self.driver=webdriver.PhantomJS()self.count=0#测试方法必须有text字样开头def testdouyu(self):self.driver.get("https://www.douyu.com/directory/all")while True:soup=bs(self.driver.page_source,"lxml")#房间名，返回列表names=soup.find_all("h3",{"class":"ellipsis"})#观众人数，返回列表numbers=soup.find_all("span",{"class":"dy-num fr"})#zip(names,numbers)将names和numbers这两个列表合并为一个元组:[(1,2),(3,4)....]for name,number in zip(names,numbers):print ("观众人数: "+number.get_text().strip()+"  "+"\t房间名: "+name.get_text().strip()).encode("gb18030")self.count=self.count+1#如果在页面上源代码里面找到“下一页”为隐藏的标签，就退出循环（因为“下一页”就按不动例如）if self.driver.page_source.find("shark-pager-disable-next")!=-1:break;#一直点击下一页self.driver.find_element_by_class_name("shark-pager-next").click()#测试结束执行的方法def tearDown(self):print u"此网站的视频"+str(self.count)#退出PhantomJS浏览器self.driver.quit()if __name__=="__main__":unittest.main()

爬斗鱼页面上每个房间，和每个房间的访问人数。每次按下一页的时候都要得到