实现用火狐进行动态网页抓取

# -*- coding: utf-8 -*-import sys,refrom lxml import etreeimport  timefrom selenium import webdriverimport osimport pymysqlhost = '服务器IP'dbName = '数据库名'user = '数据库登陆用户名'password = '数据库登陆密码'db = pymysql.connect(host, user, password, dbName, charset='utf8')cursor = db.cursor()cursor.execute("DROP TABLE IF EXISTS EMPLOYEE")  sql = """CREATE TABLE EMPLOYEE (SYMBOL CHAR(6) NOT NULL,NAME CHAR(10),PRICE CHAR(10))"""cursor.execute(sql)driver = webdriver.Firefox()  #打开火狐浏览器driver.set_window_position(x=70,y=30)   #设置火狐浏览器位置driver.set_window_size(width=1064, height=524) #设置火狐浏览器窗口大小f = open("symbol.txt", 'r')   # 以只读方式打开当前路径下的symbol.txt文件for line in f.readlines():   line = line.replace("\n", "")  # 提取每行数据，去掉换行符   url = "url1"+line+"url2"  # url拼接   driver.get(url)  #输入需要访问的URL   time.sleep(3)    #等待3秒   content = driver.page_source.encode('utf-8') #网页内容转码utf8   html = etree.HTML(content)   price = html.xpath('.//*[@class="ico_data col_open"]/text()') #提取价格   name = html.xpath('.//*[@class="ico_name label_open"]/text()') #提取名称   symbolname = ''.join(name)   price = ''.join(price)   print(line, symbolname, price)   cursor = db.cursor()   sql1 = ("INSERT INTO EMPLOYEE(SYMBOL, NAME, PRICE) VALUES ('%s', '%s', '%s')" % (str(line), symbolname, str(price)))   cursor.execute(sql1)   db.commit()  #提交数据db.close() #关闭数据库f.close()  #关闭symbol文件driver.quit() #退出浏览器