开发记录---python爬虫之gooseeker尝试

首先下载gooseeker抓取环境，到官网http://www.gooseeker.com/下载相关环境，根据教程文档http://www.gooseeker.com/tuto/tutorial.html创建抓取规则，并保存。这里尝试爬取的是某猫的搜索“笔记本”的网页https://list.tmall.com/search_product.htm?type=pc&q=%B1%CA%BC%C7%B1%BE&totalPage=1&sort=s&style=g&from=mallfp..pc_1_searchbutton

参数totalPage是页数，总共100页，我这是利用一个循环爬取每页的数据，亦可通过 教程中的翻页和层级 爬取所有数据

我的python环境是2.7的，使用下面2.7的的gooseeker的接口，返回出来的是定义规则的xml列表

#!/usr/bin/python# -*- coding: utf-8 -*-# 模块名: gooseeker_py2# 类名: GsExtractor# Version: 2.0# 适配Python版本: 2.7# 说明: html内容提取器# 功能: 使用xslt作为模板，快速提取HTML DOM中的内容。# released by 集搜客(http://www.gooseeker.com) on May 18, 2016# github: https://github.com/FullerHua/jisou/core/gooseeker_py2.pyfrom urllib2 import urlopenfrom urllib import quotefrom lxml import etreeclass GsExtractor(object):    def _init_(self):        self.xslt = ""    # 从文件读取xslt    def setXsltFromFile(self , xsltFilePath):        file = open(xsltFilePath , 'r')        try:            self.xslt = file.read()        finally:            file.close()    # 从字符串获得xslt    def setXsltFromMem(self , xsltStr):        self.xslt = xsltStr    # 通过GooSeeker API接口获得xslt    def setXsltFromAPI(self , APIKey , theme, middle=None, bname=None):        apiurl = "http://www.gooseeker.com/api/getextractor?key="+ APIKey +"&theme="+quote(theme)        if (middle):            apiurl = apiurl + "&middle="+quote(middle)        if (bname):            apiurl = apiurl + "&bname="+quote(bname)        apiconn = urlopen(apiurl)        self.xslt = apiconn.read()    # 返回当前xslt    def getXslt(self):        return self.xslt    # 提取方法，入参是一个HTML DOM对象，返回是提取结果    def extract(self , html):        xslt_root = etree.XML(self.xslt)        transform = etree.XSLT(xslt_root)        result_tree = transform(html)        return result_tree

通过上述接口获取xml数据，使用 import xml.dom.minidom解析xml数据，如下代码

# -*- encoding=utf-8 -*-# crawler_gooseeker_bbs.py# 版本: V1.0## 设置xslt抓取规则，第一个参数是app key  817b2794557252f2f46fb88733079c07，请到会员中心申请  规则名## xml解析教程  http://www.runoob.com/python/python-xml.htmlfrom urllib2 import urlopenfrom lxml import etreeimport tmallInfoimport GsExtractorimport xml.dom.minidomimport MySQLdbconn = MySQLdb.connect(    host='localhost',    port=3306,    user='root',    passwd='*****',    #mysql的密码    db='*****',    #mysql的数据库名    charset="utf8"  #这里设置格式，报这个错mySql默认设置 latin-1 ,UnicodeEncodeError: 'latin-1' codec can't encode characters in position 0-3: ordinal not in range(256))cur = conn.cursor()cur.execute(    "create table if not exists tmallInfo(id int unsigned not null auto_increment primary key,pcName varchar(100),pcPrice varchar(30),storeName varchar(30),shopLink varchar(250),monthlyTurnover varchar(30),imgSrc varchar(150))")tmallInfo.conn = conn#这里可用线程优化 爬虫速度 http://www.runoob.com/python/python-multithreading.htmlfor n in range(1,101,1):    url = "https://list.tmall.com/search_product.htm?type=pc&q=%B1%CA%BC%C7%B1%BE&totalPage=" + str(n) + "&sort=s&style=g&from=mallfp..pc_1_searchbutton"    conn = urlopen(url)    doc = etree.HTML(unicode(conn.read(), "gbk"))  # 乱码    bbsExtra = GsExtractor()    bbsExtra.setXsltFromAPI("817b2794557252f2f46fb88733079c07", "lyTmall")  # 设置xslt抓取规则，第一个参数是app key，请到会员中心申请    result = bbsExtra.extract(doc)  # 调用extract方法提取所需内容    DOMTREE = xml.dom.minidom.parseString(str(result))    pcList = DOMTREE.documentElement    items = pcList.getElementsByTagName("item")    for item in items:        pcName = item.getElementsByTagName('pcName')[0].childNodes[0].data        pcPrice = item.getElementsByTagName('pcPrice')[0].childNodes[0].data        storeName = item.getElementsByTagName('StoreName')[0].childNodes[0].data        shopLink = item.getElementsByTagName('ShopLink')[0].childNodes[0].data        monthlyTurnover = item.getElementsByTagName('monthlyTurnover')[0].childNodes[0].data        imgSrcs = item.getElementsByTagName('imgSrc')[0]        if len(imgSrcs.childNodes) > 0:            imgSrc = imgSrcs.childNodes[0].data        else:            imgSrc = "null"        info = tmallInfo(pcName, pcPrice, storeName, shopLink, monthlyTurnover, imgSrc)        info.save()    tmallInfo.conn.commit()    print ntmallInfo.conn.close()

将爬取的数据保存到mysql数据库，总共5500多条数据，在保存时创建一个bean对象，方便保存，如下：

# -*- coding:utf-8 -*-#myspl环境下载   https://pypi.python.org/pypi/MySQL-python#downloadsimport MySQLdbclass tmallInfo:    conn = None;    def __init__(self,pcName,pcPrice,storeName,shopLink,monthlyTurnover,imgSrc):        self.pcName = pcName        self.pcPrice = pcPrice        self.storeName = storeName        self.shopLink = shopLink        self.monthlyTurnover = monthlyTurnover        self.imgSrc = imgSrc    def save(self):        cur = tmallInfo.conn.cursor()        #  %s 为字符串占位符   name age为表中的属性字段，写死        sql = "insert into tmallInfo(pcName,pcPrice,storeName,shopLink,monthlyTurnover,imgSrc) values(%s,%s,%s,%s,%s,%s)"        param = (self.pcName,self.pcPrice,self.storeName,self.shopLink,self.monthlyTurnover,self.imgSrc)        n = cur.execute(sql, param)

先新建一个对象，再调用save将其保存到数据库中

info = tmallInfo(pcName, pcPrice, storeName, shopLink, monthlyTurnover, imgSrc)info.save()

将上述三个py文件放到一个目录中即可使用

本次爬虫使用到的基本技术：

• python操作数据库
• python xml解析
• gooseeker的网页采集器的使用