编写自动爬取网页的爬虫

   一、item的编写

# -*- coding: utf-8 -*-# Define here the models for your scraped items## See documentation in:# http://doc.scrapy.org/en/latest/topics/items.htmlimport scrapyclass AutopjtItem(scrapy.Item):    # define the fields for your item here like:    # name = scrapy.Field()#定义好name用来存储商品名    name=scrapy.Field()#定义好price用来存储商品价格    price=scrapy.Field()#定义好link用来存储商品链接    link=scrapy.Field()#定义好comnum用来存储商品评论数    comnum=scrapy.Field()

  二、pipeline的编写

# -*- coding: utf-8 -*-import codecsimport json# Define your item pipelines here## Don't forget to add your pipeline to the ITEM_PIPELINES setting# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.htmlclass AutopjtPipeline(object):    def __init__(self):    #打开mydata.json文件        self.file = codecs.open("F:/mydata3.json", "wb", encoding="utf-8")    def process_item(self, item, spider):        # item=dict(item)        # print(len(item["name"]))        # 每一页中包含多个商品信息，所以可以通过循环，每一次处理一个商品        # 其中len(item["name"])为当前页中商品的总数，依次遍历        for j in range(0, len(item["name"])):            # 将当前页的第j个商品的名称赋值给变量name            name = item["name"][j]            price = item["price"][j]            comnum = item["comnum"][0]            link = item["link"][j]            # 将当前页下第j个商品的name、price、comnum、link等信息处理一下            # 重新组合成一个字典            goods = {"name": name, "price": price, "comnum": comnum, "link": link}            #goods = {"name": name, "price": price,  "link": link}            # 将组合后的当前页中第j个商品的数据写入json文件            i = json.dumps(dict(goods), ensure_ascii=False)            line = i + '\n'            self.file.write(line)         # 返回item        return item    def close_spider(self,spider):#关闭mydata.json文件        self.file.close()

  三、setting的设置

#开启pipeline功能

ITEM_PIPELINES = {  'autopjt.pipelines.AutopjtPipeline': 300,}

#禁用cookie

COOKIES_ENABLED = False
#不遵守机器人协议
ROBOTSTXT_OBEY = False
 
 四、爬虫的编写
  (1)分析当当网页面
  
  
  (2)爬虫编写
# -*- coding: utf-8 -*-import scrapyfrom autopjt.items import AutopjtItemfrom scrapy.http import Requestclass AutospdSpider(scrapy.Spider):    name = 'autospd'    allowed_domains = ['dangdang.com']    start_urls = ['http://category.dangdang.com/pg1-cid4002203.html']    def parse(self, response):        pass    def parse(self, response):        item = AutopjtItem()        # 通过各Xpath表达式分别提取商品的名称、价格、链接、评论数等信息        item["name"] = response.xpath("//a[@class='pic']/@title").extract()        item["price"] = response.xpath("//span[@class='price_n']/text()").extract()        item["link"] = response.xpath("//a[@class='pic']/@href").extract()        item["comnum"] = response.xpath("//a[@name='itemlist-review']/text()").extract()        # 提取完后返回item        yield item        # 接下来很关键，通过循环自动爬取75页的数据        for i in range(1, 76):            # 通过上面总结的网址格式构造要爬取的网址            url = "http://category.dangdang.com/pg" + str(i) + "-cid4002203.html"            # 通过yield返回Request，并指定要爬取的网址和回调函数            # 实现自动爬取            yield Request(url, callback=self.parse)
   
 五、爬取结果