scrapy框架学习-爬取腾讯社招信息-tencent.py

功能：设置起始URL和爬取范围，设置要提取的数据路径，返回item或下一个URL地址

# -*- coding: utf-8 -*-import scrapyfrom Tenxun.items import TenxunItemclass TencentSpider(scrapy.Spider):    name = 'tencent'    allowed_domains = ['tencent.com']    start_urls = ['http://hr.tencent.com/position.php?&start=0']    def parse(self, response):        nodelist = response.xpath("//tr[@class='even'] | //tr[@class='odd']")        for node in nodelist:            item = TenxunItem()            item['job_name'] = node.xpath("./td[1]/a/text()").extract()[0]            item['job_link'] = node.xpath("./td[1]/a/@href").extract()[0]            if len(node.xpath("./td[2]/text()")):                item['job_type'] = node.xpath("./td[2]/text()").extract()[0]            else:                item['job_type'] = "NULL"            item['job_people_num'] = node.xpath("./td[3]/text()").extract()[0]            item['job_site'] = node.xpath("./td[4]/text()").extract()[0]            item['publish_time'] = node.xpath("./td[5]/text()").extract()[0]            yield item        if len(response.xpath("//a[@class='noactive' and @id='next']")) == 0:            url = response.xpath("//a[@id='next']/@href").extract()[0]            yield scrapy.Request("http://hr.tencent.com/" + url, callback=self.parse)