Webmagic爬虫--②爬自己的CSDN博客列表

1.创建Maven项目，添加依赖

<dependency>  <groupId>us.codecraft</groupId>  <artifactId>webmagic-core</artifactId>  <version>0.7.3</version></dependency><dependency>  <groupId>us.codecraft</groupId>  <artifactId>webmagic-extension</artifactId>  <version>0.7.3</version></dependency>

2.PageProcessor代码

package com.xt;import us.codecraft.webmagic.Page;import us.codecraft.webmagic.Site;import us.codecraft.webmagic.Spider;import us.codecraft.webmagic.processor.PageProcessor;/** * Created by XT on 2017/11/24. */public class MyCSND implements PageProcessor {    //http://blog.csdn.net/weixin_35852328/article/list/1    public static final String URL_LIST = "http://blog\\.csdn\\.net/weixin_35852328/article/list/\\d{1}";    //http://blog.csdn.net/weixin_35852328/article/details/78144353    public static final String URL_POST = "http://blog\\.csdn\\.net/weixin_35852328/article/details/\\d{8}";    private static int count = 0;    private Site site = Site            .me()            .setDomain("blog.csdn.net")            .setRetryTimes(3).setSleepTime(100);    public void process(Page page) {        //列表页        if (page.getUrl().regex(URL_LIST).match()) {            page.addTargetRequests(page.getHtml().xpath("//span[@class=link_title]/a/@href").all());            page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());            //文章页        } else {            count++;            page.putField("title", page.getHtml().xpath("//span[@class='link_title']/a/text()"));            page.putField("view", page.getHtml().xpath("//span[@class='link_view']/text()"));            page.putField("date",                    page.getHtml().xpath("//span[@class='link_postdate']/text()"));        }    }    public Site getSite() {        return site;    }    public static void main(String[] args) {        long startTime, endTime;        System.out.println("开始爬取...");        startTime = System.currentTimeMillis();        Spider.create(new MyCSND()).addUrl("http://blog.csdn.net/weixin_35852328/article/list/2").thread(5).run();        endTime = System.currentTimeMillis();        System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000) + "秒，抓取了" + count + "条记录");    }}

3.看控制台，貌似还可以涨访问量哈哈