一个简单网络爬虫示例

在学生时期，可能听到网络爬虫这个词会觉得很高大上，但是它的简单实现可能学生都不难懂。网络爬虫应用，就是把整个互联网真的就当做一张网，像蜘蛛网那样，应用就像一个虫子，在网上面按照一定的规则爬动。现在互联网应用最广的就是http(s)协议了，本文例子就是基于使用http(s)协议的，只作为示例，不涉及复杂的算法（实际上是最重要的）。

设计思路：
程序入口从一个或多个url开始，通过http(s)获取url的内容，对获取到内容处理，获取内容中需要爬取的信息，获取到内容中的url链接，再重复以上步骤。
不多说，详情看代码已经注释：

/** * 功能概要：主程序 * * @author hwz */public class MainApp {    private Integer corePoolSize = 10;    private Integer maxPoolSize = 20;    private ThreadPoolExecutor executor;    /** 工作队列 */    private SpiderQueue workQueue;    public void start(String url) throws Exception {        //初始化线程池        LinkedBlockingDeque<Runnable> executorQueue = new LinkedBlockingDeque<Runnable>(maxPoolSize);        executor = new ThreadPoolExecutor(corePoolSize, maxPoolSize, 60L, TimeUnit.SECONDS,                 executorQueue);        workQueue = new SpiderQueue(1024);        SpiderUrl spiderUrl = new SpiderUrl(url, 0);        try {            workQueue.add(spiderUrl);        }        catch (Exception e) {            System.out.println("insert url into workQueue error,url=" + url);            e.printStackTrace();        }        //提交第一个执行任务       executor.submit(new SimpleSpider(workQueue, "thread-" + "main"));       int i=0;       int idle = 0;       while(true) {           //判断是否增加更多线程执行任务           if (workQueue.size() > 20 && executor.getActiveCount() < maxPoolSize) {               idle = 0;               System.out.println("submit new thread,workQueue.size=" + workQueue.size() +                        ",executorQueue.activeCount=" + executor.getActiveCount() + ",i=" + i);               executor.submit(new SimpleSpider(workQueue, "thread-" + i++));               Thread.sleep(500);           }           else if (workQueue.size() == 0){               idle++;               System.out.println("main method, idle times=" + idle);               //主线程空闲20次，结束运行               if (idle > 20) {                   System.out.println("main method, idle times=" + idle + ",end!");                   break;               }               Thread.sleep(1000);           }           else {               Thread.sleep(2000);           }       }       System.out.println("End!,workQueue.size=" + workQueue.size() +                        ",executorQueue.activeCount=" + executor.getActiveCount() + ",executorQueue.CompletedTaskCount" +               executor.getCompletedTaskCount() +  ",i=" + i);       workQueue.printAll();       executor.shutdown();       System.exit(0);    }    public static void main(String[] args) throws Exception {        MainApp app = new MainApp();        app.start("http://www.csdn.net/");    }}

/** *  * 功能概要：自定义爬虫工作同步队列，使用ArrayList实现 * * @author hwz */public class SpiderQueue {    /** 存储器 */    private List<SpiderUrl> queue;    public SpiderQueue(int size) {        queue = new ArrayList<SpiderUrl>(size);    }    public synchronized void add(SpiderUrl spiderUrl) {        queue.add(spiderUrl);    }    public synchronized SpiderUrl poll() {        if (queue.isEmpty()) {            return null;        }        //控制台打印结果，方便查看        SpiderUrl spiderUrl = queue.remove(0);        System.out.println("SpiderQueue,poll,SpiderUrl=" + spiderUrl.toString() + ",remain size=" + queue.size());        return spiderUrl;    }    public synchronized SpiderUrl peek() {        if (queue.isEmpty()) {            return null;        }        return queue.get(0);    }    public synchronized boolean isExsit(SpiderUrl spiderUrl) {        return queue.contains(spiderUrl);    }    public synchronized int size() {        return queue.size();    }    public void printAll() {        System.out.println("Enter printAll.");        for (SpiderUrl spiderUrl : queue) {            System.out.println(spiderUrl);        }    }}

/** *  * 功能概要：爬虫工作的url * * @author hwz */public class SpiderUrl {    /** http(s) url */    private String url;    /** 该url是入口url的第几层  */    private int deep;    public SpiderUrl(String url, int deep) {        this.url = url;        this.deep = deep;    }    public String getUrl() {        return url;    }    public void setUrl(String url) {        this.url = url;    }    public int getDeep() {        return deep;    }    public void setDeep(int deep) {        this.deep = deep;    }    @Override    public boolean equals(Object obj) {        if (!(obj instanceof SpiderUrl)) {            return false;        }        SpiderUrl oth = (SpiderUrl) obj;        return this.url.equals(oth.getUrl());    }    @Override    public int hashCode() {        return url.hashCode();    }    @Override    public String toString() {        return getClass().toString() + "[url:" + url + ",deep:" + deep +"]";    }}

/** *  * 功能概要：爬虫工作类，主要实现类 * * @author hwz */public class SimpleSpider implements Runnable{    private String threadName;    private SpiderUrl url;    private SpiderQueue workQueue;    public SimpleSpider(SpiderQueue workQueue, String threadName) {        this.workQueue = workQueue;        this.threadName = threadName;    }    @Override    public void run() {        System.out.println(threadName + " start run");        //连续空闲10次循环，结束任务        int idle = 0;        while (idle < 10) {            url = workQueue.poll();            if (url != null) {                //url 解析                parseUrl(url);                idle = 0;            }            else {                System.out.println(threadName + " idle...,times=" + idle++);                try {                    Thread.sleep(1000);                }                catch (InterruptedException e) {                    e.printStackTrace();                }            }        }        System.out.println(threadName + " end run...");    }    /**     * url解析     * @param url     * @return void     */    private void parseUrl(SpiderUrl url) {        if (url == null) {            return;        }        try {            int deep = url.getDeep() + 1;            URL netUrl = new URL(url.getUrl());            URLConnection connection = netUrl.openConnection();            String contentType = connection.getContentType();            //获取内容            String resource = getResource(connection);            //获取标题            String title = getTitle(resource);            //获取链接            List<String> urls = getUrls(resource);            System.out.println(threadName +  ",parseUrl url=" + url + ",contentType=" + contentType + ",title=" + title + ",urls=" + urls);            //控制爬取链接层数，如果获取到的url全部加入工作队列，将会是指数级增加，最后程序挂掉            if (deep < 3) {                SpiderUrl newUrl;                for (String u : urls) {                    newUrl = new SpiderUrl(u,deep);                    if(!workQueue.isExsit(newUrl)) {                        workQueue.add(newUrl);                    }                }            }        }        catch (IOException e) {            e.printStackTrace();        }    }    /**     * 读取http url 内容     * @param connection     * @return     * @return String     */    private String getResource(URLConnection connection) {        if (connection == null) {            return null;        }        StringBuilder sb = new StringBuilder();        try {            InputStream inputStream = connection.getInputStream();            InputStreamReader isr = new InputStreamReader(inputStream, "UTF-8");            int input;            while ( (input = isr.read()) != -1) {                sb.append((char)input);            }        }        catch (IOException e) {            System.out.println(threadName + ",get resource error,connection=" + connection);        }        return sb.toString();    }    /**     * 从url内容获取标题     * @param content     * @return     * @return String     */    private  String getTitle(String content) {        if (content == null) {            return null;        }        Pattern pattern = Pattern.compile("(<title>.{1,}</title>)");        Matcher matcher = pattern.matcher(content);        String title = null;        if (matcher.find()) {            title = matcher.group(0).replaceAll("<title>", "").replaceAll("</title>", "");        }        return title;    }    /**     * 从url内容中获取存在的url链接     * @param content     * @return     * @return List<String>     */    private  List<String> getUrls(String content) {        if (content == null) {            return null;        }        Pattern pattern = Pattern.compile("(<a.{1,}?href=['\"]?[a-zA-z]+:\\/\\/[^\\s]*?[\\s>]{1})");        Matcher matcher = pattern.matcher(content);        String a;        String lastChar;        List<String> links = new ArrayList<String>();        while (matcher.find()) {            a = matcher.group(0).replaceAll("<a.{1,}?href=['\"]?", "");            a = a.trim();            lastChar = a.substring(a.length()-1);            if (lastChar.equals("'") || lastChar.equals("\"") || lastChar.equals(">")) {                a = a.substring(0,a.length()-1);            }            links.add(a);        }        return links;    }}

该代码示例，旨在说明一个简单的爬虫，关于多线程和http的处理没有过多考虑，如存在错误，请指出。