图片爬虫程序 JAVA

package com.liyiwen.Crawler;import java.io.*;import java.net.MalformedURLException;import java.net.URL;import java.net.URLConnection;import java.util.ArrayList;import java.util.Collection;import java.util.SplittableRandom;import java.util.concurrent.*;import java.util.regex.Matcher;import java.util.regex.Pattern;/** * Created by dell on 2015/7/18. *//** * 正则表达式中\表示转义，java字符出串\也表示转义 */public class ImgCrawler implements Runnable {    public static void main(String[] urls) throws FileNotFoundException, InterruptedException, IOException, ExecutionException{        ImgCrawler imgCrawler = new ImgCrawler();        ArrayList<Thread> threads = new ArrayList<Thread>();        for (int i = 0; i < ImgCrawler.threads; ++i){            Thread thread = new Thread(imgCrawler);            thread.start();            threads.add(thread);        }        Thread.currentThread().sleep(10000);        for (Thread t : threads){            t.interrupt();        }        System.out.println(imgCrawler.getWantedUrls().toString() + "李意文");        printToFile(imgCrawler.getWantedUrls());        System.out.println("succeed");    }    public static void printToFile(Collection<String> strs) throws FileNotFoundException, IOException{        File file = new File("crawler.html");        if (!file.exists()){            file.createNewFile();        }         Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "utf-8"));        writer.write(1);        for (String str : strs){            writer.append(str + "\r\n");        }        writer.flush();    }    static private int threads;    static {        threads = Runtime.getRuntime().availableProcessors();    }    public BlockingQueue<String> getWantedUrls() {        return wantedUrls;    }    private BlockingQueue<String> wantedUrls;    private BlockingQueue<String> handledUrls;    public ImgCrawler(){        wantedUrls = new LinkedBlockingQueue<String>();        handledUrls = new LinkedBlockingQueue<String>();        handledUrls.add("http://www.dedeshe.com/html/article/2015-7/index26785.html");    }    @Override    public void run() {        try{            for (int i = 0; i < 1000; i++ ){                String url = null;                url = handledUrls.take();                crawl(url);                if (Thread.currentThread().isInterrupted()){                    break;                }            }        }        catch (Throwable throwable){            System.out.println(throwable.getMessage());        }    }    private void crawl(String url) throws InterruptedException{        try {            URL crawledUrl = new URL(url);            URLConnection urlConnection = crawledUrl.openConnection();            BufferedReader reader = new BufferedReader(new InputStreamReader(urlConnection.getInputStream(), "utf-8"));            String line = null;            ArrayList<String> matchUrls = null;            ArrayList<String> hrefs = null;            while ((line = reader.readLine()) != null) {               // System.out.println(line);                matchUrls = matchWantedUrl(line);                if (null != matchUrls && !matchUrls.isEmpty()) {                    for (String matchUrl : matchUrls) {                        wantedUrls.put(matchUrl);                    }                }                hrefs = matchHref(line, url);                if (null != hrefs && !hrefs.isEmpty()) {                    for (String href : hrefs) {                        handledUrls.put(href);                    }                }            }        }        catch (MalformedURLException e){            System.out.println(e.getMessage());            System.out.println("url 错误");        }        catch (IOException e){            System.out.println("不能打开连接");            System.out.println(e.getMessage());        }    }    private ArrayList<String> matchWantedUrl(String line){        ArrayList<String> wantedUrsl = new ArrayList<String>();        Pattern pattern = Pattern.compile("<img .*?src=\"http.*?\".*?>");        Matcher matcher = pattern.matcher(line);        while (matcher.find()){            wantedUrsl.add(matcher.group(0));        }        return wantedUrsl;    }    private ArrayList<String> matchHref(String line,  String url){        String rootURL = null;        Pattern rootURLPattern = Pattern.compile("(http.+com).*");        Matcher rootURLMatcher = rootURLPattern.matcher(url);        if (rootURLMatcher.find()){             rootURL = rootURLMatcher.group(1);            System.out.println("根目录： " + rootURL);        }        ArrayList<String> wantedHrefs = new ArrayList<String>();        Pattern pattern = Pattern.compile("<a href=\"(.+?)\"");        Matcher matcher = pattern.matcher(line);        while (matcher.find()){            if (!matcher.group(1).startsWith("http") && rootURL != null){                wantedHrefs.add(rootURL + matcher.group(1));                System.out.println(rootURL + matcher.group(1));            }else{                wantedHrefs.add(matcher.group(1));                System.out.println(matcher.group(1));            }        }        return wantedHrefs;    }}