java多线程网页下载代码

来源：互联网发布：酷我h1头戴评测知乎编辑：程序博客网时间：2024/06/07 12:05

1.使用了java.util.concurrent包里的线程池，可以飙升到满带宽，在100M带宽上，可以达到10MB/s。

2.使用了java.nio里的channels，性能比自己缓冲有一些提高。

import java.io.FileOutputStream;import java.io.FileOutputStream; import java.io.InputStream;import java.net.URL; import java.net.URLConnection; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.ReadableByteChannel; import java.util.Calendar; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; public class HttpDownloader implements Callable<String> {     URLConnection connection;     FileChannel outputChann;     public static volatile int count = 0;      public static void main(String[] args) throws Exception {          ExecutorService poll = Executors.newFixedThreadPool(100);         for (int i = 0; i < 100; i++) {             Calendar now = Calendar.getInstance();             String fileName = "../data/" + now.get(Calendar.YEAR) + "年"                     + (now.get(Calendar.MONTH) + 1) + "月"                     + now.get(Calendar.DAY_OF_MONTH) + "日--" + i + ".txt";             poll.submit(new HttpDownloader("http://www.sina.com",                     (new FileOutputStream(fileName)).getChannel()));         }          poll.shutdown();          long start = System.currentTimeMillis();         while (!poll.isTerminated()) {             Thread.sleep(1000);             System.out.println("已运行"                     + ((System.currentTimeMillis() - start) / 1000) + "秒，"                     + HttpDownloader.count + "个任务还在运行");         }     }      public HttpDownloader(String url, FileChannel fileChannel) throws Exception {        synchronized (HttpDownloader.class) {            count++;         }         connection = (new URL(url)).openConnection();         this.outputChann = fileChannel;    }      @Override     public String call() throws Exception {         connection.connect();         InputStream inputStream = connection.getInputStream();         ReadableByteChannel rChannel = Channels.newChannel(inputStream);        outputChann.transferFrom(rChannel, 0, Integer.MAX_VALUE);        // System.out.println(Thread.currentThread().getName() + " completed!");         inputStream.close();         outputChann.close();         synchronized (HttpDownloader.class) {             count--;         }         return null;     } }

<pre style="color: rgb(51, 51, 51); font-size: 13px; line-height: 24px; background-color: rgb(214, 211, 214);">

import java.io.FileOutputStream;import java.io.InputStream;import java.net.URL;import java.net.URLConnection;import java.nio.channels.Channels;import java.nio.channels.FileChannel;import java.nio.channels.ReadableByteChannel;import java.util.Calendar;import java.util.concurrent.Callable;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors; public class HttpDownloader implements Callable<String> {     URLConnection connection;     FileChannel outputChann;     public static volatile int count = 0;      public static void main(String[] args) throws Exception {          ExecutorService poll = Executors.newFixedThreadPool(100);          for (int i = 0; i < 100; i++) {             Calendar now = Calendar.getInstance();             String fileName = "../data/" + now.get(Calendar.YEAR) + "年"                     + (now.get(Calendar.MONTH) + 1) + "月"                     + now.get(Calendar.DAY_OF_MONTH) + "日--" + i + ".txt";             poll.submit(new HttpDownloader("http://www.sina.com",                     (new FileOutputStream(fileName)).getChannel()));         }          poll.shutdown();          long start = System.currentTimeMillis();         while (!poll.isTerminated()) {             Thread.sleep(1000);             System.out.println("已运行"                     + ((System.currentTimeMillis() - start) / 1000) + "秒，"                     + HttpDownloader.count + "个任务还在运行");         }     }      public HttpDownloader(String url, FileChannel fileChannel) throws Exception {         synchronized (HttpDownloader.class) {             count++;         }         connection = (new URL(url)).openConnection();         this.outputChann = fileChannel;     }      @Override     public String call() throws Exception {         connection.connect();         InputStream inputStream = connection.getInputStream();         ReadableByteChannel rChannel = Channels.newChannel(inputStream);         outputChann.transferFrom(rChannel, 0, Integer.MAX_VALUE);         // System.out.println(Thread.currentThread().getName() + " completed!");         inputStream.close();         outputChann.close();         synchronized (HttpDownloader.class) {             count--;         }         return null;     } }

import java.io.FileOutputStream;import java.io.InputStream;import java.net.URL;import java.net.URLConnection;import java.nio.channels.Channels;import java.nio.channels.FileChannel;import java.nio.channels.ReadableByteChannel;import java.util.Calendar;import java.util.concurrent.Callable;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;public class HttpDownloader implements Callable<String> {URLConnection connection;FileChannel outputChann;public static volatile int count = 0;public static void main(String[] args) throws Exception {ExecutorService poll = Executors.newFixedThreadPool(100);for (int i = 0; i < 100; i++) {Calendar now = Calendar.getInstance();String fileName = "../data/" + now.get(Calendar.YEAR) + "年"+ (now.get(Calendar.MONTH) + 1) + "月"+ now.get(Calendar.DAY_OF_MONTH) + "日--" + i + ".txt";poll.submit(new HttpDownloader("http://www.sina.com",(new FileOutputStream(fileName)).getChannel()));}poll.shutdown();long start = System.currentTimeMillis();while (!poll.isTerminated()) {Thread.sleep(1000);System.out.println("已运行"+ ((System.currentTimeMillis() - start) / 1000) + "秒，"+ HttpDownloader.count + "个任务还在运行");}}public HttpDownloader(String url, FileChannel fileChannel) throws Exception {synchronized (HttpDownloader.class) {count++;}connection = (new URL(url)).openConnection();this.outputChann = fileChannel;}@Overridepublic String call() throws Exception {connection.connect();InputStream inputStream = connection.getInputStream();ReadableByteChannel rChannel = Channels.newChannel(inputStream);outputChann.transferFrom(rChannel, 0, Integer.MAX_VALUE);// System.out.println(Thread.currentThread().getName() + " completed!");inputStream.close();outputChann.close();synchronized (HttpDownloader.class) {count--;}return null;}}

转自：http://www.cnblogs.com/windydays/archive/2011/11/15/2298102.html

0 0