多线程断点续传--java

来源：互联网发布：青骑士知乎编辑：程序博客网时间：2024/06/06 19:39

以前看过一个前辈写的断点续传的文章，记得当时没看懂，就扔那了。昨天翻了出来，自己也仿照写了一个，不过感觉没人家写的好，有点乱，希望大家能来批评、指正，给点意见！

功能很简单，就是启动多个线程分别从给定的地址下载数据，用RandomAccessFile写到目标文件。实现思路是：

1、获得连接的长度（即要下载的文件大小），除以设定的线程数，即得到每个线程要下载的大小。

2、记录临时文件，文件中记录每个线程的编号（id），该线程要下载的起始位置、终止位置和当前位置（当前位置在首次下载时与起始位置相同）。

3、启动具体执行下载任务的线程，并等待其结束。

4、下载完成，删除临时文件。

代码如下:

主线程及测试的main方法

Java代码

package com.why.download.test;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
*
* @author why
*
*/
public class DownLoad {
//文件目录、文件名
public String fileDir = "E:/MyDownLoad";
public String fileName;
//超时重连时间
public long reconnectTime = 5;
//线程数
private int poolSize = 5;
//每个线程的缓冲区大小
public int bufferSize = 1024;
//url地址
private String urlLocation = null;
public DownLoad(){}
public DownLoad(String url){
this.urlLocation = url;
}
public void downLoad(){
if(this.urlLocation == null || "".equals(this.urlLocation))return;
downLoad(this.urlLocation);
}
public void downLoad(String urlLocation){
File file = null;
File tempFile = null;
CountDownLatch latch;
URL url = null;
ExecutorService pool = Executors.newCachedThreadPool();
long contentLength = 0;
long threadLength = 0;
try {
//如果未指定名称，则从url中获得下载的文件格式与名字
if(fileName == null || "".equals(fileName)){
this.fileName = urlLocation.substring(urlLocation.lastIndexOf("/") + 1,
urlLocation.lastIndexOf("?") > 0 ? urlLocation.lastIndexOf("?")
: urlLocation.length());
if ("".equalsIgnoreCase(this.fileName)) {
this.fileName = UUID.randomUUID().toString();
}
}
new File(fileDir).mkdirs();
file = new File(fileDir + File.separator + fileName);
tempFile = new File(fileDir + File.separator + fileName + "_temp");
url = new URL(urlLocation);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
setHeader(conn);
//得到content的长度
contentLength = conn.getContentLength();
System.out.println("total length=" + contentLength);
//把context分为poolSize段，计算每段的长度。
// threadLength = contentLength / this.poolSize;
BigDecimal b1 = new BigDecimal(Double.toString(contentLength));
BigDecimal b2 = new BigDecimal(Double.toString(this.poolSize));
threadLength = b1.divide(b2, 0, BigDecimal.ROUND_HALF_UP).longValue();
if(file.exists() && tempFile.exists()){
//如果文件已存在，根据临时文件中记载的线程数量，继续上次的任务
latch = new CountDownLatch((int)tempFile.length()/28);
for(int i=0;i<tempFile.length()/28;i++){
pool.submit(new DownLoadTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize));
}
}else{
//如果下载的目标文件不存在，则创建新文件
latch = new CountDownLatch(poolSize);
file.createNewFile();
tempFile.createNewFile();
DataOutputStream os = new DataOutputStream(new FileOutputStream(tempFile));
for(int i=0;i<this.poolSize;i++){
os.writeInt(i+1);
os.writeLong(i*threadLength);
if(i==this.poolSize-1){//最后一个线程的结束位置应为文件末端
os.writeLong(contentLength);
}else{
os.writeLong((i+1)*threadLength);
}
os.writeLong(i*threadLength);
pool.submit(new DownLoadTask(file, tempFile, url, i+1,latch,reconnectTime,bufferSize));
}
os.close();
}
//等待下载任务完成
latch.await();
//删除临时文件
tempFile.delete();
} catch (IOException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
} finally{
pool.shutdown();
}
}
private void setHeader(URLConnection conn) {
conn.setRequestProperty("User-Agent",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");
conn.setRequestProperty("Accept-Encoding", "aa");
conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
conn.setRequestProperty("Keep-Alive", "300");
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");
conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");
conn.setRequestProperty("Cache-Control", "max-age=0");
conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html");
}
public String getFileDir() {
return fileDir;
}
public void setFileDir(String fileDir) {
this.fileDir = fileDir;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public long getReconnectTime() {
return reconnectTime;
}
public void setReconnectTime(long reconnectTime) {
this.reconnectTime = reconnectTime;
}
public int getPoolSize() {
return poolSize;
}
public void setPoolSize(int poolSize) {
this.poolSize = poolSize;
}
public int getBufferSize() {
return bufferSize;
}
public void setBufferSize(int bufferSize) {
this.bufferSize = bufferSize;
}
/**
* @param args
*/
public static void main(String[] args) {
DownLoad dl = new DownLoad();
dl.setFileDir("E:/MyDownLoad/music/");
dl.setFileName("大笑江湖.mp3");
dl.setPoolSize(20);
long beginTime = System.currentTimeMillis();
dl.downLoad("http://mh.163k.com/UploadFile/video/2010/12-13/201012131213448942190.mp3");
long endTime = System.currentTimeMillis();
BigDecimal b1 = new BigDecimal(endTime - beginTime);
BigDecimal b2 = new BigDecimal(1000);
double cost = b1.divide(b2, 2, BigDecimal.ROUND_HALF_UP).doubleValue();
System.out.println("Time cost:" + cost + "s");
}
}

执行下载任务的线程：

Java代码

package com.why.download.test;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
*
* @author why
*
*/
public class DownLoadTask implements Callable<String>{
//超时重连时间
private long reconnectTime = 5;
//缓冲区大小
private int bufferSize = 1024;
private CountDownLatch latch;
private RandomAccessFile file = null;
private RandomAccessFile tempFile = null;
private URL url = null;
private int id;
private long startPosition;
private long endPosition;
private long currentPosition ;
public DownLoadTask(File file,File tempFile,URL url,int id,CountDownLatch latch,long reconnectTime,int bufferSize){
try {
this.file = new RandomAccessFile(file, "rw");
this.tempFile = new RandomAccessFile(tempFile, "rw");
} catch (FileNotFoundException e) {
e.printStackTrace();
}
this.url = url;
this.id = id;
this.latch = latch;
}
public String call(){
try {
tempFile.seek((id-1)*28);
tempFile.readInt();
this.startPosition = tempFile.readLong();
this.endPosition = tempFile.readLong();
this.currentPosition = tempFile.readLong();
} catch (IOException e) {
e.printStackTrace();
}
System.out.println("Thread " + id + " begin!");
HttpURLConnection conn = null;
InputStream inputStream = null;
while(true){
try {
tempFile.seek(id*28 - 8);
// 打开URLConnection
conn = (HttpURLConnection) this.url.openConnection();
setHeader(conn);
// 设置连接超时时间为10000ms
conn.setConnectTimeout(10000);
// 设置读取数据超时时间为10000ms
conn.setReadTimeout(10000);
if (currentPosition < endPosition) {
// 设置下载数据的起止区间
conn.setRequestProperty("Range", "bytes=" + currentPosition + "-" + endPosition);
System.out.println("Thread " + id + " startPosition=" + startPosition
+ ",endPosition=" + endPosition + ",currentPosition=" + currentPosition);
file.seek(currentPosition);
// 判断http status是否为HTTP/1.1 206 Partial Content或者200 OK
// 如果不是以上两种状态，把status改为STATUS_HTTPSTATUS_ERROR
if (conn.getResponseCode() != HttpURLConnection.HTTP_OK
&& conn.getResponseCode() != HttpURLConnection.HTTP_PARTIAL) {
System.out.println("Thread " + id + ": code = " + conn.getResponseCode() + ", status = " + conn.getResponseMessage());
file.close();
conn.disconnect();
System.out.println("Thread " + id + " finished.");
break;
}
inputStream = conn.getInputStream();
int len = 0;
byte[] b = new byte[bufferSize];
while ((len = inputStream.read(b)) != -1) {
file.write(b, 0, len);
currentPosition += len;
// set tempFile now position
tempFile.seek(id*28 - 8);
tempFile.writeLong(currentPosition);
}
file.close();
tempFile.close();
inputStream.close();
conn.disconnect();
}
System.out.println("Thread " + id + " finished.");
break;
} catch (IOException e) {
try {
TimeUnit.SECONDS.sleep(getReconnectTime());
} catch (InterruptedException e1) {
e1.printStackTrace();
}
continue;
}
}
latch.countDown();
return "finish";
}
private void setHeader(URLConnection conn) {
conn.setRequestProperty("User-Agent",
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");
conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");
conn.setRequestProperty("Accept-Encoding", "aa");
conn.setRequestProperty("Accept-Charset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
conn.setRequestProperty("Keep-Alive", "300");
conn.setRequestProperty("Connection", "keep-alive");
conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");
conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");
conn.setRequestProperty("Cache-Control", "max-age=0");
conn.setRequestProperty("Referer","http://www.skycn.com/soft/14857.html");
}
public long getReconnectTime() {
return reconnectTime;
}
public void setReconnectTime(long reconnectTime) {
this.reconnectTime = reconnectTime;
}
public int getBufferSize() {
return bufferSize;
}
public void setBufferSize(int bufferSize) {
this.bufferSize = bufferSize;
}
}

最近一直在测试，写文档，好久没敲代码了。手有点痒，写着玩的，代码写的不是很工整，纯属娱乐！

转载自：http://wuhongyu.iteye.com/blog/869109