java 之 断点续传和高速缓存

1.文件操作类，负责文件写入：

import java.io.IOException;import java.io.RandomAccessFile;import java.io.Serializable;public class FileAccess implements Serializable  {private RandomAccessFile oSavedFile;private long nPos;public FileAccess() throws IOException {this("",0);}public FileAccess(String sName, long nPos) throws IOException {this.oSavedFile = new RandomAccessFile(sName, "rw");this.nPos = nPos;this.oSavedFile.seek(nPos);}//不需要同步，写得位置不一样public  int write(byte[] b, int nStart, int nLen) {int n = -1;try {this.oSavedFile.write(b,nStart,nLen);n = nLen;} catch(IOException e) {e.printStackTrace();}return n;}}

2.文件下载器，负责启动 多线程下载：

import java.io.DataInputStream;  import java.io.DataOutputStream;  import java.io.File;  import java.io.FileInputStream;  import java.io.FileNotFoundException;import java.io.FileOutputStream;  import java.io.IOException;  import java.net.HttpURLConnection;  import java.net.URL;      public class FileDownloader {      //文件信息Bean       private SiteInfoBean siteInfoBean = null;            //多个线程       private long[] nStartPos;//开始位置       private long[] nEndPos;//结束位置             //子线程对象       private FileSplitterFetch[] fileSplitterFetch;      //文件长度       private long nFileLength;       //是否第一次读取文件       private boolean bFirst = true;      //停止标志       //boolean bStop = false;       //文件下载的临时信息，保存下载信息       private File tmpFile;            //输出到文件的输出流       private DataOutputStream output;        //下载完成的线程数量    private int count;        public FileDownloader() {        }      public FileDownloader(SiteInfoBean bean) {          // TODO Auto-generated constructor stub           this.siteInfoBean = bean;          this.tmpFile = new File(bean.getsFilePath()+File.separator+bean.getsFileName()+".info");                System.out.println(tmpFile.getAbsolutePath());        if(this.tmpFile.exists()) {              this.bFirst = false;              //读取下载信息               read_pos();          } else {          //splitter 文件被分割的份数            this.nStartPos = new long[this.siteInfoBean.getnSplitter()];              this.nEndPos = new long[this.siteInfoBean.getnSplitter()];          }      }      public void read_pos() {          // TODO Auto-generated method stub           try {              DataInputStream input = new DataInputStream(new FileInputStream(tmpFile));              //第一个数字，线程数量            int nCount = input.readInt();                            this.nStartPos = new long[nCount];              this.nEndPos = new long[nCount];                            for(int i=0;i<this.nStartPos.length;i++) {                  this.nStartPos[i] = input.readLong();                  this.nEndPos[i] = input.readLong();              }                            input.close();          } catch(Exception e) {              e.printStackTrace();          }      }            public void download() throws IOException, InterruptedException {          //如果是第一次下载，没有info文件，则分配复制 起始 端点下载量           if(this.bFirst) {              //获取文件长度               this.nFileLength = getFileSize();              System.out.println("------------fileLen-----------------"+this.nFileLength);              if(this.nFileLength==-1) {                  System.err.println("File length is not know");              } else if(this.nFileLength==-2){                  System.err.println("File is not access!");              } else {                                    //block开头                   for(int i=0;i<this.nStartPos.length;i++) {                      //nStartPos.length是自己规定的                       this.nStartPos[i] = (long)(i*(this.nFileLength/this.nStartPos.length));                  }                  //block末尾                   for(int i=0;i<this.nEndPos.length-1;i++) {                      this.nEndPos[i] = this.nStartPos[i+1];                  }                  this.nEndPos[this.nEndPos.length-1] = this.nFileLength;                                //分配好之后，创建断点记录文件                try {        this.output = new DataOutputStream(new FileOutputStream(tmpFile));        } catch (FileNotFoundException e) {        e.printStackTrace();        }              }          }                    //启动子线程                     this.fileSplitterFetch = new FileSplitterFetch[this.nStartPos.length];          System.out.println("---------------------------------");          for(int i=0;i<this.nStartPos.length;i++) {              this.fileSplitterFetch[i] = new FileSplitterFetch(this.siteInfoBean.getsSiteURL(),                      this.siteInfoBean.getsFilePath()+File.separator+this.siteInfoBean.getsFileName(),                      this.nStartPos[i],this.nEndPos[i],i);                            System.out.println("Thread "+i+" , nStartPos= "+this.nStartPos[i]+" , nEndPos="+this.nEndPos[i]);              this.fileSplitterFetch[i].start();          }                              //是否结束while循环           boolean breakWhile = false;          while(!breakWhile) {              //               System.out.println("开始while。。。。。。。。");              //write_nPos();               //Utility.sleep(1000);               //Thread.sleep(1000);           //断点 记录            write_nPos();          //必须sleep，否则记录文件会被清空，没有数据 0kb            Utility.sleep(500);                        breakWhile = true;     /*---------------------+++++++++++++++++++++++++++*/            for(int i=0;i<this.nStartPos.length;i++) {                  //System.out.println("i++++++++++++++++++++++++++++++++++"+i);                   //还没下载完成，检查如果有一个没有下载完就设置 breakWhile=false                   if(!this.fileSplitterFetch[i].bDownOver) {                      breakWhile = false;                                          break;                  //单个线程 下载完成                   } else if(!this.fileSplitterFetch[i].bStop) {                      System.out.println("---------ok------------------"+"threadID-------------------"+i+this.fileSplitterFetch[i].bStop);                      //停止线程                       this.fileSplitterFetch[i].splitterStop();                        System.out.println("---------ok------------------"+"threadID-------------------"+i+this.fileSplitterFetch[i].bStop);                      count++;                     System.out.println("-------------------------count-----------------"+count);                     //如果5个线程都下载完成，停止while.                       //有bug，如果接着上次的下载，count又重新计数了，没有意义了                       if(count>=this.fileSplitterFetch.length) breakWhile = true;                   }              }              System.out.println("break while---------------------------"+breakWhile);              //breakWhile=false如果没有下载完成，各个线程               if(breakWhile) {                  System.out.println("文件下载结束！");                  for(int i=0;i<nStartPos.length;i++)                        fileSplitterFetch[i].splitterStop();                   break;              }              //Thread.sleep(1000);          }      }          //保存下载信息（文件指针位置)           private  void write_nPos() {              // TODO Auto-generated method stub               try {                    this.output = new DataOutputStream(new FileOutputStream(tmpFile));                        //下载线程数量                   this.output.writeInt(this.nStartPos.length);                                    for(int i=0;i<this.nStartPos.length;i++) {                      this.output.writeLong(this.fileSplitterFetch[i].nStartPos);                      this.output.writeLong(this.fileSplitterFetch[i].nEndPos);                  }                  this.output.flush();                this.output.close();              } catch(Exception e) {                  e.printStackTrace();              }          }                //获得文件长度           private long getFileSize() {              // TODO Auto-generated method stub               int nFileLength = -1;                            try {                  URL url = new URL(this.siteInfoBean.getsSiteURL());                  HttpURLConnection conn = (HttpURLConnection) url.openConnection();                  conn.setRequestProperty("User-Agent",                            "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");                                      int responseCode = conn.getResponseCode();                                    if(responseCode>=400) {                      processErrorCode(responseCode);                      return -2; //-2 represents access is error                   }                  /*String sHeader;                 for(int i=1; ; i++) {                     sHeader = conn.getHeaderFieldKey(i);                     if(sHeader!=null) {                         if(sHeader.equals("Content-Length")) {                             nFileLength = Integer.parseInt(conn.getHeaderField(sHeader));                             break;                         }                     } else {                         break;                     }                 }*/                                    nFileLength = conn.getContentLength();              } catch(Exception e) {                  e.printStackTrace();              }              return nFileLength;          }          private void processErrorCode(int responseCode) {              // TODO Auto-generated method stub               System.err.println("Error Code: "+responseCode);          }                    public static void main(String args[]) throws IOException, InterruptedException {              SiteInfoBean bean = new SiteInfoBean("http://127.0.0.1:8080/fileTest/download/a.zip","d:/test","a.zip",1);              //SiteInfoBean bean1 = new SiteInfoBean("http://localhost/download/friends.rmvb","C:\\xampp\\htdocs\\test","hello.rmvb",5);              FileDownloader downloader = new FileDownloader(bean);              downloader.download();          }  }  

3. 负责部分文件的抓取的线程：

import java.io.IOException;import java.io.InputStream;import java.net.HttpURLConnection;import java.net.URL;/** * 负责部分文件的抓取, 多线程 * @author ETHAN * */public class FileSplitterFetch extends Thread {private String sURL; //file url    long nStartPos;//file snippet start position    long nEndPos; //file snippet end positionprivate int nThreadID;boolean bDownOver = false;//downing is over;boolean bStop = false; //stop identical相同private FileAccess fileAccess = null; // file access interfacepublic FileSplitterFetch(String sURL,String sName,long nStart,long nEnd,int id) throws IOException {this.sURL = sURL;this.nStartPos = nStart;this.nEndPos = nEnd;this.nThreadID = id;this.fileAccess = new FileAccess(sName,this.nStartPos);}public void run() {while(this.nStartPos<this.nEndPos&&!this.bStop) {System.out.println("-----------------------------------"+this.nThreadID+"---------开始写数据");try {URL url = new URL(sURL);HttpURLConnection conn = (HttpURLConnection) url.openConnection();conn.setRequestProperty("User-Agent",                          "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");  String sProperty = "bytes="+this.nStartPos+"-";conn.setRequestProperty("range", sProperty);Utility.log(sProperty);conn.connect();InputStream input = conn.getInputStream();//高速缓存，设置缓冲区byte[] b = new byte[1024];int nRead;while((nRead=input.read(b,0,1024))>0&&this.nStartPos<this.nEndPos) {System.out.println("================================="+this.nThreadID);this.nStartPos += this.fileAccess.write(b, 0, nRead);}Utility.log("Thread "+nThreadID+" is over!");this.bDownOver = true;//一定要注意释放和connect，很关键input.close();conn.disconnect();} catch(Exception e) {e.printStackTrace();}}}//打印回应的头信息public void logResponseHead(HttpURLConnection conn) {for(int i=1; ; i++) {String headerKey = conn.getHeaderFieldKey(i);//key 不为空if(headerKey!=null) {Utility.log(headerKey+" : "+conn.getHeaderField(headerKey));} else {break;}}}public void splitterStop() {this.bStop = true;}}

4.要抓取的文件的信息，如文件保存的目录，名字，抓取文件的 URL 等：

/** *  要抓取的文件的信息，如文件保存的目录，名字，抓取文件的 URL 等。  * @author ETHAN * */public class SiteInfoBean {private String sSiteURL;//site's urlprivate String sFilePath; //saved file's pathprivate String sFileName; //saved file's nameprivate int nSplitter; //count of splited downloading filepublic SiteInfoBean() {this("","","",5); //default value of nSplitter is 5}public SiteInfoBean(String sURL, String sPath, String sName, int nSplitter) {// TODO Auto-generated constructor stubthis.sSiteURL = sURL;this.sFileName = sName;this.sFilePath = sPath;this.nSplitter = nSplitter;}public String getsSiteURL() {return sSiteURL;}public void setsSiteURL(String sSiteURL) {this.sSiteURL = sSiteURL;}public String getsFilePath() {return sFilePath;}public void setsFilePath(String sFilePath) {this.sFilePath = sFilePath;}public String getsFileName() {return sFileName;}public void setsFileName(String sFileName) {this.sFileName = sFileName;}public int getnSplitter() {return nSplitter;}public void setnSplitter(int nSplitter) {this.nSplitter = nSplitter;}}

5.Utility.java

/*  **Utility.java  */  public class Utility {  public Utility()  {  }  public static void sleep(int nSecond)  {  try{  Thread.sleep(nSecond);  }  catch(Exception e)  {  e.printStackTrace ();  }  }  public static void log(String sMsg)  {  System.err.println(sMsg);  }  public static void log(int sMsg)  {  System.err.println(sMsg);  }  } 

 

经过测试，下载完全成功！中间有一些打印输出，没有注释掉。部分打印如下：

=================================2
=================================2
=================================2
=================================2
=================================2
=================================2
=================================2
Thread 2 is over!
---------ok------------------threadID-------------------0false
---------ok------------------threadID-------------------0true
---------ok------------------threadID-------------------1false
---------ok------------------threadID-------------------1true
---------ok------------------threadID-------------------2false
---------ok------------------threadID-------------------2true
---------ok------------------threadID-------------------3false
---------ok------------------threadID-------------------3true
---------ok------------------threadID-------------------4false
---------ok------------------threadID-------------------4true
break while---------------------------true
文件下载结束！