Java文件切片 & 切片合并代码
来源：互联网 发布：科目二约车软件 编辑：程序博客网 时间：2024/05/16 18:11
研究了下大文件的断点上传，写了个文件切片和切片合并的代码，使用了RandomAccessFile和MappedByteBuffer。
测试了下性能，速度还可以。测试中使用了1.18G的待分割文件，每片最大1M，切片+合并测试了几次，平均耗时为4.8S左右，用的机械硬盘。
代码共享如下，需要的同学拿走。
文件分割器：
package com.lonzh.fileutils;import java.io.File;import java.io.FileInputStream;import java.io.IOException;import java.io.RandomAccessFile;import java.nio.MappedByteBuffer;import java.nio.channels.FileChannel;import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.ThreadPoolExecutor;import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.Logger;/** * * @author Keven split a src-filt into sevral small pieces. The src-file's size * if required to be smaller than miPartSize * Integer.MAX_VALUE */public class FileSplitter { private final String msSrcFile, msDstDir, msOutFileBaseName; private int miPartSize = 1024 * 1024; private int miMaxThrPoolSize = 10; private ThreadPoolExecutor moThrPool; private FileInputStream moInput; private FileChannel moInChannel; /** * Constructor * * @param psSrcFile * @param psDstDir * @param psOutFileBaseName the output files will be named by */ public FileSplitter(String psSrcFile, String psDstDir, String psOutFileBaseName) { msSrcFile = psSrcFile; msDstDir = psDstDir; msOutFileBaseName = psOutFileBaseName; } /** * Constructor * * @param psSrcFile * @param psDstDir * @param psOutFileBaseName * @param piPartSize the split-out file's max-size */ public FileSplitter(String psSrcFile, String psDstDir, String psOutFileBaseName, int piPartSize) { msSrcFile = psSrcFile; msDstDir = psDstDir; msOutFileBaseName = psOutFileBaseName; miPartSize = piPartSize; } /** * Constructor * * @param psSrcFile * @param psDstDir * @param psOutFileBaseName * @param piPartSize * @param piMaxThrPool the max-size of thread pool */ public FileSplitter(String psSrcFile, String psDstDir, String psOutFileBaseName, int piPartSize, int piMaxThrPool) { msSrcFile = psSrcFile; msDstDir = psDstDir; msOutFileBaseName = psOutFileBaseName; miPartSize = piPartSize; miMaxThrPoolSize = piMaxThrPool; } /** * Judge if the splitter is working * * @return */ public boolean isBusy() { return moThrPool != null && !moThrPool.isTerminated(); } /** * Start splitting * * @return the array of the names of split-out files * @throws IOException */ public String[] start() throws IOException { File loFile = new File(msSrcFile); if (!loFile.exists()) { throw new IOException("Src-File not found:" + msSrcFile); } long liFileLen = loFile.length(); long liPartCnt = liFileLen / miPartSize + (liFileLen % miPartSize == 0 ? 0 : 1); if (liPartCnt > Integer.MAX_VALUE) { throw new IOException("Src-File too large"); } moInput = new FileInputStream(msSrcFile); moInChannel = moInput.getChannel(); moThrPool = new ThreadPoolExecutor(miMaxThrPoolSize, miMaxThrPoolSize, 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>()); String[] laOutFiles = new String[(int) liPartCnt]; for (int i = 0; i < (int) liPartCnt; i++) { moThrPool.execute(new SplitThread(i, i == liPartCnt - 1 ? liFileLen % miPartSize : miPartSize)); laOutFiles[i] = msDstDir + File.separator + msOutFileBaseName + "." + i + ".part"; } moThrPool.shutdown(); // Monitor new Thread() { @Override public void run() { while (true) { if (moThrPool.isTerminated()) { if (moInChannel != null) { try { moInChannel.close(); } catch (IOException ex) { Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex); } } if (moInput != null) { try { moInput.close(); } catch (IOException ex) { Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex); } } break; } try { Thread.sleep(100); } catch (InterruptedException ex) { Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex); } } } }.start(); return laOutFiles; } /** * Splitting Thread */ private class SplitThread implements Runnable { private final int miPartIndex; private final long miSize; public SplitThread(int piPartIndex, long piSize) { miPartIndex = piPartIndex; miSize = piSize; } @Override public void run() { String lsDstPartFile = msDstDir + File.separator + msOutFileBaseName + "." + miPartIndex + ".part"; System.out.println("FileSplitter: splitting " + lsDstPartFile); long liStartTime = System.currentTimeMillis(); // output vars File loDstFile = new File(lsDstPartFile); RandomAccessFile loOutput = null; FileChannel loOutChannel = null; try { // Input Map MappedByteBuffer loInBuf = moInChannel.map(FileChannel.MapMode.READ_ONLY, miPartSize * miPartIndex, miSize); // Output Map if (!loDstFile.exists()) { loDstFile.createNewFile(); } loOutput = new RandomAccessFile(loDstFile, "rw"); loOutChannel = loOutput.getChannel(); MappedByteBuffer loOutBuf = loOutChannel.map(FileChannel.MapMode.READ_WRITE, 0, miSize); byte liByte; while (loInBuf.hasRemaining()) { liByte = loInBuf.get(); loOutBuf.put(liByte); } } catch (IOException ex) { System.out.print("part_index:" + miPartIndex + " boom"); Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex); } finally { // close output if (loOutChannel != null) { try { loOutChannel.close(); } catch (IOException ex) { Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex); } } if (loOutput != null) { try { loOutput.close(); } catch (IOException ex) { Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex); } } } System.out.println("Splitting Task " + lsDstPartFile + " Terminated. Spend " + (System.currentTimeMillis() - liStartTime) + " milsecs"); } }}package com.lonzh.fileutils;import java.io.File;import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io.IOException;import java.io.RandomAccessFile;import java.nio.MappedByteBuffer;import java.nio.channels.FileChannel;import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.ThreadPoolExecutor;import java.util.concurrent.TimeUnit;import java.util.logging.Level;import java.util.logging.Logger;/** * * @author Keven */public class FileUnifier { private final String[] maSrcFiles; private final String msDstFile; private int miMaxThrPoolSize = 10; private ThreadPoolExecutor moThrPool; private RandomAccessFile moOutput; private FileChannel moOutputChannel; /** * Constructor * * @param paSrcFiles * @param psDstFile */ public FileUnifier(String[] paSrcFiles, String psDstFile) { maSrcFiles = paSrcFiles; msDstFile = psDstFile; } /** * Constructor * * @param paSrcFiles * @param psDstFile * @param piMaxThrPool */ public FileUnifier(String[] paSrcFiles, String psDstFile, int piMaxThrPool) { maSrcFiles = paSrcFiles; msDstFile = psDstFile; miMaxThrPoolSize = piMaxThrPool; } /** * Judge if the unifier is working * * @return */ public boolean isBusy() { return moThrPool != null && !moThrPool.isTerminated(); } /** * Start unifying * * @throws java.io.IOException */ public void start() throws IOException { File loOutFile = new File(msDstFile); if (!loOutFile.exists()) { loOutFile.createNewFile(); } moOutput = new RandomAccessFile(loOutFile, "rw"); moThrPool = new ThreadPoolExecutor(miMaxThrPoolSize, miMaxThrPoolSize, 100, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>()); // get total size long[][] laOffsetSizes = new long[maSrcFiles.length][2]; long liOffset = 0; for (int i = 0; i < maSrcFiles.length; i++) { String lsSrcFile = maSrcFiles[i]; File loFile = new File(lsSrcFile); if (!loFile.exists()) { throw new IOException("Src-File not exist:" + lsSrcFile); } laOffsetSizes[i][0] = liOffset; laOffsetSizes[i][1] = loFile.length(); liOffset += loFile.length(); } moOutput.setLength(liOffset); moOutputChannel = moOutput.getChannel(); for (int i = 0; i < maSrcFiles.length; i++) { moThrPool.execute(new UnifyThread(maSrcFiles[i], laOffsetSizes[i][0], laOffsetSizes[i][1])); } moThrPool.shutdown(); // Start monitor Thread new Thread() { @Override public void run() { while (true) { if (moThrPool.isTerminated()) { if (moOutputChannel != null) { try { moOutputChannel.close(); } catch (IOException ex) { Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex); } } if (moOutput != null) { try { moOutput.close(); } catch (IOException ex) { Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex); } } break; } try { Thread.sleep(100); } catch (InterruptedException ex) { Logger.getLogger(FileSplitter.class.getName()).log(Level.SEVERE, null, ex); } } } }.start(); } /** * Unifying thread */ private class UnifyThread implements Runnable { private final String msSrcFile; private final long miOffset, miSize; public UnifyThread(String psSrcFile, long piOffset, long piSize) { msSrcFile = psSrcFile; miOffset = piOffset; miSize = piSize; } @Override public void run() { File loSrcFile = new File(msSrcFile); FileInputStream loInput = null; FileChannel loInChannel = null; try { System.out.println("FileUnifier: start unifying " + msSrcFile); long liStartTime = System.currentTimeMillis(); loInput = new FileInputStream(loSrcFile); loInChannel = loInput.getChannel(); MappedByteBuffer loInBuf = loInChannel.map(FileChannel.MapMode.READ_ONLY, 0, miSize); MappedByteBuffer loOutBuf = moOutputChannel.map(FileChannel.MapMode.READ_WRITE, miOffset, miSize); while (loInBuf.hasRemaining()) { loOutBuf.put(loInBuf.get()); } System.out.println("FileUnifier: unify " + msSrcFile + " complete. Spend " + (System.currentTimeMillis() - liStartTime) + "milsecs"); } catch (FileNotFoundException ex) { Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex); } catch (IOException ex) { Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex); } finally { // Close input if (loInChannel != null) { try { loInChannel.close(); } catch (IOException ex) { Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex); } } if (loInput != null) { try { loInput.close(); } catch (IOException ex) { Logger.getLogger(FileUnifier.class.getName()).log(Level.SEVERE, null, ex); } } } } }}测试代码：
package test_java;import com.lonzh.fileutils.FileSplitter;import com.lonzh.fileutils.FileUnifier;import java.io.File;import java.io.IOException;/** * * @author Keven */public class Test_java { /** * @param args the command line arguments * @throws java.io.IOException * @throws java.lang.InterruptedException */ public static void main(String[] args) throws IOException, InterruptedException { String lsSrcFile = "D:\\奔腾年代.rmvb"; String lsDstDir = "D:\\parts"; String lsDstFile = "D:\\parts\\dst.rmvb"; File loDstFile = new File(lsDstDir); if (!loDstFile.exists()) { loDstFile.mkdir(); } long liStartTime = System.currentTimeMillis(); FileSplitter loSplitter = new FileSplitter(lsSrcFile, lsDstDir, "test_cut"); String[] laPartFiles = loSplitter.start(); while (loSplitter.isBusy()) { Thread.sleep(200); } FileUnifier loUnifier = new FileUnifier(laPartFiles, lsDstFile); loUnifier.start(); while (loUnifier.isBusy()) { Thread.sleep(200); } System.out.println("done, spend " + (System.currentTimeMillis() - liStartTime) + " milsecs"); }} 3 0
- Java文件切片 & 切片合并代码
- 切片
- 切片
- 切片
- 切片
- 切片
- 切片
- 切片
- 切片
- 切片与合并
- tiff切片、合并、转换
- Hbase region切片 reqion切片合并操作
- 切片文件发布成切片服务
- python切片代码[收藏]
- 文件上传之切片
- 文件切片上传
- SRS 代码分析【HLS切片】
- c语言把文件切片
- Hibernate实体关系关联映射中配置文件的编写
- BookList
- CUDA-CODE4-并行编程-blockIDx
- 二叉树遍历111. Minimum Depth of Binary Tree
- input type=number验证问题
- Java文件切片 & 切片合并代码
- python os 模块的应用
- java 取目前操作系统桌面路径
- Razor引擎学习：RenderBody，RenderPage和RenderSection
- 导入excel数据到数据库
- JQuery学习&资料整理
- redis集合set的相关命令
- 134. Gas Station
- 上传windows上的项目到github
