多线程io分离与合并（大文件压缩包解密）

需求：用户上传了一个大文件压缩包，压缩包是加密的，需要在后台进行解密操作，文件大小（1G）左右，解密过程需要1分钟。。。需要提速

ok，直接用java多线程的方式来解决吧，先读取文件，然后将io流切分，每段io开启一个线程进行解密，最后按顺序将解密后的io片段进行合并

闲话不说，直接上代码，不需要导入其他jar包

首先是工具

ioutil，就是关闭流而已

package com.noryar.filesystem.util;import java.io.Closeable;import java.io.IOException;/** * IOUtil. * @author Leon Lee. */public class IOUtil {/** * close IO resource. * @param closeables :IO resource, implements Closable * @throws IOException :Exception */public static void close(Closeable... closeables) throws IOException {if (closeables != null) {for (Closeable closeable : closeables) {if (closeable != null) {closeable.close();}}}}/** * close IO resource. do not throw Exception. * @param closeables : IO resource, implements Closable */public static void closeQuietly(Closeable... closeables) {try {close(closeables);} catch (IOException e) {// do nothing}}}

加密解密工具类

package com.noryar.filesystem.util;import java.io.ByteArrayOutputStream;import java.io.File;import java.io.FileInputStream;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import java.security.SecureRandom;import javax.crypto.Cipher;import javax.crypto.CipherInputStream;import javax.crypto.CipherOutputStream;import javax.crypto.KeyGenerator;import javax.crypto.SecretKey;import javax.crypto.spec.SecretKeySpec;/** * SecurityFileUtil. * @author Leon Lee. */public class SecurityUtil {    /**     * init AES Cipher.     * @param keySource keySource     * @param cipherMode keyMode     * @return Cipher     */    private static Cipher initAESCipher(final String keySource, final int cipherMode) {        KeyGenerator keyGenerator = null;        Cipher cipher = null;        try {            keyGenerator = KeyGenerator.getInstance("AES");            // init 128 key with import random source[key]            keyGenerator.init(128, new SecureRandom(keySource.getBytes()));            // create key            SecretKey secretKey = keyGenerator.generateKey();            // get key of byte[]            byte[] codeFormat = secretKey.getEncoded();            SecretKeySpec key = new SecretKeySpec(codeFormat, "AES");            cipher = Cipher.getInstance("AES");            cipher.init(cipherMode, key);        } catch (Exception e) {            e.printStackTrace();        }        return cipher;    }    /**     * encryptFile.<br>     * decryptFile will named [en+source file name]. And in same folder     * @param sourceFile sourceFile     * @param keySource keySource     * @return boolean     */    public static boolean encryptFile(final File sourceFile, final String keySource) {        boolean flag = false;        String path = sourceFile.getAbsolutePath();        InputStream inputStream = null;        OutputStream outputStream = null;        CipherInputStream cipherInputStream = null;        try {            inputStream = new FileInputStream(sourceFile);            File encrypfile = new File(path.replaceFirst(sourceFile.getName(), "en"+sourceFile.getName()));            outputStream = new FileOutputStream(encrypfile);            Cipher cipher = initAESCipher(keySource, Cipher.ENCRYPT_MODE);            cipherInputStream = new CipherInputStream(inputStream, cipher);            byte[] cache = new byte[1024];            int nRead = 0;            while ((nRead = cipherInputStream.read(cache)) != -1) {                outputStream.write(cache, 0, nRead);                outputStream.flush();            }            flag = true;        } catch (IOException e) {            e.printStackTrace();        } finally {            try {                if (cipherInputStream != null) {                    cipherInputStream.close();                }                if (inputStream != null) {                    inputStream.close();                }                if (outputStream != null) {                    outputStream.close();                }            } catch (IOException e) {                e.printStackTrace();            }        }        return flag;    }    /**     * decryptFile.     * @param encryptFile encryptFile     * @param decryptFile decryptFile     * @param keySource keySource     * @return decryptFile     */    public static File decryptFile(final File encryptFile, final File decryptFile, final String keySource) {        InputStream inputStream = null;        OutputStream outputStream = null;        try {            Cipher cipher = initAESCipher(keySource, Cipher.DECRYPT_MODE);            inputStream = new FileInputStream(encryptFile);            outputStream = new FileOutputStream(decryptFile);            CipherOutputStream cipherOutputStream = new CipherOutputStream(outputStream, cipher);            byte[] buffer = new byte[1024];            int r;            while ((r = inputStream.read(buffer)) >= 0) {                cipherOutputStream.write(buffer, 0, r);            }            cipherOutputStream.close();        } catch (IOException e) {            e.printStackTrace();        } finally {            try {                inputStream.close();            } catch (IOException e) {                e.printStackTrace();            }            try {                outputStream.close();            } catch (IOException e) {                e.printStackTrace();            }        }        return decryptFile;    }    /**     * decryptFileAsByte[] input InputStream.     * @param sourceFile sourceFile     * @param keySource keySource     * @return byte[]     */    public static byte[] decryptFile(final InputStream inputStream, final String keySource) {        ByteArrayOutputStream bos = new ByteArrayOutputStream();        try {            Cipher cipher = initAESCipher(keySource, Cipher.DECRYPT_MODE);            CipherOutputStream cipherOutputStream = new CipherOutputStream(bos, cipher);            byte[] buffer = new byte[1024];            int r;            while ((r = inputStream.read(buffer)) >= 0) {                cipherOutputStream.write(buffer, 0, r);            }            cipherOutputStream.close();        } catch (Exception e) {            e.printStackTrace();        } finally {            try {                inputStream.close();                bos.close();            } catch (IOException e) {            }        }        return bos.toByteArray();    }}

多线程解密类

package com.noryar.filesystem.thread;import java.io.InputStream;import com.noryar.filesystem.util.SecurityUtil;/** * Parallel decrypt thread.<br> * run every thread must create new Instance * @author Leon Lee. */public class ParallelDecryptThread implements Runnable{private InputStream in;private byte[] byteArray;public byte[] getByteArray() {return byteArray.clone();}private String keySource;public ParallelDecryptThread(InputStream in, String keySource) {this.in  = in;this.keySource = keySource;}public void run() {byteArray = SecurityUtil.decryptFile(in, keySource);}}

OK,最后上测试类

package com.noryar.test;import java.io.BufferedOutputStream;import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;import java.io.File;import java.io.FileInputStream;import java.io.FileOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;import com.noryar.filesystem.thread.ParallelDecryptThread;import com.noryar.filesystem.util.IOUtil;import com.noryar.filesystem.util.SecurityUtil;/** * decrypt test. * @author Leon Lee. */public class Test {/** * do test. * @param args * @throws IOException * @throws InterruptedException */public static void main(String[] args) throws IOException,InterruptedException {// encrypt file// encryptFile();// decrypt file with parallelparallelDecryptTest();System.out.println("############");// decrypt file with single threadsingleDecryptTest();}/** * encrypt file. */public static void encryptFile(){File sourceFile = new File("/Users/leon/Downloads/mysql.zip");SecurityUtil.encryptFile(sourceFile, "123");}/** * ParallelDecryptTest. * @throws IOException * @throws InterruptedException */public static void parallelDecryptTest() throws IOException,InterruptedException {File encryptFile = new File("/Users/leon/Downloads/enmysql.zip");File decryptFile = new File("/Users/leon/Downloads/demysql.zip");System.out.println("parallel decrypt start");long start = System.currentTimeMillis();// init rule.long cut = 30 * 1024 * 1024; // 30MB;long srcSize = encryptFile.length();int number = (int) (srcSize / cut);number = srcSize % cut == 0 ? number : number + 1;System.out.println("create thread num: " + number);Thread[] allThread = new Thread[number];ParallelDecryptThread[] pdu = new ParallelDecryptThread[number];// IO start, with threadInputStream is = new FileInputStream(encryptFile);ByteArrayOutputStream baos = null;byte[] rb = null;for (int i = 0; i < number; i++) {rb = new byte[1024];baos = new ByteArrayOutputStream();int len = -1;int cnt = 0;while ((len = is.read(rb)) != -1) {baos.write(rb, 0, len);cnt += len;if (cnt >= cut) {pdu[i] = new ParallelDecryptThread(new ByteArrayInputStream(baos.toByteArray()), "123");allThread[i] = new Thread(pdu[i]);allThread[i].start();break;}}// lastif (i + 1 == number && cnt < cut) {pdu[i] = new ParallelDecryptThread(new ByteArrayInputStream(baos.toByteArray()), "123");allThread[i] = new Thread(pdu[i]);allThread[i].start();}}IOUtil.close(is, baos);// Thread joinfor (Thread tr : allThread) {tr.join();}// writeOutputStream os = new BufferedOutputStream(new FileOutputStream(decryptFile));rb = new byte[1024];InputStream lastIn = null;for (ParallelDecryptThread parallelDecryptUtil : pdu) {lastIn = new ByteArrayInputStream(parallelDecryptUtil.getByteArray());while (lastIn.read(rb) != -1) {os.write(rb);}}IOUtil.close(lastIn, os);System.out.println("parallel decrypt end, cost " + (System.currentTimeMillis() - start)/ 1000 + " seconds");}/** * SingleDecryptTest. */public static void singleDecryptTest() {File encryptFile = new File("/Users/leon/Downloads/enmysql.zip");File decryptFile = new File("/Users/leon/Downloads/demysql.zip");System.out.println("single decrypt start");long start = System.currentTimeMillis();SecurityUtil.decryptFile(encryptFile, decryptFile, "123");System.out.println("single decrypt end, cost " + (System.currentTimeMillis() - start)/ 1000 + " seconds");}}

需要注意的是，这里的多线程解密方式是将文件分段成字节数组流进行解密，整个过程完全在内存中进行，如果文件太大，或者并发量大，或者线程切分数量太少（字节数组太大），均非常容易出现内存溢出，即使增加jvm内存大小也不能很好解决，因此可以尝试将文件输入流切分以后输出成小文件，然后解密的时候读取所有分段文件进行解密，解密完成以后删除分段文件～

另一方面，需要合理考虑io流的切分数量，考虑服务区硬盘是否为机械硬盘或固态，否则容易出现多线程解密反而更耗时的情况。

接下来附上本人测试结果，分别为机械硬盘和固态硬盘结果，测试文件为mysql安装包，打成了zip，文件大小在 350 MB左右。

机械硬盘测试结果：

parallel decrypt start

create thread num: 12

parallel decrypt end, cost 4 seconds

############

single decrypt start

single decrypt end, cost 8 seconds

固态硬盘测试结果：

parallel decrypt start

create thread num: 12

parallel decrypt end, cost 2 seconds

############

single decrypt start

single decrypt end, cost 1 seconds