Java文件Queue实现

这是一个新的系列，将会记录在工作之外写的一些小的程序，而且是有一定难度并且实用的小代码，相信会给自己和给读者一些收获。

这个系列的原因是有的时候回想自己的工作历程经常想不起曾经实现过哪些有意思的东西，总结提炼过哪些模型，这个系列将会记录这一切。

 

在前段时间的工作中曾经有个需求是要对一个内存中的Queue做持久化。组内的大牛使用了文件完成了一个FileBasedQueue，很有意思，把代码简化和重写了后下放出来：

 

实现要点：

1.使用了NIO、FileChannel、MappedByteBuffer来加快读写文件的速度

2.一次创建一个128M的文件Page，如果不够用再开辟下一个文件Page

3.单独使用一个Index文件记录当前读和写分别是在哪个文件Page中

4.使用一个Type位标记文件中对象的状态：EMPTY、FULL、ROTATE

5.用sun的cleaner来完成MappedByteBuffer的unmap

 

import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;import java.io.Closeable;import java.io.File;import java.io.FileNotFoundException;import java.io.IOException;import java.io.ObjectInputStream;import java.io.ObjectOutputStream;import java.io.RandomAccessFile;import java.io.Serializable;import java.nio.ByteBuffer;import java.nio.MappedByteBuffer;import java.nio.channels.FileChannel;import com.sun.corba.se.impl.ior.WireObjectKeyTemplate;import sun.nio.ch.DirectBuffer;import static java.nio.channels.FileChannel.MapMode.READ_WRITE;public class FileQueue<T extends Serializable> {public static final int PAGE_SIZE = 128 * 1024 * 1024;public static final int SIZE_OF_INT = 4;public static String FILE_NAME = "data";public static String FILE_SUFFIX = ".dat";public static String INDEX_NAME = "data.inx";public String fileDir;private RandomAccessFile readFile;private RandomAccessFile writeFile;private RandomAccessFile indexFile;private FileChannel readChannel;private FileChannel writeChannel;private FileChannel indexChannel;private MappedByteBuffer readMbb;private MappedByteBuffer writeMbb;private MappedByteBuffer indexMbb;private static final int INDEX_SIZE = SIZE_OF_INT + SIZE_OF_INT;private static final int HEADER_SIZE = SIZE_OF_INT+ SIZE_OF_INT;private static final int ENDER_SIZE = HEADER_SIZE;private ByteBuffer headerBb = ByteBuffer.allocate(HEADER_SIZE);private int readIndexFile;private int writeIndexFile;private enum ITEM_TYPE {EMPTY, FILL, ROTATE}public FileQueue(String fileDir) throws IOException {if (fileDir == null || fileDir.trim().length() == 0) {throw new IllegalArgumentException("filename illegal");}if (!fileDir.endsWith("/")) {fileDir += File.separator;}File dir = new File(fileDir);if (!dir.exists()) {dir.mkdirs();}this.fileDir = fileDir;indexFile = new RandomAccessFile(fileDir + INDEX_NAME, "rw");indexChannel = indexFile.getChannel();indexMbb = indexChannel.map(READ_WRITE, 0, INDEX_SIZE);readIndexFile = indexMbb.getInt();writeIndexFile = indexMbb.getInt();readFile = new RandomAccessFile(fileDir + FILE_NAME + readIndexFile + FILE_SUFFIX, "rw");readChannel = readFile.getChannel();writeFile = new RandomAccessFile(fileDir + FILE_NAME + writeIndexFile + FILE_SUFFIX, "rw");writeChannel = writeFile.getChannel();readMbb = readChannel.map(READ_WRITE, 0, PAGE_SIZE);writeMbb = writeChannel.map(READ_WRITE, 0, PAGE_SIZE);initWriteMbb();initReadMbb();}private void initReadMbb(){int currentPos = readMbb.position();int type = readMbb.getInt();int length = readMbb.getInt();while(type == ITEM_TYPE.EMPTY.ordinal() && length > 0){readMbb.position(currentPos + HEADER_SIZE + length);currentPos = readMbb.position();type = readMbb.getInt();length = readMbb.getInt();}readMbb.position(currentPos);}private void initWriteMbb(){int currentPos = writeMbb.position();int type = writeMbb.getInt();int length = writeMbb.getInt();while(length > 0){writeMbb.position(currentPos + HEADER_SIZE + length);currentPos = writeMbb.position();type = writeMbb.getInt();length = writeMbb.getInt();}writeMbb.position(currentPos);}public synchronized void product(T item) throws Exception {if (item == null) {throw new IllegalArgumentException("item is null");}byte[] contents = toBytes(item);int length = contents.length;int writePos = writeMbb.position();// if reach the button of the filequeueif (writePos + length + ENDER_SIZE + HEADER_SIZE >= PAGE_SIZE) {writeIndexFile += 1;writeMbb.putInt(ITEM_TYPE.ROTATE.ordinal());writeMbb.force();unmap(writeMbb);closeResource(writeChannel);closeResource(writeFile);writeFile = new RandomAccessFile(fileDir + FILE_NAME + writeIndexFile + FILE_SUFFIX, "rw");writeChannel = writeFile.getChannel();writeMbb = writeChannel.map(READ_WRITE, 0, PAGE_SIZE);indexMbb.putInt(Integer.SIZE, writeIndexFile);}headerBb.clear();headerBb.putInt(ITEM_TYPE.FILL.ordinal());headerBb.putInt(length);headerBb.flip();writeMbb.put(headerBb);writeMbb.put(contents);}private byte[] toBytes(T item) throws IOException {ByteArrayOutputStream baos = null;ObjectOutputStream oos = null;try {baos = new ByteArrayOutputStream();oos = new ObjectOutputStream(baos);oos.writeObject((Object) item);oos.flush();return baos.toByteArray();} finally {closeResource(baos);closeResource(oos);}}public synchronized T comsume() throws Exception {int readPos = readMbb.position();int type = readMbb.getInt();int length = readMbb.getInt();if(type == ITEM_TYPE.ROTATE.ordinal()){readIndexFile += 1;readMbb.putInt(ITEM_TYPE.ROTATE.ordinal());readMbb.force();unmap(readMbb);closeResource(readChannel);closeResource(readFile);readFile = new RandomAccessFile(fileDir + FILE_NAME + readIndexFile + FILE_SUFFIX, "rw");readChannel = readFile.getChannel();readMbb = readChannel.map(READ_WRITE, 0, PAGE_SIZE);indexMbb.putInt(0, readIndexFile);type = readMbb.getInt();length = readMbb.getInt();}if(type == ITEM_TYPE.EMPTY.ordinal() || length <= 0){readMbb.position(readPos);return null;}byte[] contents = new byte[length];readMbb.get(contents);readMbb.putInt(readPos, ITEM_TYPE.EMPTY.ordinal());T object = toObject(contents);return object;}private T toObject(byte[] content) throws IOException,ClassNotFoundException {ByteArrayInputStream bais = null;ObjectInputStream ois = null;try {bais = new ByteArrayInputStream(content);ois = new ObjectInputStream(bais);return (T) ois.readObject();} finally {closeResource(bais);closeResource(ois);}}private void closeResource(Closeable c) throws IOException {if (c != null) {c.close();}}private static void unmap(MappedByteBuffer buffer){if (buffer == null) return;    sun.misc.Cleaner cleaner = ((DirectBuffer) buffer).cleaner();    if (cleaner != null) {    cleaner.clean();    }}public void shutdown() throws IOException {if (writeMbb != null) {writeMbb.force();unmap(writeMbb);}if (readMbb != null) {readMbb.force();unmap(readMbb);}if (indexMbb != null) {indexMbb.force();unmap(indexMbb);}closeResource(readChannel);closeResource(readFile);closeResource(writeChannel);closeResource(writeFile);closeResource(indexChannel);closeResource(indexFile);}}