一个文件级循环队列的Journal实现
来源：互联网 发布：手机自动安装软件 编辑：程序博客网 时间：2024/05/29 10:09
准备做一个比较靠谱的下载组件，这是第一步，Journal。
最开始想的比较多，希望新增和更新单个消息都尽量少的进行磁盘操作（后来想想，特别是看了Android官方的下载之后，觉得完全没必要，数据库足矣），不希望用数据库（理论上，没有index，会读全部数据）。
想来想去，做了一个循环队列的方法，尽量减少文件操作，当然肯定还是不如数据库+index来的快。是个思路，也是个教训。
思路
基本与最最简单的磁盘块管理相同，因为没有文件名、文件夹导航之类的问题，就是一个数据块的表格，读、写基本都可以保证很少次数的操作。
- 想要减少磁盘IO操作，肯定是要有index的，而手写又很麻烦。有一个简单的办法，就是把每块数据的大小固定，构成一个定长队列，使用定距的RandomAccessFile.seek来在不同的记录中切换。这样就保证了，不想读的数据，完全不用磁盘IO，seek就能搞定
- 给每个数据块做标记（一个char），在seek到当前位置时，只要读一个字符就能判断当前块是否可用，在delete、add时很快
代码
- 数据块
封装了很多文件类的操作，主要是要定长
package gt.research.losf.journal.file;import android.text.TextUtils;import java.io.IOException;import java.io.RandomAccessFile;import java.util.Arrays;import gt.research.losf.journal.IBlockInfo;import gt.research.losf.util.TypeUtils;/** * Created by GT on 2016/3/16. */public class FileBlockInfo implements IBlockInfo { public static final int STATE_LENGTH = 1; public static final int URI_LENGTH = 256; public static final int BLOCK_LENGTH = 10; public static final int OFFSET_LENGTH = 10; public static final char STATE_PROGRESS = 'p'; public static final char STATE_DELETE = 'd'; public static final char STATE_NEW = 'n'; public static final int LENGTH = STATE_LENGTH + URI_LENGTH + BLOCK_LENGTH + OFFSET_LENGTH; private char mState; private String mUri; private int mBlockId; private int mOffset;//offset of this block in file (next reading start) public FileBlockInfo() { } public FileBlockInfo(String uri, int blockId, int offset) { mState = STATE_NEW; mUri = uri; mBlockId = blockId; mOffset = offset; } public FileBlockInfo(char[] raw) { fromChars(raw); } public FileBlockInfo(RandomAccessFile file) throws Exception { byte[] bytes = new byte[LENGTH]; int read = file.read(bytes, 0, LENGTH); if (LENGTH != read) { throw new Exception("Illegal Length " + read); } fromChars(TypeUtils.bytesToChars(bytes)); } @Override public char getState() { return mState; } @Override public String getUri() { return mUri; } @Override public int getBlockId() { return mBlockId; } @Override public int getOffset() { return mOffset; } @Override public boolean isLegal() { boolean result = STATE_PROGRESS == mState || STATE_DELETE == mState || STATE_NEW == mState; result &= !TextUtils.isEmpty(mUri); result &= mBlockId > 0; result &= mOffset > 0; return result; } public void fromChars(char[] raw) { int offset = STATE_LENGTH; mState = raw[0]; mUri = readValue(raw, offset, totalLength(STATE_LENGTH, URI_LENGTH)); offset += URI_LENGTH; mBlockId = Integer.valueOf(readValue(raw, offset, totalLength(STATE_LENGTH, URI_LENGTH, BLOCK_LENGTH))); offset += BLOCK_LENGTH; mOffset = Integer.valueOf(readValue(raw, offset, totalLength(LENGTH))); } public char[] toRaw() { int offset = STATE_LENGTH; char[] raw = new char[LENGTH]; raw[0] = mState; offset = fillSpace(raw, mUri, offset, totalLength(STATE_LENGTH, URI_LENGTH)); offset = fillSpace(raw, String.valueOf(mBlockId), offset, totalLength(STATE_LENGTH, URI_LENGTH, BLOCK_LENGTH)); fillSpace(raw, String.valueOf(mOffset), offset, totalLength(LENGTH)); return raw; } public void writeToFile(RandomAccessFile file) throws IOException { byte[] bytes = TypeUtils.charsToBytes(toRaw()); file.write(bytes); } public void setStateProgressAndFile(RandomAccessFile file) throws IOException { setStateAndFile(file, STATE_PROGRESS); } public void setStateDeleteAndFile(RandomAccessFile file) throws IOException { setStateAndFile(file, STATE_DELETE); } public void setStateNewAndFile(RandomAccessFile file) throws IOException { setStateAndFile(file, STATE_NEW); } private void setStateAndFile(RandomAccessFile file, char state) throws IOException { mState = state; file.writeChar(mState); } private int fillSpace(char[] raw, String value, int offset, int end) { System.arraycopy(value.toCharArray(), 0, raw, offset, value.length()); offset += value.length(); if (offset >= end) { return end; } Arrays.fill(raw, offset, Math.min(end + 1, raw.length), ' '); return end; } private String readValue(char[] raw, int offset, int end) { int i = end - 1; for (; i >= offset; --i) { if (' ' != raw[i]) { break; } } return String.valueOf(raw, offset, i - offset + 1); } private int totalLength(int... lengths) { int sum = 0; for (int length : lengths) { sum += length; } return sum; }}- 单个循环队列文件
维护一个文件级的循环队列。最重要的是做到遍历足够快、尽量减少IO，还要保证状态与文件同步。
package gt.research.losf.journal.file;import java.io.File;import java.io.IOException;import java.io.RandomAccessFile;import java.util.LinkedList;import java.util.List;import gt.research.losf.common.Reference;import gt.research.losf.journal.IBlockInfo;import gt.research.losf.journal.IJournal;import gt.research.losf.journal.util.BlockUtils;import gt.research.losf.util.LogUtils;/** * Created by GT on 2016/3/16. */public class FileJournal implements IJournal { private static final int sCount = 10;//100 lines of journal private static final int sLength = FileBlockInfo.LENGTH * sCount; private RandomAccessFile mFile; // the position after this insertion, may be occupied. private int mLastIndex = -1; // block info count private int mSize = 0; public FileJournal(String name) throws IOException { this(new File(name)); } public FileJournal(File file) throws IOException { if (!file.exists()) { try { file.createNewFile(); } catch (IOException e) { LogUtils.exception(this, e); } } mFile = new RandomAccessFile(file, "rwd"); mFile.setLength(sLength); readStateFromFile(); } private void readStateFromFile() { iterateOverBlocks(new BlockIterateCallback() { @Override public boolean onBlock(RandomAccessFile file, int index) throws Exception { if (!BlockUtils.isCurrentInfoVailable(file)) { ++mSize; } return false; } }, false); } @Override public int addBlock(FileBlockInfo info) { moveToNextEmptyIndex(); if (mLastIndex >= 0) { try { info.writeToFile(mFile); ++mSize; return RESULT_SUCCESS; } catch (IOException e) { LogUtils.exception(this, e); } } return RESULT_FAIL; } @Override public int addBlock(int id, String uri, int offset) { return addBlock(new FileBlockInfo(uri, id, offset)); } @Override public int deleteBlock(final int id) { int lastSize = mSize; iterateOverBlocks(new BlockIterateCallback() { @Override public boolean onBlock(RandomAccessFile file, int index) throws Exception { if (BlockUtils.isCurrentInfoVailable(file)) { return false; } FileBlockInfo blockInfo = new FileBlockInfo(file); if (id == blockInfo.getBlockId()) { blockInfo.setStateDeleteAndFile(file); --mSize; return true; } return false; } }, false); return lastSize == mSize ? RESULT_FAIL : RESULT_SUCCESS; } @Override public int deleteBlock(final String uri) { int lastSize = mSize; iterateOverBlocks(new BlockIterateCallback() { @Override public boolean onBlock(RandomAccessFile file, int index) throws Exception { if (BlockUtils.isCurrentInfoVailable(file)) { return false; } FileBlockInfo blockInfo = new FileBlockInfo(file); if (uri.equals(blockInfo.getUri())) { blockInfo.setStateDeleteAndFile(file); --mSize; } return false; } }, false); return lastSize == mSize ? RESULT_FAIL : RESULT_SUCCESS; } @Override public boolean isFull() { return mSize == sCount; } @Override public int getSize() { return mSize; } @Override public IBlockInfo getBlock(final int id) { final Reference<FileBlockInfo> ref = new Reference<>(); iterateOverBlocks(new BlockIterateCallback() { @Override public boolean onBlock(RandomAccessFile file, int index) throws Exception { if (BlockUtils.isCurrentInfoVailable(file)) { return false; } FileBlockInfo blockInfo = new FileBlockInfo(file); if (id == blockInfo.getBlockId()) { ref.ref = blockInfo; return true; } return false; } }, true); return ref.ref; } @Override public List<IBlockInfo> getBlocks(final String uri) { final LinkedList<IBlockInfo> list = new LinkedList<>(); iterateOverBlocks(new BlockIterateCallback() { @Override public boolean onBlock(RandomAccessFile file, int index) throws Exception { if (BlockUtils.isCurrentInfoVailable(file)) { return false; } FileBlockInfo blockInfo = new FileBlockInfo(file); if (uri.equals(blockInfo.getUri())) { list.add(blockInfo); } return false; } }, true); return list.isEmpty() ? null : list; } private void moveToNextEmptyIndex() { iterateOverBlocks(new BlockIterateCallback() { @Override public boolean onBlock(RandomAccessFile file, int index) throws IOException { return BlockUtils.isCurrentInfoVailable(file); } }, false); } private void iterateOverBlocks(BlockIterateCallback callback, boolean restoreFileSeeker) { if (null == callback) { return; } int startIndex = mLastIndex; try { int count = 0; do { mLastIndex = ++mLastIndex % sCount; long offset = mLastIndex * FileBlockInfo.LENGTH; mFile.seek(offset); ++count; } while (!callback.onBlock(mFile, mLastIndex) && count < sCount); } catch (Exception e) { LogUtils.exception(this, e); mLastIndex = RESULT_INDEX_EXCEPTION; } if (startIndex == mLastIndex) { mLastIndex = RESULT_INDEX_FULL; } if (restoreFileSeeker) { mLastIndex = startIndex; try { long pointer = mLastIndex * FileBlockInfo.LENGTH; if (pointer < 0) { pointer = 0; } mFile.seek(pointer); } catch (IOException e) { LogUtils.exception(this, e); } } } private interface BlockIterateCallback { boolean onBlock(RandomAccessFile file, int index) throws Exception; }}- 总管理器
因为单个Journal队列的大小是固定的，可能要维护多个Journal队列。当然如果能做的一致性Hash，还可以进一步减少文件量，不过太麻烦了。不是关注点，有时间再试吧。
因为这段代码是不使用的，所以没有测试，只是记个思路。
package gt.research.losf.journal.file;import android.util.ArrayMap;import java.io.File;import java.io.FilenameFilter;import java.io.IOException;import java.util.LinkedList;import java.util.List;import java.util.Random;import gt.research.losf.journal.IBlockInfo;import gt.research.losf.journal.IJournal;import gt.research.losf.util.LogUtils;/** * Created by GT on 2016/3/16. * manages a particular type of journal */public class FileJournalManager implements IJournal { public static class Factory { private static final ArrayMap<String, FileJournalManager> sInstances = new ArrayMap<>(); private static final Random mRandom = new Random(System.currentTimeMillis()); public static FileJournalManager getJournal(String name) { if (null == sInstances.get(name)) { synchronized (sInstances) { if (null == sInstances.get(name)) { sInstances.put(name, new FileJournalManager(name)); } } } return sInstances.get(name); } public static int generateId() { return mRandom.nextInt(); } } private LinkedList<IJournal> mJournals = new LinkedList<>(); private int mLastSuffix = 0; private String mParentPath; private String mNamePrefix; private FileJournalManager(String name) { final File file = new File(name); ensurePath(file); loadExistingJournals(file); } @Override public int addBlock(FileBlockInfo info) { for (IJournal journal : mJournals) { if (!journal.isFull()) { return journal.addBlock(info); } } IJournal journal = expandJournals(); if (null != journal) { journal.addBlock(info); } return RESULT_FAIL; } @Override public int addBlock(int id, String uri, int offset) { return addBlock(new FileBlockInfo(uri, id, offset)); } @Override public int deleteBlock(int id) { for (IJournal journal : mJournals) { if (0 != journal.getSize()) { int result = journal.deleteBlock(id); if (RESULT_SUCCESS == result) { return RESULT_SUCCESS; } } } return RESULT_FAIL; } @Override public int deleteBlock(String uri) { boolean deleted = false; for (IJournal journal : mJournals) { if (0 != journal.getSize()) { int result = journal.deleteBlock(uri); if (RESULT_SUCCESS == result) { deleted = true; } } } return deleted ? RESULT_SUCCESS : RESULT_FAIL; } @Override public boolean isFull() { return false; } @Override public int getSize() { int size = 0; for (IJournal journal : mJournals) { size += journal.getSize(); } return size; } @Override public IBlockInfo getBlock(int id) { IBlockInfo blockInfo = null; for (IJournal journal : mJournals) { blockInfo = journal.getBlock(id); if (null != blockInfo) { return blockInfo; } } return null; } @Override public List<IBlockInfo> getBlocks(String uri) { List<IBlockInfo> resultList = new LinkedList<>(); for (IJournal journal : mJournals) { List<IBlockInfo> list = journal.getBlocks(uri); if (null == list) { continue; } resultList.addAll(list); } return resultList.isEmpty() ? null : resultList; } private void ensurePath(File file) { File parent = file.getParentFile(); if (null == parent) { return; } parent.mkdirs(); mParentPath = parent.getAbsolutePath(); } private void loadExistingJournals(File file) { File parent = file.getParentFile(); mNamePrefix = file.getName(); File[] journals = parent.listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String filename) { return filename.startsWith(mNamePrefix); } }); if (null == journals || 0 == journals.length) { journals = new File[]{new File(file.getParent(), file.getName() + 0)}; } for (File journalFile : journals) { try { IJournal journal = new FileJournal(journalFile); if (0 == journal.getSize()) { journalFile.delete(); continue; } mJournals.add(new FileJournal(journalFile)); int count = Integer.valueOf(journalFile.getName().substring(mNamePrefix.length())); if (count > mLastSuffix) { mLastSuffix = count; } } catch (IOException e) { LogUtils.exception(this, e); } } } private IJournal expandJournals() { ++mLastSuffix; try { IJournal journal = new FileJournal(new File(mParentPath, mNamePrefix + mLastSuffix)); mJournals.add(journal); return journal; } catch (IOException e) { LogUtils.exception(this, e); } return null; }} 0 0
- 一个文件级循环队列的Journal实现
- 循环队列的一个简单实现
- 如何实现一个循环队列
- 如何实现一个循环队列
- 如何实现一个循环队列
- 如何实现一个循环队列
- 如何实现一个循环队列
- 如何实现一个循环队列
- 如何实现一个循环队列
- 循环数组实现一个队列
- 队列----循环队列的实现
- 循环队列的实现
- 循环队列的实现
- 循环队列的实现
- 循环队列的实现
- 循环队列的实现
- 循环队列的实现
- 循环队列的实现
- ChatOps如何变革企业业务
- V4l2 capture时 USERPTR和MMAP的区别
- VC 读写注册表
- Retrofit2与RxJava用法大全
- mssql的object_id
- 一个文件级循环队列的Journal实现
- sdut 3253 Game! 博弈
- 微信报40029错误
- android 框架 MVP
- xpath和dom4j解析xml
- Android Matrix对图像简单介绍
- 关于使用jsp:include标签及<%@ include标签时要注意的事项
- HDU2222 AC自动机模板
- Junit参数化设置
