多线程从MongoDB读取数据，并以固定大小写入HDFS

从Mongo里多线程取数据，我一开始尝试了用Executors.newFixedThreadPool线程池来实现。实际操作中，发现只有第一个线程会正常取数据，后建立的线程访问Mongo的cursor会报错。
后来改变了实现方式，通过Thread.start()来进行多线程取数据。代码如下：

RsUserTagsRunner job = new RsUserTagsRunner(start, end, subOver);for (int i = 0; i < ConstantTool.MONGO_THREADS; i++) {    new Thread(job).start();}

由于是用Thread.start()方法来执行的，而且是多线程共享一个RsUserTagsRunner 实例，所以逻辑上要与Runnable.run()方法不同。

多线程中，Thread.start()和Runnable.run()的不同请自行百度，资料很多，这里不再赘述。

所以就按照Mongo数据库中数据的时间来进行分片，进行分片读取数据。代码上就是通过getStart方法获取每次访问数据库时的时间分片。用synchronized 来保证多线程下，每次取时间分片时的数据一致性。

以下代码实现了多线程从MongoDB读取数据，并以固定大小写入HDFS。

public class RsUserTagsRunner implements Runnable {    static Logger log = Logger.getLogger(RsUserTagsRunner.class);    private long end;    private long nextStart;    private static final long timeClock = 1000 * 60 * 60;    private StringBuffer stringBuffer = new StringBuffer();    public static Boolean successFlag = null;    public static AtomicLong size = new AtomicLong(0L);    private long num = 1;    private long CurrentThread = 0;    private CountDownLatch over;    public RsUserTagsRunner(long start, long end, CountDownLatch over) {        this.end = end;        this.nextStart = start;        this.over = over;    }    synchronized private long getStart() {        long temp = nextStart;        nextStart = nextStart + timeClock;        return temp;    }    synchronized private long getNum() {        return num++;    }    @Override    public void run() {        MongoCursor<Document> cursor = null;        try {            CurrentThread++;            log.info("##### rs_user_tags 子线程获取数据开始");            List<String> index = TableIndex.map.get("rs_user_tags");            while (true) {                long tempStart = getStart();                if (tempStart < end) {                    long tempEnd = tempStart + timeClock;                    if (tempEnd > end) {                        tempEnd = end;                    }                    log.info("##### rs_user_tags 子线程获取分片数据 [" + tempStart + ", " + tempEnd + ") 开始");                    cursor = MongoDBDao.getRsUserTags(tempStart, tempEnd);                    while (cursor.hasNext()) {                        Document document = cursor.next();                        size.addAndGet(1L);                        String line = document.get(index.get(0)).toString();                        for (int i = 1; i < index.size(); i++) {                            Object columnOb = document.get(index.get(i));                            String columnValue = "NULL";                            if (columnOb != null) {                                columnValue = columnOb.toString();                            }                            line = line + '\t' + columnValue;                        }                        write(line + '\n');                    }                    log.info("##### rs_user_tags 子线程获取分片数据 [" + tempStart + ", " + tempEnd + ") 结束，截止目前已取得 " + size                            + " 条数据");                } else {                    break;                }            }            end();        } catch (Exception e) {            successFlag = false;            log.error("##### rs_user_tags 获取数据失败");            log.error(e.getMessage(), e);        } finally {            if (cursor != null) {                cursor.close();            }            if (successFlag != null) {                over.countDown();            }        }    }    synchronized private void write(String line) {        stringBuffer.append(line);        if (stringBuffer.length() > ConstantTool.HDFS_FLUSH_SIZE || line.isEmpty()) {            final String context = stringBuffer.toString();            new Runnable() {                public void run() {                    long localNum = getNum();                    log.info("##### 数据写入HDFS " + ConstantTool.OUTPUT_PATH + "/rs_user_tags_inc/tmp-" + localNum);                    try {                        Path path = new Path(ConstantTool.OUTPUT_PATH + "/rs_user_tags_inc/tmp-" + localNum);                        FileSystem fs = HDFSTool.getFS();                        if (fs.exists(path)) {                            fs.delete(path, true);                        }                        InputStream in = new ByteArrayInputStream(context.getBytes());                        OutputStream out = fs.create(path);                        IOUtils.copyBytes(in, out, context.length(), true);                        in.close();                        out.close();                        if (successFlag == null) {                            successFlag = true;                        }                    } catch (IOException e) {                        e.printStackTrace();                    }                }            }.run();            stringBuffer = new StringBuffer();        }    }    synchronized private void end() {        CurrentThread--;        if (CurrentThread == 0) {            write("");        }    }

注意用synchronized 方法修饰符对方法进行修饰，例如写方法，不synchronized 的话，多线程运行会造成结果数据与实际数据不一致。