Java 多线程写同一个文件实现

最近项目中需要从网站上抓取大量的数据，采用了多线程技术，每个线程抓取的数据都需要保存到一个文件中，避免消耗大量的内存。

思路：多个访问线程将需要写入到文件中的数据先保存到一个队列里面，然后由专门的 写出线程负责从队列中取出数据并写入到文件中。

WriterQueue.java 存放要输出的数据队列

package com.yulore.write;import java.util.LinkedList;import java.util.concurrent.locks.Condition;import java.util.concurrent.locks.Lock;import java.util.concurrent.locks.ReentrantLock;public class WriterQueue {private static final int MAX_QUEUE_SIZE = 5000;private LinkedList<String> queue = new LinkedList<String>();private Lock lock = new ReentrantLock();private Condition notFull = lock.newCondition();private Condition notEmpty = lock.newCondition();private static WriterQueue manager = new WriterQueue();private WriterQueue(){}public static WriterQueue getQueue(){return manager;}public void put(String phone){lock.lock();try {while (queue.size() == MAX_QUEUE_SIZE) {System.out.println("warning: data queue is full!");notFull.await();}queue.addFirst(phone);notEmpty.signal();} catch (InterruptedException e) {e.printStackTrace();} finally{lock.unlock();}}public LinkedList<String> takeAll(){LinkedList<String> retVal = new LinkedList<String>();lock.lock();try {while (queue.size() == 0) {System.out.println("warning: data queue is empty!");notEmpty.await();}retVal.addAll(queue);//for(String str : queue){//retVal.add(str);//}//清空队列queue.clear();notFull.signal();} catch (InterruptedException e) {e.printStackTrace();} finally{lock.unlock();}return retVal;}}

WriteTask_New.java 模拟产生数据的线程类

package com.yulore.write;public class WriteTask_New implements Runnable {@Overridepublic void run() {for(int i=0;i<20;i++){//try {//sleep(100);//} catch (InterruptedException e) {//e.printStackTrace();//}WriterQueue.getQueue().put("for:"+i+" thread:"+Thread.currentThread().getName());}}private void sleep(int millis) throws InterruptedException {Thread.sleep(millis);}}

OutputTask.java 负责将数据写入到文件中

package com.yulore.write;import java.io.BufferedWriter;import java.io.File;import java.io.FileOutputStream;import java.io.IOException;import java.io.OutputStreamWriter;import java.util.LinkedList;public class OutputTask implements Runnable {private String fileName;public OutputTask(String fileName) {this.fileName = fileName;}@Overridepublic void run() {while(true){try {sleep(5000);} catch (InterruptedException e) {e.printStackTrace();}LinkedList<String> list = WriterQueue.getQueue().takeAll();write2Disk(list);list = null;}}private void write2Disk(LinkedList<String> list) {if(list==null ||list.size()==0){System.out.println("no data...");return;}System.out.println("开始序列化数据 "+fileName);String path = "D:/fbb/myWorkSpace_DW07/";File outputFile = new File(path+fileName);if(outputFile==null ||!outputFile.exists()){try {outputFile.createNewFile();} catch (IOException e) {e.printStackTrace();}}FileOutputStream out = null;OutputStreamWriter writer = null;BufferedWriter bw = null;try {out = new FileOutputStream(outputFile, true);writer = new OutputStreamWriter(out);bw = new BufferedWriter(writer);for(String content : list){bw.write(content);bw.newLine();bw.flush();}} catch (IOException e) {e.printStackTrace();}finally{try {if(bw!=null)bw.close();} catch (IOException e) {e.printStackTrace();}}}private void sleep(int millis) throws InterruptedException {Thread.sleep(millis);}}

测试类

package com.yulore.write;public class TestWrite {/** * @param args */public static void main(String[] args) {//test();test02();}private static void test02() {WriteTask_New write = new WriteTask_New();for(int i=0;i<4;i++){new Thread(write).start();}OutputTask output = new OutputTask("abc.txt");new Thread(output).start();}private static void test() {WriteTask write = new WriteTask("abc.txt");for(int i=0;i<5;i++){new Thread(write).start();}}}