Spark--Scala Actor统计多文件WordCount

来源:互联网 发布:mac mini 发布 编辑:程序博客网 时间:2024/06/05 11:33
import scala.actors.{Future, Actor}import scala.collection.mutableimport scala.collection.mutable.ArrayBufferimport scala.io.Source/** * Created by Administrator on 2017/7/30. */class MyTask extends Actor{  def act(): Unit ={    loop{      react{        case SubmitTask(fileName) =>{          val result = Source.fromFile(fileName).getLines().flatMap(_.split(" ")).map((_,1)).toList.groupBy(_._1).mapValues(_.size)          sender ! ResultTask(result)        }        case StopTask =>{          exit()        }      }    }  }}case class SubmitTask(fileName:String)case class ResultTask(result:Map[String,Int])case object StopTaskobject ActorWordCount extends App{  val replySet = new mutable.HashSet[Future[Any]]()  val resultList = new ArrayBuffer[ResultTask]()  val files = Array[String]("E:\\hadoopSoftWare\\data\\wordCnt.txt","E:\\hadoopSoftWare\\data\\wordCount.txt")  for(file <- files){    val task = new MyTask    val reply = task.start()!!SubmitTask(file)    replySet += reply  }  while(replySet.size>0){    val toCompute = replySet.filter(_.isSet)    for(r <- toCompute){      val result = r.apply().asInstanceOf[ResultTask] //r.apply或者r()取出返回值Future中所谓any的值      resultList += result      replySet -= r    }  }  val value = resultList.toList.flatMap(_.result).groupBy(_._1).mapValues(_.foldLeft(0)(_+_._2));  println(value)}
原创粉丝点击