Spark问题11之广播失败

更多代码请见：https://github.com/xubo245/SparkLearning

Spark生态之Alluxio学习 版本：alluxio（tachyon） 0.7.1，spark-1.5.2,hadoop-2.6.0

1.问题描述

运行是出现Error cleaning broadcast 7

2.运行记录：

17/02/28 08:28:48 ERROR ContextCleaner: Error cleaning broadcast 7java.lang.InterruptedException    at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1325)    at scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:208)    at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)    at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223)    at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107)at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)at scala.concurrent.Await$.result(package.scala:107)at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcEnv.scala:241)at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:101)at org.apache.spark.rpc.RpcEndpointRef.askWithRetry(RpcEndpointRef.scala:77)at org.apache.spark.storage.BlockManagerMaster.removeBroadcast(BlockManagerMaster.scala:128)at org.apache.spark.broadcast.TorrentBroadcast$.unpersist(TorrentBroadcast.scala:228)at org.apache.spark.broadcast.TorrentBroadcastFactory.unbroadcast(TorrentBroadcastFactory.scala:45)at org.apache.spark.broadcast.BroadcastManager.unbroadcast(BroadcastManager.scala:67)at org.apache.spark.ContextCleaner.doCleanupBroadcast(ContextCleaner.scala:214)at org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1$$anonfun$apply$mcV$sp$2.apply(ContextCleaner.scala:170)    at org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1$$anonfun$apply$mcV$sp$2.apply(ContextCleaner.scala:161)    at scala.Option.foreach(Option.scala:236)    at org.apache.spark.ContextCleaner$$anonfun$org$apache$spark$ContextCleaner$$keepCleaning$1.apply$mcV$sp(ContextCleaner.scala:161)    at org.apache.spark.util.Utils$.tryOrStopSparkContext(Utils.scala:1136)    at org.apache.spark.ContextCleaner.org$apache$spark$ContextCleaner$$keepCleaning(ContextCleaner.scala:154)    at org.apache.spark.ContextCleaner$$anon$3.run(ContextCleaner.scala:67)17/02/28 08:28:48 ERROR LiveListenerBus: SparkListenerBus has already stopped! Dropping event SparkListenerBlockUpdated(BlockUpdatedInfo(BlockManagerId(1, Mcnode2, 43825),broadcast_7_piece0,StorageLevel(false, false, false, false, 1),0,0,0))17/02/28 08:28:48 ERROR LiveListenerBus: SparkListenerBus has already stopped! Dropping event SparkListenerBlockUpdated(BlockUpdatedInfo(BlockManagerId(6, Mcnode3, 41749),broadcast_7_piece0,StorageLevel(false, false, false, false, 1),0,0,0))17/02/28 08:28:48 ERROR LiveListenerBus: SparkListenerBus has already stopped! Dropping event SparkListenerBlockUpdated(BlockUpdatedInfo(BlockManagerId(4, Mcnode7, 45730),broadcast_7_piece0,StorageLevel(false, false, false, false, 1),0,0,0))java.util.concurrent.RejectedExecutionException: Task scala.concurrent.impl.CallbackRunnable@407b9c6d rejected from java.util.concurrent.ThreadPoolExecutor@735ad92d[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 79]at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)at scala.concurrent.impl.ExecutionContextImpl$$anon$1.execute(ExecutionContextImpl.scala:133)    at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)    at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)    at scala.concurrent.Promise$class.complete(Promise.scala:55)    at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:153)    at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324)at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324)    at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)    at org.spark-project.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:293)    at scala.concurrent.impl.ExecutionContextImpl$$anon$1.execute(ExecutionContextImpl.scala:133)at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)at scala.concurrent.Promise$class.complete(Promise.scala:55)at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:153)at scala.concurrent.Future$$anonfun$map$1.apply(Future.scala:235)    at scala.concurrent.Future$$anonfun$map$1.apply(Future.scala:235)at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.processBatch$1(Future.scala:643)    at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.apply$mcV$sp(Future.scala:658)at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.apply(Future.scala:635)    at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.apply(Future.scala:635)at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)at scala.concurrent.Future$InternalCallbackExecutor$Batch.run(Future.scala:634)at scala.concurrent.Future$InternalCallbackExecutor$.scala$concurrent$Future$InternalCallbackExecutor$$unbatchedExecute(Future.scala:694)    at scala.concurrent.Future$InternalCallbackExecutor$.execute(Future.scala:685)    at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)    at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)    at scala.concurrent.Promise$class.complete(Promise.scala:55)    at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:153)    at scala.concurrent.Future$$anonfun$flatMap$1.apply(Future.scala:249)at scala.concurrent.Future$$anonfun$flatMap$1.apply(Future.scala:249)    at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)    at org.spark-project.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:293)    at scala.concurrent.impl.ExecutionContextImpl$$anon$1.execute(ExecutionContextImpl.scala:133)at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)at akka.pattern.PromiseActorRef$$anonfun$1.apply$mcV$sp(AskSupport.scala:334)    at akka.actor.Scheduler$$anon$7.run(Scheduler.scala:117)at akka.actor.LightArrayRevolverScheduler$TaskHolder.run(Scheduler.scala:476)at akka.actor.LightArrayRevolverScheduler$$anonfun$close$1.apply(Scheduler.scala:282)    at akka.actor.LightArrayRevolverScheduler$$anonfun$close$1.apply(Scheduler.scala:281)at scala.collection.Iterator$class.foreach(Iterator.scala:727)at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)at scala.collection.AbstractIterable.foreach(Iterable.scala:54)at akka.actor.LightArrayRevolverScheduler.close(Scheduler.scala:280)at akka.actor.ActorSystemImpl.stopScheduler(ActorSystem.scala:689)at akka.actor.ActorSystemImpl$$anonfun$liftedTree2$1$1.apply$mcV$sp(ActorSystem.scala:618)    at akka.actor.ActorSystemImpl$$anonfun$liftedTree2$1$1.apply(ActorSystem.scala:618)at akka.actor.ActorSystemImpl$$anonfun$liftedTree2$1$1.apply(ActorSystem.scala:618)    at akka.actor.ActorSystemImpl$$anon$3.run(ActorSystem.scala:642)at akka.actor.ActorSystemImpl$TerminationCallbacks$$anonfun$run$1.runNext$1(ActorSystem.scala:809)    at akka.actor.ActorSystemImpl$TerminationCallbacks$$anonfun$run$1.apply$mcV$sp(ActorSystem.scala:812)    at akka.actor.ActorSystemImpl$TerminationCallbacks$$anonfun$run$1.apply(ActorSystem.scala:805)    at akka.actor.ActorSystemImpl$TerminationCallbacks$$anonfun$run$1.apply(ActorSystem.scala:805)    at akka.util.ReentrantGuard.withGuard(LockUtil.scala:15)    at akka.actor.ActorSystemImpl$TerminationCallbacks.run(ActorSystem.scala:805)    at akka.actor.ActorSystemImpl$$anonfun$terminationCallbacks$1.apply(ActorSystem.scala:639)at akka.actor.ActorSystemImpl$$anonfun$terminationCallbacks$1.apply(ActorSystem.scala:639)    at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)    at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55)    at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:91)    at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)    at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)    at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)    at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:90)    at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:40)    at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:397)    at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)    at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)    at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)    at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

参考

【1】http://spark.apache.org/docs/1.5.2/programming-guide.html【2】https://github.com/xubo245/SparkLearning