spark 2.1 TaskResult
来源:互联网 发布:知天命尽人事的网名 编辑:程序博客网 时间:2024/06/06 07:35
TaskResult
// Task result. Also contains updates to accumulator variables.private[spark] sealed trait TaskResult[T]
IndirectTaskResult
/** A reference to a DirectTaskResult that has been stored in the worker's BlockManager. */private[spark] case class IndirectTaskResult[T](blockId: BlockId, size: Int) extends TaskResult[T] with Serializable
DirectTaskResult
/** A TaskResult that contains the task's return value and accumulator updates. */private[spark] class DirectTaskResult[T]( var valueBytes: ByteBuffer, var accumUpdates: Seq[AccumulatorV2[_, _]]) extends TaskResult[T] with Externalizable { private var valueObjectDeserialized = false private var valueObject: T = _ def this() = this(null.asInstanceOf[ByteBuffer], null) override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException { out.writeInt(valueBytes.remaining) Utils.writeByteBuffer(valueBytes, out) out.writeInt(accumUpdates.size) accumUpdates.foreach(out.writeObject) } override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException { val blen = in.readInt() val byteVal = new Array[Byte](blen) in.readFully(byteVal) valueBytes = ByteBuffer.wrap(byteVal) val numUpdates = in.readInt if (numUpdates == 0) { accumUpdates = Seq() } else { val _accumUpdates = new ArrayBuffer[AccumulatorV2[_, _]] for (i <- 0 until numUpdates) { _accumUpdates += in.readObject.asInstanceOf[AccumulatorV2[_, _]] } accumUpdates = _accumUpdates } valueObjectDeserialized = false } /** * When `value()` is called at the first time, it needs to deserialize `valueObject` from * `valueBytes`. It may cost dozens of seconds for a large instance. So when calling `value` at * the first time, the caller should avoid to block other threads. * * After the first time, `value()` is trivial and just returns the deserialized `valueObject`. */ def value(resultSer: SerializerInstance = null): T = { if (valueObjectDeserialized) { valueObject } else { // This should not run when holding a lock because it may cost dozens of seconds for a large // value val ser = if (resultSer == null) SparkEnv.get.serializer.newInstance() else resultSer valueObject = ser.deserialize(valueBytes) valueObjectDeserialized = true valueObject } }}
阅读全文
0 0
- spark 2.1 TaskResult
- spark 2.1 spark.yarn.services
- Spark 2.1.0 -- Spark Streaming Programming Guide
- spark 2.1 spark-shell Startup Process
- spark 2.1 spark executor topology information
- spark-2.1.0安装
- spark(2.1.0)
- Spark 2.1 structured streaming
- Spark 2.1 CallSite
- spark 2.1 ConfigProvider
- Spark ML 2.1 --Pipelines
- spark-2.1.0
- spark 2.1 SparkContext postEnvironmentUpdate
- spark 2.1 StorageLevel
- spark 2.1 BlockManagerId
- spark 2.1 BlockManagerMasterEndpoint
- spark 2.1 BlockManagerInfo
- spark 2.1 BlockManagerSlaveEndpoint
- socket系列(一)——socket实现推送
- android18
- css之元素的显示方式
- tensorflow实现对图片的读取
- 正则语言引擎:一个简单LEX和YACC结合运用的实例
- spark 2.1 TaskResult
- postfix
- android18
- 用c语言链表实现通讯录
- 列出联通集
- android18
- 数据结构基本概念
- Android-mvp&mvvm分别实现列表加载-demo
- 常见的正则表达式