spark 2.1 ResultTask run process

ResultTask.runTask

  override def runTask(context: TaskContext): U = {    // Deserialize the RDD and the func using the broadcast variables.    val threadMXBean = ManagementFactory.getThreadMXBean    val deserializeStartTime = System.currentTimeMillis()    val deserializeStartCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) {      threadMXBean.getCurrentThreadCpuTime    } else 0L    val ser = SparkEnv.get.closureSerializer.newInstance()    val (rdd, func) = ser.deserialize[(RDD[T], (TaskContext, Iterator[T]) => U)](      ByteBuffer.wrap(taskBinary.value), Thread.currentThread.getContextClassLoader)    _executorDeserializeTime = System.currentTimeMillis() - deserializeStartTime    _executorDeserializeCpuTime = if (threadMXBean.isCurrentThreadCpuTimeSupported) {      threadMXBean.getCurrentThreadCpuTime - deserializeStartCpuTime    } else 0L    func(context, rdd.iterator(partition, context))  }

RDD.iterator(partition, context)

  /**   * Internal method to this RDD; will read from cache if applicable, or otherwise compute it.   * This should ''not'' be called by users directly, but is available for implementors of custom   * subclasses of RDD.   */  final def iterator(split: Partition, context: TaskContext): Iterator[T] = {    if (storageLevel != StorageLevel.NONE) {      getOrCompute(split, context)    } else {      computeOrReadCheckpoint(split, context)    }  }

RDD.computeOrReadCheckpoint

  /**   * Compute an RDD partition or read it from a checkpoint if the RDD is checkpointing.   */  private[spark] def computeOrReadCheckpoint(split: Partition, context: TaskContext): Iterator[T] =  {    if (isCheckpointedAndMaterialized) {      firstParent[T].iterator(split, context)    } else {      compute(split, context)    }  }

ShuffleRDD.compute

 override def compute(split: Partition, context: TaskContext): Iterator[(K, C)] = {    val dep = dependencies.head.asInstanceOf[ShuffleDependency[K, V, C]]    SparkEnv.get.shuffleManager.getReader(dep.shuffleHandle, split.index, split.index + 1, context)      .read()      .asInstanceOf[Iterator[(K, C)]]  }

SortShuffleManager.getReader

 /**   * Get a reader for a range of reduce partitions (startPartition to endPartition-1, inclusive).   * Called on executors by reduce tasks.   */  override def getReader[K, C](      handle: ShuffleHandle,      startPartition: Int,      endPartition: Int,      context: TaskContext): ShuffleReader[K, C] = {    new BlockStoreShuffleReader(      handle.asInstanceOf[BaseShuffleHandle[K, _, C]], startPartition, endPartition, context)  }

BlockStoreShuffleReader

/** * Fetches and reads the partitions in range [startPartition, endPartition) from a shuffle by * requesting them from other nodes' block stores. */private[spark] class BlockStoreShuffleReader[K, C](    handle: BaseShuffleHandle[K, _, C],    startPartition: Int,    endPartition: Int,    context: TaskContext,    serializerManager: SerializerManager = SparkEnv.get.serializerManager,    blockManager: BlockManager = SparkEnv.get.blockManager,    mapOutputTracker: MapOutputTracker = SparkEnv.get.mapOutputTracker)  extends ShuffleReader[K, C] with Logging {  private val dep = handle.dependency

BlockStoreShuffleReader.read

/** Read the combined key-values for this reduce task */  override def read(): Iterator[Product2[K, C]] = {    val blockFetcherItr = new ShuffleBlockFetcherIterator(      context,      blockManager.shuffleClient,      blockManager,      mapOutputTracker.getMapSizesByExecutorId(handle.shuffleId, startPartition, endPartition),      // Note: we use getSizeAsMb when no suffix is provided for backwards compatibility      SparkEnv.get.conf.getSizeAsMb("spark.reducer.maxSizeInFlight", "48m") * 1024 * 1024,      SparkEnv.get.conf.getInt("spark.reducer.maxReqsInFlight", Int.MaxValue))    // Wrap the streams for compression and encryption based on configuration    val wrappedStreams = blockFetcherItr.map { case (blockId, inputStream) =>      serializerManager.wrapStream(blockId, inputStream)    }    val serializerInstance = dep.serializer.newInstance()    // Create a key/value iterator for each stream    val recordIter = wrappedStreams.flatMap { wrappedStream =>      // Note: the asKeyValueIterator below wraps a key/value iterator inside of a      // NextIterator. The NextIterator makes sure that close() is called on the      // underlying InputStream when all records have been read.      serializerInstance.deserializeStream(wrappedStream).asKeyValueIterator    }    // Update the context task metrics for each record read.    val readMetrics = context.taskMetrics.createTempShuffleReadMetrics()    val metricIter = CompletionIterator[(Any, Any), Iterator[(Any, Any)]](      recordIter.map { record =>        readMetrics.incRecordsRead(1)        record      },      context.taskMetrics().mergeShuffleReadMetrics())    // An interruptible iterator must be used here in order to support task cancellation    val interruptibleIter = new InterruptibleIterator[(Any, Any)](context, metricIter)    val aggregatedIter: Iterator[Product2[K, C]] = if (dep.aggregator.isDefined) {      if (dep.mapSideCombine) {        // We are reading values that are already combined        val combinedKeyValuesIterator = interruptibleIter.asInstanceOf[Iterator[(K, C)]]        dep.aggregator.get.combineCombinersByKey(combinedKeyValuesIterator, context)      } else {        // We don't know the value type, but also don't care -- the dependency *should*        // have made sure its compatible w/ this aggregator, which will convert the value        // type to the combined type C        val keyValuesIterator = interruptibleIter.asInstanceOf[Iterator[(K, Nothing)]]        dep.aggregator.get.combineValuesByKey(keyValuesIterator, context)      }    } else {      require(!dep.mapSideCombine, "Map-side combine without Aggregator specified!")      interruptibleIter.asInstanceOf[Iterator[Product2[K, C]]]    }    // Sort the output if there is a sort ordering defined.    dep.keyOrdering match {      case Some(keyOrd: Ordering[K]) =>        // Create an ExternalSorter to sort the data. Note that if spark.shuffle.spill is disabled,        // the ExternalSorter won't spill to disk.        val sorter =          new ExternalSorter[K, C, C](context, ordering = Some(keyOrd), serializer = dep.serializer)        sorter.insertAll(aggregatedIter)        context.taskMetrics().incMemoryBytesSpilled(sorter.memoryBytesSpilled)        context.taskMetrics().incDiskBytesSpilled(sorter.diskBytesSpilled)        context.taskMetrics().incPeakExecutionMemory(sorter.peakMemoryUsedBytes)        CompletionIterator[Product2[K, C], Iterator[Product2[K, C]]](sorter.iterator, sorter.stop())      case None =>        aggregatedIter    }  }

ShuffleBlockFetcherIterator

/** * An iterator that fetches multiple blocks. For local blocks, it fetches from the local block * manager. For remote blocks, it fetches them using the provided BlockTransferService. * * This creates an iterator of (BlockID, InputStream) tuples so the caller can handle blocks * in a pipelined fashion as they are received. * * The implementation throttles the remote fetches so they don't exceed maxBytesInFlight to avoid * using too much memory. * * @param context [[TaskContext]], used for metrics update * @param shuffleClient [[ShuffleClient]] for fetching remote blocks * @param blockManager [[BlockManager]] for reading local blocks * @param blocksByAddress list of blocks to fetch grouped by the [[BlockManagerId]]. *                        For each block we also require the size (in bytes as a long field) in *                        order to throttle the memory usage. * @param maxBytesInFlight max size (in bytes) of remote blocks to fetch at any given point. * @param maxReqsInFlight max number of remote requests to fetch blocks at any given point. */private[spark]final class ShuffleBlockFetcherIterator(    context: TaskContext,    shuffleClient: ShuffleClient,    blockManager: BlockManager,    blocksByAddress: Seq[(BlockManagerId, Seq[(BlockId, Long)])],    maxBytesInFlight: Long,    maxReqsInFlight: Int)  extends Iterator[(BlockId, InputStream)] with Logging {  import ShuffleBlockFetcherIterator._

ShuffleBlockFetcherIterator.fields

/**   * Total number of blocks to fetch. This can be smaller than the total number of blocks   * in [[blocksByAddress]] because we filter out zero-sized blocks in [[initialize]].   *   * This should equal localBlocks.size + remoteBlocks.size.   */  private[this] var numBlocksToFetch = 0  /**   * The number of blocks processed by the caller. The iterator is exhausted when   * [[numBlocksProcessed]] == [[numBlocksToFetch]].   */  private[this] var numBlocksProcessed = 0  private[this] val startTime = System.currentTimeMillis  /** Local blocks to fetch, excluding zero-sized blocks. */  private[this] val localBlocks = new ArrayBuffer[BlockId]()  /** Remote blocks to fetch, excluding zero-sized blocks. */  private[this] val remoteBlocks = new HashSet[BlockId]()  /**   * A queue to hold our results. This turns the asynchronous model provided by   * [[org.apache.spark.network.BlockTransferService]] into a synchronous model (iterator).   */  private[this] val results = new LinkedBlockingQueue[FetchResult]  /**   * Current [[FetchResult]] being processed. We track this so we can release the current buffer   * in case of a runtime exception when processing the current buffer.   */  @volatile private[this] var currentResult: FetchResult = null  /**   * Queue of fetch requests to issue; we'll pull requests off this gradually to make sure that   * the number of bytes in flight is limited to maxBytesInFlight.   */  private[this] val fetchRequests = new Queue[FetchRequest]  /** Current bytes in flight from our requests */  private[this] var bytesInFlight = 0L  /** Current number of requests in flight */  private[this] var reqsInFlight = 0  private[this] val shuffleMetrics = context.taskMetrics().createTempShuffleReadMetrics()  /**   * Whether the iterator is still active. If isZombie is true, the callback interface will no   * longer place fetched blocks into [[results]].   */  @GuardedBy("this")  private[this] var isZombie = false  initialize()

ShuffleBlockFetcherIterator.initialize

private[this] def initialize(): Unit = {    // Add a task completion callback (called in both success case and failure case) to cleanup.    context.addTaskCompletionListener(_ => cleanup())    // Split local and remote blocks.    val remoteRequests = splitLocalRemoteBlocks()    // Add the remote requests into our queue in a random order    fetchRequests ++= Utils.randomize(remoteRequests)    assert ((0 == reqsInFlight) == (0 == bytesInFlight),      "expected reqsInFlight = 0 but found reqsInFlight = " + reqsInFlight +      ", expected bytesInFlight = 0 but found bytesInFlight = " + bytesInFlight)    // Send out initial requests for blocks, up to our maxBytesInFlight    fetchUpToMaxBytes()    val numFetches = remoteRequests.size - fetchRequests.size    logInfo("Started " + numFetches + " remote fetches in" + Utils.getUsedTimeMs(startTime))    // Get Local Blocks    fetchLocalBlocks()    logDebug("Got local blocks in " + Utils.getUsedTimeMs(startTime))  }

ShuffleBlockFetcherIterator.splitLocalRemoteBlocks

 private[this] def splitLocalRemoteBlocks(): ArrayBuffer[FetchRequest] = {    // Make remote requests at most maxBytesInFlight / 5 in length; the reason to keep them    // smaller than maxBytesInFlight is to allow multiple, parallel fetches from up to 5    // nodes, rather than blocking on reading output from one node.    val targetRequestSize = math.max(maxBytesInFlight / 5, 1L)    logDebug("maxBytesInFlight: " + maxBytesInFlight + ", targetRequestSize: " + targetRequestSize)    // Split local and remote blocks. Remote blocks are further split into FetchRequests of size    // at most maxBytesInFlight in order to limit the amount of data in flight.    val remoteRequests = new ArrayBuffer[FetchRequest]    // Tracks total number of blocks (including zero sized blocks)    var totalBlocks = 0    for ((address, blockInfos) <- blocksByAddress) {      totalBlocks += blockInfos.size      if (address.executorId == blockManager.blockManagerId.executorId) {        // Filter out zero-sized blocks        localBlocks ++= blockInfos.filter(_._2 != 0).map(_._1)        numBlocksToFetch += localBlocks.size      } else {        val iterator = blockInfos.iterator        var curRequestSize = 0L        var curBlocks = new ArrayBuffer[(BlockId, Long)]        while (iterator.hasNext) {          val (blockId, size) = iterator.next()          // Skip empty blocks          if (size > 0) {            curBlocks += ((blockId, size))            remoteBlocks += blockId            numBlocksToFetch += 1            curRequestSize += size          } else if (size < 0) {            throw new BlockException(blockId, "Negative block size " + size)          }          if (curRequestSize >= targetRequestSize) {            // Add this FetchRequest            remoteRequests += new FetchRequest(address, curBlocks)            curBlocks = new ArrayBuffer[(BlockId, Long)]            logDebug(s"Creating fetch request of $curRequestSize at $address")            curRequestSize = 0          }        }        // Add in the final request        if (curBlocks.nonEmpty) {          remoteRequests += new FetchRequest(address, curBlocks)        }      }    }    logInfo(s"Getting $numBlocksToFetch non-empty blocks out of $totalBlocks blocks")    remoteRequests  }

ShuffleBlockFetcherIterator.fetchUpToMaxBytes

 private def fetchUpToMaxBytes(): Unit = {    // Send fetch requests up to maxBytesInFlight    while (fetchRequests.nonEmpty &&      (bytesInFlight == 0 ||        (reqsInFlight + 1 <= maxReqsInFlight &&          bytesInFlight + fetchRequests.front.size <= maxBytesInFlight))) {      sendRequest(fetchRequests.dequeue())    }  }

private[this] def sendRequest(req: FetchRequest) {    logDebug("Sending request for %d blocks (%s) from %s".format(      req.blocks.size, Utils.bytesToString(req.size), req.address.hostPort))    bytesInFlight += req.size    reqsInFlight += 1    // so we can look up the size of each blockID    val sizeMap = req.blocks.map { case (blockId, size) => (blockId.toString, size) }.toMap    val remainingBlocks = new HashSet[String]() ++= sizeMap.keys    val blockIds = req.blocks.map(_._1.toString)    val address = req.address    shuffleClient.fetchBlocks(address.host, address.port, address.executorId, blockIds.toArray,      new BlockFetchingListener {        override def onBlockFetchSuccess(blockId: String, buf: ManagedBuffer): Unit = {          // Only add the buffer to results queue if the iterator is not zombie,          // i.e. cleanup() has not been called yet.          ShuffleBlockFetcherIterator.this.synchronized {            if (!isZombie) {              // Increment the ref count because we need to pass this to a different thread.              // This needs to be released after use.              buf.retain()              remainingBlocks -= blockId              results.put(new SuccessFetchResult(BlockId(blockId), address, sizeMap(blockId), buf,                remainingBlocks.isEmpty))              logDebug("remainingBlocks: " + remainingBlocks)            }          }          logTrace("Got remote block " + blockId + " after " + Utils.getUsedTimeMs(startTime))        }        override def onBlockFetchFailure(blockId: String, e: Throwable): Unit = {          logError(s"Failed to get block(s) from ${req.address.host}:${req.address.port}", e)          results.put(new FailureFetchResult(BlockId(blockId), address, e))        }      }    )  }