TaskScheduler原理剖析与源码分析

来源：互联网发布：sql 优化方面的书籍编辑：程序博客网时间：2024/06/13 08:58

这里写图片描述

 def resourceOffers(offers: Seq[WorkerOffer]): Seq[Seq[TaskDescription]] = synchronized {    var newExecAvail = false    for (o <- offers) {      executorIdToHost(o.executorId) = o.host      executorIdToTaskCount.getOrElseUpdate(o.executorId, 0)      if (!executorsByHost.contains(o.host)) {        executorsByHost(o.host) = new HashSet[String]()        executorAdded(o.executorId, o.host)        newExecAvail = true      }      for (rack <- getRackForHost(o.host)) {        hostsByRack.getOrElseUpdate(rack, new HashSet[String]()) += o.host      }    }

这里写图片描述

 val tasks = shuffledOffers.map(o => new ArrayBuffer[TaskDescription](o.cores))    val availableCpus = shuffledOffers.map(o => o.cores).toArray

这里写图片描述

    val sortedTaskSets = rootPool.getSortedTaskSetQueue    for (taskSet <- sortedTaskSets) {      logDebug("parentName: %s, name: %s, runningTasks: %s".format(        taskSet.parent.name, taskSet.name, taskSet.runningTasks))      if (newExecAvail) {        taskSet.executorAdded()      }    }

这里写图片描述

    var launchedTask = false    for (taskSet <- sortedTaskSets; maxLocality <- taskSet.myLocalityLevels) {      do {

这里写图片描述

//resourceOfferSingleTaskSet（）方法遍历所有executor，调用manager的resoucerOffer方法，找到合适的本地化级别找到可以启动的task执行task。       launchedTask = resourceOfferSingleTaskSet(            taskSet, maxLocality, shuffledOffers, availableCpus, tasks)      } while (launchedTask)    }

    if (tasks.size > 0) {      hasLaunchedTask = true    }    return tasks

阅读全文

0 0