Master原理剖析与源码分析：Master状态改变处理机制原理剖析与源码分析

1、主备切换机制原理剖析与源码分析
2、注册机制原理剖析与源码分析

3、状态改变处理机制源码分析

4、资源调度机制源码分析（schedule()，两种资源调度算法）

// Master中Driver状态改变的的信息  private def removeDriver(      driverId: String,      finalState: DriverState,      exception: Option[Exception]) {    drivers.find(d => d.id == driverId) match {      case Some(driver) =>        logInfo(s"Removing driver: $driverId")        //将Driver从缓存中移除        drivers -= driver        if (completedDrivers.size >= RETAINED_DRIVERS) {          val toRemove = math.max(RETAINED_DRIVERS / 10, 1)          completedDrivers.trimStart(toRemove)        }        //将Driver标记为已完成        completedDrivers += driver        //使用持久化引擎去除Driver得持久化信息        persistenceEngine.removeDriver(driver)        driver.state = finalState        driver.exception = exception        //将Driver所在的worker去除Driver。        driver.worker.foreach(w => w.removeDriver(driver))        schedule()      case None =>        logWarning(s"Asked to remove unknown driver: $driverId")    }  }

---

Master中Executor状态改变case ExecutorStateChanged(appId, execId, state, message, exitStatus) => {      val execOption = idToApp.get(appId).flatMap(app => app.executors.get(execId))      execOption match {        case Some(exec) => {          val appInfo = idToApp(appId)          val oldState = exec.state          exec.state = state          if (state == ExecutorState.RUNNING) {            assert(oldState == ExecutorState.LAUNCHING,              s"executor $execId state transfer from $oldState to RUNNING is illegal")            appInfo.resetRetryCount()          }            //向Driver发送ExecutorUpdated信息          exec.application.driver.send(ExecutorUpdated(execId, state, message, exitStatus))            //如果ExecutorState.isFinished          if (ExecutorState.isFinished(state)) {            // Remove this executor from the worker and app            logInfo(s"Removing executor ${exec.fullId} because it is $state")            // If an application has already finished, preserve its            // state to display its information properly on the UI            if (!appInfo.isFinished) {            //从app的缓存中移除executor              appInfo.removeExecutor(exec)            }            exec.worker.removeExecutor(exec)            //判断，如果Executor退出试是非正常的            val normalExit = exitStatus == Some(0)            // 判断application的重试次数是否达到了最大值            if (!normalExit) {              if (appInfo.incrementRetryCount() < ApplicationState.MAX_NUM_RETRY) {              //重新调度                schedule()              } else {                val execs = appInfo.executors.values                if (!execs.exists(_.state == ExecutorState.RUNNING)) {                  logError(s"Application ${appInfo.desc.name} with ID ${appInfo.id} failed " +                    s"${appInfo.retryCount} times; removing it")                    //Executor反复调度都是失败的话，那么Application任务也会失败，就要进行removeApplication 的操作。                  removeApplication(appInfo, ApplicationState.FAILED)                }              }            }          }        }        case None =>          logWarning(s"Got status update for unknown executor $appId/$execId")      }