3.Master注册机制源码分析和状态改变机制源码分析

master注册机制原理图如下 , 说明了就是将Application信息 , Driver信息和所有的Worker信息加入缓存队列中

1. Application的注册其实在master.scala的代码中就一段代码 : 

/**
     * 处理Application注册请求信息
     */
    case RegisterApplication(description) => {
      // 如果当前master为standByMaster , 不是ActiveMaster , 那么Application来注册则什么都不会做
      if (state == RecoveryState.STANDBY) {
        // ignore, don't send response
      } else {
        logInfo("Registering app " + description.name)
        // 通过接收到的application desc信息创建Application对象
        val app = createApplication(description, sender)
        
        // 注册Application对象
        registerApplication(app)
        logInfo("Registered app " + description.name + " with ID " + app.id)
        
        // 持久化Application信息
        persistenceEngine.addApplication(app)
        
        // 向master发送注册Application的信息 , 也就是反向向SparkDeploySchedulerBackend的AppClient的ClientActor发型已经注册的RegisteredApplication消息
        sender ! RegisteredApplication(app.id, masterUrl)
        
        // 开始资源调度
        schedule()
      }
    }

创建Application代码如下 :

def createApplication(desc: ApplicationDescription, driver: ActorRef): ApplicationInfo = {
    val now = System.currentTimeMillis()
    val date = new Date(now)
    new ApplicationInfo(now, newApplicationId(date), desc, date, driver, defaultCores)
  }

注册Application对象如下 :

/**
   * 注册Application信息 , 将相关的Application信息加入缓存队列中
   */
  def registerApplication(app: ApplicationInfo): Unit = {
    val appAddress = app.driver.path.address
    if (addressToApp.contains(appAddress)) {
      logInfo("Attempted to re-register application at same address: " + appAddress)
      return
    }
    
    // 这里将相关的Application信息加入缓存队列中 
    applicationMetricsSystem.registerSource(app.appSource)
    apps += app
    idToApp(app.id) = app
    actorToApp(app.driver) = app
    addressToApp(appAddress) = app
    
    // 这里将Application加入等待调度的队列中 , waitingApps其实就是一个ArrayBuffer
    waitingApps += app
  }

这其中关于ApplicationDescription的代码如下 :

private[spark] class ApplicationDescription(
    val name: String, // Application的名称
    val maxCores: Option[Int], //Application的最大使用cou core
    val memoryPerSlave: Int, //Application需要的每个节点使用的内存
    val command: Command, //命令
    var appUiUrl: String, //Application所在节点的ui URL
    val eventLogDir: Option[String] = None,
    // short name of compression codec used when writing event logs, if any (e.g. lzf)
    val eventLogCodec: Option[String] = None)

2.Worker的注册原理和Application相似 , 代码如下 : 

case RegisterWorker(id, workerHost, workerPort, cores, memory, workerUiPort, publicAddress) =>
    {
      logInfo("Registering worker %s:%d with %d cores, %s RAM".format(
        workerHost, workerPort, cores, Utils.megabytesToString(memory)))
        
      // 检查Master的状态
      if (state == RecoveryState.STANDBY) {
        // ignore, don't send response
      } else if (idToWorker.contains(id)) {
        sender ! RegisterWorkerFailed("Duplicate worker ID")
      } else {
        
        // 创建WorkerInfo信息对象 , 封装相关的worker信息
        val worker = new WorkerInfo(id, workerHost, workerPort, cores, memory,
          sender, workerUiPort, publicAddress)
        
        // 注册worker , 调用registerWorker方法进行注册 , 若注册成功进行worker信息持久化并向master发送注册的消息
        if (registerWorker(worker)) {
          // worker信息持久化
          persistenceEngine.addWorker(worker)
          // 向master发送注册消息
          sender ! RegisteredWorker(masterUrl, masterWebUiUrl)
          // 开始资源调度
          schedule()
        } else {
          // 注册失败的话向master发送注册失败的消息
          val workerAddress = worker.actor.path.address
          logWarning("Worker registration failed. Attempted to re-register worker at same " +
            "address: " + workerAddress)
          sender ! RegisterWorkerFailed("Attempted to re-register worker at same address: "
            + workerAddress)
        }
      }
    }

注册worker的相关信息源码如下 : 

/**
   * 注册worker信息
   */
  def registerWorker(worker: WorkerInfo): Boolean = {
    // There may be one or more refs to dead workers on this same node (w/ different ID's),
    // remove them.
    // 这里过滤掉已经死掉的worker , 将他们从缓存队列中移除
    workers.filter { w =>
      (w.host == worker.host && w.port == worker.port) && (w.state == WorkerState.DEAD)
    }.foreach { w =>
      workers -= w
    }
    // 获取worker的url地址
    val workerAddress = worker.actor.path.address
    // 检查worker的地址缓存队列中是否已经有了该worker的地址信息
    if (addressToWorker.contains(workerAddress)) {
      // 从worker的地址缓存队列(HashMap)中获取已经存在的worker的地址信息 , 称之为oldworker
      val oldWorker = addressToWorker(workerAddress)
      // 若是oldworker为UNKNOW状态的话需要将其从缓存队列中移除
      if (oldWorker.state == WorkerState.UNKNOWN) {
        // A worker registering from UNKNOWN implies that the worker was restarted during recovery.
        // The old worker must thus be dead, so we will remove it and accept the new worker.
        removeWorker(oldWorker)
      } else {
        logInfo("Attempted to re-register worker at same address: " + workerAddress)
        return false
      }
    }
    // 将新增加的worker加入缓存队列HashSet中
    workers += worker
    // worker的id信息加入id缓存队列
    idToWorker(worker.id) = worker
    // 将worker的地址加入address缓存队列中
    addressToWorker(workerAddress) = worker
    true
  }

3.Dirver的注册源码也相似 , 代码如下 :

    

/**
     * 注册Driver
     */
    case RequestSubmitDriver(description) => {
      // 检查master状态
      if (state != RecoveryState.ALIVE) {
        val msg = s"Can only accept driver submissions in ALIVE state. Current state: $state."
        sender ! SubmitDriverResponse(false, None, msg)
      } else {
        logInfo("Driver submitted " + description.command.mainClass)
        
        // 根据DriverDescription创建Driver
        val driver = createDriver(description)
        // 持久化Driver信息
        persistenceEngine.addDriver(driver)
        // 将Driver加入等待调度的缓存队列中
        waitingDrivers += driver
        // 将Driver加入缓存队列
        drivers.add(driver)
        // 开始调度
        schedule()
        // TODO: It might be good to instead have the submission client poll the master to determine
        //       the current status of the driver. For now it's simply "fire and forget".
        sender ! SubmitDriverResponse(true, Some(driver.id),
          s"Driver successfully submitted as ${driver.id}")
      }
    }

 关于DriverDescription.scala的部分代码如下 :

private[spark] class DriverDescription(
    val jarUrl: String, // jar包的名称
    val mem: Int, // Dirver所需要的内存
    val cores: Int, // Driver所需要的cpu core数量
    val supervise: Boolean, // Driver是否被master监控
    val command: Command) // 相关命令
  extends Serializable {
  def copy(
      jarUrl: String = jarUrl,
      mem: Int = mem,
      cores: Int = cores,
      supervise: Boolean = supervise,
      command: Command = command): DriverDescription =
    new DriverDescription(jarUrl, mem, cores, supervise, command)
  override def toString: String = s"DriverDescription (${command.mainClass})"
}

同时呢在看一下Driver状态改变的代码 : 

/**
     * Driver的状态改变时需要做的曹组
     */
    case DriverStateChanged(driverId, state, exception) => {
      state match {
        // 如果Driver的状态为ERROR,FINISHED,KILLED,FAILED 那么都会将Driver杀掉
        case DriverState.ERROR | DriverState.FINISHED | DriverState.KILLED | DriverState.FAILED =>
          removeDriver(driverId, state, exception)
        case _ =>
          throw new Exception(s"Received unexpected state update for driver $driverId: $state")
      }
    }

 重点查看Driver移除的代码 :

/**
   * 移除Driver
   */
  def removeDriver(driverId: String, finalState: DriverState, exception: Option[Exception]) {
    // 通过scala的find调度找到缓存队列中的driver并进行匹配
    drivers.find(d => d.id == driverId) match {
      case Some(driver) =>
        logInfo(s"Removing driver: $driverId")
        // 从缓存队列Driver中移除
        drivers -= driver
        if (completedDrivers.size >= RETAINED_DRIVERS) {
          val toRemove = math.max(RETAINED_DRIVERS / 10, 1)
          completedDrivers.trimStart(toRemove)
        }
        // 加入已经完成的缓存driver队列中
        completedDrivers += driver
        // 持久化缓存信息
        persistenceEngine.removeDriver(driver)
        // 更改缓存状态
        driver.state = finalState
        driver.exception = exception
        driver.worker.foreach(w => w.removeDriver(driver))
        schedule()
      case None =>
        logWarning(s"Asked to remove unknown driver: $driverId")
    }
  }

关于Executor的状态改变如下 :

/**
     * Executor状态改变所需要的操作
     */
    case ExecutorStateChanged(appId, execId, state, message, exitStatus) => {
      // 找到executor对应app,然后反过来通过app对应的executor缓存获取executor信息
      val execOption = idToApp.get(appId).flatMap(app => app.executors.get(execId))
      execOption match {
        // 成功找到
        case Some(exec) => {
          // 设置executor的当前状态
          val appInfo = idToApp(appId)
          exec.state = state
          if (state == ExecutorState.RUNNING) { appInfo.resetRetryCount() }
          // 向driver同步发送executorUpdate的信息
          exec.application.driver ! ExecutorUpdated(execId, state, message, exitStatus)
           
          // 判断 , 如果executor已经完成了
          if (ExecutorState.isFinished(state)) {
            // Remove this executor from the worker and app
            logInfo(s"Removing executor ${exec.fullId} because it is $state")
            
            // 从Application中移除掉executor
            appInfo.removeExecutor(exec)
            // 从worker中移除exec
            exec.worker.removeExecutor(exec)
            // 如果executor的状态退出异常
            val normalExit = exitStatus == Some(0)
            // Only retry certain number of times so we don't go into an infinite loop.
            if (!normalExit) {
              
              // 判断Application当前的重试次数是否达到了最大值 , 最大值默认为10
              if (appInfo.incrementRetryCount() < ApplicationState.MAX_NUM_RETRY) {
                // 没有达到最大值 则继续调度
                schedule()
              } else {
                
                // 没有达到最大值那就认为executor调度失败 , 并同时认为Application也是失败了 , 将Application也从缓存队列移除掉
                val execs = appInfo.executors.values
                if (!execs.exists(_.state == ExecutorState.RUNNING)) {
                  logError(s"Application ${appInfo.desc.name} with ID ${appInfo.id} failed " +
                    s"${appInfo.retryCount} times; removing it")
                  
                  // 移除掉executor所在的Application
                  removeApplication(appInfo, ApplicationState.FAILED)
                }
              }
            }
          }
        }
        case None =>
          logWarning(s"Got status update for unknown executor $appId/$execId")
      }
    }