Worker原理剖析与源码分析

• Master会让worker启动一个Driver Runnner线程，启动Driver线程（launchDriver）。
• Master让Worker启动一个ExecutorRunner，去启动Executor进程并向Driver反向注册（launchExecutor）。

case LaunchDriver(driverId, driverDesc) => {      logInfo(s"Asked to launch driver $driverId")      //创建一个Driver Runner，管理Driver 进程。      val driver = new DriverRunner(        conf,        driverId,        workDir,        sparkHome,        driverDesc.copy(command = Worker.maybeUpdateSSLSettings(driverDesc.command, conf)),        self,        workerUri,        securityMgr)      drivers(driverId) = driver      driver.start()      coresUsed += driverDesc.cores      memoryUsed += driverDesc.mem

Driver Runner机制

private[worker] def start() = {//这里创建了一个JAVA的线程    new Thread("DriverRunner for " + driverId) {      override def run() {        try {          //创建一个Driver目录          val driverDir = createWorkingDirectory()          //获取JOb需要的jar包          val localJarFilename = downloadUserJar(driverDir)          def substituteVariables(argument: String): String = argument match {            case "{{WORKER_URL}}" => workerUrl            case "{{USER_JAR}}" => localJarFilename            case other => other          }          // 构建了ProessBuilder          //启动Driver，并为其分布资源          val builder = CommandUtils.buildProcessBuilder(driverDesc.command, securityManager,            driverDesc.mem, sparkHome.getAbsolutePath, substituteVariables)            //ProessBuilder启动Driver          launchDriver(builder, driverDir, driverDesc.supervise)        }        catch {          case e: Exception => finalException = Some(e)        }        val state =        //对Driver的退出状态做一些处理           if (killed) {            DriverState.KILLED          } else if (finalException.isDefined) {            DriverState.ERROR          } else {            finalExitCode match {              case Some(0) => DriverState.FINISHED              case _ => DriverState.FAILED            }          }        finalState = Some(state)        //Driver向worker发送一个DriverStateChanged事件        worker.send(DriverStateChanged(driverId, state, finalException))      }    }.start()  }