spark 2.2 源码分析 Spark-submit 篇

spark 2.2 源码分析 Spark-submit 篇

本文主要分析spark的第一步spark-submit类。之前shell阶段主要就是环境变量的加载，而个人认为spark-submit才是spark程序的真正步。由上文可知org.apache.spark.launcher.Main启动了 org.apache.spark.deploy.SparkSubmit。我们来看一下submit的main入口函数：

override def main(args: Array[String]): Unit = {  // Initialize logging if it hasn't been done yet. Keep track of whether logging needs to  // be reset before the application starts.  val uninitLog = initializeLogIfNecessary(true, silent = true)  // SparkSubmitArguments封装spark-submit传入的参数 例如/bin/spark-submit --master yarn-cluster --num-executors 10  //SparkSubmitArguments中的参数比较多,详情请见源码变量列表  val appArgs: SparkSubmitArguments = new SparkSubmitArguments(args)  // be reset before the application starts.  // 判断的是否启用类debug模式  if (appArgs.verbose) {    // scalastyle:off println    printStream.println(appArgs)    // scalastyle:on println  }  appArgs.action match {      // 调用类submit的入口函数 默认    case SparkSubmitAction.SUBMIT => submit(appArgs, uninitLog)    case SparkSubmitAction.KILL => kill(appArgs)    case SparkSubmitAction.REQUEST_STATUS => requestStatus(appArgs)  }}

在这里对sparksubmitArguments说明一下，由于里面参数较多，请自行百度，如下面，主要是loadEnvriomentArguments函数进行赋值

private def loadEnvironmentArguments(): Unit = {  master = Option(master)    .orElse(sparkProperties.get("spark.master"))    .orElse(env.get("MASTER"))    .orNull  driverExtraClassPath = Option(driverExtraClassPath)    .orElse(sparkProperties.get("spark.driver.extraClassPath"))    .orNull  driverExtraJavaOptions = Option(driverExtraJavaOptions)    .orElse(sparkProperties.get("spark.driver.extraJavaOptions"))    .orNull  driverExtraLibraryPath = Option(driverExtraLibraryPath)    .orElse(sparkProperties.get("spark.driver.extraLibraryPath"))    .orNull  driverMemory = Option(driverMemory)    .orElse(sparkProperties.get("spark.driver.memory"))    .orElse(env.get("SPARK_DRIVER_MEMORY"))    .orNull

现在我们看一下submit的函数

private def submit(args: SparkSubmitArguments, uninitLog: Boolean): Unit = {  // childMainClass就是我们要运行的主类，而根据spark的集群模式（--master）和集群driver部署模式（--deploy-mode）不同  // 对childMainClass进行不同的赋值。目前大部公司都采用yarn模式，我们真对yarn的两种部署模式进行分析。  val (childArgs, childClasspath, sysProps, childMainClass) = prepareSubmitEnvironment(args)  def doRunMain(): Unit = {    if (args.proxyUser != null) {      val proxyUser = UserGroupInformation.createProxyUser(args.proxyUser,        UserGroupInformation.getCurrentUser())      try {        proxyUser.doAs(new PrivilegedExceptionAction[Unit]() {          override def run(): Unit = {                        runMain(childArgs, childClasspath, sysProps, childMainClass, args.verbose)          }        })      } catch {        case e: Exception =>          // Hadoop's AuthorizationException suppresses the exception's stack trace, which          // makes the message printed to the output by the JVM not very helpful. Instead,          // detect exceptions with empty stack traces here, and treat them differently.          if (e.getStackTrace().length == 0) {            // scalastyle:off println            printStream.println(s"ERROR: ${e.getClass().getName()}: ${e.getMessage()}")            // scalastyle:on println            exitFn(1)          } else {            throw e          }      }    } else {      runMain(childArgs, childClasspath, sysProps, childMainClass, args.verbose)    }  }

如上图，对环境变量准备和main函数指定后，运行runmain函数进行反射调用。

private def runMain(    childArgs: Seq[String],    childClasspath: Seq[String],    sysProps: Map[String, String],    childMainClass: String,    verbose: Boolean): Unit = {  // scalastyle:off println  if (verbose) {    printStream.println(s"Main class:\n$childMainClass")    printStream.println(s"Arguments:\n${childArgs.mkString("\n")}")    // sysProps may contain sensitive information, so redact before printing    printStream.println(s"System properties:\n${Utils.redact(sysProps).mkString("\n")}")    printStream.println(s"Classpath elements:\n${childClasspath.mkString("\n")}")    printStream.println("\n")  }  // scalastyle:on println  val loader =    if (sysProps.getOrElse("spark.driver.userClassPathFirst", "false").toBoolean) {      new ChildFirstURLClassLoader(new Array[URL](0),        Thread.currentThread.getContextClassLoader)    } else {      new MutableURLClassLoader(new Array[URL](0),        Thread.currentThread.getContextClassLoader)    }  Thread.currentThread.setContextClassLoader(loader)  for (jar <- childClasspath) {    // 添加jar依赖    addJarToClasspath(jar, loader)  }  for ((key, value) <- sysProps) {    System.setProperty(key, value)  }  var mainClass: Class[_] = null  try {    // 通过反射获得main函数的class    mainClass = Utils.classForName(childMainClass)  } catch {    case e: ClassNotFoundException =>      e.printStackTrace(printStream)      if (childMainClass.contains("thriftserver")) {        // scalastyle:off println        printStream.println(s"Failed to load main class $childMainClass.")        printStream.println("You need to build Spark with -Phive and -Phive-thriftserver.")        // scalastyle:on println      }      System.exit(CLASS_NOT_FOUND_EXIT_STATUS)    case e: NoClassDefFoundError =>      e.printStackTrace(printStream)      if (e.getMessage.contains("org/apache/hadoop/hive")) {        // scalastyle:off println        printStream.println(s"Failed to load hive class.")        printStream.println("You need to build Spark with -Phive and -Phive-thriftserver.")        // scalastyle:on println      }      System.exit(CLASS_NOT_FOUND_EXIT_STATUS)  }  // SPARK-4170  if (classOf[scala.App].isAssignableFrom(mainClass)) {    printWarning("Subclasses of scala.App may not work correctly. Use a main() method instead.")  }  // 获取mainclass的main函数的方法  val mainMethod = mainClass.getMethod("main", new Array[String](0).getClass    //main函数必须要是静态的方法，进行检查  if (!Modifier.isStatic(mainMethod.getModifiers)) {    throw new IllegalStateException("The main method in the given main class must be static")  }  @tailrec  def findCause(t: Throwable): Throwable = t match {    case e: UndeclaredThrowableException =>      if (e.getCause() != null) findCause(e.getCause()) else e    case e: InvocationTargetException =>      if (e.getCause() != null) findCause(e.getCause()) else e    case e: Throwable =>      e  }  try {    // 调用main函数进行执行    mainMethod.invoke(null, childArgs.toArray)  } catch {    case t: Throwable =>      findCause(t) match {        case SparkUserAppException(exitCode) =>          System.exit(exitCode)        case t: Throwable =>          throw t      }  }}

目前就可以开始执行用户的提交的main的函数的程序了，进行sparkSession的创建，以及后面rdd的执行等待。

值得注意的是spark-shell模式 其实是org.apache.spark.repl.Main进行相关的创建。

同时，client和cluster的模式 会有所不同。