spark 2.0 NettyStreamManager -- StreamManager的实现

来源：互联网发布：unity3d 寻路算法编辑：程序博客网时间：2024/05/20 23:34

NettyStreamManager是StreamManager的一种实现，它从NettyRpcEnv中提供文件服务。

这个管理器可以有三种资源，背后都是物理文件。

第一种：“/files”: 文件列表，为SparkContext.addFile提供支持。

第二种：“/jars”: 一个文件列表，为SparkContexst.addJar提供支持。

第三种：任意目录，在这个目录下的所有的文件都会变得可用。

这个管理器只支持openStream操作。

/** * StreamManager implementation for serving files from a NettyRpcEnv. * * Three kinds of resources can be registered in this manager, all backed by actual files: * * - "/files": a flat list of files; used as the backend for [[SparkContext.addFile]]. * - "/jars": a flat list of files; used as the backend for [[SparkContext.addJar]]. * - arbitrary directories; all files under the directory become available through the manager, *   respecting the directory's hierarchy. * * Only streaming (openStream) is supported. */private[netty] class NettyStreamManager(rpcEnv: NettyRpcEnv)  extends StreamManager with RpcEnvFileServer {  private val files = new ConcurrentHashMap[String, File]()  private val jars = new ConcurrentHashMap[String, File]()  private val dirs = new ConcurrentHashMap[String, File]()  override def getChunk(streamId: Long, chunkIndex: Int): ManagedBuffer = {    throw new UnsupportedOperationException()  }  override def openStream(streamId: String): ManagedBuffer = {    val Array(ftype, fname) = streamId.stripPrefix("/").split("/", 2)    val file = ftype match {      case "files" => files.get(fname)      case "jars" => jars.get(fname)      case other =>        val dir = dirs.get(ftype)        require(dir != null, s"Invalid stream URI: $ftype not found.")        new File(dir, fname)    }    if (file != null && file.isFile()) {      new FileSegmentManagedBuffer(rpcEnv.transportConf, file, 0, file.length())    } else {      null    }  }  override def addFile(file: File): String = {    val existingPath = files.putIfAbsent(file.getName, file)    require(existingPath == null || existingPath == file,      s"File ${file.getName} was already registered with a different path " +        s"(old path = $existingPath, new path = $file")    s"${rpcEnv.address.toSparkURL}/files/${Utils.encodeFileNameToURIRawPath(file.getName())}"  }  override def addJar(file: File): String = {    val existingPath = jars.putIfAbsent(file.getName, file)    require(existingPath == null || existingPath == file,      s"File ${file.getName} was already registered with a different path " +        s"(old path = $existingPath, new path = $file")    s"${rpcEnv.address.toSparkURL}/jars/${Utils.encodeFileNameToURIRawPath(file.getName())}"  }  override def addDirectory(baseUri: String, path: File): String = {    val fixedBaseUri = validateDirectoryUri(baseUri)    require(dirs.putIfAbsent(fixedBaseUri.stripPrefix("/"), path) == null,      s"URI '$fixedBaseUri' already registered.")    s"${rpcEnv.address.toSparkURL}$fixedBaseUri"  }}

1 0