spark 2.1 StorageLevel

class StorageLevel

class StorageLevel private(    private var _useDisk: Boolean,    private var _useMemory: Boolean,    private var _useOffHeap: Boolean,    private var _deserialized: Boolean,    private var _replication: Int = 1)  extends Externalizable {  // TODO: Also add fields for caching priority, dataset ID, and flushing.  private def this(flags: Int, replication: Int) {    this((flags & 8) != 0, (flags & 4) != 0, (flags & 2) != 0, (flags & 1) != 0, replication)  }  def this() = this(false, true, false, false)  // For deserialization  def useDisk: Boolean = _useDisk  def useMemory: Boolean = _useMemory  def useOffHeap: Boolean = _useOffHeap  def deserialized: Boolean = _deserialized  def replication: Int = _replication  assert(replication < 40, "Replication restricted to be less than 40 for calculating hash codes")  if (useOffHeap) {    require(!deserialized, "Off-heap storage level does not support deserialized storage")  }  private[spark] def memoryMode: MemoryMode = {    if (useOffHeap) MemoryMode.OFF_HEAP    else MemoryMode.ON_HEAP  }  override def clone(): StorageLevel = {    new StorageLevel(useDisk, useMemory, useOffHeap, deserialized, replication)  }  override def equals(other: Any): Boolean = other match {    case s: StorageLevel =>      s.useDisk == useDisk &&      s.useMemory == useMemory &&      s.useOffHeap == useOffHeap &&      s.deserialized == deserialized &&      s.replication == replication    case _ =>      false  }  def isValid: Boolean = (useMemory || useDisk) && (replication > 0)  def toInt: Int = {    var ret = 0    if (_useDisk) {      ret |= 8    }    if (_useMemory) {      ret |= 4    }    if (_useOffHeap) {      ret |= 2    }    if (_deserialized) {      ret |= 1    }    ret  }  override def writeExternal(out: ObjectOutput): Unit = Utils.tryOrIOException {    out.writeByte(toInt)    out.writeByte(_replication)  }  override def readExternal(in: ObjectInput): Unit = Utils.tryOrIOException {    val flags = in.readByte()    _useDisk = (flags & 8) != 0    _useMemory = (flags & 4) != 0    _useOffHeap = (flags & 2) != 0    _deserialized = (flags & 1) != 0    _replication = in.readByte()  }  @throws(classOf[IOException])  private def readResolve(): Object = StorageLevel.getCachedStorageLevel(this)  override def toString: String = {    val disk = if (useDisk) "disk" else ""    val memory = if (useMemory) "memory" else ""    val heap = if (useOffHeap) "offheap" else ""    val deserialize = if (deserialized) "deserialized" else ""    val output =      Seq(disk, memory, heap, deserialize, s"$replication replicas").filter(_.nonEmpty)    s"StorageLevel(${output.mkString(", ")})"  }  override def hashCode(): Int = toInt * 41 + replication  def description: String = {    var result = ""    result += (if (useDisk) "Disk " else "")    if (useMemory) {      result += (if (useOffHeap) "Memory (off heap) " else "Memory ")    }    result += (if (deserialized) "Deserialized " else "Serialized ")    result += s"${replication}x Replicated"    result  }}

object StorageLevel

/** * Various [[org.apache.spark.storage.StorageLevel]] defined and utility functions for creating * new storage levels. */object StorageLevel {  val NONE = new StorageLevel(false, false, false, false)  val DISK_ONLY = new StorageLevel(true, false, false, false)  val DISK_ONLY_2 = new StorageLevel(true, false, false, false, 2)  val MEMORY_ONLY = new StorageLevel(false, true, false, true)  val MEMORY_ONLY_2 = new StorageLevel(false, true, false, true, 2)  val MEMORY_ONLY_SER = new StorageLevel(false, true, false, false)  val MEMORY_ONLY_SER_2 = new StorageLevel(false, true, false, false, 2)  val MEMORY_AND_DISK = new StorageLevel(true, true, false, true)  val MEMORY_AND_DISK_2 = new StorageLevel(true, true, false, true, 2)  val MEMORY_AND_DISK_SER = new StorageLevel(true, true, false, false)  val MEMORY_AND_DISK_SER_2 = new StorageLevel(true, true, false, false, 2)  val OFF_HEAP = new StorageLevel(true, true, true, false, 1)  /**   * :: DeveloperApi ::   * Return the StorageLevel object with the specified name.   */  @DeveloperApi  def fromString(s: String): StorageLevel = s match {    case "NONE" => NONE    case "DISK_ONLY" => DISK_ONLY    case "DISK_ONLY_2" => DISK_ONLY_2    case "MEMORY_ONLY" => MEMORY_ONLY    case "MEMORY_ONLY_2" => MEMORY_ONLY_2    case "MEMORY_ONLY_SER" => MEMORY_ONLY_SER    case "MEMORY_ONLY_SER_2" => MEMORY_ONLY_SER_2    case "MEMORY_AND_DISK" => MEMORY_AND_DISK    case "MEMORY_AND_DISK_2" => MEMORY_AND_DISK_2    case "MEMORY_AND_DISK_SER" => MEMORY_AND_DISK_SER    case "MEMORY_AND_DISK_SER_2" => MEMORY_AND_DISK_SER_2    case "OFF_HEAP" => OFF_HEAP    case _ => throw new IllegalArgumentException(s"Invalid StorageLevel: $s")  }  /**   * :: DeveloperApi ::   * Create a new StorageLevel object.   */  @DeveloperApi  def apply(      useDisk: Boolean,      useMemory: Boolean,      useOffHeap: Boolean,      deserialized: Boolean,      replication: Int): StorageLevel = {    getCachedStorageLevel(      new StorageLevel(useDisk, useMemory, useOffHeap, deserialized, replication))  }  /**   * :: DeveloperApi ::   * Create a new StorageLevel object without setting useOffHeap.   */  @DeveloperApi  def apply(      useDisk: Boolean,      useMemory: Boolean,      deserialized: Boolean,      replication: Int = 1): StorageLevel = {    getCachedStorageLevel(new StorageLevel(useDisk, useMemory, false, deserialized, replication))  }  /**   * :: DeveloperApi ::   * Create a new StorageLevel object from its integer representation.   */  @DeveloperApi  def apply(flags: Int, replication: Int): StorageLevel = {    getCachedStorageLevel(new StorageLevel(flags, replication))  }  /**   * :: DeveloperApi ::   * Read StorageLevel object from ObjectInput stream.   */  @DeveloperApi  def apply(in: ObjectInput): StorageLevel = {    val obj = new StorageLevel()    obj.readExternal(in)    getCachedStorageLevel(obj)  }  private[spark] val storageLevelCache = new ConcurrentHashMap[StorageLevel, StorageLevel]()  private[spark] def getCachedStorageLevel(level: StorageLevel): StorageLevel = {    storageLevelCache.putIfAbsent(level, level)    storageLevelCache.get(level)  }}