spark2.0 Java序列化和Kyro序列化测试

来源：互联网发布：五线谱转吉他谱软件编辑：程序博客网时间：2024/05/30 23:04

先列出测试结果，后面附有测试代码，可以发现，改成Kyro序列化之后，可以节约大量空间。

JavaSerialKyroSerialInt812empty string73string with 1 character84string with 2 character94string with 10 character1712string with 20 character2722

测试代码

import org.apache.spark._import java.nio.ByteBufferimport org.apache.spark.serializer._;object TestDataSerialize {  def main(args: Array[String]) = {    // testKyro();    testJavaSerial()    testKyroSerial()  }    def testSerial(serialInst: SerializerInstance) = {    println("test IntObject")    for (i <- 1 to 1) {      val byteBuf = serialInst.serialize(i)      println(byteBuf.limit())    }    println("test empty string")    for (i <- 1 to 1) {      val byteBuf = serialInst.serialize("")      println(byteBuf.limit())    }    println("test  string with 1 character")    for (i <- 1 to 1) {      val byteBuf = serialInst.serialize("a")      println(byteBuf.limit())    }    println("test  string with 2 character")    for (i <- 1 to 1) {      val byteBuf = serialInst.serialize("ab")      println(byteBuf.limit())    }    println("test  string with 10 character")    for (i <- 1 to 1) {      val byteBuf = serialInst.serialize("abcdefghij")      println(byteBuf.limit())    }    println("test  string with 20 character")    for (i <- 1 to 1) {      val byteBuf = serialInst.serialize("abcdefghijabcdefghij")      println(byteBuf.limit())    }    println("test SerObject")    for (i <- 1 to 1) {      val byteBuf = serialInst.serialize(new SerObject(i, "testname", i + 1))      println(byteBuf.limit())    }    println("\n\n")  }    def testKyroSerial() = {    println("testKyroSerial")    val conf = new SparkConf().setAppName("spark util test").setMaster("local[2]");    val kryoSerializer = new KryoSerializer(conf)    val serialInst = kryoSerializer.newInstance()    testSerial(serialInst)  }  def testJavaSerial() = {    println("testJavaSerial")    val conf = new SparkConf().setAppName("spark util test").setMaster("local[2]");    val javaSerializer = new org.apache.spark.serializer.JavaSerializer(conf)    val serialInst = javaSerializer.newInstance()    testSerial(serialInst)  }}

0 0