pyspark.sql.DataFrame与pandas.DataFrame之间的相互转换

代码如下，步骤流程在代码注释中可见：

# -*- coding: utf-8 -*-import pandas as pdfrom pyspark.sql import SparkSessionfrom pyspark.sql import SQLContextfrom pyspark import SparkContext#初始化数据#初始化pandas DataFramedf = pd.DataFrame([[1, 2, 3], [4, 5, 6]], index=['row1', 'row2'], columns=['c1', 'c2', 'c3'])#打印数据print df#初始化spark DataFramesc = SparkContext()if __name__ == "__main__":    spark = SparkSession\        .builder\        .appName("testDataFrame")\        .getOrCreate()sentenceData = spark.createDataFrame([    (0.0, "I like Spark"),    (1.0, "Pandas is useful"),    (2.0, "They are coded by Python ")], ["label", "sentence"])#显示数据sentenceData.select("label").show()#spark.DataFrame 转换成 pandas.DataFramesqlContest = SQLContext(sc)spark_df = sqlContest.createDataFrame(df)#显示数据spark_df.select("c1").show()# pandas.DataFrame 转换成 spark.DataFramepandas_df = sentenceData.toPandas()#打印数据print pandas_df

程序结果