pyspark.sql.DataFrame与pandas.DataFrame之间的相互转换实例

# -*- coding: utf-8 -*-
import pandas as pd
from pyspark.sql import SparkSession
from pyspark.sql import SQLContext
from pyspark import SparkContext
 
#初始化数据
 
#初始化pandas Data 
df = pd.Data ([[1, 2, 3], [4, 5, 6]], index=['row1', 'row2'], columns=['c1', 'c2', 'c3'])
 
#打印数据
print df
 
#初始化spark Data 
sc = SparkContext()
if __name__ == "__main__":
 spark = SparkSession\
  .builder\
  .appName("testData ")\
  .getOrCreate()
 
sentenceData = spark.createData ([
 (0.0, "I like Spark"),
 (1.0, "Pandas is useful"),
 (2.0, "They are coded by Python ")
], ["label", "sentence"])
 
#显示数据
sentenceData.select("label").show()
 
#spark.Data  转换成 pandas.Data 
sqlContest = SQLContext(sc)
spark_df = sqlContest.createData (df)
 
#显示数据
spark_df.select("c1").show()
 
 
# pandas.Data  转换成 spark.Data 
pandas_df = sentenceData.toPandas()
 
#打印数据
print pandas_df