pyspark写法总结

往hive表中插入数据

法1：dataFrame数据写入hive表

def log2Hive():

log=hiveContext.createDataFrame([{"dt":dt,"types":types,"message":msg,"currtime":currTime}]).coalesce(1)#types: "INFO" ,"ERROR"

log.write.mode("append").insertInto("app.app_tion_log")

 

法2：dataFrame数据写入hive表

temp_vl_org=self.vl_data.join(self.order_slice,["po_no"]).coalesce(10)

os.system("hadoop fs -rm -r -skipTrash dev.db/" + str.lower(temp_vl))

hiveContext.sql("drop table if exists dev."+temp_vl)

temp_vlt_org.write.saveAsTable("dev."+temp_vl)

hiveContext写法

1.

order_slice= hiveContext.table("app.app_ rage").coalesce(10).where(col("dt")==self.order_new_dt).select("po_no","goods_no",col("distribution_no").alias("dc_id")).distinct().coalesce(10)

 

2.

log =hiveContext.createDataFrame([{"dt":dt,"types":types,"message":msg,"currtime":currTime}]).coalesce(1) #types:"INFO" ,"ERROR"

3.

hiveContext.sql("showpartitions app.app_ rage").select(func.max("partition").alias("partition")).rdd.map(lambda x:str(x[0])).take(1)[0]

4.dataFrame数据写入hive表

temp_vl_org=self.vl_data.join(self.order_slice,["po_no"]).coalesce(10)

os.system("hadoop fs -rm -r -skipTrash dev.db/" +str.lower(temp_vl))

hiveContext.sql("drop table if exists dev."+temp_vl)

temp_vlt_org.write.saveAsTable("dev."+temp_vl)

python 中执行hadoop命令

#删除表

os.system("hadoop fs -rm -r -skipTrashdev.db/" + str.lower(temp_vl))

 

#从集市1路径localFolde拷贝数据到集市2Hive表分区数据tbFolder

 

localFolder="app.db/app_filter /"

partitonName="dt="+self.table_new_dt

tbFolder= "hdfs://102.1.1.1:8080/user/cm_pc/app.db/app_filter/"

os.system("hadoop fs -rm -r -skipTrash"+tbFolder+partitonName)

os.system("hadoop distcp"+localFolder+partitonName+" "+tbFolder+partitonName)

# localFolder+partitonName为源路径，tbFolder+partitonName为目标路径

dataFrame和rdd互转,存入Hive表

#以下返回dataFrame类型值

df = hc.sql(sql).coalesce(5)

 

#dataFrame转rdd

r2 = df.rdd.map(lambda row :((row[0],row[1]),(row[2],row[3],row[4],row[5]))).groupByKey().map(lambda(k, v): sub_process(k, v))

cond=["rowkey","top_10","dt"]

 

#rdd转dataFrame

result=r2.toDF(cond)

 

#dataframe存入hive表

result.write.mode("append").insertInto(table_name)

 

创建dataFrame方法

法1：

#rdd转dataFrame

result=r2.toDF(cond)

法2：

order_slice= hiveContext.table("app.apprage").coalesce(10).where(col("dt")==self.ord_new_dt).select("po_no","goods_no",col("distribution_no").alias("dc_id")).distinct().coalesce(10)

 

法3：

log =hiveContext.createDataFrame([{"dt":dt,"types":types,"message":msg,"currtime":currTime}]).coalesce(1) #types:"INFO" ,"ERROR"

法4：

hiveContext.sql("showpartitions app.apprage")

 

法5：

从hdfs中读取数据变成dataFrame

input_path为hdfs路径

textFile=sc.textFile(input_path).map(lambdax: x.split("\x01"))

df = textFile.toDF(["seller_no","seller_name", "dept_no", "goods_no","shop_id", "sp_goods_no", "dc_id","erp_warehouse_no","real_outstore_qty", "sp_create_time"]).groupby(["seller_no","seller_name", "dept_no", "goods_no","dc_id", "erp_warehouse_no","sp_create_time"]).agg(func.sum("real_outstore_qty").alias("real_outstore_qty")).where((col("sp_create_time")< endDate) & (col("sp_create_time") >"2015-01-01"))

pyspark when

dataDf = hiveContext.table(self.table_name).where((col("dt")==self.table_new_dt)& (col("so_status")<> "10028") &(col("so_status")<> "10009"))\

           .select("seller_no","seller_name","dept_no",

          "goods_no",col("shop_id").cast("int"),F.when(func.isnull("sp_goods_no"),-1).otherwise(col("sp_goods_no").cast("int")).alias("sp_goods_no"),

          col("distribution_no").cast("int").alias("dc_id"),col("erp_warehouse_no").cast("int"),

          col("sp_create_time").cast("date"),F.when(func.isnull("apply_out_qty"),1).otherwise(col("apply_out_qty")).alias("apply_qty"),).coalesce(10)

 

 

dataFrame添加列

法1,2

self.groupedDf=dataDf.groupBy("seller_no","dept_no","goods_no","shop_id","sp_goods_no","dc_id","erp_warehouse_no","sp_create_time")\

            #添加两列

           .agg(func.max("seller_name").alias("seller_name"),func.sum("apply_outstore_qty").alias("apply_outstore_qty"))\

           .withColumn("dt",lit(self.table_new_dt))

 

 

判断是否执行成功

if __name__ == "__main__":

    main()

    r = os.system("hadoop fs -test-e /tmp/for/result/e_cast/_SUCCESS")

    if r != 0:

        raise Exception("1")

    else:

        print ("执行spark获取eclp预测数据成功！！")

 

从hdfs导入数据到hive

def push2hive(self):

        hiveContext.sql("LOAD DATA IN PATH '/tmp/for/result/ec_forecast' OVERWRITE INTO TABLE app.app_ ver1 PARTITION( dt  = '"+ _today+"')")

 

读取全路径下的Hive表

hdfs_path_ver1 ="/user/cmo_ipc/app.db/app_ ver1/dt=" + _today

   

df1=sc.textFile(hdfs_path_ver1).map(lambda x:x.split("\t")).map(lambdaline:((line[0],line[1],line[2],line[3]),line[8])).groupByKey().map(lambda(k,v): sales_nation_ver1(k,v))