Spark--python开发实例

来源：互联网发布：张宇唱功知乎编辑：程序博客网时间：2024/04/29 07:40

-spark python开发---------------
cond.py
def isFirstMinute(line) :
return line.split('\t')[0] < '00:01:00'

---------------
sort.py
from pyspark import SparkContext

sc = SparkContext("spark://server1:8888", "Python Sort", pyFiles=['cond.py'])
data = sc.textFile("hdfs://server1:9000/user/cc/reduced/")

print data.filter(lambda line : len(line.split('\t')) == 5).map(lambda line : (line.split('\t')[1],1)).reduceByKey(lambda x , y : x + y ).map(lambda pair : (pair[1],pair[0])).sortByKey(False).map(lambda pair : (pair[1],pair[0])).take(10) #.saveAsTextFile("hdfs://server1:9000/result")
------------------
wc.py
from pyspark import SparkContext
from cond import isFirstMinute

sc = SparkContext("spark://server1:8888", "Python Analysis", pyFiles=['cond.py'])
data = sc.textFile("hdfs://server1:9000/user/cc/reduced/")
#fltData = data.filter(lambda line : line.split('\t')[0] < '00:01:00')
fltData = data.filter(lambda line : isFirstMinute(line))

print 'first minute : ' + str(fltData.count())
-----------------------------------------

0 0