Spark--python开发实例

-spark python开发---------------

cond.py

def isFirstMinute(line) :

return line.split('\t')[0] < '00:01:00'

---------------

sort.py

from pyspark import SparkContext

sc = SparkContext("spark://server1:8888", "Python Sort", pyFiles=['cond.py'])

data = sc.textFile("hdfs://server1:9000/user/cc/reduced/")

print data.filter(lambda line : len(line.split('\t')) == 5).map(lambda line : (line.split('\t')[1],1)).reduceByKey(lambda x , y : x + y ).map(lambda pair : (pair[1],pair[0])).sortByKey(False).map(lambda pair : (pair[1],pair[0])).take(10) #.saveAsTextFile("hdfs://server1:9000/result")

------------------

wc.py

from pyspark import SparkContext

from cond import isFirstMinute

sc = SparkContext("spark://server1:8888", "Python Analysis", pyFiles=['cond.py'])

data = sc.textFile("hdfs://server1:9000/user/cc/reduced/")

#fltData = data.filter(lambda line : line.split('\t')[0] < '00:01:00')

fltData = data.filter(lambda line : isFirstMinute(line))

print 'first minute : ' + str(fltData.count())

-----------------------------------------