spark | 做基本计算和批量提交

#以我的为例，我的用户名是hadoop
su hadoop 
#这里会提示输入密码
＃切换到spark目录下
＃cd /home/hadoop/spark

./bin/pyspark

# -*- coding:utf-8 -*- 
from pyspark import SparkContext

＃定义SparkContext对象,2个线程，命名为First Spark App
sc = SparkContext("local[2]", "First Spark App")
＃读数据，放在spark/data下
data = sc.textFile("data/UserPurchaseHistory.csv").map(lambda line: line.split(",")).map(lambda record: (record[0], record[1], record[2]))
＃计算总购买次数
numPurchases = data.count()
＃计算有多少不同客户购买过商品
uniqueUsers = data.map(lambda record: record[0]).distinct().count()
＃计算总收入
totalRevenue = data.map(lambda record: float(record[2])).sum()
＃计算最畅销的产品
products = data.map(lambda record: (record[1], 1.0)).reduceByKey(lambda a, b: a + b).collect()
mostPopular = sorted(products, key=lambda x: x[1], reverse=True)[0]

＃结果打印出来
print ("Total purchases: %d" % numPurchases)
print ("Unique users: %d" % uniqueUsers)
print ("Total revenue: %2.2f" % totalRevenue)
print ("Most popular product: %s with %d purchases" % (mostPopular[0], mostPopular[1]))

sc.stop()

/home/hadoop/spark/bin/spark-submit pythonapp.py