spark学习（五）——分区数据

#!/usr/bin/python
# -*- coding: UTF-8 -*-
from pyspark import SparkContext, SparkConf


def main():
    logFile = "/user/root/spark/sparkstudy03.txt"
    master = 'yarn-client'
    appName = 'Simple App spark study03'
    conf = SparkConf().setAppName(appName).setMaster(master)
    sc = SparkContext(conf=conf)
    logData = sc.textFile(logFile)
    logData.cache()
    print("partitions is :%s" % logData.getNumPartitions())
    logList = logData.glom().collect()
    print("glom result is :%s" % logList)
    logDataPartition4 = logData.repartition(4)
    print("repartition num :%s", logDataPartition4.getNumPartitions())
    print("repartition result :%s", logDataPartition4.glom().collect())
    logListCoalesce = logDataPartition4.coalesce(3, True).glom().collect()
    print("coalesce shuffle true result is :%s" % logListCoalesce)
    logListCoalesce2 = logDataPartition4.coalesce(3, False).glom().collect()
    print("coalesce shuffle false result is :%s" % logListCoalesce2)
    logData.unpersist()


if __name__ == '__main__':
    main()

1
2
3
4
5
6
7

partitions is :2
glom result is :[[u'1', u'2', u'3', u'4'], [u'5', u'6', u'7']]
('repartition num :%s', 4)
('repartition result :%s', [[u'2', u'6'], [u'3', u'7'], [u'4'], [u'1', u'5']])
coalesce shuffle true result is :[[u'1'], [u'2', u'3', u'5'], [u'6', u'7', u'4']]
coalesce shuffle false result is :[[u'2', u'6'], [u'3', u'7'], [u'4', u'1', u'5']]