问spark:对RDDs进行加扰并压缩

EN

x = sc.parallelize(['A','B','C','D'])
y = sc.parallelize(['W','X','Y','Z'])
x = x.takeSample(False, 4)
y = y.takeSample(False, 4)
combine = zip(x,y)
combine
>> [('D', 'Z'), ('B', 'X'), ('A', 'W'), ('C', 'Y')]

from pyspark.sql.functions import rand

s = lambda x: (x[1], x[0])

def shuffle(rdd):
    return rdd.map(lambda x: (x, )) \
              .toDF(["data"]).withColumn("rand", rand()) \
              .orderBy("rand") \
              .rdd.map(lambda x: x.data)

shuffle(rdd1).zipWithIndex().map(s).join(rdd2.zipWithIndex().map(s)).values()