Spark入门_2_LoadSaveData

hdfs://namenodehost/parent/child
hdfs://parent/child
file://parent/child
sc.textFile("hdfs://host:port_no/data/searches")

#读单个数据
input = sc.textFile("file:///home/holden/repos/spark/README.md")
input = sc.textFile("README.md")
input3 = sc.textFile("hdfs://Master:50070/test/sample.txt")
#主机名和端口号在hadoop的core-site.xml中查看

#读目录数据
input = sc.wholeTextFile("file:///home/holden/repos/spark/")

#写数据
result.saveAsTextFile(outputFile)

import json
data = input.map(lambda x: json.loads(x))

(data.filter(lambda x: x['lovesPandas']).map(lambda x: json.dumps(x))
.saveAsTextFile(outputFile))

import csv
import StringIO
...
def loadRecord(line):
    """Parse a CSV line"""
    input = StringIO.StringIO(line)
    reader = csv.DictReader(input, fieldnames=["name", "favouriteAnimal"])
    return reader.next()
input = sc.textFile(inputFile).map(loadRecord)

def loadRecords(fileNameContents):
    """Load all the records in a given file"""
    input = StringIO.StringIO(fileNameContents[1])
    reader = csv.DictReader(input, fieldnames=["name", "favoriteAnimal"])
    return reader
fullFileData = sc.wholeTextFiles(inputFile).flatMap(loadRecords)

def writeRecords(records):
    """Write out CSV lines"""
    output = StringIO.StringIO()
    writer = csv.DictWriter(output, fieldnames=["name", "favoriteAnimal"])
    for record in records:
        writer.writerow(record)
    return [output.getvalue()]
pandaLovers.mapPartitions(writeRecords).saveAsTextFile(outputFile)

val rdd = sc.textFile("file:///home/holden/happypandas.gz")

 hdfs://master:port/path