提交任务到集群

import org.apache.spark.{SparkConf, SparkContext}
object WordCount{
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("wordcount")
    val sc = new SparkContext(conf)
    // 接收文件参数
    val input=sc.textFile(args(0))
    // flatMap展平返回list
    val lines=input.flatMap(x=>x.split("[ ,.'?/\\|><:;\"-+_=()*&^%$#@!`~]+"))
    val count=lines.map(word=>(word,1)).reduceByKey{(x,y)=>x+y}
    // 保存到目录
    val output=count.saveAsTextFile(args(1))
  }
}

scp /opt/spark-2.2.0-bin-hadoop2.7 dc2-user@116.85.9.118:

spark-submit --master spark://114.55.246.88:7077 --class \
WordCount  wordcount.jar  \
hdfs://Master:9000/Hadoop/Input/Jane.txt  \
hdfs://Master:9000/Hadoop/Output