问火花远程开发环境

EN

    String appName = "test" + new Date(System.currentTimeMillis());
    String master = "spark://*:6066";
    String host = "*";
    String jar = "C:\\Users\\default.DESKTOP-0BP338U\\Desktop\\workspace\\workspace_study\\spark-start-on-yarn\\target\\spark-start-on-yarn-0.0.1-SNAPSHOT.jar";

    SparkConf conf = new SparkConf().setAppName(appName).setMaster(master)
            .set("spark.driver.host",  host)
            .setJars(new String[]{jar});
    JavaSparkContext sc = new JavaSparkContext(conf);

    SparkConf conf = new SparkConf()
            .setAppName(new Date(System.currentTimeMillis()).toString())
            .setMaster("yarn");
    JavaSparkContext sc = new JavaSparkContext(conf);


    List<Integer> data = Arrays.asList(1,2,3,4,1,2,3,4,5,1,4,1,1,1,4,2,2,4,1,1,3,4,2,3);
    JavaRDD<Integer> distData = sc.parallelize(data);

    JavaPairRDD<Integer, Integer> pairs = distData.mapToPair(s -> new Tuple2<Integer, Integer>(s, 1));
    JavaPairRDD<Integer, Integer> counts = pairs.reduceByKey((a, b) -> a + b);

    System.out.println("================= " + counts);

    sc.close();
    sc.stop();