Hadoop入门 WordCount案例和Echarts

原创

Emperor_LawD

修改于 2021-08-21 10:26:27

7520

修改于 2021-08-21 10:26:27

文章被收录于专栏：LawD的技术专栏LawD的技术专栏

WordCount案例

新建文件

在java文件夹下的com.syh中新建一个java文件

在WordCount.java中写入

package com.syh;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * 词频统计
 */
public class WordCountApp {
    /**
     * map 阶段
     */
    public static class MyMapper extends Mapper<LongWritable, Text, Text, LongWritable> {
        LongWritable one = new LongWritable(1);

        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            // 分
            String line = value.toString();
            // 拆分
            String[] s = line.split(" ");
            for (String word : s) {
                // 输出
                context.write(new Text(word), one);
            }
        }
    }

    /**
     * reduce 阶段
     */
    public static class MyReducer extends Reducer<Text, LongWritable, Text, LongWritable> {
        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
            long sum = 0;
            // 合并统计
            for (LongWritable value : values) {
                // 求和
                sum += value.get();
            }
            context.write(key, new LongWritable(sum));
        }
    }

    public static void main(String[] args) throws Exception {
        Configuration configuration = new Configuration();
        Job job = Job.getInstance(configuration, "wordcount");
        job.setJarByClass(WordCountApp.class);

        // 设置 map 相关参数
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        job.setMapperClass(MyMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);

        // 设置 reduce 相关参数
        job.setReducerClass(MyReducer.class);
        job.setOutputKeyClass(MyReducer.class);
        job.setOutputValueClass(LongWritable.class);

        Path outPath = new Path(args[1]);
        FileSystem fileSystem = FileSystem.get(configuration);
        if (fileSystem.exists(outPath)) {
            // 删除文件
            fileSystem.delete(outPath, true);
            System.out.println("输出路径已存在, 已被删除");
        }
        FileOutputFormat.setOutputPath(job, outPath);

        // 控制台输出详细信息
        // 输出：1  不输出：0
        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }
}

打包程序

Maven -> hadoopdemo -> Lifecycle -> package

点击package后开始打包

上传jar包

将打包好的jar拖入到虚拟机中

通过shell方式将输出文件夹删除

 hadoop fs -rm -r /output/wc

  语法：
  hadoop jar 主函数全限定名 输入 输出
  示例：
  hadoop jar hadoopdemo-1.0-SNAPSHOT.jar com.syh.WordCountApp hdfs://hadoop000:8020/WordCount.txt hdfs://hadoop000:8020/output/wc

完成作业

查看统计结果

通过shell方式查看

  hadoop fs -cat /output/wc/part-r-00000

Echarts

新建文件

创建一个resources文件夹用来存放js和html文件

bar.html（柱状图）

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Title</title>
    <script src="./js/echarts.min.js"></script>
</head>
<body>
    <!-- preparing a DOM with width and height for ECharts -->
    <div id="main" style="width:600px; height:400px;"></div>
<script>
    var main = document.getElementById('main');
    var myChart = echarts.init(main);
    // 指定图表的配置项和数据
    var option = {
        // 标题
        title: {
            text: 'ECharts 入门示例'
        },
        // 工具箱
        toolbox: {
            show: true,
            feature: {
                saveAsImage: {
                    show: true
                }
            }
        },
        // 图例
        legend: {
            data: ['销量']
        },
        // x轴（可修改）
        xAxis: {
            data: ["衬衫", "羊毛衫", "雪纺衫", "裤子", "高跟鞋", "袜子"]
        },
        yAxis: {},
        // 数据（可修改）
        series: [{
            name: '销量',
            type: 'bar', // 修改图标样式
            data: [5, 20, 36, 10, 10, 20]
        }]
    };
    myChart.setOption(option);
</script>
</body>
</html>

bar.html（线性图）

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>line</title>
    <script src="./js/echarts.min.js"></script>
</head>
<body>
<!-- preparing a DOM with width and height for ECharts -->
<div id="main" style="width:600px; height:400px;"></div>
<script>
    var main = document.getElementById('main');
    var myChart = echarts.init(main);
    // 指定图表的配置项和数据
    var option = {
        // 标题
        title: {
            text: 'ECharts 入门示例'
        },
        // 工具箱
        toolbox: {
            show: true,
            feature: {
                saveAsImage: {
                    show: true
                }
            }
        },
        // 图例
        legend: {
            data: ['销量']
        },
        // x轴（可修改）
        xAxis: {
            data: ["衬衫", "羊毛衫", "雪纺衫", "裤子", "高跟鞋", "袜子"]
        },
        yAxis: {},
        // 数据（可修改）
        series: [{
            name: '销量',
            type: 'line', // 修改图标样式
            data: [5, 20, 36, 10, 10, 20]
        }]
    };
    myChart.setOption(option);
</script>
</body>
</html>

查看结果

点击Chrome查看结果

显示结果（柱状图）

显示结果（线性图）

toolbox相关配置

toolbox.html

// 工具箱
toolbox: {
    show: true,
        feature: {
            saveAsImage: {
                show: true
            },
            dataView: { // 数据预览
                show:true
            },
            restore: { // 刷新
                show:true
            },
            dataZoom: {
                show:true
            },
            magicType: {
                type: ['line', 'bar']
        }
    }
},

效果

饼图

pie.html

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>pie</title>
    <script src="./js/echarts.min.js"></script>
</head>
<body>
<!-- preparing a DOM with width and height for ECharts -->
<div id="main" style="width:600px; height:400px;"></div>
<script>
    var main = document.getElementById('main');
    var myChart = echarts.init(main);
    // 指定图表的配置项和数据
    var option = {
        // 标题
        title: {
            text: '某网站用户访问来源数据'
        },
        // 显示百分比
        tooltip: {
            trigger: 'item',
            formatter: "{a} <br/> {b} : {c} ({d}%)"
        },
        legend: {
            orient: 'vertical',
            left: 'right',
            data: ['谷歌', '火狐', 'Safari', '360', 'QQ浏览器']
        },
        series: [
            {
                name: '访问来源',
                type: 'pie',
                radius: '55%',
                center: ['50%', '60%'],
                data: [
                    {value: 335, name: '谷歌'},
                    {value: 310, name: '火狐'},
                    {value: 234, name: 'Safari'},
                    {value: 135, name: '360'},
                    {value: 1548, name: 'QQ浏览器'}
                ]
            }
        ]
    };
    myChart.setOption(option);
</script>
</body>
</html>