MapReduce 编程不可怕，一篇文章搞定它

全栈程序员站长

发布于 2022-07-02 10:53:58

1980

发布于 2022-07-02 10:53:58

文章被收录于专栏：全栈程序员必看全栈程序员必看

大家好，又见面了，我是你们的朋友全栈君。

前言

本文隶属于专栏《1000个问题搞定大数据技术体系》，该专栏为笔者原创，引用请注明来源，不足和错误之处请在评论区帮忙指出，谢谢！

本专栏目录结构和参考文献请见1000个问题搞定大数据技术体系

正文

需求： Word Count，大数据领域的 Hello World。

Mapper

package com.shockang.study.bigdata.mapreduce;

import java.io.IOException;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class WordCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> { 
   
    protected void map(LongWritable key, Text value, Context context)
            throws IOException, InterruptedException { 
   
        String[] words = value.toString().split(" ");
        for (String word : words) { 
   
            // 每个单词出现１次，作为中间结果输出
            context.write(new Text(word), new IntWritable(1));
        }
    }
}

Reducer

package com.shockang.study.bigdata.mapreduce;

import java.io.IOException;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class WordCountReducer extends Reducer<Text, IntWritable, Text, IntWritable> { 
   
    /* key: hello value: List(1, 1, ...) */
    protected void reduce(Text key, Iterable<IntWritable> values,
                          Context context) throws IOException, InterruptedException { 
   
        int sum = 0;

        for (IntWritable count : values) { 
   
            sum = sum + count.get();
        }
        context.write(key, new IntWritable(sum));
    };
}

Main

package com.shockang.study.bigdata.mapreduce;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

public class WordCountMain { 
   
    public static void main(String[] args) throws IOException,
            ClassNotFoundException, InterruptedException { 
   
        if (args.length != 2) { 
   
            System.out.println("please input Path!");
            System.exit(0);
        }

        Configuration configuration = new Configuration();
        Job job = Job.getInstance(configuration, WordCountMain.class.getSimpleName());

        // 打jar包
        job.setJarByClass(WordCountMain.class);

        // 通过job设置输入/输出格式，默认的就是 TextInputFormat/TextOutputFormat
        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        // 设置输入/输出路径
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        // 设置处理Map/Reduce阶段的类
        job.setMapperClass(WordCountMapper.class);
        job.setReducerClass(WordCountReducer.class);
        //如果map、reduce的输出的kv对类型一致，直接设置reduce的输出的kv对就行
        //如果不一样，需要分别设置map, reduce的输出的kv类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        // 设置最终输出key/value的类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        // 提交作业
        job.waitForCompletion(true);

    }

}

Combiner

package com.shockang.study.bigdata.mapreduce;


import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;


public class WordCountMainWithCombiner { 
   
    public static void main(String[] args) throws IOException,
            ClassNotFoundException, InterruptedException { 
   
        if (args.length != 2) { 
   
            System.out.println("please input Path!");
            System.exit(0);
        }

        Configuration configuration = new Configuration();
        Job job = Job.getInstance(configuration, WordCountMainWithCombiner.class.getSimpleName());

        // 打jar包
        job.setJarByClass(WordCountMainWithCombiner.class);

        // 通过job设置输入/输出格式
        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        // 设置输入/输出路径
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        // 设置处理Map/Reduce阶段的类
        job.setMapperClass(WordCountMapper.class);
        job.setCombinerClass(WordCountReducer.class);
        job.setReducerClass(WordCountReducer.class);
        //如果map、reduce的输出的kv对类型一致，直接设置reduce的输出的kv对就行
        // 如果不一样，需要分别设置map, reduce的输出的kv类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        // 设置最终输出key/value的类型m
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        // 提交作业
        job.waitForCompletion(true);

    }
}

二次排序

package com.shockang.study.bigdata.mapreduce;

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/** * 现有需求，需要自定义key类型，并自定义key的排序规则，如按照人的salary降序排序，若相同，则再按age升序排序 */
public class Person implements WritableComparable<Person> { 
   
    private String name;
    private int age;
    private int salary;

    public Person() { 
   
    }

    public Person(String name, int age, int salary) { 
   
        this.name = name;
        this.age = age;
        this.salary = salary;
    }

    public String getName() { 
   
        return name;
    }

    public void setName(String name) { 
   
        this.name = name;
    }

    public int getAge() { 
   
        return age;
    }

    public void setAge(int age) { 
   
        this.age = age;
    }

    public int getSalary() { 
   
        return salary;
    }

    public void setSalary(int salary) { 
   
        this.salary = salary;
    }

    @Override
    public String toString() { 
   
        return this.salary + " " + this.age + " " + this.name;
    }

    public int compareTo(Person o) { 
   
        //先比较salary，高的排序在前；若相同，age小的在前
        int compareResult1 = this.salary - o.salary;
        if (compareResult1 != 0) { 
   
            return -compareResult1;
        } else { 
   
            return this.age - o.age;
        }
    }

    public void write(DataOutput dataOutput) throws IOException { 
   
        //序列化，将NewKey转化成使用流传输的二进制
        dataOutput.writeUTF(name);
        dataOutput.writeInt(age);
        dataOutput.writeInt(salary);
    }

    public void readFields(DataInput dataInput) throws IOException { 
   
        //使用in读字段的顺序，要与write方法中写的顺序保持一致
        this.name = dataInput.readUTF();
        this.age = dataInput.readInt();
        this.salary = dataInput.readInt();
    }
}

自定义分区

package com.shockang.study.bigdata.mapreduce;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;

import java.util.HashMap;

public class CustomPartitioner extends Partitioner<Text, IntWritable> { 
   
    public static HashMap<String, Integer> dict = new HashMap<>();

    static { 
   
        dict.put("Dear", 0);
        dict.put("Bear", 1);
        dict.put("River", 2);
        dict.put("Car", 3);
    }

    public int getPartition(Text text, IntWritable intWritable, int i) { 
   
        return dict.get(text.toString());
    }
}

数据倾斜处理

当遇到数据倾斜的时候，我们可以在 Reducer 中日志记录哪些超过阈值的 key

package com.shockang.study.bigdata.mapreduce;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;
import java.util.Iterator;

public class WordCountReducerWithDataSkew extends Reducer<Text, IntWritable, Text, IntWritable> { 
   

    public static final String MAX_VALUES = "skew.maxvalues";
    private int maxValueThreshold;

    @Override
    protected void setup(Context context) throws IOException, InterruptedException { 
   
        Configuration conf = context.getConfiguration();
        maxValueThreshold = Integer.parseInt(conf.get(MAX_VALUES));
    }

    /* key: hello value: List(1, 1, ...) */
    protected void reduce(Text key, Iterable<IntWritable> values,
                          Context context) throws IOException, InterruptedException { 
   
        int i = 0;
        for (IntWritable value : values) { 
   
            System.out.println(value);
            i++;
        }

        if (++i > maxValueThreshold) { 
   
            System.out.println("Received " + i + " values for key " + key);
        }
    }
}

发布者：全栈程序员栈长，转载请注明出处：https://javaforall.cn/147546.html原文链接：https://javaforall.cn

本文参与腾讯云自媒体同步曝光计划，分享自作者个人站点/博客。

如有侵权请联系 cloudcommunity@tencent.com 删除

大数据