为什么要知道Hadoop机架感知？

/**
 * 机架感知类
 */
public class MyRackAware implements DNSToSwitchMapping {
    /**
     * 根据需求，将不同的主机划分到不同的机架上
     * @param names 数据节点主机的集合
     * @return 机架感知的集合
     */
    public List<String> resolve(List<String> names) {
        List<String> list = new ArrayList<String>();
        try {
            //将原始信息输出到目录，方便查看
            FileWriter fw = new FileWriter("/home/centos/rackaware.txt");
            for (String host : names) {
                //将输入的原始host写入文件
                fw.append(host+"/r/n");
 
                //进行原始的host进行分机架
                // IP形式
                if (host.startsWith("192")) {
                    String ipEnd = host.substring(host.lastIndexOf(".") + 1);
                    if (Integer.parseInt(ipEnd) <= 103) { //s102,s103 在一个机架
                        list.add("/rack1/" + ipEnd);
                    } else {                              //s104 在一个机架
                        list.add("/rack2/" + ipEnd);
                    }
                }
                //主机名形式
                else if (host.startsWith("s")) {
                    String ipEnd = host.substring(1);
                    if (Integer.parseInt(ipEnd) <= 103) { //s102,s103 在一个机架
                        list.add("/rack1/" + ipEnd);
                    } else {                              //s104 在一个机架
                        list.add("/rack2/" + ipEnd);
                    }
                }
            }
            fw.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
 
        return list;
    }
 
    public void reloadCachedMappings() {
 
    }
 
    public void reloadCachedMappings(List<String> names) {
 
    }
}

<property>
        <name>net.topology.node.switch.mapping.impl</name>
        <value>com.fresher.hdfs.rackaware.MyRackAware</value>
</property>

#!/usr/bin/python3
# this script makes assumptions about the physical environment.
#  1) each rack is its own layer 3 network with a /24 subnet, which
# could be typical where each rack has its own
#     switch with uplinks to a central core router.
#
#             +-----------+
#             |core router|
#             +-----------+
#            /             \
#   +-----------+        +-----------+
#   |rack switch|        |rack switch|
#   +-----------+        +-----------+
#   | data node |        | data node |
#   +-----------+        +-----------+
#   | data node |        | data node |
#   +-----------+        +-----------+
#
# 2) topology script gets list of IP's as input, calculates network address, and prints '/network_address/ip'.

import netaddr
import sys
sys.argv.pop(0)                                                  # discard name of topology script from argv list as we just want IP addresses

netmask = '255.255.255.0'                                        # set netmask to what's being used in your environment.  The example uses a /24

for ip in sys.argv:                                              # loop over list of datanode IP's
    address = '{0}/{1}'.format(ip, netmask)                      # format address string so it looks like 'ip/netmask' to make netaddr work
    try:
        network_address = netaddr.IPNetwork(address).network     # calculate and print network address
        print("/{0}".format(network_address))
    except:
        print("/rack-unknown")                                   # print catch-all value if unable to calculate network address

#!/usr/bin/env bash
# Here's a bash example to show just how simple these scripts can be
# Assuming we have flat network with everything on a single switch, we can fake a rack topology.
# This could occur in a lab environment where we have limited nodes,like 2-8 physical machines on a unmanaged switch.
# This may also apply to multiple virtual machines running on the same physical hardware.
# The number of machines isn't important, but that we are trying to fake a network topology when there isn't one.
#
#       +----------+    +--------+
#       |jobtracker|    |datanode|
#       +----------+    +--------+
#              \        /
#  +--------+  +--------+  +--------+
#  |datanode|--| switch |--|datanode|
#  +--------+  +--------+  +--------+
#              /        \
#       +--------+    +--------+
#       |datanode|    |namenode|
#       +--------+    +--------+
#
# With this network topology, we are treating each host as a rack.  This is being done by taking the last octet
# in the datanode's IP and prepending it with the word '/rack-'.  The advantage for doing this is so HDFS
# can create its 'off-rack' block copy.
# 1) 'echo $@' will echo all ARGV values to xargs.
# 2) 'xargs' will enforce that we print a single argv value per line
# 3) 'awk' will split fields on dots and append the last field to the string '/rack-'. If awk
#    fails to split on four dots, it will still print '/rack-' last field value

echo $@ | xargs -n 1 | awk -F '.' '{print "/rack-"$NF}'