Hadoop的安装和使用

$ sudo useradd -m hadoop -s /bin/bash

$ sudo passwd hadoop
Enter new UNIX password: 
Retype new UNIX password: 
passwd: password updated successfully

$ sudo adduser hadoop sudo
Adding user `hadoop' to group `sudo' ...
Adding user hadoop to group sudo
Done.

$ sudo apt-get install default-jre default-jdk

$ vim ~/.bashrc

$ source ~/.bashrc

$ echo $JAVA_HOME
/usr/lib/jvm/default-java
$ java -version
openjdk version "1.8.0_191"
OpenJDK Runtime Environment (build 1.8.0_191-8u191-b12-0ubuntu0.16.04.1-b12)
OpenJDK 64-Bit Server VM (build 25.191-b12, mixed mode)

$ sudo apt-get install openssh-server

$ ssh localhost
The authenticity of host 'localhost (127.0.0.1)' can't be established.
ECDSA key fingerprint is SHA256:MCT7ubGt3sPlkvS9v//KhAoa7vBO+EVPJN/JXenC8XM.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'localhost' (ECDSA) to the list of known hosts.
hadoop@localhost's password: 
Welcome to Ubuntu 16.04.3 LTS (GNU/Linux 4.10.0-42-generic x86_64)

 * Documentation:  https://help.ubuntu.com
 * Management:     https://landscape.canonical.com
 * Support:        https://ubuntu.com/advantage

243 packages can be updated.
11 updates are security updates.

$ cd ~/.ssh/
$ ssh-keygen -t rsa  #出现提示直接按enter
Generating public/private rsa key pair.
Enter file in which to save the key (/home/hadoop/.ssh/id_rsa): 
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /home/hadoop/.ssh/id_rsa.
Your public key has been saved in /home/hadoop/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:FaavA0T6j8XH0clbVu0pq5hkad7kADUBibL/76I2U00 hadoop@ubuntu
The key's randomart image is:
+---[RSA 2048]----+
|      o.o.+     o|
|   . + . = + . ..|
|    + . o + + o..|
|   . o o E . = ..|
|    . o S = . o  |
|     . * X . .   |
|      + O B .    |
|     + o = +     |
|    ..+ +o       |
+----[SHA256]-----+
$ cat ./id_rsa.pub >> ./authorized_keys  #加入授权

$ sudo tar -zxvf hadoop-2.7.1.tar.gz -C /usr/local
$ cd /usr/local/
$ sudo mv ./hadoop-2.7.1/ ./hadoop      # 将文件夹名改为hadoop
$ sudo chown -R hadoop ./hadoop       # 修改文件权限

$ cd /usr/local/hadoop/bin
$ ./hadoop version
Hadoop 2.7.1
Subversion https://git-wip-us.apache.org/repos/asf/hadoop.git -r 15ecc87ccf4a0228f35af08fc56de536e6ce657a
Compiled by jenkins on 2015-06-29T06:04Z
Compiled with protoc 2.5.0
From source with checksum fc0a1a23fc1868e4d5ee7fa2b28a58a
This command was run using /usr/local/hadoop/share/hadoop/common/hadoop-common-2.7.1.jar

$ ./hadoop jar ../share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar
An example program must be given as the first argument.
Valid program names are:
  aggregatewordcount: An Aggregate based map/reduce program that counts the words in the input files.
  aggregatewordhist: An Aggregate based map/reduce program that computes the histogram of the words in the input files.
  bbp: A map/reduce program that uses Bailey-Borwein-Plouffe to compute exact digits of Pi.
  dbcount: An example job that count the pageview counts from a database.
  distbbp: A map/reduce program that uses a BBP-type formula to compute exact bits of Pi.
  grep: A map/reduce program that counts the matches of a regex in the input.
  join: A job that effects a join over sorted, equally partitioned datasets
  multifilewc: A job that counts words from several files.
  pentomino: A map/reduce tile laying program to find solutions to pentomino problems.
  pi: A map/reduce program that estimates Pi using a quasi-Monte Carlo method.
  randomtextwriter: A map/reduce program that writes 10GB of random textual data per node.
  randomwriter: A map/reduce program that writes 10GB of random data per node.
  secondarysort: An example defining a secondary sort to the reduce.
  sort: A map/reduce program that sorts the data written by the random writer.
  sudoku: A sudoku solver.
  teragen: Generate data for the terasort
  terasort: Run the terasort
  teravalidate: Checking results of terasort
  wordcount: A map/reduce program that counts the words in the input files.
  wordmean: A map/reduce program that counts the average length of the words in the input files.
  wordmedian: A map/reduce program that counts the median length of the words in the input files.
  wordstandarddeviation: A map/reduce program that counts the standard deviation of the length of the words in the input files.

$ cd /usr/local/hadoop
$ mkdir input
$ cp ./etc/hadoop/*.xml ./input   # 将配置文件复制到input目录下
$ ./bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar grep ./input ./output 'dfs[a-z.]+'
$ cat ./output/*          # 查看运行结果
1       dfsadmin

<configuration>
</configuration>

<configuration>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/usr/local/hadoop/tmp</value>
        <description>Abase for other temporary directories.</description>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:/usr/local/hadoop/tmp/dfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:/usr/local/hadoop/tmp/dfs/data</value>
    </property>
</configuration>

$ cd /usr/local/hadoop
$ ./bin/hdfs namenode -format

19/01/11 18:38:02 ERROR namenode.NameNode: Failed to start namenode.
java.lang.IllegalArgumentException: URI has an authority component
    at java.io.File.<init>(File.java:423)
    at org.apache.hadoop.hdfs.server.namenode.NNStorage.getStorageDirectory(NNStorage.java:329)
    at org.apache.hadoop.hdfs.server.namenode.FSEditLog.initJournals(FSEditLog.java:276)
    at org.apache.hadoop.hdfs.server.namenode.FSEditLog.initJournalsForWrite(FSEditLog.java:247)
    at org.apache.hadoop.hdfs.server.namenode.NameNode.format(NameNode.java:985)
    at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1429)
    at org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:1554)
19/01/11 18:38:02 INFO util.ExitUtil: Exiting with status 1
19/01/11 18:38:02 INFO namenode.NameNode: SHUTDOWN_MSG: 
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at ubuntu/127.0.1.1
************************************************************/

19/01/11 18:46:35 INFO common.Storage: Storage directory /usr/local/hadoop/tmp/dfs/name has been successfully formatted.
19/01/11 18:46:36 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0
19/01/11 18:46:36 INFO util.ExitUtil: Exiting with status 0

$ cd /usr/local/hadoop
$ ./sbin/start-dfs.sh

Starting namenodes on [localhost]
localhost: Error: JAVA_HOME is not set and could not be found.
localhost: Error: JAVA_HOME is not set and could not be found.
Starting secondary namenodes [0.0.0.0]

$ echo $JAVA_HOME
/usr/lib/jvm/default-java

$ jps
4821 Jps
4459 DataNode
4348 NameNode
4622 SecondaryNameNode

$ ./sbin/stop-dfs.sh   # 关闭
$ rm -r ./tmp     # 删除 tmp 文件，注意这会删除 HDFS中原有的所有数据
$ ./bin/hdfs namenode -format   # 重新格式化名称节点
$ ./sbin/start-dfs.sh  # 重启

$ cd /usr/local/hadoop
$ ./bin/hdfs dfs -mkdir -p /user/hadoop # 在HDFS中创建用户目录
$ ./bin/hdfs dfs -mkdir input  #在HDFS中创建hadoop用户对应的input目录
$ ./bin/hdfs dfs -put ./etc/hadoop/*.xml input  #把本地文件复制到HDFS中
$ ./bin/hdfs dfs -ls input #查看文件列表
Found 8 items
-rw-r--r--   1 hadoop supergroup       4436 2019-01-11 19:35 input/capacity-scheduler.xml
-rw-r--r--   1 hadoop supergroup       1075 2019-01-11 19:35 input/core-site.xml
-rw-r--r--   1 hadoop supergroup       9683 2019-01-11 19:35 input/hadoop-policy.xml
-rw-r--r--   1 hadoop supergroup       1130 2019-01-11 19:35 input/hdfs-site.xml
-rw-r--r--   1 hadoop supergroup        620 2019-01-11 19:35 input/httpfs-site.xml
-rw-r--r--   1 hadoop supergroup       3518 2019-01-11 19:35 input/kms-acls.xml
-rw-r--r--   1 hadoop supergroup       5511 2019-01-11 19:35 input/kms-site.xml
-rw-r--r--   1 hadoop supergroup        690 2019-01-11 19:35 input/yarn-site.xml
$ ./bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar grep input output 'dfs[a-z.]+'
....
$ ./bin/hdfs dfs -cat output/* #查看运行结果
1   dfsadmin
1   dfs.replication
1   dfs.namenode.name.dir
1   dfs.datanode.data.dir

$ ./bin/hdfs dfs -rm -r output    # 删除 output 文件夹

./sbin/stop-dfs.sh

$ cd /usr/local/hadoop
$ ./bin/hdfs namenode -format #格式化名称节点 （这个命令只需只需一次）
$ ./sbin/start-dfs.sh  #启动Hadoop
$ jps #查看Hadoop是否成功启动
$ ./sbin/stop-dfs.sh   # 关闭Hadoop
$ rm -r ./tmp     # 删除 tmp 文件，注意这会删除 HDFS中原有的所有数据
$ ./sbin/start-dfs.sh  # 重启