su passwd root
su
mkdir /opt/software /opt/module
cd /opt/software
wget "https://mirrors.tuna.tsinghua.edu.cn/Adoptium/8/jdk/x64/linux/OpenJDK8U-jdk_x64_linux_hotspot_8u392b08.tar.gz"
tar -xzvf /opt/software/OpenJDK8U-jdk_x64_linux_hotspot_8u392b08.tar.gz -C /opt/module
tar -xzvf /opt/software/hadoop-3.1.3.tar.gz -C /opt/module
vi /etc/profile
add following lines:
export JAVA_HOME="/opt/module/jdk8u392-b08"
export PATH=$PATH:$JAVA_HOME/bin
export HADOOP_HOME="/opt/module/hadoop-3.1.3"
export PATH=$PATH:$HADOOP_HOME/bin
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
load file:
source /etc/profile
vi /etc/hostname
# modify hostname to master
vi /etc/hosts
# add:
# 192.168.56.104 master
# 192.168.56.105 slave1
ping slave1
to test connection
cd ~/.ssh
ssh-keygen -t rsa
cat id_rsa.pub >> authorized_keys
ssh master
# rsa auth, no need to type in password
need to config hadoop-env
and then workers
and then 4 xml files: core, hdfs, mapreduce, yarn
cd $HADOOP_HOME
vi etc/hadoop/hadoop-env.sh
add lines:
export JAVA_HOME="/opt/module/jdk8u392-b08"
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_NODEMANAGER_USER=root
export YARN_RESOURCEMANAGER_USER=root
vi etc/hadoop/workers
drop localhost and add lines:
master
slave1
vi etc/hadoop/core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:9000</value>
</property>
</configuration>
place directories for config:
mkdir -p /opt/data/nameNode /opt/data/dataNode
vi etc/hadoop/hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/opt/data/nameNode</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/opt/data/dataNode</value>
</property>
</configuration>
vi etc/hadoop/mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.application.classpath</name>
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
</property>
</configuration>
vi etc/hadoop/yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.env-whitelist</name>
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
</property>
</configuration>
ssh slave1
mkdir /opt/module
modify hostname to slave1
vi /etc/hostname
# add slave1
exit
from slave1 to proceed scp
scp -r /opt/module/hadoop-3.1.3 slave1:/opt/module
hdfs namenode -format
$HADOOP_HOME/sbin/start-dfs.sh
namenode will run on master, and datanodes will run on master and slave1
1) use jps
to check namenode and datanode
2) ssh slave1
and use jps
to chek datanode
hdfs dfs -mkdir /test-path
hdfs dfs -ls /
# test with file:
wget -O ~/alice.txt "https://www.gutenberg.org/files/11/11-0.txt"
hdfs dfs -put ~/alice.txt /test-path
hdfs dfs -ls /test-path
more ops, view https://sparkbyexamples.com/apache-hadoop/hadoop-hdfs-dfs-commands-and-starting-hdfs-dfs-services/
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。