toc
此篇博客主要记录集群安装Hadoop环境的流程及配置讲解。这里的IP已换成192.168.0.*
集群一共五台机器,各节点的安装服务如下:
IP:192.168.0.1
namenode(active) journalnode nodemanager zkfc hbase-master zookeeper
IP:192.168.0.2
namenode(standby) journalnode nodemanager zkfc hbase-master zookeeper
IP:192.168.0.3
spark journalnode nodemanager resourcemanager(active) hbase-regionserver zookeeper
IP:192.168.0.4
spark journalnode nodemanager resourcemanager(standby) hbase-regionserver zookeeper
IP:192.168.0.5
spark journalnode nodemanager resourcemanager(standby) hbase-regionserver zookeeper
#wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz"
#tar xf /usr/local/jdk-8u141-linux-x64.tar.gz;mv /usr/local/jdk1.8.0_141 /usr/local/jdk1.8;echo -e "export JAVA_HOME=/usr/local/jdk1.8 \nexport PATH=\$PATH:\$JAVA_HOME/bin" >> #/etc/profile;source /etc/profile
192.168.0.1:
#hostnamectl set-hostname hdfs01.hdfscluster
192.168.0.2:
#hostnamectl set-hostname hdfs02.hdfscluster
192.168.0.3:
#hostnamectl set-hostname hdfs03.hdfscluster
192.168.0.4:
#hostnamectl set-hostname hdfs04.hdfscluster
192.168.0.5:
#hostnamectl set-hostname hdfs05.hdfscluster
vi /etc/hosts
192.168.0.1 hdfs01 hdfs01.hdfscluster
192.168.0.2 hdfs02 hdfs02.hdfscluster
192.168.0.3 hdfs03 hdfs03.hdfscluster
192.168.0.4 hdfs04 hdfs04.hdfscluster
192.168.0.5 hdfs05 hdfs05.hdfscluster
# ssh-keygen -t rsa
#ssh-copy-id -i /root/.ssh/id_rsa.pub 192.168.0.1
#ssh-copy-id -i /root/.ssh/id_rsa.pub 192.168.0.2
#ssh-copy-id -i /root/.ssh/id_rsa.pub 192.168.0.3
#ssh-copy-id -i /root/.ssh/id_rsa.pub 192.168.0.4
#ssh-copy-id -i /root/.ssh/id_rsa.pub 192.168.0.5
# cd /usr/local;wget
#mkdir /data/zookeeper/{logs,data}
#tar xf /usr/local/zookeeper-latest.tar.gz -C /usr/local/
#ln -sv /usr/local/zookeeper-3.4.5-cdh5.16.2 /usr/local/zookeeper
#vi /etc/profile #追加配置
export ZOOKEEPER_HOME=/usr/local/zookeeper
export PATH=$ZOOKEEPER_HOME/bin:$PATH
souce /etc/profile
#vi $ZOOKEEPER_HOME/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/logs
clientPort=2181
server.1=192.168.0.1:2888:3888
server.2=192.168.0.2:2888:3888
server.3=192.168.0.3:2888:3888
server.4=192.168.0.4:2888:3888
server.5=192.168.0.5:2888:3888
#scp -r /usr/local/zookeeper 192.168.0.2:/usr/local/
#scp -r /usr/local/zookeeper 192.168.0.3:/usr/local/
#scp -r /usr/local/zookeeper 192.168.0.4:/usr/local/
#scp -r /usr/local/zookeeper 192.168.0.5:/usr/local/
#scp /etc/profile 192.168.0.2:/etc/
#scp /etc/profile 192.168.0.3:/etc/
#scp /etc/profile 192.168.0.4:/etc/
#scp /etc/profile 192.168.0.5:/etc/
192.168.0.1:
#echo "1" > /data/zookeeper/data/myid
192.168.0.2:
#echo "2" > /data/zookeeper/data/myid
192.168.0.3:
#echo "3" > /data/zookeeper/data/myid
192.168.0.4:
#echo "4" > /data/zookeeper/data/myid
192.168.0.5:
#echo "5" > /data/zookeeper/data/myid
#/usr/local/zookeeper/bin/zkServer.sh start
#/usr/local/zookeeper/bin/zkServer.sh status
#cd /usr/local/
#wget https://archive.cloudera.com/cdh5/cdh/5/hbase-latest.tar.gz
#wget https://archive.cloudera.com/cdh5/cdh/5/hadoop-latest.tar.gz
#wget https://archive.cloudera.com/cdh5/cdh/5/spark-latest.tar.gz
#cd /usr/local
#tar xf hadoop-latest.tar.gz ;mv hadoop-2.6.0-cdh5.9.3 hadoop;
#tar xf hbase-latest.tar.gz; mv hbase-1.2.0-cdh5.16.2 hbase
#tar xf spark-latest.tar.gz
#vi /etc/profile
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export HBASE_HOME=/usr/local/hbase
export PATH=$PATH:$HBASE_HOME/bin
#source /etc/profile
#mkdir /data/hadoop/{data,name,tmp,log,checkpoint,journalnode} -p
192.168.0.3 14 15机器创建目录:
#mkdir /data{1,2,3,4}/hadoop/{data,name,tmp,log,checkpoint,journalnode} -p
先配置主节点192.168.0.1然后scp其它节点
core-site.xml
#vi $HADOOP_HOME/etc/hadoop/core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://hdfscluster</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/data/hadoop/journalnode</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/data/hadoop/tmp</value>
</property>
<property>
<name>fs.trash.interval</name>
<value>1440</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131702</value>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>192.168.0.1:2181,192.168.0.2:2181,192.168.0.3:2181,192.168.0.4:2181,192.168.0.5:2181</value>
</property>
</configuration>
fs.defaultFS
:表示集群namenode的地址,对于namenode HA来说,需要取一个cluster id来作区分,以便于区分跟hadoop联邦的其它namenode,这里取hdcluster作为该集群的IDdfs.journalnode.edits.dir
、hadoop.tmp.dir
:表示journalnode的数据文件路径、临时文件路径fs.trash.interval
:表示回收站的保留时间(分钟),也即hdfs的文件删除后,在回收站里面保留的时长io.file.buffer.size
:表示读取文件的字节数(byte)ha.zookeeper.quorum
:表示zookeeper集群的主机与端口hdfs-site.xml
#vi $HADOOP_HOME/etc/hadoop/hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/data/hadoop/name</value>
</property>
<property>
<name>dfs.blocksize</name>
<value>134217728</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/data1/hadoop/data,/data2/hadoop/data,/data3/hadoop/data,/data4/hadoop/data</value>
</property>
<property>
<name>dfs.namenode.checkpoint.dir</name>
<value>/data/hadoop/checkpoint</value>
</property>
<property>
<name>dfs.namenode.handler.count</name>
<value>10</value>
</property>
<property>
<name>dfs.datanode.handler.count</name>
<value>10</value>
</property>
<property>
<name>dfs.nameservices</name>
<value>hdfscluster</value>
</property>
<property>
<name>dfs.ha.namenodes.hdfscluster</name>
<value>nn1,nn2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.hdfscluster.nn1</name>
<value>hdfs01:9000</value>
</property>
<property>
<name>dfs.namenode.rpc-address.hdfscluster.nn2</name>
<value>hdfs02:9000</value>
</property>
<property>
<name>dfs.namenode.http-address.hdfscluster.nn1</name>
<value>hdfs01:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.hdfscluster.nn2</name>
<value>hdfs02:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hdfs01:8485;hdfs02:8485;hdfs03:8485;hdfs04:8485;hdfs05:8485/hdfscluster</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.hdfscluster</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>shell(/bin/true)</value>
</property>
</configuration>
配置说明:
dfs.replication
:表示hdfs的数据块备份数量,默认是3dfs.namenode.name.dir
,dfs.datanode.data.dir
,dfs.namenode.checkpoint.dir
:表示namenode、datanode、checkpoint的数据路径dfs.blocksize
:表示数据块的大小,默认为64M,可根据需要改为128M,甚至256Mdfs.namenode.handler.count
、dfs.datanode.handler.count
:表示在namenode、datanode的进程数dfs.nameservices
:表示集群namenode的ID,这里命名为hdcluster,注意要跟 core-size.xml 里面的配置项 fs.defaultFS 中的集群ID一致dfs.ha.namenodes.hdcluster
:表示namenode的id,这里有两个namenode节点,因此使用nn1,nn2命名dfs.namenode.rpc-address.hdcluster.nn1
,dfs.namenode.rpc-address.hdcluster.nn2
:表示nn1、nn2的远程调用主机和端口dfs.namenode.http-address.hdcluster.nn1
,dfs.namenode.http-address.hdcluster.nn2
:表示 nn1、nn2的所在节点http服务和端口dfs.namenode.shared.edits.dir
:表示namenode共享的元数据路径,在配置HA时,使用journalnode来保存元数据,维持namenode元数据的一致性dfs.client.failover.proxy.provider.hdcluster
:表示HDFS客户端连接到Active NameNode的一个java类(默认)dfs.ha.automatic-failover.enabled.hdcluster
:表示当namenode ha的active namenode出现故障时,是否自动切换dfs.ha.fencing.methods
:表示故障时自动切换使用的方法mapred-site.xml
#vi $HADOOP_HOME/etc/hadoop/mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
该配置表示使用yarn框架
yarn-site.xml
#vi $HADOOP_HOME/etc/hadoop/yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>hdfscluster</value>
</property>
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2,rm3</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>hdfs03</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>hdfs04</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm3</name>
<value>hdfs05</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm1</name>
<value>hdfs03:8088</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm2</name>
<value>hdfs04:8088</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address.rm3</name>
<value>hdfs05:8088</value>
</property>
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>192.168.0.1:2181,192.168.0.2:2181,192.168.0.3:2181,192.168.0.4:2181,192.168.0.5:2181</value>
</property>
<property>
<name>yarn.nodemanager.resource.memory-mb</name>
<value>51200</value>
</property>
<property>
<name>yarn.scheduler.minimum-allocation-mb</name>
<value>1024</value>
</property>
<property>
<name>yarn.scheduler.maximum-allocation-mb</name>
<value>51200</value>
</property>
<property>
<name>yarn.app.mapreduce.am.resource.mb</name>
<value>2048</value>
</property>
</configuration>
yarn.nodemanager.aux-services
:NodeManager上运行的附属服务,需配置成mapreduce_shuffle,才可运行MapReduce程序,否则会报错yarn.resourcemanager.ha.enabled
:表示启动resourcemanager HA高可用yarn.resourcemanager.cluster-id
:表示resourcemanager的集群ID,不要与别的集群混淆即可,这里命名为hdclusteryarn.resourcemanager.ha.rm-ids
:表示resourcemanager的节点id,这里有两个节点,使用rm1,rm2作为IDyarn.resourcemanager.hostname.rm1
,yarn.resourcemanager.hostname.rm2
:表示rm1,rm2的主机,这里取hd3、hd4yarn.resourcemanager.webapp.address.rm1
,yarn.resourcemanager.webapp.address.rm2
:表示rm1,rm2的网页访问地址和端口,也即通过该地址和端口可访问作业情况yarn.resourcemanager.zk-address
:表示使用zookeeper来协助管理resourcemanager主备的zookeeper集群主机与端口其他配置:
#vi $HADOOP_HOME/etc/hadoop/slaves
hdfs03
hdfs04
hdfs05
#vi $HADOOP_HOME/etc/hadoop/log4j.properties
hadoop.log.dir=/data/hadoop/log
hadoop.log.file=hadoop.log
修改日志的路径
#vi $HADOOP_HOME/etc/hadoop/hadoop-env.sh
export JAVA_HOME=/usr/local/jdk1.8
#cd $HADOOP_HOME/etc/hadoop/
#scp hadoop-env.sh core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml slaves log4j.properties 192.168.0.2:`pwd`
#scp hadoop-env.sh core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml slaves log4j.properties 192.168.0.3:`pwd`
#scp hadoop-env.sh core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml slaves log4j.properties 192.168.0.4:`pwd`
#scp hadoop-env.sh core-site.xml hdfs-site.xml mapred-site.xml yarn-site.xml slaves log4j.properties 192.168.0.5:`pwd`
启动 journalnode(hd1,hd2,hd3,hd4,hd5)
#hadoop-daemon.sh start journalnode
格式化 namenode(hd1)
#hdfs namenode -format
初始化 HA 状态到 zk(hd1)
#hdfs zkfc -formatZK
启动 namenode active(hd1)
#hadoop-daemon.sh start namenode
同步 namenode(hd2)
#hdfs namenode -bootstrapStandby
启动 namenode standby(hd2)
#hadoop-daemon.sh start namenode
启动ZookeeperFailoverController(hd1,hd2)
#hadoop-daemon.sh start zkfc
启动 datanode(hd3,hd4,hd5)
#hadoop-daemon.sh start datanode
启动 resourcemanager(hd3,hd4,hd5)
#yarn-daemon.sh start resourcemanager
启动 nodemanager(hd3,hd4,hd5)
#yarn-daemon.sh start nodemanager
#jps
浏览器访问方式该地址时需要进行隧道配置,本地50070端口映射192.168.0.1的50070端口:
访问方式:http://127.0.0.1:50070
http://192.168.0.1:50070
#cp $HADOOP_HOME/etc/hadoop/hdfs-site.xml $HBASE_HOME/conf/
#vi $HBASE_HOME/conf/hbase-site.xml
<configuration>
<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>2181</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>192.168.0.1,192.168.0.2,192.168.0.3,192.168.0.4,192.168.0.5</value>
<description>The directory shared by RegionServers.
</description>
</property>
<property>
<name>hbase.zookeeper.property.dataDir</name>
<value>/data/zookeeper/data</value>
<description>
注意这里的zookeeper数据目录与hadoop ha的共用,也即要与 zoo.cfg 中配置的一致
Property from ZooKeeper config zoo.cfg.
The directory where the snapshot is stored.
</description>
</property>
<property>
<name>hbase.rootdir</name>
<value>hdfs://hdfs01:9000/hbase</value>
<description>
The directory shared by RegionServers.
官网多次强调这个目录不要预先创建,hbase会自行创建,否则会做迁移操作,引发错误
至于端口,有些是8020,有些是9000,看 $HADOOP_HOME/etc/hadoop/hdfs-site.xml 里面的配置,本实验配置的是
dfs.namenode.rpc-address.hdfscluster.nn1 , dfs.namenode.rpc-address.hdfscluster.nn2
</description>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
<description>
分布式集群配置,这里要设置为true,如果是单节点的,则设置为false
The mode the cluster will be in. Possible values are
false: standalone and pseudo-distributed setups with managed ZooKeeper
true: fully-distributed with unmanaged ZooKeeper Quorum (see hbase-env.sh)
</description>
</property>
</configuration>
region server 配置
#vi $HBASE_HOME/conf/regionservers
hdfs03
hdfs04
hdfs05
master backup 配置
#vi $HBASE_HOME/conf/backup-masters
hdfs02
环境配置
#vi $HBASE_HOME/conf/hbase-env.sh
export JAVA_HOME=/usr/local/jdk1.8
export HBASE_MANAGES_ZK=false
#cd $HBASE_HOME/conf/
#scp hdfs-site.xml hbase-site.xml regionservers backup-masters hbase-env.sh 192.168.0.5:`pwd`
#scp hdfs-site.xml hbase-site.xml regionservers backup-masters hbase-env.sh 192.168.0.4:`pwd`
#scp hdfs-site.xml hbase-site.xml regionservers backup-masters hbase-env.sh 192.168.0.3:`pwd`
#scp hdfs-site.xml hbase-site.xml regionservers backup-masters hbase-env.sh 192.168.0.2:`pwd`
启动Hbase master(hdfs01)
#hbase-daemon.sh start master
启动Hbase regionserver(hdfs03、hdfs04、hdfs05)
#hbase-daemon.sh start regionserver
启动Hbase backup-master(hdfs02)
#hbase-daemon.sh start master --backup
#jps
#hbase shell
>status
1 active master, 1 backup masters, 3 servers, 0 dead, 25.3333 average load
spark只利用三台主机部署:
hdfs03 hdfs04 hdfs05
#cd /usr/local/
#tar xf spark-latest.tar.gz
#mv spark-1.6.0-cdh5.16.2 spark
#cd /usr/local/spark
#cp spark-env.sh.template spark-env.sh
vim spark-env.sh
export JAVA_HOME=/usr/local/jdk1.8
export SCALA_HOME=/home/software/scala2.11
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop
#vi spark-defaults.conf
spark.yarn.jars=hdfs://hdfs01:9000/spark_jars/*
#vi slaves
hdfs04
hdfs05
在HDFS上,创建一个目录,用来存放spark的依赖jar包。
此目录是spark-defaults.conf目录中配置的目录名称,命令如下:
#hadoop fs -mkdir /spark_jars
进入spark安装目录的jars目录,执行:
#hadoop fs -put ./* /spark_jars
进入Spark安装目录的bin目录,执行如下命令:
#sh spark-shell --master yarn-client
有些关于Spark on YARN部署的博客,实际上介绍的是Spark的 standalone运行模式。如果启动Spark的master和worker服务,这是Spark的 standalone运行模式,不是Spark on YARN运行模式。
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。
原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。
如有侵权,请联系 cloudcommunity@tencent.com 删除。