namenode resourcemanager JobHistoryServer HistoryServer
datanode nodemanager
docker宿主机(virtualbox) Intellij idea spark client hdfs client
sudo route -n add 172.18.0.0/24 192.168.99.100
docker network create --subnet=172.18.0.0/16 hadoopnet
hadoop文件夹
说明:每个文件夹包含一个启动脚本和一个hdfs挂载的共享卷data
etc/hadoop
说明:本地的hadoop目录会挂载到docker中hadoop/etc/hadoop配置文件目录。
docker run --name namenode \
--hostname namenode \
--network hadoopnet \
--ip 172.18.0.11 \
-d \
-v $PWD/data:/opt/tmp \
-v /Users/wangsen/hadoop/datanode/hadoop:/opt/hadoop-2.7.3/etc/hadoop \
-v $PWD/spark-2.1.1-bin-hadoop2.7:/opt/spark \
--rm dbp/hadoop
dbp/hadoop是docker镜像的名字,共加载了3个共享卷(文件夹)
在创建镜像的时候没有装载spark,hadoop是通过Dockerfile创建dbp/hadoop时,装载到镜像中的;设置spark采用装载模式,也可以重新commit或build dockerfile生成包含spark的镜像。
docker run --name datanode1 --hostname datanode1 --network hadoopnet --ip 172.18.0.13 -d -v $PWD/data:/opt/tmp -v /Users/wangsen/hadoop/datanode/hadoop:/opt/hadoop-2.7.3/etc/hadoop --rm dbp/hadoop
docker run --name datanode2 --hostname datanode2 --network hadoopnet --ip 172.18.0.14 -d -v $PWD/data:/opt/tmp -v /Users/wangsen/hadoop/datanode/hadoop:/opt/hadoop-2.7.3/etc/hadoop --rm dbp/hadoop
## 配置HDFS路径
<property>
<name>fs.defaultFS</name>
<value>hdfs://namenode:9000</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/opt/tmp</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/opt/tmp</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>namenode:18040</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>namenode:18030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>namenode:18025</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>namenode:18141</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>namenode:18088</value>
</property>
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
</property>
<property>
<name>yarn.log.server.url</name>
<value>http://namenode:19888/jobhistory/logs</value>
</property>
<property>
<name>yarn.nodemanager.vmem-check-enabled</name>
<value>false</value>
</property>
<property>
<name>yarn.nodemanager.pmem-check-enabled</name>
<value>false</value>
</property>
export HADOOP_CONF_DIR=/opt/hadoop-2.7.3/etc/hadoop
## 配置spark ui 页面,通过yarn history服务查看spark任务运行结果
## hdfs:///tmp/spark/events是hdfs上的路径,保存spark运行信息
spark.master=local
spark.yarn.historyServer.address=namenode:18080
spark.history.ui.port=18080
spark.eventLog.enabled=true
spark.eventLog.dir=hdfs:///tmp/spark/events
spark.history.fs.logDirectory=hdfs:///tmp/spark/events
修改JAVA_HOME,填写java_home的绝对路径
spark history
如果你希望按作者的思路,搭建自己的spark docker集群,那么你可以从Dockerfile 创建image开始。
FROM ubuntu:16.04
MAINTAINER wsn
RUN apt-get update
RUN apt-get install -y openjdk-8-jdk
RUN apt-get install -y vim
RUN apt install -y net-tools
RUN apt install -y iputils-ping
RUN apt-get install -y openssh-server
RUN mkdir /var/run/sshd
RUN echo 'root:root' |chpasswd
RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config
RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
RUN sed -ri 's/# StrictHostKeyChecking ask/StrictHostKeyChecking no/' /etc/ssh/ssh_config
RUN mkdir /root/.ssh
RUN ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa
RUN cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
ENV JRE_HOME /usr/lib/jvm/java-8-openjdk-amd64/jre
ENV PATH /opt/hadoop-2.7.3/bin:/opt/hadoop-2.7.3/sbin:/usr/lib/jvm/java-8-openjdk-amd64/bin:$PATH
ENV CLASSPATH ./:/usr/lib/jvm/java-8-openjdk-amd64/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib
ADD hadoop-2.7.3.tar.gz /opt/
EXPOSE 22
CMD ["/usr/sbin/sshd", "-D"]