(1)到官网http://hadoop.apache.org/releases.html下载2.7.3的source源码包
(2)解压缩
tar -zxvf hadoop-2.7.3-src.tar.gz -C /opt
(3)打开解压目录下的BUILDING.txt,编译过程和需要的软件其实就是根据这个文档里的描述来的
[root@cyq opt]# cd hadoop-2.7.3-src/
[root@cyq hadoop-2.7.3-src]# ls
BUILDING.txt hadoop-build-tools hadoop-dist hadoop-maven-plugins hadoop-project-dist LICENSE.txt README.txt
dev-support hadoop-client hadoop-hdfs-project hadoop-minicluster hadoop-tools NOTICE.txt
hadoop-assemblies hadoop-common-project hadoop-mapreduce-project hadoop-project hadoop-yarn-project pom.xml
[root@cyq hadoop-2.7.3-src]#
[root@cyq hadoop-2.7.3-src]# cat BUILDING.txt
其中一部分
Requirements:
* Unix System
* JDK 1.7+
* Maven 3.0 or later
* Findbugs 1.3.9 (if running findbugs)
* ProtocolBuffer 2.5.0
* CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac
* Zlib devel (if compiling native code)
* openssl devel ( if compiling native hadoop-pipes and to get the best HDFS encryption performance )
* Linux FUSE (Filesystem in Userspace) version 2.6 or above ( if compiling fuse_dfs )
* Internet connection for first build (to fetch all Maven and Hadoop dependencies)
(0)更新RPM源
wget -O/etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum clean all
yum makecache
(1)JDK7
tar -zxvf jdk-7u80-linux-x64.tar.gz -C /opt
(2)Maven
[root@hadron opt]# tar -zxvf apache-maven-3.3.9-bin.tar.gz
[root@hadron opt]# cd apache-maven-3.3.9/conf
[root@hadron opt]# vi settings.xml
<mirrors>
<!-- mirror
| Specifies a repository mirror site to use instead of a given repository. The repository that
| this mirror serves has an ID that matches the mirrorOf element of this mirror. IDs are used
| for inheritance and direct lookup purposes, and must be unique across the set of mirrors.
|-->
<!-- 阿里云仓库 -->
<mirror>
<id>alimaven</id>
<mirrorOf>central</mirrorOf>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/repositories/central/</url>
</mirror>
</mirrors>
虽然换源了,但是还是可以看到hadoop项目向http://repository.jboss.org这个网址下载jar,所以我在hadoop源码项目下的pom.xml文件也进行修改 更新源码pom.xml 添加
<repository>
<id>alimaven</id>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/repositories/central/</url>
</repository>
</distributionManagement>
<repositories>
<repository>
<id>alimaven</id>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/repositories/central/</url>
</repository>
<repository>
<id>${distMgmtSnapshotsId}</id>
<name>${distMgmtSnapshotsName}</name>
<url>${distMgmtSnapshotsUrl}</url>
</repository>
<repository>
<id>repository.jboss.org</id>
<url>http://repository.jboss.org/nexus/content/groups/public/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
(3)Findbugs 1.3.9
[root@hadron opt]# tar -zxvf protobuf-2.5.0.tar.gz
[root@hadron opt]# cd protobuf-2.5.0/
[root@hadron opt]# ./configure
[root@hadron opt]# make
[root@hadron opt]# make install
(4)ProtocolBuffer 2.5.0
[root@hadron opt]# tar -zxvf findbugs-1.3.9.tar.gz
[root@cyq opt]# vi /etc/profile
export JAVA_HOME=/opt/jdk1.7.0_80
export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib
export PATH=$PATH:$JAVA_HOME/bin
export MAVEN_HOME=/opt/apache-maven-3.3.9
export MAVEN_OPTS="-Xms256m -Xmx512m"
export PATH=$PATH:$MAVEN_HOME/bin
export FINDBUGS_HOME=/opt/findbugs-1.3.9
export PATH=$PATH:/opt/findbugs-1.3.9/bin
[root@cyq opt]# source /etc/profile
[root@cyq opt]# java -version
java version "1.7.0_80"
Java(TM) SE Runtime Environment (build 1.7.0_80-b15)
Java HotSpot(TM) 64-Bit Server VM (build 24.80-b11, mixed mode)
[root@hadron opt]# mvn -version
Apache Maven 3.3.9 (bb52d8502b132ec0a5a3f4c09453c07478323dc5; 2015-11-11T00:41:47+08:00)
Maven home: /opt/apache-maven-3.3.9
Java version: 1.7.0_80, vendor: Oracle Corporation
Java home: /opt/jdk1.7.0_80/jre
Default locale: zh_CN, platform encoding: UTF-8
OS name: "linux", version: "3.10.0-514.2.2.el7.x86_64", arch: "amd64", family: "unix"
[root@hadron opt]# protoc --version
libprotoc 2.5.0
[root@hadron opt]# findbugs -version
1.3.9
[root@hadron ~]# yum -y install svn ncurses-devel gcc*
[root@hadron ~]# yum -y install lzo-devel zlib-devel autoconf automake libtool cmake openssl-devel
[root@cyq opt]# cd /opt/hadoop-2.7.3-src/
[root@cyq hadoop-2.7.3-src]# mvn package -Pdist,native -DskipTests -Dtar
.....
main:
[exec] $ tar cf hadoop-2.7.3.tar hadoop-2.7.3
[exec] $ gzip -f hadoop-2.7.3.tar
[exec]
[exec] Hadoop dist tar available at: /opt/hadoop-2.7.3-src/hadoop-dist/target/hadoop-2.7.3.tar.gz
[exec]
[INFO] Executed tasks
[INFO]
[INFO] --- maven-javadoc-plugin:2.8.1:jar (module-javadocs) @ hadoop-dist ---
[INFO] Building jar: /opt/hadoop-2.7.3-src/hadoop-dist/target/hadoop-dist-2.7.3-javadoc.jar
[INFO] ------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Apache Hadoop Main ................................. SUCCESS [ 0.620 s]
[INFO] Apache Hadoop Build Tools .......................... SUCCESS [ 0.417 s]
[INFO] Apache Hadoop Project POM .......................... SUCCESS [ 0.541 s]
[INFO] Apache Hadoop Annotations .......................... SUCCESS [ 1.417 s]
[INFO] Apache Hadoop Assemblies ........................... SUCCESS [ 0.106 s]
[INFO] Apache Hadoop Project Dist POM ..................... SUCCESS [ 1.264 s]
[INFO] Apache Hadoop Maven Plugins ........................ SUCCESS [ 1.650 s]
[INFO] Apache Hadoop MiniKDC .............................. SUCCESS [ 2.462 s]
[INFO] Apache Hadoop Auth ................................. SUCCESS [ 2.363 s]
[INFO] Apache Hadoop Auth Examples ........................ SUCCESS [ 1.449 s]
[INFO] Apache Hadoop Common ............................... SUCCESS [ 38.059 s]
[INFO] Apache Hadoop NFS .................................. SUCCESS [ 2.390 s]
[INFO] Apache Hadoop KMS .................................. SUCCESS [ 6.722 s]
[INFO] Apache Hadoop Common Project ....................... SUCCESS [ 0.038 s]
[INFO] Apache Hadoop HDFS ................................. SUCCESS [01:05 min]
[INFO] Apache Hadoop HttpFS ............................... SUCCESS [ 22.294 s]
[INFO] Apache Hadoop HDFS BookKeeper Journal .............. SUCCESS [ 9.266 s]
[INFO] Apache Hadoop HDFS-NFS ............................. SUCCESS [ 1.825 s]
[INFO] Apache Hadoop HDFS Project ......................... SUCCESS [ 0.027 s]
[INFO] hadoop-yarn ........................................ SUCCESS [ 0.020 s]
[INFO] hadoop-yarn-api .................................... SUCCESS [ 56.840 s]
[INFO] hadoop-yarn-common ................................. SUCCESS [ 21.115 s]
[INFO] hadoop-yarn-server ................................. SUCCESS [ 0.019 s]
[INFO] hadoop-yarn-server-common .......................... SUCCESS [ 5.407 s]
[INFO] hadoop-yarn-server-nodemanager ..................... SUCCESS [ 8.420 s]
[INFO] hadoop-yarn-server-web-proxy ....................... SUCCESS [ 2.156 s]
[INFO] hadoop-yarn-server-applicationhistoryservice ....... SUCCESS [ 3.662 s]
[INFO] hadoop-yarn-server-resourcemanager ................. SUCCESS [ 8.969 s]
[INFO] hadoop-yarn-server-tests ........................... SUCCESS [ 3.045 s]
[INFO] hadoop-yarn-client ................................. SUCCESS [ 3.022 s]
[INFO] hadoop-yarn-server-sharedcachemanager .............. SUCCESS [ 1.535 s]
[INFO] hadoop-yarn-applications ........................... SUCCESS [ 0.015 s]
[INFO] hadoop-yarn-applications-distributedshell .......... SUCCESS [ 1.247 s]
[INFO] hadoop-yarn-applications-unmanaged-am-launcher ..... SUCCESS [ 1.011 s]
[INFO] hadoop-yarn-site ................................... SUCCESS [ 0.015 s]
[INFO] hadoop-yarn-registry ............................... SUCCESS [ 2.210 s]
[INFO] hadoop-yarn-project ................................ SUCCESS [ 2.045 s]
[INFO] hadoop-mapreduce-client ............................ SUCCESS [ 0.086 s]
[INFO] hadoop-mapreduce-client-core ....................... SUCCESS [ 8.907 s]
[INFO] hadoop-mapreduce-client-common ..................... SUCCESS [ 7.925 s]
[INFO] hadoop-mapreduce-client-shuffle .................... SUCCESS [ 1.848 s]
[INFO] hadoop-mapreduce-client-app ........................ SUCCESS [ 4.112 s]
[INFO] hadoop-mapreduce-client-hs ......................... SUCCESS [ 2.555 s]
[INFO] hadoop-mapreduce-client-jobclient .................. SUCCESS [ 3.649 s]
[INFO] hadoop-mapreduce-client-hs-plugins ................. SUCCESS [ 1.079 s]
[INFO] Apache Hadoop MapReduce Examples ................... SUCCESS [ 2.752 s]
[INFO] hadoop-mapreduce ................................... SUCCESS [ 1.833 s]
[INFO] Apache Hadoop MapReduce Streaming .................. SUCCESS [ 2.369 s]
[INFO] Apache Hadoop Distributed Copy ..................... SUCCESS [ 9.073 s]
[INFO] Apache Hadoop Archives ............................. SUCCESS [ 1.321 s]
[INFO] Apache Hadoop Rumen ................................ SUCCESS [ 2.303 s]
[INFO] Apache Hadoop Gridmix .............................. SUCCESS [ 1.825 s]
[INFO] Apache Hadoop Data Join ............................ SUCCESS [ 1.090 s]
[INFO] Apache Hadoop Ant Tasks ............................ SUCCESS [ 1.108 s]
[INFO] Apache Hadoop Extras ............................... SUCCESS [ 1.184 s]
[INFO] Apache Hadoop Pipes ................................ SUCCESS [ 4.356 s]
[INFO] Apache Hadoop OpenStack support .................... SUCCESS [ 1.968 s]
[INFO] Apache Hadoop Amazon Web Services support .......... SUCCESS [ 13.292 s]
[INFO] Apache Hadoop Azure support ........................ SUCCESS [ 3.469 s]
[INFO] Apache Hadoop Client ............................... SUCCESS [ 4.902 s]
[INFO] Apache Hadoop Mini-Cluster ......................... SUCCESS [ 0.422 s]
[INFO] Apache Hadoop Scheduler Load Simulator ............. SUCCESS [ 2.130 s]
[INFO] Apache Hadoop Tools Dist ........................... SUCCESS [ 4.845 s]
[INFO] Apache Hadoop Tools ................................ SUCCESS [ 0.016 s]
[INFO] Apache Hadoop Distribution ......................... SUCCESS [ 21.432 s]
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 06:33 min
[INFO] Finished at: 2017-07-27T10:46:24+08:00
[INFO] Final Memory: 181M/474M
[INFO] ------------------------------------------------------------------------
[root@cyq hadoop-2.7.3-src]#
[root@cyq hadoop-2.7.3-src]# cd hadoop-dist/target/
[root@cyq target]# ls
antrun dist-tar-stitching.sh hadoop-dist-2.7.3.jar hadoop-dist-2.7.3-test-sources.jar maven-shared-archive-resources
classes hadoop-2.7.3 hadoop-dist-2.7.3-javadoc.jar javadoc-bundle-options test-classes
dist-layout-stitching.sh hadoop-2.7.3.tar.gz hadoop-dist-2.7.3-sources.jar maven-archiver test-dir
[root@cyq target]#
软件包hadoop-2.7.3.tar.gz即是我们编译得到成果!
main:
[mkdir] Created dir: /opt/hadoop-2.7.3-src/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads
[get] Getting: http://archive.apache.org/dist/tomcat/tomcat-6/v6.0.44/bin/apache-tomcat-6.0.44.tar.gz
[get] To: /opt/hadoop-2.7.3-src/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads/apache-tomcat-6.0.44.tar.gz
..............................................................................................................................................................................................................................................
解决办法 单独下载apache-tomcat-6.0.44.tar.gz
[root@hadron ~]# mkdir -p /opt/hadoop-2.7.3-src/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads/
[root@hadron ~]# mv apache-tomcat-6.0.44.tar.gz /opt/hadoop-2.7.3-src/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads/
[root@hadron ~]#
<mirrors>
<!-- mirror
| Specifies a repository mirror site to use instead of a given repository. The repository that
| this mirror serves has an ID that matches the mirrorOf element of this mirror. IDs are used
| for inheritance and direct lookup purposes, and must be unique across the set of mirrors.
|-->
<!-- 阿里云仓库 -->
<mirror>
<id>alimaven</id>
<mirrorOf>central</mirrorOf>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/repositories/central/</url>
</mirror>
</mirrors>
虽然换源了,但是还是可以看到hadoop项目向http://repository.jboss.org这个网址下载jar,所以我在hadoop源码项目下的pom.xml文件也进行修改
[root@cyq hadoop-2.7.3-src]# vi pom.xml
</distributionManagement>
<repositories>
<repository>
<id>alimaven</id>
<name>aliyun maven</name>
<url>http://maven.aliyun.com/nexus/content/repositories/central/</url>
</repository>
<repository>
<id>${distMgmtSnapshotsId}</id>
<name>${distMgmtSnapshotsName}</name>
<url>${distMgmtSnapshotsUrl}</url>
</repository>
<repository>
<id>repository.jboss.org</id>
<url>http://repository.jboss.org/nexus/content/groups/public/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>