如何使用HiBench进行基准测试

[root@ip-172-31-30-69 ~]# java -version
java version "1.8.0_131"
Java(TM) SE Runtime Environment (build 1.8.0_131-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.131-b11, mixed mode)
[root@ip-172-31-30-69 ~]# 

export JAVA_HOME=/usr/java/jdk1.8.0_131/
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tool.jar:$CLASSPATH

[root@ip-172-31-30-69 ~]# source /etc/profile
[root@ip-172-31-30-69 ~]# java -version

http://maven.apache.org/download.cgi

tar -zxvf apache-maven-3.5.0-bin.tar.gz -C /usr/local/maven/

export MVN_HOME=/usr/local/apache-maven-3.5.2
export PATH=$MVN_HOME/bin:$PATH

[root@ip-172-31-30-69 ~]# source /etc/profile

[root@ip-172-31-30-69 ~]# mvn -v
Apache Maven 3.5.2 (138edd61fd100ec658bfa2d307c43b76940a5d7d; 2017-10-18T03:58:13-04:00)
Maven home: /usr/local/apache-maven-3.5.2
Java version: 1.8.0_131, vendor: Oracle Corporation
Java home: /usr/java/jdk1.8.0_131/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "linux", version: "3.10.0-514.el7.x86_64", arch: "amd64", family: "unix"
[root@ip-172-31-30-69 ~]# 

[root@ip-172-31-30-69 ~]# git clone https://github.com/intel-hadoop/HiBench.git

[root@ip-172-31-30-69 ~]# cd HiBench/
[root@ip-172-31-30-69 HiBench]# mvn -Dscala=2.10 clean package

[root@ip-172-31-30-69 HiBench]# mvn -Dspark=1.6 clean package

[root@ip-172-31-30-69 ~]# cd HiBench/
[root@ip-172-31-30-69 HiBench]# ll

[root@ip-172-31-30-69 HiBench]# mvn -Dspark=1.6 -Dscala=2.10 clean package

[root@ip-172-31-30-69 HiBench]# mvn -Phadoopbench -Dspark=1.6 -Dscala=2.10 clean package

[root@ip-172-31-30-69 HiBench]# mvn –Phadoopbench -Psparkbench -Dspark=1.6 -Dscala=2.10 clean package

mvn -Psparkbench -Dmodules -Psql -Dspark=1.6 -Dscala=2.10 clean package

mvn -Psparkbench -Dmodules -PstructuredStreaming clean package

[root@ip-172-31-30-69 conf]# vim benchmarks.lst 
micro.sleep
micro.sort
micro.terasort
micro.wordcount
micro.dfsioe
sql.aggregation
sql.join
sql.scan
websearch.nutchindexing
websearch.pagerank
ml.bayes
ml.kmeans
ml.lr
ml.als
ml.pca
ml.gbt
ml.rf
ml.svd
ml.linear
ml.lda
ml.svm
graph.nweight

[root@ip-172-31-30-69 conf]# cp hadoop.conf.template hadoop.conf
[root@ip-172-31-30-69 conf]# ll hadoop.conf
-rw-r--r-- 1 root root 448 Apr 21 10:58 hadoop.conf
[root@ip-172-31-30-69 conf]#

[root@ip-172-31-30-69 conf]# vim hadoop.conf
# Hadoop home
hibench.hadoop.home     /opt/cloudera/parcels/CDH/lib/hadoop
# The path of hadoop executable
hibench.hadoop.executable     ${hibench.hadoop.home}/bin/hadoop
# Hadoop configraution directory
hibench.hadoop.configure.dir  ${hibench.hadoop.home}/etc/hadoop
# The root HDFS path to store HiBench data
hibench.hdfs.master       hdfs://nameservice1:8020
# Hadoop release provider. Supported value: apache, cdh5, hdp
hibench.hadoop.release    cdh5

hibench.hdfs.master       hdfs://nameservice1:8020

[root@ip-172-31-30-69 conf]# cp spark.conf.template spark.conf
[root@ip-172-31-30-69 conf]# ll spark.conf
-rw-r--r-- 1 root root 1655 Apr 21 11:05 spark.conf
[root@ip-172-31-30-69 conf]# 

[root@ip-172-31-30-69 conf]# vim spark.conf
# Spark home
hibench.spark.home     /opt/cloudera/parcels/CDH/lib/spark
# Spark master
#   standalone mode: spark://xxx:7077
#   YARN mode: yarn-client
hibench.spark.master    yarn-client
# executor number and cores when running on Yarn
hibench.yarn.executor.num     2
hibench.yarn.executor.cores   4
# executor and driver memory in standalone & YARN mode
spark.executor.memory  4g
spark.driver.memory    4g

[root@ip-172-31-30-69 conf]# vim hibench.conf 
# Data scale profile. Available value is tiny, small, large, huge, gigantic and bigdata.
# The definition of these profiles can be found in the workload's conf file i.e. conf/workloads/micro/wordc
ount.conf
hibench.scale.profile                tiny
# Mapper number in hadoop, partition number in Spark
hibench.default.map.parallelism         8
# Reducer nubmer in hadoop, shuffle partition number in Spark
hibench.default.shuffle.parallelism     8
#======================================================
# Report files
#======================================================
# default report formats
hibench.report.formats          "%-12s %-10s %-8s %-20s %-20s %-20s %-20s\n"
# default report dir path
hibench.report.dir              ${hibench.home}/report
# default report file name
hibench.report.name             hibench.report
# input/output format settings. Available formats: Text, Sequence.
sparkbench.inputformat         Sequence
sparkbench.outputformat        Sequence
# hibench config folder
hibench.configure.dir           ${hibench.home}/conf

hibench.dfsioe.mybigdata.read.number_of_files     5120
hibench.dfsioe.mybigdata.read.file_size           1024
hibench.dfsioe.mybigdata.write.number_of_files    5120
hibench.dfsioe.mybigdata.write.file_size          1024

hibench.wordcount.mybigdata.datasize          5497558138880

/**
 * Generate the official GraySort input data set.
 * The user specifies the number of rows and the output directory and this
 * class runs a map/reduce program to generate the data.
 * The format of the data is:
 * <ul>
 * <li>(10 bytes key) (constant 2 bytes) (32 bytes rowid)
 *     (constant 4 bytes) (48 bytes filler) (constant 4 bytes)
 * <li>The rowid is the right justified row id as a hex number.
 * </ul>
 * <p>
 * To run the program:
 * <b>bin/hadoop jar hadoop-*-examples.jar teragen 10000000000 in-dir</b>
 */

hibench.terasort.mybigdata.datasize   54975581389

hibench.join.bigdata.uservisits                 18000000000
hibench.join.bigdata.pages                      450000000

[INFO] ------------------------------------------------------------------------
[ERROR] Failed to execute goal net.alchim31.maven:scala-maven-plugin:3.2.0:compile (scala-compile-first) on project hibench-common: wrap: org.apache.commons.exec.ExecuteException: Process exited with an error: 1 (Exit value: 1) -> [Help 1]
[ERROR] 
[ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch.
[ERROR] Re-run Maven using the -X switch to enable full debug logging.
[ERROR] 
[ERROR] For more information about the errors and possible solutions, please read the following articles:
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoExecutionException
[ERROR] 
[ERROR] After correcting the problems, you can resume the build with the command
[ERROR]   mvn <goals> -rf :hibench-common