Alluxio升级方案（1.8.1-->2.5.0)

wget https://***/packages/29/alluxio-2.5.0-bin.tar.gz （请联系作者询问cos地址）
# md5
[root@172 ~]# md5sum alluxio-2.5.0-bin.tar.gz 
127b0ea8616ed14bb68a4be7a5ffd0a2  alluxio-2.5.0-bin.tar.gz

# 备份181
mv /usr/local/service/alluxio{,.181}
# 解压250
tar -xvf /home/hadoop/alluxio-2.5.0-bin.tar.gz -C /usr/local/service/
mv /usr/local/service/alluxio-2.5.0/ /usr/local/service/alluxio
# 替换配置
cd /usr/local/service/; mv alluxio/conf{,.bak}; cp -r alluxio.181/conf alluxio/; cd
mkdir /etc/alluxio
ln -s /usr/local/service/alluxio/conf/alluxio-site.properties /etc/alluxio/alluxio-site.properties
chown -R hadoop:hadoop /etc/alluxio

# 修改/usr/local/service/alluxio/bin/alluxio-start.sh
######################################
510     master)
511       start_master "${FORMAT}"
512       start_job_master	# 此行为新增
513       ;;

537     worker)
538       start_worker "${MOPT}"
539       start_job_worker	# 此行为新增
540       ;;
######################################
# 修改/usr/local/service/alluxio/bin/alluxio-stop.sh
######################################
 59 stop_master() {
 60   if [[ ${ALLUXIO_MASTER_SECONDARY} == "true" ]]; then
 61     ${LAUNCHER} "${BIN}/alluxio" "killAll" "alluxio.master.AlluxioSecondaryMaster"
 62   else
 63     ${LAUNCHER} "${BIN}/alluxio" "killAll" "alluxio.master.AlluxioMaster"
 64     stop_job_master	# 此行为新增
 65   fi
 66 }

 81 stop_worker() {
 82   ${LAUNCHER} "${BIN}/alluxio" "killAll" "alluxio.worker.AlluxioWorker"
 83   stop_job_worker	# 此行为新增
 84 }
######################################

# 升级其他组件依赖的alluxio client
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | awk -F"alluxio-" '{print $1}' > /root/path.log
while read line; do cp /usr/local/service/alluxio/client/alluxio-2.5.0-client.jar $line; chown -R hadoop:hadoop $line; done < /root/path.log
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | xargs rm -rf
find /usr/local/service/ -name alluxio-*-client.jar

wget https://woodpecker-cos-apps-bj-1259353343.cos.ap-beijing.myqcloud.com/packages/29/tez-0.9.2.tar.gz

# md5
[root@172 ~]# md5sum tez-0.9.2.tar.gz 
47fc7b120b31bc473d62d18ecc73d746  tez-0.9.2.tar.gz

# 替换
mv /usr/local/service/tez/tez-0.9.2.tar.gz{,.bak}
cp /root/tez-0.9.2.tar.gz /usr/local/service/tez/
chown -R hadoop:hadoop /usr/local/service/tez/tez-0.9.2.tar.gz
find /usr/local/service/ -name tez-0.9.2.tar.gz | xargs ls -l

mv /usr/local/service/alluxio{,.181}
mv /usr/local/service/tez/tez-0.9.2.tar.gz{,.bak}

scp -r /usr/local/service/alluxio $OTHER_NODE:/usr/local/service/
scp /usr/local/service/tez/tez-0.9.2.tar.gz $OTHER_NODE:/usr/local/service/tez/

mkdir /etc/alluxio
ln -s /usr/local/service/alluxio/conf/alluxio-site.properties /etc/alluxio/alluxio-site.properties
chown -R hadoop:hadoop /usr/local/service/tez/tez-0.9.2.tar.gz /etc/alluxio /usr/local/service/alluxio
# 升级其他组件依赖的 alluxio client
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | awk -F"alluxio-" '{print $1}' > /root/path.log
while read line; do cp /usr/local/service/alluxio/client/alluxio-2.5.0-client.jar $line; chown -R hadoop:hadoop $line; done < /root/path.log
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | xargs rm -rf
find /usr/local/service/ -name alluxio-*-client.jar

# 首先需将除本台master之外的其他master、core节点ip写入 /root/nodelist 中（换行分割）
while read line; do ssh-copy-id $line; done < /root/nodelist 

for line in `cat /root/nodelist`
do

ssh $line "mv /usr/local/service/alluxio{,.181}; mv /usr/local/service/tez/tez-0.9.2.tar.gz{,.bak}; mkdir /etc/alluxio"
echo $line
scp -r /usr/local/service/alluxio $line:/usr/local/service/
scp /usr/local/service/tez/tez-0.9.2.tar.gz $line:/usr/local/service/tez/
ssh $line "ln -s /usr/local/service/alluxio/conf/alluxio-site.properties /etc/alluxio/alluxio-site.properties; chown -R hadoop:hadoop /usr/local/service/tez/tez-0.9.2.tar.gz /etc/alluxio /usr/local/service/alluxio; find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 > /root/path.log"

done


# 替换alluxio-*-client.jar
while read line; do targetPath=`echo ${line%/*}`; cp /usr/local/service/alluxio/client/alluxio-2.5.0-client.jar $targetPath; chown -R hadoop:hadoop $targetPath; done < /root/path.log
find /usr/local/service/ -name alluxio-1.8.1-client.jar | grep -v 181 | xargs rm -rf
find /usr/local/service/ -name alluxio-*-client.jar

#EMR PROFILE ENV
source /etc/profile

# Common properties
alluxio.master.hostname=172.21.16.35
alluxio.underfs.address=hdfs://HDFS84486

# Security properties
# alluxio.security.authorization.permission.enabled=true
# alluxio.security.authentication.type=SIMPLE

# Worker properties
alluxio.worker.memory.size=8GB
alluxio.worker.tieredstore.levels=1
alluxio.worker.tieredstore.level0.alias=MEM
alluxio.worker.tieredstore.level0.dirs.path=/mnt/alluxio_used_and_do_not_delete_me
alluxio.worker.tieredstore.level0.watermark.high.ratio=0.95
alluxio.worker.tieredstore.level0.watermark.low.ratio=0.7

alluxio.worker.evictor.class=alluxio.worker.block.evictor.LRUEvictor
alluxio.worker.evictor.lrfu.attenuation.factor=1
alluxio.worker.evictor.lrfu.step.factor=0.25

# User properties
alluxio.user.file.readtype.default=CACHE_PROMOTE
alluxio.user.block.size.bytes.default=64M
alluxio.user.file.writetype.default=MUST_CACHE	# ALLUXIO_USER_FILE_WRITETYPE_DEFAULT
alluxio.user.file.copyfromlocal.write.location.policy.class=alluxio.client.file.policy.RoundRobinPolicy
alluxio.user.file.write.location.policy.class=alluxio.client.file.policy.LocalFirstPolicy
alluxio.user.ufs.block.read.location.policy=alluxio.client.file.policy.LocalFirstPolicy
alluxio.user.ufs.block.read.location.policy.deterministic.hash.shards=1
alluxio.user.file.passive.cache.enabled=true	# ALLUXIO_USER_FILE_PASSIVE_CACHE_ENABLED

alluxio.zookeeper.enabled=true
alluxio.zookeeper.address=172.21.16.49:2181,172.21.16.42:2181,172.21.16.88:2181
alluxio.master.journal.folder=hdfs://HDFS84486/data/alluxio/journal

# 修改的配置项
# User properties
alluxio.user.file.writetype.default=ASYNC_THROUGH	# ALLUXIO_USER_FILE_WRITETYPE_DEFAULT
alluxio.user.file.passive.cache.enabled=false	# ALLUXIO_USER_FILE_PASSIVE_CACHE_ENABLED

# 新增的配置项
# User properties
alluxio.job.worker.threadpool.size=32
alluxio.locality.compare.node.ip=true
alluxio.master.journal.type=UFS
alluxio.master.persistence.blacklist=.staging,_temporary,.sparkStaging
alluxio.underfs.hdfs.configuration=/usr/local/service/hadoop/etc/hadoop/core-site.xml:/usr/local/service/hadoop/etc/hadoop/hdfs-site.xml
alluxio.underfs.version=2.8.5
alluxio.user.client.cache.async.write.threads=16
alluxio.user.metrics.collection.enabled=true
alluxio.user.short.circuit.preferred=true
alluxio.user.streaming.data.timeout=90sec
alluxio.worker.file.buffer.size=5MB
alluxio.worker.network.netty.boss.threads=16
alluxio.worker.network.writer.buffer.size.messages=16

# hadoop-env.sh
HADOOP_CLASSPATH=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar:$HADOOP_CLASSPATH

# hive-env.sh
EXPORT_HIVE_AUX_JARS_PATH=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar

# hive-site.xml
spark.driver.extraClassPath=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar
spark.executor.extraClassPath=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar

# spark-defaults.conf
spark.driver.extraClassPath=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar
spark.executor.extraClassPath=/usr/local/service/alluxio/client/alluxio-2.5.0-client.jar

alluxio fsadmin report