centos7源码编译和安装lustre

$ uname -r
3.10.0-1160.42.2.el7.x86_64

$ cat /etc/redhat-release 
CentOS Linux release 7.9.2009 (Core)

// lustre安装版本需要和https://downloads.whamcloud.com/public/lustre/lustre-2.12.7/el7.9.2009/server/RPMS/ 查看对应的版本

$ yum install asciidoc audit-libs-devel automake bc binutils-devel \ bison device-mapper-devel elfutils-devel elfutils-libelf-devel expect \ flex gcc gcc-c++ git glib2 glib2-devel hmaccalc keyutils-libs-devel \ krb5-devel ksh libattr-devel libblkid-devel libselinux-devel libtool \ libuuid-devel libyaml-devel lsscsi make ncurses-devel net-snmp-devel \ net-tools newt-devel numactl-devel parted patchutils pciutils-devel \ perl-ExtUtils-Embed pesign python-devel redhat-rpm-config rpm-build \ systemd-devel tcl tcl-devel tk tk-devel wget xmlto yum-utils zlib-devel libaio-devel -y
$ yum install kernel-devel -y

$ git clone https://github.com/zfsonlinux/spl.git && cd spl&& git checkout spl-0.7.13

$ sh autogen.sh
$ ./configure --with-spec=redhat  --with-linux=/usr/src/kernels/3.10.0-1160.42.2.el7.x86_64
$ make pkg-utils pkg-kmod 
$ mkdir -p releases/zfs-spl && mv *.rpm releases/zfs-spl

$ ls -l releases/zfs-spl/
total 2804
-rw-r--r--. 1 root root  111440 Sep 24 05:33 kmod-spl-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root   57624 Sep 24 05:33 kmod-spl-devel-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root  575218 Sep 24 05:32 spl-0.7.13-1.el7.src.rpm
-rw-r--r--. 1 root root   29260 Sep 24 05:32 spl-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root   34296 Sep 24 05:32 spl-debuginfo-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root  572485 Sep 24 05:32 spl-kmod-0.7.13-1.el7.src.rpm
-rw-r--r--. 1 root root 1473348 Sep 24 05:33 spl-kmod-debuginfo-0.7.13-1.el7.x86_64.rpm

$ yum localinstall -y {spl-[0-9].*,kmod-spl-[0-9].*,kmod-spl-devel-[0-9].*}.x86_64.rpm源码编译

// 下载zfs源代码
$ git clone https://github.com/openzfs/zfs.git && cd zfs && git checkout zfs-0.7.13
$ sh autogen.sh

// 编译之前必须安装spl
$ ./configure --with-spec=redhat --with-spl=/usr/src/spl-0.7.13 --with-linux=/usr/src/kernels/3.10.0-1160.42.2.el7.x86_64 

$ make pkg-utils pkg-kmod 
$ mkdir -p zfs-rpm &&  mv *.rpm zfs-rpm/
$ ls zfs-rpm/ -l
total 31836
-rw-r--r--. 1 root root  872764 Sep 24 05:49 kmod-zfs-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root  283612 Sep 24 05:49 kmod-zfs-devel-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root   30984 Sep 24 05:46 libnvpair1-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root   36088 Sep 24 05:46 libuutil1-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root  133804 Sep 24 05:46 libzfs2-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root  322832 Sep 24 05:46 libzfs2-devel-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root  608308 Sep 24 05:46 libzpool2-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 6719217 Sep 24 05:45 zfs-0.7.13-1.el7.src.rpm
-rw-r--r--. 1 root root  425236 Sep 24 05:46 zfs-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 4073668 Sep 24 05:46 zfs-debuginfo-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root   15388 Sep 24 05:46 zfs-dracut-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 6716596 Sep 24 05:46 zfs-kmod-0.7.13-1.el7.src.rpm
-rw-r--r--. 1 root root 9644824 Sep 24 05:49 zfs-kmod-debuginfo-0.7.13-1.el7.x86_64.rpm
-rw-r--r--. 1 root root 2694520 Sep 24 05:46 zfs-test-0.7.13-1.el7.x86_64.rpm

// 如果libnvpair* libuutil*已经安装就先卸载
$ yum localinstall  -y  libuutil*.rpm libnvpair*.rpm {zfs-[0-9].*,zfs-dracut-[0-9].*,kmod-zfs-[0-9].*,kmod-zfs-devel-[0-9].*,lib*}.x86_64.rpm
$ modprobe zfs

$ git clone git://git.whamcloud.com/fs/lustre-release.git
cd lustre-release && git checkout origin/b2_12

$ sh autogen.sh

// 编译命令如下
$ cd lustre-release
$ sh autogen.sh
$ make distclean

$ ./configure --enable-server  --enable-quota \
--with-linux=/usr/src/kernels/3.10.0-1160.42.2.el7.x86_64 \
--with-zfs=/usr/src/zfs-0.7.13 \
--with-spl=/usr/src/spl-0.7.13 

$ make rpms &&mkdir -p server-rpm &&mv *.rpm server-rpm/


// 安装lustre服务端的组件
$ yum localinstall -y kmod-lustre-2.12.7_13_g67a24ac-1.el7.x86_64.rpm
	kmod-lustre-osd-zfs-2.12.7_13_g67a24ac-1.el7.x86_64.rpm   \
	lustre-osd-zfs-mount-2.12.7_13_g67a24ac-1.el7.x86_64.rpm	\
	kmod-lustre-tests-2.12.7_13_g67a24ac-1.el7.x86_64.rpm     \
	lustre-2.12.7_13_g67a24ac-1.src.rpm                       \
	lustre-2.12.7_13_g67a24ac-1.el7.x86_64.rpm				\
	lustre-debuginfo-2.12.7_13_g67a24ac-1.el7.x86_64.rpm lustre-iokit-2.12.7_13_g67a24ac-1.el7.x86_64.rpm

// 编译客户端的rpm
$ ./configure --disable-server --enable-client --disable-ldiskfs \
--with-linux=/usr/src/kernels/3.10.0-1160.42.2.el7.x86_64 \
--with-zfs=/usr/src/zfs-0.7.13 \
--with-spl=/usr/src/spl-0.7.13 


$ make rpms &&mkdir -p client-rpm &&mv *.rpm client-rpm/
$ ls client-rpm/ -l

// 客户端集群上安装lustre客户端
$ yum localinstall -y kmod-lustre-client-2.12.7_13_g67a24ac-1.el7.x86_64.rpm \
	lustre-client-2.12.7_13_g67a24ac-1.el7.x86_64.rpm lustre-client-debuginfo-2.12.7_13_g67a24ac-1.el7.x86_64.rpm

// lustre服务端内核模块加载后的验证
$ modprobe  zfs
$ modprobe  lustre
$ lsmod  |grep zfs
zfs                  3564425  3 
zunicode              331170  1 zfs
zavl                   15236  1 zfs
icp                   270148  1 zfs
zcommon                73440  1 zfs
znvpair                89131  2 zfs,zcommon
spl                   102412  4 icp,zfs,zcommon,znvpair
$ lsmod  |grep lustre
lustre                776767  0 
lmv                   187141  1 lustre
mdc                   247679  1 lustre
lov                   319046  1 lustre
ptlrpc               2302209  7 fid,fld,lmv,mdc,lov,osc,lustre
obdclass             2636957  8 fid,fld,lmv,mdc,lov,osc,lustre,ptlrpc
lnet                  595508  6 lmv,osc,lustre,obdclass,ptlrpc,ksocklnd
libcfs                415905  11 fid,fld,lmv,mdc,lov,osc,lnet,lustre,obdclass,ptlrpc,ksocklnd
$ lsmod  |grep lnet
lnet                  595508  6 lmv,osc,lustre,obdclass,ptlrpc,ksocklnd
libcfs                415905  11 fid,fld,lmv,mdc,lov,osc,lnet,lustre,obdclass,ptlrpc,ksocklnd

// 部署
$ wipefs -a /dev/sdb
$ mkdir -p /mdt
$ mkfs.lustre --fsname=samplefs --mgs --mdt --index=0   --backfstype=zfs --reformat pool/mdt /dev/sdb 
$ mount.lustre pool/mdt /mdt

// 检查mgs/mds内核进程
$ ps -ef|grep mgs
root      2541     2  0 22:38 ?        00:00:00 [mgs_params_noti]
root      2542     2  0 22:38 ?        00:00:00 [ll_mgs_0000]
root      2543     2  0 22:38 ?        00:00:00 [ll_mgs_0001]
root      2544     2  0 22:38 ?        00:00:00 [ll_mgs_0002]
root      2567     2  0 22:38 ?        00:00:00 [mgs_samplefs_no]
$ ps -ef|grep mdt
root      2549     2  0 22:38 ?        00:00:00 [mdt00_000]
root      2550     2  0 22:38 ?        00:00:00 [mdt00_001]
root      2551     2  0 22:38 ?        00:00:00 [mdt00_002]
root      2552     2  0 22:38 ?        00:00:00 [mdt_rdpg00_000]
root      2553     2  0 22:38 ?        00:00:00 [mdt_rdpg00_001]
root      2554     2  0 22:38 ?        00:00:00 [mdt_attr00_000]
root      2555     2  0 22:38 ?        00:00:00 [mdt_attr00_001]
root      2556     2  0 22:38 ?        00:00:00 [mdt_out00_000]
root      2557     2  0 22:38 ?        00:00:00 [mdt_out00_001]
root      2558     2  0 22:38 ?        00:00:00 [mdt_seqs_0000]
root      2559     2  0 22:38 ?        00:00:00 [mdt_seqs_0001]
root      2560     2  0 22:38 ?        00:00:00 [mdt_seqm_0000]
root      2561     2  0 22:38 ?        00:00:00 [mdt_seqm_0001]
root      2562     2  0 22:38 ?        00:00:00 [mdt_fld_0000]
root      2563     2  0 22:38 ?        00:00:00 [mdt_fld_0001]
root      2564     2  0 22:38 ?        00:00:00 [mdt_io00_000]
root      2565     2  0 22:38 ?        00:00:00 [mdt_io00_001]
root      2566     2  0 22:38 ?        00:00:00 [mdt_io00_002]
root      2936     2  0 22:38 ?        00:00:00 [mdt00_003]

$ systemctl stop firewalld.service
$ systemctl disable firewalld.service

mkdir -p /ost1 && mkdir -p /ost2
//清除该该节点的文件系统信息
wipefs -a /dev/sdb && wipefs -a /dev/sdc


$ mkfs.lustre --ost --reformat --fsname samplefs --index 1 --mgsnode 10.211.55.15@tcp0 --backfstype zfs osspool1/ost /dev/sdb 

$ mkfs.lustre --ost --reformat --fsname samplefs --index 2 --mgsnode 10.211.55.15@tcp0 --backfstype zfs osspool2/ost /dev/sdc



$ mount.lustre osspool1/ost /ost1
$ mount.lustre osspool2/ost /ost2

// 检查ost的内核进程
$ ps -ef|grep ost
root      2764     2  0 22:55 ?        00:00:00 [ll_ost00_000]
root      2765     2  0 22:55 ?        00:00:00 [ll_ost00_001]
root      2766     2  0 22:55 ?        00:00:00 [ll_ost00_002]
root      2767     2  0 22:55 ?        00:00:00 [ll_ost_create00]
root      2768     2  0 22:55 ?        00:00:00 [ll_ost_create00]
root      2769     2  0 22:55 ?        00:00:00 [ll_ost_io00_000]
root      2770     2  0 22:55 ?        00:00:00 [ll_ost_io00_001]
root      2771     2  0 22:55 ?        00:00:00 [ll_ost_io00_002]
root      2772     2  0 22:55 ?        00:00:00 [ll_ost_seq00_00]
root      2773     2  0 22:55 ?        00:00:00 [ll_ost_seq00_00]
root      2774     2  0 22:55 ?        00:00:00 [ll_ost_out00_00]
root      2775     2  0 22:55 ?        00:00:00 [ll_ost_out00_00]
root      3241     2  0 22:55 ?        00:00:00 [ll_ost00_003]

$ yum localinstall -y kmod-lustre-client-2.12.7_13_g67a24ac-1.el7.x86_64.rpm \
	lustre-client-2.12.7_13_g67a24ac-1.el7.x86_64.rpm lustre-client-debuginfo-2.12.7_13_g67a24ac-1.el7.x86_64.rpm
	

$ modprobe -v lustre
$ mkdir -p /mnt/lustre/
$ mount -t lustre 10.211.55.15@tcp0:/samplefs /mnt/lustre/
$ df -h |grep lustre
10.211.55.15@tcp:/samplefs  123G   15M  123G   1% /mnt/lustre


//  检查客户端的内核进程
$ lsmod  |grep lustre
lustre                776767  0 
lmv                   187141  1 lustre
mdc                   247679  1 lustre
lov                   318973  1 lustre
ptlrpc               1351137  7 fid,fld,lmv,mdc,lov,osc,lustre
obdclass             1754619  8 fid,fld,lmv,mdc,lov,osc,lustre,ptlrpc
lnet                  595508  6 lmv,osc,lustre,obdclass,ptlrpc,ksocklnd
libcfs                415905  11 fid,fld,lmv,mdc,lov,osc,lnet,lustre,obdclass,ptlrpc,ksocklnd