CentOS 7.9通过ceph-ansible安装ceph集群笔记--外包精通

https://github.com/ceph/ceph-ansible/releases

[root@server0 ~]# ifconfig
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 52:54:00:9d:88:61 brd ff:ff:ff:ff:ff:ff
    inet 103.45.152.24/28 brd 103.45.152.31 scope global eth0
       valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 52:54:00:d1:77:bb brd ff:ff:ff:ff:ff:ff
    inet 10.45.152.24/28 brd 10.45.152.31 scope global eth1
       valid_lft forever preferred_lft forever
[root@server0 ~]#

[root@server0 ~]# lsblk 
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sr0     11:0    1    4M  0 rom  
vda    253:0    0   40G  0 disk 
└─vda1 253:1    0   40G  0 part /
vdb    253:16   0  100G  0 disk 
vdc    253:32   0  100G  0 disk 
vdd    253:48   0  100G  0 disk 
[root@server0 ~]# 

sudo yum -y install ntp
timedatectl set-timezone Asia/Shanghai
timedatectl 
ntpdate ntp.aliyun.com
date

firewall-cmd --zone=public --add-port=3300/tcp --permanent
firewall-cmd --zone=public --add-port=6789/tcp --permanent
firewall-cmd --zone=public --add-port=6800-7100/tcp --permanent
firewall-cmd --reload

useradd ceph
echo -e 'Defaults:ceph !requiretty\nceph ALL = (root) NOPASSWD:ALL' | tee /etc/sudoers.d/ceph
chmod 440 /etc/sudoers.d/ceph

vi /etc/hosts
10.10.10.71 server0
10.10.10.72 server1
10.10.10.73 server2

[root@server0 ~]# ssh-keygen 
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa): 
Created directory '/root/.ssh'.
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:cXcegg19YSsBNJtejhlYVNR8g2DzmgP/bXw/y5QvCbg root@server0
The key's randomart image is:
+---[RSA 2048]----+
|         oBO=o+. |
|         o.B+++o.|
|        o.* *o+..|
|         +oBo= . |
|        S +*. .  |
|          . + o .|
|           . o B.|
|          E   *.+|
|               +=|
+----[SHA256]-----+
[root@server0 ~]#  
[root@server0 ~]# for i in 0 1 2  3; do ssh-copy-id root@server$i; ssh-copy-id ceph@server$i; done

# 更换阿里源
# 阿里源官方文档：https://developer.aliyun.com/mirror/centos?spm=a2c6h.13651102.0.0.3e221b11EPcoMp

# 备份默认源
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup

#配置为阿里源
wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo

yum -y install python-pip git  epel-release  vim lrzsz
yum clean all && yum makecache
# 安装ansible
yum install ansible -y

#下载ceph-ansible
https://github.com/ceph/ceph-ansible/releases

#下载版本
[root@server0 ~]# mv ceph-ansible-4.0.45 ceph-ansible
[root@server0 ~]# ls
ceph-ansible  ceph-ansible-4.0.45.zip
[root@server0 ~]# 




#通过pip安装ansible所需要Python3
pip install --upgrade pip
cd ceph-ansible && pip install -r requirements.txt

stable-3.0 Supports Ceph versions jewel and luminous. This branch requires Ansible version 2.4.
stable-3.1 Supports Ceph versions luminous and mimic. This branch requires Ansible version 2.4.
stable-3.2 Supports Ceph versions luminous and mimic. This branch requires Ansible version 2.6.
stable-4.0 Supports Ceph version nautilus. This branch requires Ansible version 2.9.
stable-5.0 Supports Ceph version octopus. This branch requires Ansible version 2.9.
stable-6.0 Supports Ceph version pacific. This branch requires Ansible version 2.9.
master Supports the master branch of Ceph. This branch requires Ansible version 2.9.

vim /etc/ansible/hosts

#最后新增
[mons]
server[1:3]

[mgrs]
server[1:3]

[osds]
server[1:3]

[rgws]
server[1:3]

[mdss]
server[1:3]

[clients]
server[1:3]

[monitoring]
server[1:3]

[grafana-server]
server[1:3]

ansible -m ping all

cd ceph-ansible
vim roles/ceph-defaults/defaults/main.yml

#取消注释（不需要选择）
#dashboard_admin_password: p@ssw0rd
dashboard_admin_password: p@ssw0rd

#取消注释（不需要选择）
#grafana_admin_password: admin
grafana_admin_password: admin

#默认yum源下载比较慢~（阿里源、清华源二选一），国内源速度soso的
ceph_mirror: http://download.ceph.com
ceph_stable_key: https://download.ceph.com/keys/release.asc

#修改为国内清华源（阿里源、清华源二选一）
ceph_mirror: http://mirrors.tuna.tsinghua.edu.cn/ceph
ceph_stable_key: https://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc

#修改为国内阿里源
ceph_mirror: http://mirrors.aliyun.com/ceph
ceph_stable_key: https://mirrors.aliyun.com/ceph/keys/release.asc

cd ceph-ansible &&  cp site.yml.sample site.yml

[root@server0 ceph-ansible]# vim group_vars/all.yml
ceph_origin: repository
ceph_repository: community
ceph_repository_type: cdn
ceph_stable_release: Octopus
monitor_interface: eth1
radosgw_interface: eth1
journal_size: 1024
public_network: 10.45.152.17/28
cluster_network: "{{ public_network }}"

[root@server0 ceph-ansible]#

[root@server0 ceph-ansible]# vim group_vars/mdss.yml
---

dummy:

copy_admin_key: true
[root@server0 ceph-ansible]# 

[root@server0 ceph-ansible]# vim group_vars/osds.yml
---
dummy:
osd_scenario: "non-collocated"
devices:
  - /dev/vdb
  - /dev/vdc

dedicated_devices:
  - /dev/vdd
  - /dev/vdd

[root@server0 ceph-ansible]# 

[root@server0 ceph-ansible]# vim group_vars/rgws.yml
---
dummy:
copy_admin_key: true

[root@server0 ceph-ansible]# 

[root@server0 ceph-ansible]# ansible-playbook site.yml

PLAY RECAP ***************************************************************************************
server1                    : ok=750  changed=39   unreachable=0    failed=0    skipped=588  rescued=0    ignored=0   
server2                    : ok=752  changed=39   unreachable=0    failed=0    skipped=624  rescued=0    ignored=0   
server3                    : ok=712  changed=36   unreachable=0    failed=0    skipped=597  rescued=0    ignored=0   


INSTALLER STATUS *********************************************************************************
Install Ceph Monitor           : Complete (0:00:39)
Install Ceph Manager           : Complete (0:00:44)
Install Ceph OSD               : Complete (0:01:11)
Install Ceph MDS               : Complete (0:00:50)
Install Ceph RGW               : Complete (0:00:36)
Install Ceph Client            : Complete (0:00:33)
Install Ceph Dashboard         : Complete (0:00:59)
Install Ceph Grafana           : Complete (0:05:37)
Install Ceph Node Exporter     : Complete (0:00:25)

Monday 22 February 2021  13:12:35 +0800 (0:00:00.070)       0:12:48.489 ******* 
=============================================================================== 
ceph-grafana : wait for grafana to start ------------------------------------------------ 300.67s
ceph-osd : wait for all osd to be up ----------------------------------------------------- 11.68s
ceph-osd : use ceph-volume lvm batch to create bluestore osds ---------------------------- 11.66s
ceph-dashboard : copy self-signed generated certificate on mons --------------------------- 3.80s
ceph-dashboard : check if dashboard admin user exists ------------------------------------- 3.16s
ceph-infra : install firewalld python binding --------------------------------------------- 2.96s
ceph-mds : set pg_autoscale_mode value on pool(s) ----------------------------------------- 2.74s
ceph-mgr : create ceph mgr keyring(s) on a mon node --------------------------------------- 2.41s
ceph-mds : customize pool crush_rule ------------------------------------------------------ 2.05s
ceph-mds : customize pool size ------------------------------------------------------------ 2.04s
ceph-mds : customize pool min_size -------------------------------------------------------- 2.02s
ceph-facts : check if the ceph mon socket is in-use --------------------------------------- 1.86s
ceph-facts : check if the ceph mon socket is in-use --------------------------------------- 1.82s
ceph-osd : unset noup flag ---------------------------------------------------------------- 1.80s
ceph-facts : check for a ceph mon socket -------------------------------------------------- 1.77s
ceph-config : look up for ceph-volume rejected devices ------------------------------------ 1.71s
ceph-osd : set noup flag ------------------------------------------------------------------ 1.70s
ceph-config : look up for ceph-volume rejected devices ------------------------------------ 1.68s
ceph-config : look up for ceph-volume rejected devices ------------------------------------ 1.68s
ceph-osd : get keys from monitors --------------------------------------------------------- 1.66s
[root@server0 ceph-ansible]# 

[root@server1 ~]# ceph -s
  cluster:
    id:     fb0062e6-9ee3-46e8-96ae-d1470586938f
    health: HEALTH_OK
 
  services:
    mon: 3 daemons, quorum server1,server2,server3 (age 20m)
    mgr: server1(active, since 8m), standbys: server2, server3
    mds: cephfs:1 {0=server2=up:active} 2 up:standby
    osd: 4 osds: 4 up (since 18m), 4 in (since 18m)
    rgw: 3 daemons active (server1.rgw0, server2.rgw0, server3.rgw0)
 
  task status:
    scrub status:
        mds.server2: idle
 
  data:
    pools:   6 pools, 168 pgs
    objects: 214 objects, 6.3 KiB
    usage:   204 GiB used, 396 GiB / 600 GiB avail
    pgs:     168 active+clean
 
[root@server1 ~]# 

[root@server1 ~]# ceph --version
ceph version 14.2.16 (762032d6f509d5e7ee7dc008d80fe9c87086603c) nautilus (stable)
[root@server1 ~]# 

[root@server3 ceph]# ceph osd tree
ID CLASS WEIGHT  TYPE NAME        STATUS REWEIGHT PRI-AFF 
-1       0.87900 root default                             
-3       0.29300     host server1                         
 1   hdd 0.14650         osd.1        up  1.00000 1.00000 
 3   hdd 0.14650         osd.3        up  1.00000 1.00000 
-7       0.29300     host server2                         
 4   hdd 0.14650         osd.4        up  1.00000 1.00000 
 5   hdd 0.14650         osd.5        up  1.00000 1.00000 
-5       0.29300     host server3                         
 0   hdd 0.14650         osd.0        up  1.00000 1.00000 
 2   hdd 0.14650         osd.2        up  1.00000 1.00000 
[root@server3 ceph]#