Haproxy+Heartbeat 高可用集群方案操作记录

172.16.60.208(eth0)    HA主节点(ha-master)         haproxy,heartbeat
172.16.60.207(eth0)    HA备节点(ha-slave)          haproxy,heartbeat
172.16.60.229             VIP地址
172.16.60.204(eth0)    后端节点1(rs-204)           nginx/tomcat
172.16.60.205(eth0)    后端节点2(rs-205)           nginx/tomcat

1) 关闭防火墙和selinux (四台节点机都操作)
[root@ha-master ~]# /etc/init.d/iptables stop
[root@ha-master ~]# setenforce 0
[root@ha-master ~]# vim /etc/sysconfig/selinux
SELINUX=disabled
 
2) 设置主机名和绑定hosts (两台HA节点机器都操作)
主节点操作
[root@ha-master ~]# hostname ha-master
[root@ha-master ~]# vim /etc/sysconfig/network
HOSTNAME=ha-master
[root@ha-master ~]# vim /etc/hosts
172.16.60.208 ha-master
172.16.60.207 ha-slave
 
备节点操作
[root@ha-slave ~]# hostname ha-slave
[root@ha-slave ~]# vim /etc/sysconfig/network
HOSTNAME=ha-slave
[root@ha-slave ~]# vim /etc/hosts
172.16.60.208 ha-master
172.16.60.207 ha-slave

采用yum方式在两台realserver节点上安装nginx (先安装nginx的yum源)
[root@rs-204 ~]# rpm -ivh http://nginx.org/packages/centos/6/noarch/RPMS/nginx-release-centos-6-0.el6.ngx.noarch.rpm
[root@rs-204 ~]# yum install -y nginx
   
rs-204的nginx配置
[root@rs-204 ~]# cd /etc/nginx/conf.d/
[root@rs-204 conf.d]# cat default.conf
[root@rs-204 conf.d]# >/usr/share/nginx/html/index.html
[root@rs-204 conf.d]# vim /usr/share/nginx/html/index.html
this is test page of realserver01:172.16.60.204
   
[root@rs-204 conf.d]# /etc/init.d/nginx start
Starting nginx:                                            [  OK  ]
[root@rs-204 conf.d]# lsof -i:80
COMMAND   PID  USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
nginx   31944  root    6u  IPv4  91208      0t0  TCP *:http (LISTEN)
nginx   31945 nginx    6u  IPv4  91208      0t0  TCP *:http (LISTEN)
   
rs-205的nginx配置
[root@rs-205 src]# cd /etc/nginx/conf.d/
[root@rs-205 conf.d]# cat default.conf
[root@rs-205 conf.d]# >/usr/share/nginx/html/index.html
[root@rs-205 conf.d]# vim /usr/share/nginx/html/index.html
this is test page of realserver02:172.16.60.205
 
[root@rs-205 conf.d]# /etc/init.d/nginx start
Starting nginx:                                            [  OK  ]
[root@rs-205 conf.d]# lsof -i:80
COMMAND   PID  USER   FD   TYPE    DEVICE SIZE/OFF NODE NAME
nginx   20839  root    6u  IPv4 289527645      0t0  TCP *:http (LISTEN)
nginx   20840 nginx    6u  IPv4 289527645      0t0  TCP *:http (LISTEN)
   
访问http://172.16.60.204/, 访问结果为"this is test page of realserver01:172.16.60.204"
访问http://172.16.60.205/, 访问结果为"this is test page of realserver02:172.16.60.205"

1) 先安装haproxy
[root@ha-master ~]# yum install gcc gcc-c++ make openssl-devel kernel-devel
[root@ha-master ~]# cd /usr/local/src/                              #下载haproxy软件到/usr/local/src目录下
[root@ha-master src]# ls haproxy-1.8.12.tar.gz 
haproxy-1.8.12.tar.gz
[root@ha-master src]# tar -zvxf haproxy-1.8.12.tar.gz
[root@ha-master src]# cd haproxy-1.8.12
[root@ha-master haproxy-1.8.12]# make TARGET=linux26 CPU=x86_64 PREFIX=/usr/local/haprpxy USE_OPENSSL=1 ADDLIB=-lz

参数说明：
TARGET=linux26      #使用 uname -r 查看内核，如：2.6.32-642.el6.x86_64，此时该参数就为linux26
CPU=x86_64          #使用 uname -r 查看系统信息，如 x86_64 GNU/Linux，此时该参数就为 x86_64
PREFIX=/usr/local/haprpxy      #haprpxy 安装路径

[root@ha-master haproxy-1.8.12]# ldd haproxy | grep ssl
        libssl.so.10 => /usr/lib64/libssl.so.10 (0x00000031d0400000)

[root@ha-master haproxy-1.8.12]# make install PREFIX=/usr/local/haproxy

[root@ha-master haproxy-1.8.12]# mkdir -p /usr/local/haproxy/conf
[root@ha-master haproxy-1.8.12]# mkdir -p /etc/haproxy
[root@ha-master haproxy-1.8.12]# cp /usr/local/src/haproxy-1.8.12/examples/option-http_proxy.cfg /usr/local/haproxy/conf/haproxy.cfg
[root@ha-master haproxy-1.8.12]# ln -s /usr/local/haproxy/conf/haproxy.cfg /etc/haproxy/haproxy.cfg
[root@ha-master haproxy-1.8.12]# cp -r /usr/local/src/haproxy-1.8.12/examples/errorfiles  /usr/local/haproxy/errorfiles
[root@ha-master haproxy-1.8.12]# ln -s /usr/local/haproxy/errorfiles /etc/haproxy/errorfiles
[root@ha-master haproxy-1.8.12]# mkdir -p /usr/local/haproxy/log
[root@ha-master haproxy-1.8.12]# touch /usr/local/haproxy/log/haproxy.log
[root@ha-master haproxy-1.8.12]# ln -s /usr/local/haproxy/log/haproxy.log /var/log/haproxy.log
[root@ha-master haproxy-1.8.12]# cp /usr/local/src/haproxy-1.8.12/examples/haproxy.init /etc/rc.d/init.d/haproxy
[root@ha-master haproxy-1.8.12]# chmod +x /etc/rc.d/init.d/haproxy
[root@ha-master haproxy-1.8.12]# chkconfig haproxy on
[root@ha-master haproxy-1.8.12]# ln -s /usr/local/haproxy/sbin/haproxy /usr/sbin

2) haroxy.cfg文件进行负载配置 
[root@ha-master haproxy-1.8.12]# cd /usr/local/haproxy/conf/
[root@ha-master conf]# cp haproxy.cfg haproxy.cfg.bak
[root@ha-master conf]# > haproxy.cfg
[root@ha-master conf]# vim haproxy.cfg
global                              
   log 127.0.0.1 local3 info        
   maxconn 65535                     
   chroot /usr/local/haproxy        
   uid 99                            
   gid 99                          
   daemon                          
 
defaults
   log global
   mode http                        
   retries 3                         
   option redispatch               
   stats uri /haproxy          
   stats refresh 30s                 
   stats realm haproxy-status        
   stats auth admin:dxInCtFianKtL]36   
   stats hide-version                 
   maxconn 65535                    
   timeout connect 5000           
   timeout client 50000            
   timeout server 50000             

frontend http-in                    
   mode http                         
   maxconn 65535                    
   bind :80                          
   log global                       
   option httplog                   
   option httpclose                  
   acl is_01 hdr_beg(host) www.kevin.com       
   use_backend web-server if is_01    
 
backend web-server
   mode http
   balance roundrobin                
   cookie SERVERID insert indirect nocache
   option httpclose
   option forwardfor
   server web01 172.16.60.204:80 weight 1 cookie 3 check inter 2000 rise 2 fall 5
   server web02 172.16.60.205:80 weight 1 cookie 4 check inter 2000 rise 2 fall 5

3) 配置HAProxy日志 
[root@ha-master conf]# vim /etc/rsyslog.conf
.......
$ModLoad imudp                        #取消注释 ，这一行不注释，日志就不会写
$UDPServerRun 514                    #取消注释 ，这一行不注释，日志就不会写
.......
local3.*                                                /var/log/haproxy.log      #这一行必须要写,因为在haproxy.cfg里global全局定义好的日志级别

[root@ha-master conf]# vim /etc/sysconfig/rsyslog
SYSLOGD_OPTIONS="-r -m 0"           #接收远程服务器日志

重启syslog日志服务
[root@ha-master conf]# service rsyslog restart
Shutting down system logger:                               [  OK  ]
Starting system logger:                                         [  OK  ]

4) 设置haproxy负载均衡的最大并发连接数
查看内核
[root@ha-master conf]# sysctl -a | grep file
fs.file-nr = 992        0       386459
fs.file-max = 386459

查看应用层面的需求
[root@ha-master conf]# cat /usr/local/haproxy/conf/haproxy.cfg
global                                                        #全局参数设置
   maxconn 65535                                      #设置最大连接数

更改系统层面
[root@ha-master conf]# vim /etc/security/limits.conf                    #最后一行增加
*                -       nofile          65535

5) 重启两台HA机器的haproxy
[root@ha-master conf]# /etc/init.d/haproxy start
Starting haproxy:                                          [  OK  ]

[root@ha-master conf]# ps -ef|grep haproxy
nobody   13080     1  0 16:43 ?        00:00:00 /usr/sbin/haproxy -D -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid
root     13083 11940  0 16:43 pts/0    00:00:00 grep haproxy

[root@ha-master conf]# lsof -i:80
COMMAND   PID   USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
haproxy 13080 nobody    4u  IPv4 428975      0t0  TCP *:http (LISTEN)

将www.kevin.com域名解析到两个HA节点上, 即172.16.60.208 和 172.16.60.207上
接着访问http://www.kevin.com/, 则发现访问结果是"this is test page of realserver01:172.16.60.204" , 不断刷新, 访问结果也是这个.
只有当172.16.60.204这个节点的nginx挂了, 访问结果才变成"this is test page of realserver02:172.16.60.205", 即请求转发到正常的realserver节点上.

从haproxy.cfg文件中可以看出, 虽然配置了"balance roundrobin"这个选项, 即客户端每一次访问, 都跳转到后端不同的服务器上. 但是并没有生效!
因为又配置了"cookie SERVERID insert indirect nocache", 即保持客户端session会话同步的配置, 所以客户端请求会一直转发到同一个realserver节点上,直至
这个节点发生故障才会转发到另外正常的节点上.

把"cookie SERVERID insert indirect nocache" 这个配置去掉或注释掉, 再次访问http://www.kevin.com/, 就会发现每刷新一次, 请求都会转发到不同的realserver
节点上, 即"balance roundrobin" 配置生效!

访问http://www.kevin.com/haproxy, 输入haproxy.cfg文件中配置的用户名和密码admin:dxInCtFianKtL]36， 即可打开haproxy监控页面