Nginx配合keepalived实现双主负载均衡

前言：由于之前一直使用的是主从高可用加后端负载，随着业务量的增大，前端一台服务在高峰时期有些吃力，所有对之前架构进行了一点改造，把主从升级成了双主，可以更充分的利用现有服务器资源，由于实验环境有限，后端的测试仅仅使用了一个静态页面，没有搭建动态环境，也没用使用数据库，如有需要可以参考其他文档。

一、架构规划

1、服务器IP地址规划

VIP1：192.168.1.149
VIP2：192.168.1.150
Keepalived1:192.168.1.151
Keepalived2:192.168.1.152
WebServer1:192.168.1.201
WebServer2:192.168.1.202

2、服务器操作系统

所使用的操作系统均为CentOS release 6.6 (Final) x86_64，最小化安装。

3、网络拓扑图

二、配置Nginx代理服务器

此部分Node1与Node2的配置完全相同。

1，准备编译环境

# yum –y install gccgcc-c++ pcre-devel openssl openssl-devel wget

2，编译安装nginx

# ./configure \
 --prefix=/usr/local/nginx \
 --sbin-path=/usr/local/nginx/sbin/nginx \
 --conf-path=/etc/nginx/nginx.conf \
 --error-log-path=/var/log/nginx/error.log \
 --http-log-path=/var/log/nginx/access.log \
 --pid-path=/var/run/nginx/nginx.pid \
 --lock-path=/var/lock/nginx.lock \
 --user=nginx \
 --group=nginx \
 --with-http_ssl_module \
 --with-pcre
# make && make install

3,为nginx提供SysV init脚本:

# vi /etc/rc.d/init.d/nginx

添加如下内容

#!/bin/sh
#
# nginx - this script starts and stopsthe nginx daemon
#
# chkconfig:   - 85 15 
# description:  Nginx is an HTTP(S) server, HTTP(S) reverse \
#               proxy and IMAP/POP3 proxy server
# processname: nginx
# config:      /etc/nginx/nginx.conf
# config:      /etc/sysconfig/nginx
# pidfile:     /var/run/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no"] && exit 0
nginx="/usr/local/nginx/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/etc/nginx/nginx.conf"
[ -f /etc/sysconfig/nginx ] && ./etc/sysconfig/nginx
lockfile=/var/lock/subsys/nginx
make_dirs() {
  # make required directories
  user=`nginx -V 2>&1 | grep "configure arguments:" | sed's/[^*]*--user=\([^ ]*\).*/\1/g' -`
  options=`$nginx -V 2>&1 | grep 'configure arguments:'`
  for opt in $options; do
      if [ `echo $opt | grep '.*-temp-path'` ]; then
           value=`echo $opt | cut -d"=" -f 2`
           if [ ! -d "$value" ]; then
               # echo "creating"$value
               mkdir -p $value && chown-R $user $value
           fi
      fi
  done
}
start() {
   [ -x $nginx ] || exit 5
   [ -f $NGINX_CONF_FILE ] || exit 6
   make_dirs
   echo -n $"Starting $prog: "
   daemon $nginx -c $NGINX_CONF_FILE
   retval=$?
   echo
   [ $retval -eq 0 ] && touch $lockfile
   return $retval
}
stop() {
   echo -n $"Stopping $prog: "
   killproc $prog -QUIT
   retval=$?
   echo
   [ $retval -eq 0 ] && rm -f $lockfile
   return $retval
}
restart() {
   configtest || return $?
   stop
   sleep 1
   start
}
reload() {
   configtest || return $?
   echo -n $"Reloading $prog: "
   killproc $nginx -HUP
   RETVAL=$?
   echo
}
force_reload() {
   restart
}
configtest() {
 $nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
   status $prog
}
rh_status_q() {
   rh_status >/dev/null 2>&1
}
case "$1" in
   start)
       rh_status_q && exit 0
       $1
       ;;
   stop)
       rh_status_q || exit 0
       $1
       ;;
   restart|configtest)
       $1
       ;;
   reload)
       rh_status_q || exit 7
       $1
       ;;
   force-reload)
       force_reload
        ;;
   status)
       rh_status
       ;;
   condrestart|try-restart)
       rh_status_q || exit 0
            ;;
   *)
       echo $"Usage: $0{start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
       exit 2
esac

而后为此脚本赋予执行权限：

# chmod +x /etc/rc.d/init.d/nginx

添加至服务管理列表，并让其开机自动启动：

# chkconfig --add nginx
# chkconfig nginx on

4，配置Nginx代理

# cat nginx.conf

user nginx nginx;
    worker_processes 1;
    pid /var/run/nginx/nginx.pid;
    worker_rlimit_nofile 51200;
    events
    {
    use epoll;
    worker_connections 51200;
    }
    http{
    include       mime.types;
    default_type application/octet-stream;
    server_names_hash_bucket_size 128;
    client_header_buffer_size 32k;
    large_client_header_buffers 4 32k;
    client_max_body_size 8m;
    sendfile on;
    tcp_nopush     on;
    keepalive_timeout 60;
    tcp_nodelay on;
    fastcgi_connect_timeout 300;
    fastcgi_send_timeout 300;
    fastcgi_read_timeout 300;
    fastcgi_buffer_size 64k;
    fastcgi_buffers 4 64k;
    fastcgi_busy_buffers_size 128k;
    fastcgi_temp_file_write_size 128k;
    gzip on;
    gzip_min_length 1k;
    gzip_buffers     4 16k;
    gzip_http_version 1.0;
    gzip_comp_level 2;
    gzip_types       text/plain application/x-javascript text/css application/xml;
    gzip_vary on;
    upstream backend
    {
    ip_hash;
    server 192.168.1.201:80;
    server 192.168.1.202:80;
    }
    log_format access '$remote_addr - $remote_user [$time_local] "$request" '
    '$status $body_bytes_sent "$http_referer" '
    '"$http_user_agent" $http_x_forwarded_for';
    access_log /var/log/nginx/access.log access;
    server {
    listen 80;
    server_name www.test.com;
    location / {
    root /var/www/html ;
    index index.php index.htm index.html;
    proxy_redirect off;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_pass http://backend;
    }
    location /nginx {
    access_log off;
    auth_basic "NginxStatus";
    }
    }
}

三、安装与配置keepalived

# wgethttp://www.keepalived.org/software/keepalived-1.2.16.tar.gz  
# yum -y install libnl-devel
# ./configure --prefix=/usr/local/keepalived  
# make && make install  
# cp /usr/local/keepalived/sbin/keepalived /usr/sbin/ 
# cp /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/ 
# cp /usr/local/keepalived/etc/rc.d/init.d/keepalived /etc/init.d/ 
# mkdir /etc/keepalived  
# cd /etc/keepalived/
# vim keepalived.conf

以上步骤在两台keepalived机器上都需要进行

下面分别是两节点的配置文件

节点一

! Configuration File for keepalived  
  
global_defs {  
   notification_email {  
         ganen2008@126.com #接收警报的email地址,可以添加多个
   }  
   notification_email_from ganen201405@126.com #发件人地址
   smtp_connect_timeout 3  #超时时间
   smtp_server 127.0.0.1  #发送邮件的服务器
   router_id LVS_DEVEL  #load balancer的标识ID,用于email警报
}  
vrrp_instance VI_1 {  
    state MASTER
    interface eth0
    virtual_router_id 51 
    priority 100    # 权值要比 back 高
    advert_int 1
    authentication {  
        auth_type PASS  
        auth_pass 123456
    }  
    virtual_ipaddress {  
        192.168.1.149 #vip的地址
  }
}  
vrrp_instance VI_2 {  
    state BACKUP
    interface eth0  
    virtual_router_id 52 
    priority 90
    advert_int 1 
    authentication {  
        auth_type PASS  
        auth_pass 123456 
    }  
    virtual_ipaddress {  
        192.168.1.150
    } 
}

节点二

! Configuration File for keepalived
global_defs {
   notification_email {
         ganen2008@126.com
   }
   notification_email_from ganen201405@126.com
   smtp_connect_timeout 3
   smtp_server 127.0.0.1
   router_id LVS_DEVEL
}
vrrp_instance VI_1 {
    state BACKUP
    interface eth1
    virtual_router_id 51
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.1.149
  }
}
vrrp_instance VI_2 { 
    state MASTER
    interface eth1 
    virtual_router_id 52
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.1.150
    }
}

为两节点添加nginx状态监控脚本，由于keepalived本身不能检测到nginx的存活状态，需要借助于第三方脚本来实现，下面是出自余洪春前辈的一个检测脚本，在这里借用一下。

# vim /home/nginx_chk.sh

#!/bin/bash
while  :
do
nginxpid=`ps -C nginx --no-header | wc -l`
 if [ $nginxpid -eq 0 ];then
  /usr/local/nginx/sbin/nginx
  sleep 5
nginxpid=`ps -C nginx --no-header | wc -l`
  echo $nginxpid
    if [ $nginxpid -eq 0 ];then
 /etc/init.d/keepalived stop
   fi
 fi
 sleep 5
done

# chmod +x /home/nginx_chk.sh

后台执行该脚本

# nohup sh /home/nginx_chk.sh &

四、安装web server

这里为了测试我直接使用yum安装两台服务器上的nginx服务。

增加额外资源库

# yum -y install yum-priorities
# rpm -Uvh http://mirrors.yun-idc.com/epel/6/x86_64/epel-release-6-8.noarch.rpm 
# rpm -Uvh http://rpms.famillecollet.com/enterprise/remi-release-6.rpm 
# yum -y install nginx
# /etc/init.d/nginx start

五、启动测试

首先测试两台WebServer的可用性