keepalived+haproxy搭建LNMP架构并做数据同步

准备环境

# 数字8表示CentOS8系统数字7表示CentOS7系统（建议使用相同的系统，比如CentOS8）
虚拟VIP：172.31.0.188
172.31.0.27 keepalived + haproxy master
172.31.0.37 keepalived + haproxy backup
172.31.0.7 nginx + php-fpm
172.31.0.17 nginx + php-fpm
172.31.0.8 mysql-server8.0 master
172.31.0.18 mysql-server8.0 backup
172.31.0.28 nfs-utils master （CentOS7不支持安装rsync-daemon）
172.31.0.38 nfs-utils backup 

安装数据库实现主从同步

# 172.31.0.8
# 安装
[root@centos8 ~]# yum install mysql-server -y
[root@centos8 ~]# mysql

mysql> show master logs;
+------------------+-----------+-----------+
| Log_name         | File_size | Encrypted |
+------------------+-----------+-----------+
| mysql-bin.000001 |       156 | No        |
+------------------+-----------+-----------+

mysql> create database db1;
Query OK, 1 row affected (0.00 sec)

mysql> create user repluser@'172.31.0.%' identified by '123456';
Query OK, 0 rows affected (0.00 sec)

mysql> grant replication slave on *.* to repluser@'172.31.0.%';
Query OK, 0 rows affected (0.00 sec)

mysql> create database db222;
Query OK, 1 row affected (0.00 sec)

mysql> create database wordpress;
Query OK, 1 row affected (0.04 sec)

mysql> create user wordpress@'172.31.0.%' identified by '123456';
Query OK, 0 rows affected (0.00 sec)

mysql> grant all on wordpress.* to wordpress@'172.31.0.%';
Query OK, 0 rows affected (0.00 sec)

# 172.31.0.18
# 安装
[root@centos8 ~]# yum install mysql-server -y
[root@centos8 ~]# mysql

CHANGE MASTER TO
 MASTER_HOST='172.31.0.8',
 MASTER_USER='repluser',
 MASTER_PASSWORD='123456',
 MASTER_PORT=3306,
 MASTER_LOG_FILE='mysql-bin.000001',
 MASTER_LOG_POS=156;

mysql> start slave;
Query OK, 0 rows affected (0.00 sec)

mysql> show slave status\G
*************************** 1. row ***************************
               Slave_IO_State: Waiting for master to send event
                  Master_Host: 172.31.0.8
                  Master_User: repluser
                  Master_Port: 3306
                Connect_Retry: 60
              Master_Log_File: mysql-bin.000001
          Read_Master_Log_Pos: 867
               Relay_Log_File: centos8-relay-bin.000002
                Relay_Log_Pos: 1035
        Relay_Master_Log_File: mysql-bin.000001
             Slave_IO_Running: Yes
            Slave_SQL_Running: Yes

测试看数据是否同步成功

mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| db1                |
| db222              |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
| wordpress          |
+--------------------+
7 rows in set (0.00 sec)

两台nginx + php 搭建

# 172.31.0.7
# 安装 nginx
[root@centos8 ~]# yum install nginx -y

# 安装php（编译安装php）
[root@centos8 ~]# cd /usr/local/src/
[root@centos8 ~]# tar xf php-7.4.10.tar.gz
[root@centos8 ~]# cd php-7.4.10
[root@centos8 ~]# yum -y -q  install gcc make oniguruma-devel libxml2-devel bzip2-devel libmcrypt-devel libsqlite3x-devel oniguruma oniguruma-devel pcre-devel openssl-devel zlib-devel perl-ExtUtils-Embed git

[root@centos8 ~]# ./configure --prefix=/apps/php74 --enable-mysqlnd --with-mysqli=mysqlnd --with-pdo-mysql=mysqlnd --with-openssl --with-zlib  --with-config-file-path=/etc --with-config-file-scan-dir=/etc/php.d --enable-mbstring --enable-xml --enable-sockets --enable-fpm --enable-maintainer-zts --disable-fileinfo

（因为我的系统cpu是2核）
[root@centos8 ~]# make -j 2 && make install
[root@centos8 ~]# cp  sapi/fpm/php-fpm.service /usr/lib/systemd/system/
[root@centos8 ~]# cd /apps/php74/etc
[root@centos8 ~]# cp  php-fpm.conf.default  php-fpm.conf
[root@centos8 ~]# cd  php-fpm.d/
[root@centos8 ~]# cp www.conf.default www.conf
[root@centos8 ~]# sed -i.bak  -e  's/^user.*/user = nginx/' -e 's/^group.*/group = nginx/' /apps/php74/etc/php-fpm.d/www.conf
[root@centos8 ~]# systemctl daemon-reload
[root@centos8 ~]# systemctl start php-fpm
[root@centos8 ~]# ss -tanl
9000

两台都是一样的配置nginx

[root@centos8 ~]# cat /etc/nginx/conf.d/pc.conf
server{
    listen 80;
    server_name www.wrdlongxuan.vip;
    root /data/www;
    location / {
        root /data/www;
        index  index.php index.html index.htm;
    }
    location ~ \.php$|pm_status|ping {
        root /data/www; #下面的$document_root调用此行的root指令指定的目录
        #fastcgi_pass unix:/run/php-fpm/www.sock;
        fastcgi_pass 127.0.0.1:9000;
        fastcgi_index index.php;
        #fastcgi_param SCRIPT_FILENAME /data/php$fastcgi_script_name;
        #如果SCRIPT_FILENAME是上面的绝对路径则可以省略root /data/php;
        fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
        include fastcgi_params;
    }
}

做个php测试页面

[root@centos8 ~]# vim /data/www/index.php
<?php
phpinfo();
?>

创建目录

[root@centos8 ~]# mkdir /data/www -p
# 授权
[root@centos8 ~]# chown -R  nginx.nginx /data/www/

重启服务

[root@centos8 ~]# systemctl restart nginx php-fpm

搭建keepalived + haproxy

# 172.31.0.27
# 安装 （两台机器）
[root@centos8 ~]# yum install keepalived haproxy -y

在两个ka1和ka2两个节点启用内核参数

[root@centos8 ~]# cat /etc/sysctl.conf
net.ipv4.ip_nonlocal_bind = 1

[root@centos8 ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1

修改haproxy配置（两台机器）

[root@centos8 ~]# cat /etc/haproxy/haproxy.cfg

listen web-80
    bind 172.31.0.188:80
    server 172.31.0.7 172.31.0.7:80 check inter 3000 fall 2 rise 5
    server 172.31.0.17 172.31.0.17:80 check inter 3000 fall 2 rise 5

修改keepalived配置（master）

# 172.31.0.27
[root@centos8 ~]# /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL1
   vrrp_mcast_group4 224.0.100.100
}
vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"
    interval 1
    weight -30
    fall 3
    rise 2
    timeout 2
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 88
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.188/16 dev eth0 label eth0:1
    }
    track_interface {
        eth0
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    track_script {
        check_haproxy
    }
}

准备haproxy脚本

[root@centos8 ~]# cat /etc/keepalived/check_haproxy.sh
#!/bin/bash
# 为0就表示进程存在，否则表示不存在
/usr/bin/killall -0 haproxy || systemctl restart haproxy

准备notify 脚本

[root@centos8 ~]# cat /etc/keepalived/notify.sh
!/bin/bash
contact='root@localhost'
notify() {
    mailsubject="$(hostname) to be $1:vip floating"
    mailbody="$(date +'%F %T'):vrrp transition,$(hostname) change to be $1"
    echo $mailbody | mail -s "$mailsubject" $contract
}
case $1 in
master)
    notify master
    systemctl start nginx
    ;;
backup)
    notify backup
    systemctl restart nginx
    ;;
fault)
    notify fault
    ;;
*)
    echo "Usage: $(basename $0) {master|backup|fault}"
esac

脚本授权

[root@centos8 ~]# chmod +x /etc/keepalived/notify.sh
[root@centos8 ~]# chmod +x /etc/keepalived/check_haproxy.sh

修改keepalived配置（backup）

# 172.31.0.37
[root@centos8 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL2
   vrrp_mcast_group4 224.0.100.100
}
vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"
    interval 1
    weight -30
    fall 3
    rise 2
    timeout 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 88
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        172.31.0.188/16 dev eth0 label eth0:1
    }
    track_interface {
        eth0
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
    track_script {
        check_haproxy
    }
}

脚本同上，复用即可

测试

看看VIP是否会漂移到其他backup机器

[root@centos8 ~]# hostname -I
172.31.0.27 172.31.0.188

[root@centos8 ~]# hostname -I
172.31.0.37 

停止master 172.31.0.27 机器上的keepalived，效果如下

[root@centos8 ~]# hostname -I
172.31.0.27 

[root@centos8 ~]# hostname -I
172.31.0.37 172.31.0.188

搭建nfs做数据同步

# 172.31.0.28
# 安装
[root@localhost ~]# yum install -y nfs-utils
# 创建目录
[root@localhost ~]# mkdir -p /data/www
# 配置挂载目录
[root@localhost ~]# cat /etc/exports
/data/www 172.31.0.0/16(rw)
# 启动服务
[root@localhost ~]# systemctl start nfs-server

测试

[root@localhost ~]# showmount -e 172.31.0.28
Export list for 172.31.0.28:
/data/www 172.31.0.0/16

安装wrodpress软件

[root@localhost ~]# cd /usr/local/src/
# 解压
[root@localhost ~]# tar xf wordpress-5.4.2-zh_CN.tar.gz
# 拷贝数据
[root@localhost ~]# cp -a wordpress/* /data/www/
[root@localhost ~]# ll /data/www/
...
# 授权
[root@localhost ~]# chown -R nginx.nginx /data/www/

# 安装
[root@localhost ~]# yum -y install inotify-tools
# 测试
[root@localhost ~]# rsync rsync://172.31.0.38
backup
[root@localhost ~]# tar xf sersync2.5.4_64bit_binary_stable_final.tar.gz
[root@localhost ~]# cp -a GNU-Linux-x86 /usr/local/sersync
[root@localhost ~]# echo 'PATH=/usr/local/sersync:$PATH' > /etc/profile.d/sersync.sh
[root@localhost ~]# source /etc/profile.d/sersync.sh
[root@localhost ~]# rpm -q rsync &> /dev/null || yum -y install rsync
[root@localhost ~]# cp /usr/local/sersync/confxml.xml{,.bak}

改sersync配置

[root@localhost ~]# vim /usr/local/sersync/confxml.xml
[root@localhost ~]# cat /usr/local/sersync/confxml.xml
<?xml version="1.0" encoding="ISO-8859-1"?>
<head version="2.5">
    <host hostip="localhost" port="8008"></host>
    <debug start="false"/>
    <fileSystem xfs="false"/>
    <filter start="false">
        <exclude expression="(.*)\.svn"></exclude>
        <exclude expression="(.*)\.gz"></exclude>
        <exclude expression="^info/*"></exclude>
        <exclude expression="^static/*"></exclude>
    </filter>
    <inotify>
        <delete start="true"/>
        <createFolder start="true"/>
        <createFile start="false"/>
        <closeWrite start="true"/>
        <moveFrom start="true"/>
        <moveTo start="true"/>
        <attrib start="true"/> # 改这里
        <modify start="false"/>
    </inotify>

    <sersync>
            <localpath watch="/data/www"> #改这里
            <remote ip="172.31.0.38" name="backup"/> 改这里
            <!--<remote ip="192.168.8.39" name="tongbu"/>-->
            <!--<remote ip="192.168.8.40" name="tongbu"/>-->
        </localpath>
        <rsync>
            <commonParams params="-artuz"/>
            <auth start="true" users="rsyncuser" passwordfile="/etc/rsync.pas"/> # 改这里
            <userDefinedPort start="false" port="874"/><!-- port=874 -->
            <timeout start="false" time="100"/><!-- timeout=100 -->
            <ssh start="false"/>
        </rsync>
        <failLog path="/tmp/rsync_fail_log.sh" timeToExecute="60"/><!--default every 60mins execute once-->
        <crontab start="false" schedule="600"><!--600mins-->
            <crontabfilter start="false">
                <exclude expression="*.php"></exclude>
                <exclude expression="info/*"></exclude>
            </crontabfilter>
        </crontab>
        <plugin start="false" name="command"/>
    </sersync>

    <plugin name="command">
        <param prefix="/bin/sh" suffix="" ignoreError="true"/>  <!--prefix /opt/tongbu/mmm.sh suffix-->
        <filter start="false">
            <include expression="(.*)\.php"/>
            <include expression="(.*)\.sh"/>
        </filter>
    </plugin>

    <plugin name="socket">
        <localpath watch="/opt/tongbu">
            <deshost ip="192.168.138.20" port="8009"/>
        </localpath>
    </plugin>
    <plugin name="refreshCDN">
        <localpath watch="/data0/htdocs/cms.xoyo.com/site/">
            <cdninfo domainname="ccms.chinacache.com" port="80" username="xxxx" passwd="xxxx"/>
            <sendurl base="http://pic.xoyo.com/cms"/>
            <regexurl regex="false" match="cms.xoyo.com/site([/a-zA-Z0-9]*).xoyo.com/images"/>
        </localpath>
    </plugin>
</head>

添加密码

[root@localhost ~]# echo centos > /etc/rsync.pas
# 授权
[root@localhost ~]# chmod 600 /etc/rsync.pas

nfs backup机器操作

# 172.31.0.38
# 安装
[root@localhost ~]# yum install -y rsync rsync-daemon
# 创建跟master一样的目录
[root@localhost ~]# mkdir -p /data/www

# 在备份服务器启动 rsync 进程
[root@localhost ~]# rsync --daemon
873
# 检查有没有这个用户
[root@localhost ~]# id nginx
# 创建一个不能登录的系统用户
[root@localhost ~]# useradd -r -s /bin/nologin nginx
# 添加用户和密码
[root@localhost ~]# echo "rsyncuser:centos" > /etc/rsync.pas
# 授权
[root@localhost ~]# chmod 600 /etc/rsync.pas
# 重启服务
[root@localhost ~]# systemctl restart rsyncd

修改配置文件

[root@localhost ~]# cat /etc/rsyncd.conf
[backup]
uid = root
gid = root
path = /data/www/
read only = no
auth users = rsyncuser #默认anonymous可以访问rsync服务器
secrets file = /etc/rsync.pas

在master机器操作

# 测试master机操作
[root@localhost ~]# rsync rsync://172.31.0.38
backup

# 在master机器手动执行同步
[root@localhost ~]# echo "centos" > /etc/rsync.pas
[root@localhost ~]# chmod 600 /etc/rsync.pas

先把数据手工推送到back机器
[root@localhost ~]# rsync /data/www/ rsync://172.31.0.38::backup

#以后台方式执行同步
[root@localhost ~]# sersync2 -dro /usr/local/sersync/confxml.xml

# 如果同步失败,可以手动执行下面命令,观察过程
[root@localhost ~]# cd /data/www && rsync -artuz -R --delete ./ rsyncuser@172.31.0.38::backup --password-file=/etc/rsync.pas >/dev/null 2>&1

测试看看目录下有没有数据即可

[root@CentOS8 ~]# ll /data/www/

挂载（两台nginx都要）

# 这是命令临时挂载，重启机器就没有了
[root@CentOS8 ~]# mount -t nfs 172.31.0.28:/data/www /data/www/

# 永久挂载
[root@localhost ~]# vim /etc/fstab
172.31.0.28:/data/www /data/www/ nfs  _netdev 0 0

上面的方法：是先把WordPress软件安装在nfs共享服务器上在挂载

方法二：先在某一台nginx安装好wrodprees，授权，然后在把WordPress下的/wp-content/uploads 拷贝到NFS服务器上共享出来，把nginx下的WordPress文件里面的wp-content/uploads删除，把剩下的所有文件也拷贝一份到另外一台nginx服务器上，nfs服务器只共享wp-content/uploads这个目录，然后两台nginx挂载即可

# 172.31.0.7
[root@localhost ~]# mkdir /data/www -p
[root@localhost ~]# cd /usr/local/src/
# 解压
[root@localhost ~]# tar xf wordpress-5.4.2-zh_CN.tar.gz
# 拷贝数据
[root@localhost ~]# cp -a wordpress/* /data/www/
[root@localhost ~]# ll /data/www/
...
# 授权
[root@localhost ~]# chown -R nginx.nginx /data/www/
# 拷贝数据到远程nfs服务器
[root@localhost ~]# scp -r /data/www/wp-content 172.31.0.28:/data/www/

# 删除本机的wp-content/uploads
[root@localhost ~]# rm -rf /data/www/wp-content

# 172.31.0.28
# 配置挂载目录
[root@localhost ~]# cat /etc/exports
/data/www/wp-content/uploads 172.31.0.0/16(rw)
# 启动服务
[root@localhost ~]# systemctl start nfs-server
# 测试
[root@localhost ~]# showmount -e 172.31.0.28
Export list for 172.31.0.28:
/data/www/wp-content/uploads 172.31.0.0/16(rw)
# 172.31.0.17
[root@localhost ~]# mkdir /data/www -p
# 172.31.0.7
[root@localhost ~]# scp -r /data/www/* 172.31.0.17:/data/www/
# 在两台nginx挂载到nfs
临时挂载
[root@localhost ~]# mount -t nfs 172.31.0.28:/data/www/wp-content/uploads /data/www/

# 永久挂载
[root@localhost ~]# vim /etc/fstab
172.31.0.28:/data/www/wp-content/uploads /data/www/ nfs  _netdev 0 0

测试

登录浏览器172.31.0.188 或者使用域名www.wrdlongxuan.vip访问，按照提示安装WordPress即可

如果没有域名，需要自己做本地解析

登录并上传图片测试

# 172.31.0.37 当haproxy挂了看会不会出现VIP漂移，抓包分析（添加组播地址可以方便分析问题）看IP和优先级（prio）
# keepalived
[root@localhost ~]# tcpdump -i eth0 -nn host 224.0.100.100
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), capture size 262144 bytes

19:02:15.558135 IP 172.31.0.37 > 224.0.100.100: VRRPv2, Advertisement, vrid 88, prio 80, authtype simple, intvl 1s, length 20
19:02:15.558540 IP 172.31.0.27 > 224.0.100.100: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20
19:02:16.560621 IP 172.31.0.27 > 224.0.100.100: VRRPv2, Advertisement, vrid 88, prio 100, authtype simple, intvl 1s, length 20

把haproxy主停止看看VIP是否会飘移到其他backup机器，结果如下

# 172.31.0.27
[root@localhost ~]# systemctl stop haproxy
[root@localhost ~]# hostname -I
172.31.0.27

# 172.31.0.37
[root@localhost ~]# hostname -I
172.31.0.37 172.31.0.188

停止nginx，检查页面是否受到影响（http:/172.31.0.188）

# 172.31.0.7
[root@localhost ~]# killall nginx

报错

先抓包分析原因：keepalived配置文件里添加组播IP地址可以方便分析问题

vip 在一台机器重复来回漂移，原因是因为没有安装kill命令（keepalived的haproxy脚本需要killall命令），安装重启服务即可

[root@localhost ~]# while :;do hostname -I ;sleep 0.5;done
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188
172.31.0.37 172.31.0.188

# 使用
[root@localhost ~]# killall
-bash: killall: command not found
# 安装killall命令
[root@localhost ~]# yum install psmisc -y