04 . Prometheus(联邦集群)监控MySQL

CentOS7.3
alertmanager-0.19.0.linux-amd64.tar.gz
mysql-5.7.23-1.el7.x86_64.rpm-bundle.tar
mysqld_exporter-0.12.1.linux-amd64.tar.gz
prometheus-2.13.0.linux-amd64.tar.gz

1.初始化
init_security() {
systemctl stop firewalld
systemctl disable firewalld &>/dev/null
setenforce 0
sed -i '/^SELINUX=/ s/enforcing/disabled/'  /etc/selinux/config
sed -i '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
sed -i '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
systemctl enable sshd crond &> /dev/null
rpm -e postfix --nodeps
echo -e "33[32m [安全配置] ==> OK 33[0m"
}
init_security

init_yumsource() {
if [ ! -d /etc/yum.repos.d/backup ];then
    mkdir /etc/yum.repos.d/backup
fi
mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null
if ! ping -c2 www.baidu.com &>/dev/null    
then
    echo "您无法上外网，不能配置yum源"
    exit    
fi
    curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
    curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
    yum clean all
    timedatectl set-timezone Asia/Shanghai
    echo "nameserver 114.114.114.114" > /etc/resolv.conf
    echo "nameserver 8.8.8.8" >> /etc/resolv.conf
    chattr +i /etc/resolv.conf
    yum -y install ntpdate
    ntpdate -b  ntp1.aliyun.com        # 对时很重要，避免zookeeper因为时间不准找不到主机
    echo -e "33[32m [YUM　Source] ==> OK 33[0m"
}
init_yumsource
# 配置主机名解析
tail -2 /etc/hosts
172.19.0.55 prometheus-master
172.19.0.56 prometheus-slave

# 为了简化测试环境复杂度，这里使用Docker Compose定义并启动MySQL以及MySQLD Exporter：
cat docker-compose.yml 
version: '3'
services:
  mysql:
    image: daocloud.io/library/mysql:5.7
    ports:
      - "3306:3306"
    environment:
      - MYSQL_ROOT_PASSWORD=XUANji.20
      - MYSQL_DATABASE=database
  mysqlexporter:
    image: prom/mysqld-exporter
    ports:
      - "9104:9104"
    environment:
      - DATA_SOURCE_NAME=root:XUANji.20@(mysql:3306)/database
# 这里通过环境变量DATA_SOURCE_NAME方式定义监控目标,使用Docker Compose启动测试用的Mysql示例以及MySQLD Exporter

# docker-compose up -d

# 启动完成后，可以通过以下命令登录到Mysql容器当中，并执行Mysql相关的指令.
docker exec -it cb2017 mysql -uroot -pXUANji.20
mysql>

init_mysql() {
rpm -e mariadb-libs --nodeps
rm -rf /var/lib/mysql
rm -rf /etc/my.cnf
tar xvf /root/mysql-5.7.23-1.el7.x86_64.rpm-bundle.tar -C /usr/local/
cd /usr/local
rpm -ivh mysql-community-server-5.7.23-1.el7.x86_64.rpm 
mysql-community-client-5.7.23-1.el7.x86_64.rpm 
mysql-community-common-5.7.23-1.el7.x86_64.rpm 
mysql-community-libs-5.7.23-1.el7.x86_64.rpm 
rm -rf mysql-community-* 
}
changepass() {
sed -i '/[mysqld]/ a skip-grant-tables' /etc/my.cnf
systemctl restart mysqld
mysql <<EOF
        update mysql.user set authentication_string='' where user='root' and Host='localhost';
        flush privileges;
EOF
sed -i '/skip-grant/d' /etc/my.cnf
systemctl restart mysqld
yum -y install expect 
expect <<-EOF
spawn  mysqladmin -uroot -p password "ZHOUjian.20"
        expect {
                "password" { send "r"  }
}
        expect eof
EOF
systemctl restart mysqld
}
main() {
init_mysql
changepass
}
main

wget    
https://github.com/prometheus/mysqld_exporter/releases/download/v0.12.1/mysqld_exporter-0.12.1.linux-amd64.tar.gz  
tar xvf mysqld_exporter-0.10.0.linux-amd64.tar.gz -C /usr/local/  
cd /usr/local/  
mv mysqld_exporter-0.10.0.linux-amd64/ /usr/local/mysqld_exporter  
  
# 加载mysqld_exporter 添加配置文件（需要MySQL授权用户）   
# mysqld_exporter需要连接到MySQL,需要授权   
grant replication client, process on *.* to prometheus@"localhost" identified by "ZHOUjian.20";   
grant select on performance_schema.* to prometheus@"localhost";  
cat .my.cnf  
[client]  
user=prometheus  
password=ZHOUjian.20  
  
systemctl restart mysqld
nohup ./mysqld_exporter --config.my-cnf=.my.cnf &

  - job_name: 'mysqld'
    static_configs:
    - targets: ['172.19.0.27:9104']
systemctl restart prometheus

show global status like "Questions";
+---------------+-------+
| Variable_name | Value |
+---------------+-------+
| Questions     | 545   |
+---------------+-------+

rate(mysql_global_status_questions[2m])

show global status like "Com_insert";
+---------------+-------+
| Variable_name | Value |
+---------------+-------+
| Com_insert    | 0     |
+---------------+-------+

docker exec -it  cb2017 /bin/bash
create database test;
use test
create table student(id int,name varchar(50),sex enum('male','emale'),age int);
insert into student values(1,'tom','male',12),(2,'jack','male',13),(3,'alice','male',14);
for i in {1..1000}; do mysql -uroot -pZHOUjian.20 -e "use test;insert into student values(1,'tom','male',12),(2,'jack','male',13),(3,'alice','male',14);" ;done

 show variables like 'max_connections';
+-----------------+-------+
| Variable_name   | Value |
+-----------------+-------+
| max_connections | 151   |
+-----------------+-------+
1 row in set (0.00 sec)
# Mysql默认最大连接数为151，临时调整最大连接数，可以通过以下指令进行设置.

set global max_connections = 200;

mysql> show variables like 'max_connections';
+-----------------+-------+
| Variable_name   | Value |
+-----------------+-------+
| max_connections | 200   |
+-----------------+-------+

# 如果想永久化设置，则需要通过MySQL配置文件my.cnf，添加以下内容.
max_connections = 200

show global status like "Threads_connected";
+-------------------+-------+
| Variable_name     | Value |
+-------------------+-------+
| Threads_connected | 1     |
+-------------------+-------+

* mysql_global_variables_max_connections： 允许的最大连接数；
* mysql_global_status_threads_connected： 当前开放的连接；
* mysql_global_status_threads_running：当前开放的连接；
* mysql_global_status_aborted_connects：当前开放的连接；
* mysql_global_status_connection_errors_total{error="max_connections"}：由于超出最大连接数导致的错误；
* mysql_global_status_connection_errors_total{error="internal"}：由于系统内部导致的错误；

mysql_global_variables_max_connections - mysql_global_status_threads_connected

mysql_global_status_aborted_connects

show global status like "Innodb_buffer_pool_pages_total";
+--------------------------------+-------+
| Variable_name                  | Value |
+--------------------------------+-------+
| Innodb_buffer_pool_pages_total | 8191  |
+--------------------------------+-------+

a# HELP mysql_global_status_buffer_pool_pages Innodb buffer pool pages by state.
# TYPE mysql_global_status_buffer_pool_pages gauge
mysql_global_status_buffer_pool_pages{state="data"} 516
mysql_global_status_buffer_pool_pages{state="dirty"} 0
mysql_global_status_buffer_pool_pages{state="free"} 7675
mysql_global_status_buffer_pool_pages{state="misc"} 0

show global status like "Innodb_buffer_pool_read_requests";
+----------------------------------+-------+
| Variable_name                    | Value |
+----------------------------------+-------+
| Innodb_buffer_pool_read_requests | 1481  |
+----------------------------------+-------+

# HELP mysql_global_status_innodb_buffer_pool_read_requests Generic metric from SHOW GLOBAL STATUS.
# TYPE mysql_global_status_innodb_buffer_pool_read_requests untyped
mysql_global_status_innodb_buffer_pool_read_requests 1481

show global status like "Innodb_buffer_pool_reads";
+--------------------------+-------+
| Variable_name            | Value |
+--------------------------+-------+
| Innodb_buffer_pool_reads | 408   |
+--------------------------+-------+

rate(mysql_global_status_innodb_buffer_pool_reads[2m])

show variables like 'long_query_time';
+-----------------+-----------+
| Variable_name   | Value     |
+-----------------+-----------+
| long_query_time | 10.000000 |
+-----------------+-----------+

show global status like "slow_queries";
+---------------+-------+
| Variable_name | Value |
+---------------+-------+
| Slow_queries  | 0     |
+---------------+-------+

# HELP mysql_global_status_slow_queries Generic metric from SHOW GLOBAL STATUS.
# TYPE mysql_global_status_slow_queries untyped
mysql_global_status_slow_queries 0

rate(mysql_global_status_slow_queries[2m])

tar xvf prometheus-2.13.0.linux-amd64.tar.gz -C /usr/local/
cd /usr/local/
ln -s prometheus-2.13.0.linux-amd64/ prometheus

# 为了安全，我们使用普通用户来启动prometheus服务
# 作为一个时序型数据库产品，prometheus的数据默认会存放在应用目录下，我们需修改为/data/prometheus下 

useradd -s /sbin/nologin -M prometheus 
mkdir -p /data/prometheus 
chown -R prometheus:prometheus /usr/local/prometheus 
chown -R prometheus:prometheus /data/prometheus/

vim /usr/lib/systemd/system/prometheus.service
[Unit]
Description=Prometheus
Documentation=https://prometheus.io/
After=network.target
[Service]
Type=simple
User=prometheus
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml 
--storage.tsdb.path=/data/prometheus
Restart=on-failure
[Install]
WantedBy=multi-user.target

#在此配置文件里面，定义了启动的命令，定义了数据存储在/data/prometheus路径下
#否则默认会在prometheus二进制目录的data下.
systemctl start prometheus
systemctl status prometheus
systemctl enable prometheus

# 从官网下载好包后我们可以对他进行解压
tar xvf alertmanager-0.19.0.linux-amd64.tar.gz -C /usr/local/
cd /usr/local/alertmanager-0.19.0.linux-amd64/
./alertmanager &>/dev/null  &
# 接下来我们访问http://172.19.0.51:9093/#/alerts，就可以打开alertmanager的页面,

# 接下来我们编辑alertmanager.yml文件，配置邮箱账号授权码相关配置
cat alertmanager.yml
global:		# 全局配置，报警策略，报警渠道等.
  smtp_smarthost: 'smtp.163.com:25'
  smtp_from: '18621048481@163.com'
  smtp_auth_username: '18621048481@163.com'
  smtp_auth_password:  'ZHOUjian22' 		# 邮箱授权码
  smtp_require_tls: false 

route:		# 分发策略
  group_by: ['alertname'] 
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 5m
  receiver: 'email'

receivers:		# 接受者
- name: 'email'
  email_configs:
  - to: '18621048481@163.com'		# 接受的邮箱
    send_resolved: true
inhibit_rules:		# 抑制策略，当存在另一组匹配的警报，抑制规则将禁止与另一组匹配的警报.
  - source_match:
      serverity: 'critical'

cat /usr/lib/systemd/system/alertmanager.service 
[Unit]
Description=Alertmanager
After=network.target

[Service]
Type=simple
User=prometheus
ExecStart=//usr/local/alertmanager-0.19.0.linux-amd64/alertmanager 
  --config.file=/usr/local/alertmanager-0.19.0.linux-amd64/alertmanager.yml 
  --storage.path=/usr/local/alertmanager-0.19.0.linux-amd64/data
Restart=on-failure

[Install]
WantedBy=multi-user.target

systemctl restart alertmanager
# 如果服务启动失败报错，可以先systemctl daemon-reload，再重启

scrape_configs:
  - job_name: 'prometheus'

    static_configs:
    - targets: ['localhost:9090']

  - job_name: 'mysqld'
    static_configs:
    - targets: ['172.19.0.56:9104']

  - job_name: 'mysql'
    static_configs:
    - targets: ['172.19.0.27:9104']
systemctl restart prometheus

vim prometheus.yml
    - "first_rules.yml"

scrape_configs:
  - job_name: 'mysqld'
    scrape_interval: 15s
    honor_labels: true
    metrics_path: '/federate'

    params:
      'match[]':
        - '{job="prometheus"}'
        - '{__name__=~"mysql_up.*"}'
        - '{__name__=~"mysql_up.*"}'

    static_configs:
      - targets:
        - 'prometheus-slave2:9090'

  - job_name: 'mysql'
    scrape_interval: 15s
    honor_labels: true
    metrics_path: '/federate'

    params:
      'match[]':
        - '{job="prometheus"}'
        - '{__name__=~"mysql_up.*"}'
        - '{__name__=~"mysql_up.*"}'

    static_configs:
      - targets:
        - 'prometheus-slave2:9090'

vim /usr/local/prometheus/first_rules.yml
groups:
- name: mysql_up
  rules:
  - alert: mysql_up
    expr: up == 0
    for: 15s
    labels:
      severity: 1
      team: Prometheus_mysql
    annotations:
      summary: "{{$labels.instance}}Instance has been down for more than 5 minutes"
systemctl restart prometheus alertmanager