Docker 入门到实战教程(十二)ELK+Filebeat搭建日志分析系统

docker pull docker.elastic.co/elasticsearch/elasticsearch:7.6.1

vm.max_map_count=262144 # 修改或者新增

sysctl -w vm.max_map_count=262144

mkdir -p /usr/local/src/docker/es/data

/usr/local/elk/es/config/elasticsearch.yml

# 配置es的集群名称
cluster.name: "docker-cluster"
# 0.0.0.0为不限制，生产环境请设置为固定IP
network.host: 0.0.0.0
# 开启x-pack安全验证 访问时需要密码
xpack.security.enabled: true
# 关闭跨域验证（可以不开启）
http.cors.enabled: true
http.cors.allow-origin: "*"

docker run -d --name es -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -v /usr/local/src/elk/es/data:/usr/share/elasticsearch/data -v /usr/local/src/elk/es/config/elasticsearch.yml://usr/share/elasticsearch/config/elasticsearch.yml --privileged=true docker.elastic.co/elasticsearch/elasticsearch:7.3.1

把宿主机es挂载的data目录的权限开启就行了
chmod 777 /usr/local/src/elk/es/data

# docker 进入容器
docker exec -it es /bin/bash
# 修改密码
[root@g50 elasticsearch]# bin/elasticsearch-setup-passwords interactive

用户名: elastic
密码: xxxx

docker pull docker.elastic.co/logstash/logstash:7.6.1

mkdir -p /usr/local/src/elk/logstash/config
touch /usr/local/src/elk/logstash/config/logstash.conf
touch /usr/local/src/elk/logstash/config/logstash.yml

input {
    beats {
        port => 5044
	    ssl => false
    }
}

filter {
     grok {
        # 筛选过滤
        match => {
           "message" => "(?<timestamp>d{4}-d{2}-d{2}sd{2}:d{2}:d{2}.d{3}) %{GREEDYDATA:thread} %{LOGLEVEL:level} %{GREEDYDATA:class} - (?<msg>.*)"
        }
	remove_field => ["message"]
     }
	# 不匹配正则则删除，匹配正则用=~
     if [level] !~ "(ERROR|WARN|INFO)" {
         # 删除日志
         drop {}
     }
}


output {
   stdout {
    codec => rubydebug #控制台输出处理过后的数据
 }
if [fields][logtype] == "pre"{
    elasticsearch {
      hosts => ["ip:9200"]
      user => elastic
      password => xxx
      index => "pre"
    }
  }
if [fields][logtype] == "prex"{
    elasticsearch {
      hosts => ["ip:9200"]
      user => elastic
      password => xxx
      index => "prex"
    }
  }
}

grok 里边有定义好的现场的模板你可以用，但是更多的是自定义模板，规则是这样的，小括号里边包含所有一个key和value，例子：（?<key>value），比如以下的信息，第一个我定义的key是data，表示方法为：?<key> 前边一个问号，然后用<>把key包含在里边去。value就是纯正则了，这个我就不举例子了。这个有个在线的调试库，可以供大家参考，
http://grokdebug.herokuapp.com/

{
  "date": [
    [
      "2019-09-21 20:34:09.318"
    ]
  ],
  "thread": [
    [
      "[XNIO-1 task-39]"
    ]
  ],
  "level": [
    [
      "DEBUG"
    ]
  ],
  "class": [
    [
      "com.xd.pre.modules.sys.mapper.SysLogMapper.insert"
    ]
  ],
  "msg": [
    [
      "- ==> Parameters: 103.134.198.167(String), 1(Integer), admin(String), 查询用户集合(String), getList(String), /user(String), [com.baomidou.mybatisplus.extension.plugins.pagination.Page@1cb5e16a, UserDTO(userId=null, username=, password=null, deptId=0, jobId=null, phone=null, email=null, avatar=null, lockFlag=null, delFlag=null, roleList=null, deptList=null, newPassword=null, smsCode=null)](String), Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36(String), com.xd.pre.modules.sys.controller.SysUserController(String), GET(String), 2019-09-21T20:34:08.905(LocalDateTime), 2019-09-21T20:34:08.905(LocalDateTime), 0(Long)"
    ]
  ]
}

# 不受限限制访问
http.host: 0.0.0.0
# 使用了es安全认证所以要配置es相关信息
xpack.monitoring.elasticsearch.hosts:
- http://ip:9200
# 使用x-pack安全认证
xpack.monitoring.enabled: true
# es账号
xpack.monitoring.elasticsearch.username: "elastic" 
# es密码
xpack.monitoring.elasticsearch.password: "xxx"

docker run -p 5044:5044 
--name logstash -d 
-v /usr/local/src/elk/logstash/config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:rw 
-v /usr/local/src/elk/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:rw 
docker.elastic.co/logstash/logstash:7.6.1

docker logs -f logstash

docker pull docker.elastic.co/kibana/kibana:7.6.1

mkdir /usr/local/src/elk/kibana
mkdir /usr/local/src/elk/kibana/config
touch /usr/local/src/elk/kibana/config/kibana.yml

vim /usr/local/src/elk/kibana/config/kibana.yml

server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
elasticsearch.username: elastic
elasticsearch.password: xxx
xpack.monitoring.ui.container.elasticsearch.enabled: true
#汉化
i18n.locale: "zh-CN"

docker run -d -p 5601:5601 
--name=kibana 
-v /usr/local/src/elk/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml 
docker.elastic.co/kibana/kibana:7.6.1

pull docker.elastic.co/beats/filebeat:7.6.1

mkdir -p /usr/local/src/elk/filebeat/data
mkdir -p /usr/local/src/elk/filebeat/config
touch /usr/local/elk/filebeat/config/filebeat.yml

filebeat.inputs:
- input_type: log
  enable: true
  paths:  # 采集日志的路径这里是容器内的path
   - /Users/lihaodong/Desktop/log/**
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  multiline.timeout: 10s
  # 为每个项目标识,或者分组，可区分不同格式的日志
  tags: ["pre-logs"]
  # 这个文件记录日志读取的位置，如果容器重启，可以从记录的位置开始取日志
  registry_file: /usr/share/filebeat/data/
  fields:
    logsource: node1
    logtype: pre

# 输出到logstash中,logstash更换为自己的ip
output.logstash:
  enabled: true
  hosts: ["ip:5044"]

output:
    logstash:
        hosts: ["ip:5044"]
        worker: 2
        loadbalance: true
        index: filebeat
 
这里worker 的含义，是 beat 连到每个 host 的线程数。
在  loadbalance  开启的情况下，意味着有 4 个worker 轮训发送数据

docker run --name filebeat -d 
-v /Users/lihaodong/Desktop/log:/Users/lihaodong/Desktop/log 
-v /usr/local/src/elk/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml 
-v /usr/local/src/elk/filebeat/data:/usr/share/filebeat/data 
docker.elastic.co/beats/filebeat:7.6.1

docker logs -f filebeat

chmod 777 /usr/local/src/elk/filebeat/data

docker logs -f filebeat