一、通过TCP/UDP收集日志

1.配置

[root@m01 conf.d]# vim tcp.conf
input {
  tcp {
    port => 1234
    type => "tcplog"
    mode => "server"
  }
}

output {
  stdout {
    codec => rubydebug
  }
}

#启动
[root@m01 conf.d]# /usr/share/logstash/bin/logstash -f tcp.conf

2.测试

#使用telnet测试
[root@redis03 ~]# telnet 10.0.0.61 1234
Trying 10.0.0.61...
Connected to 10.0.0.61.
Escape character is '^]'.
123
qwertyuiop

#查看输出内容
{
          "host" => "10.0.0.93",
          "port" => 55112,
       "message" => "123\r",
      "@version" => "1",
    "@timestamp" => 2020-05-25T02:00:59.152Z,
          "type" => "tcplog"
}
{
          "host" => "10.0.0.93",
          "port" => 55112,
       "message" => "qwertyuiop\r",
      "@version" => "1",
    "@timestamp" => 2020-05-25T02:02:51.973Z,
          "type" => "tcplog"
}

3.使用nc命令测试

#安装
[root@m01 ~]# yum install -y nc

#使用nc传送数据
[root@redis03 ~]# nc 10.0.0.61 1234
123
[root@redis03 ~]# echo "test nc" | nc 10.0.0.61 1234

#使用nc发送文件
[root@redis03 ~]# cat /var/log/messages | nc 10.0.0.61 1234

#发送伪设备数据
[root@redis03 ~]# echo "伪设备 测试"  > /dev/tcp/10.0.0.61/1234

4.将数据存储到ES

[root@m01 conf.d]# vim tcp.conf 
input {
  tcp {
    port => 1234
    type => "tcplog"
    mode => "server"
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.91:9200"]
    index => "tcp_log-%{+YYYY-MM-dd}"
  }
}

5.到ES或者kibana查看数据

二、Filebeat学习

filebeat与logstash作用是一样的
E/L/K都是java程序写的
filebeat是golang语言写的，比较轻量

Filebeat模块很好的入门，它是轻量级单用途的日志收集工具，用于在没有安装java的服务器上专门收集日志，可以将日志转发到logstash、elasticsearch或redis等场景中进行下一步处理。

1.安装filebeat

#上传代码包
[root@redis03 ~]# rz filebeat-6.6.0-x86_64.rpm

#安装
[root@redis03 ~]# rpm -ivh filebeat-6.6.0-x86_64.rpm

2.配置文件

[root@redis03 ~]# rpm -qc filebeat
/etc/filebeat/filebeat.yml

3.日志文件

[root@m01 ~]# tail -f -n 100 /var/log/filebeat/filebeat

三、Filebeat收集单类型日志到本地文件

1.配置

#备份
[root@redis03 ~]# cp /etc/filebeat/filebeat.yml /etc/filebeat/filebeat.yml.bak

#配置
[root@m01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access_json.log

output.file:
  path: "/tmp"
  filename: "filebeat.log"

2.启动

[root@m01 ~]# systemctl start filebeat.service

#验证
[root@m01 ~]# ps -ef | grep filebeat
root       3415      1  0 11:04 ?        00:00:00 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/sharefilebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat
root       3434 125832  0 11:04 pts/0    00:00:00 grep --color=auto filebeat

3.访问页面测试

#访问nginx以后，查看/tmp目录下

[root@m01 ~]# ll /tmp/
total 52
-rw------- 1 root root  3037 May 25 11:08 filebeat.log

四、Filebeat收集单类型日志到ES

1.配置

[root@m01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access_json.log

output.elasticsearch:
  hosts: ["10.0.0.91:9200"]

2.重启

[root@m01 ~]# systemctl restart filebeat.service

3.访问nginx测试

五、修改收集的日志格式

1.配置

[root@m01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access_json.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["10.0.0.91:9200"]

2.重新配置nginx日志格式为json格式

[root@m01 ~]# vim /etc/nginx/nginx.conf
    log_format log_json '{ "time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr",'
                          '"up_host": "$upstream_http_host",'
                          '"upstream_time": "$upstream_response_time",'
                          '"request_time": "$request_time" }';

    access_log  /var/log/nginx/access_json.log  log_json;

3.访问nginx，kibana查看数据格式

六、收集日志到ES指定所有名称

1.配置

[root@m01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access_json.log
  json.keys_under_root: true
  json.overwrite_keys: true
  #以上两句是开启json格式

output.elasticsearch:
  hosts: ["10.0.0.91:9200"]
  index: "nginx_json-%{[beat.version]}-%{+yyyy.MM.dd}"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.overwrite: false
setup.template.json.enabled: true
setup.template.enabled: false
setup.ilm.enabled: false
#注意以上几行顶格

2.重启

[root@m01 ~]# systemctl restart filebeat.service

3.访问nginx查看ES数据

4.指定分片和副本数

setup.template.settings:
  index.number_of_shards: 1
  index.number_of_replicas: 1