1 查看Java格式日志

elasticsearch属于Java日志，可以收集elasticsearch作为Java日志范本

[root@node3 ~]# tail -f /usr/local/elasticsearch/logs/my-elktest-cluster.log

[2020-01-19T01:57:18,496][INFO ][o.e.t.TransportService ] [node-3] publish_address {192.168.132.133:9300}, bound_addresses {[::]:9300}
[2020-01-19T01:57:18,506][INFO ][o.e.b.BootstrapChecks ] [node-3] bound or publishing to a non-loopback address, enforcing bootstrap checks
[2020-01-19T01:57:18,531][INFO ][o.e.c.c.Coordinator ] [node-3] cluster UUID [4xt-ZTijTz2oTnlz1gMFjg]
[2020-01-19T01:57:19,195][INFO ][o.e.c.s.ClusterApplierService] [node-3] master node changed {previous [], current [{node-2}{9qVjdVSvSAGlZ7lpB9O78g}{m9_mEscHTMKQ0VwFt9YHog}{192.168.132.132}{192.168.132.132:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true}]}, added {{node-1}{KupGTMgUTTmp3poPa9iYdQ}{9vTQ4MQfRgqkB_gv4Jyp_A}{192.168.132.131}{192.168.132.131:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true},{node-2}{9qVjdVSvSAGlZ7lpB9O78g}{m9_mEscHTMKQ0VwFt9YHog}{192.168.132.132}{192.168.132.132:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true},}, term: 705, version: 875, reason: ApplyCommitRequest{term=705, version=875, sourceNode={node-2}{9qVjdVSvSAGlZ7lpB9O78g}{m9_mEscHTMKQ0VwFt9YHog}{192.168.132.132}{192.168.132.132:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true}}

2 直接使用filebeat收集

直接配置，使用filebeat收集

filebeat.inputs:
#####################################################

Nginx log

#####################################################

• type: log
  enabled: true
  paths:

  • /usr/local/nginx/logs/access.log
    json.key_under_root: true
    json.overwrite_keys: true
    tags: ["access"]

• type: log
  enabled: true
  paths:

  • /usr/local/nginx/logs/error.log
    tags: ["error"]

#####################################################

tomcat log

#####################################################

• type: log enabled: true paths:
  • /var/log/tomcat/localhost_access_log.*.txt
    json.key_under_root: true
    json.overwrite_keys: true
    tags: ["tomcat"]

#####################################################

tomcat log

#####################################################

• type: log enabled: true paths:
  • /usr/local/elasticsearch/logs/my-elktest-cluster.log
    tags: ["es-java"]
    #####################################################

Output

#####################################################
setup.kibana:
host: "192.168.132.131:5601"
output.elasticsearch:
hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]
#index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"
indices:
- index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"
when.contains:
tags: "access"
- index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"
when.contains:
tags: "error"
- index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"
when.contains:
tags: "tomcat"
- index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"
when.contains:
tags: "es-java"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.overwrite: true
setup.template.enabled: true
setup.ilm.enabled: false

[root@node3 ~]# systemctl restart filebeat

3 错误日志显示不清晰

但是错误信息分开显示

[2020-01-19T01:55:48,077][WARN ][o.e.c.NodeConnectionsService] [node-3] failed to connect to {node-2}{9qVjdVSvSAGlZ7lpB9O78g}{aFwQRTD4TWKE97npXNwSVg}{192.168.132.132}{192.168.132.132:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true} (tried [1] times)
org.elasticsearch.transport.ConnectTransportException: [node-2][192.168.132.132:9300] connect_exception
at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:976) ~[elasticsearch-7.4.2.jar:7.4.2]
at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:161) ~[elasticsearch-7.4.2.jar:7.4.2]
at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.4.2.jar:7.4.2]
at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?]
at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?]
at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?]
at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?]
at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.4.2.jar:7.4.2]
at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:500) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:493) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:472) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:413) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:538) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:531) ~[?:?]
at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:111) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:323) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:685) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:597) ~[?:?]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:551) ~[?:?]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:511) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?]
at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?]
at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:327) ~[?:?]
at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:336) ~[?:?]
… 7 more
[2020-01-19T01:55:49,727][INFO ][o.e.n.Node ] [node-3] stopping …

这是一段错误日志，需要单独收集

4 配置filebat的多行匹配

以时间作为标识，再读取一个时间标记之后，先缓存，当读到下一个时间标记，再整个发送日志，相当于多行匹配

官方文档：https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html

multiline.pattern: '^\['
multiline.negate: true
multiline.match: after

配置filebeat

filebeat.inputs:
#####################################################

Nginx log

#####################################################

• type: log
  enabled: true
  paths:

  • /usr/local/nginx/logs/access.log
    json.key_under_root: true
    json.overwrite_keys: true
    tags: ["access"]

• type: log
  enabled: true
  paths:

  • /usr/local/nginx/logs/error.log
    tags: ["error"]

#####################################################

tomcat log

#####################################################

• type: log enabled: true paths:
  • /var/log/tomcat/localhost_access_log.*.txt
    json.key_under_root: true
    json.overwrite_keys: true
    tags: ["tomcat"]

#####################################################

tomcat log

#####################################################

• type: log enabled: true paths:
  • /usr/local/elasticsearch/logs/my-elktest-cluster.log
    tags: ["es-java"]
    multiline.pattern: '^\['
    multiline.negate: true
    multiline.match: "after"
    #####################################################

Output

#####################################################
setup.kibana:
host: "192.168.132.131:5601"
output.elasticsearch:
hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"]
#index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}"
indices:
- index: "access-%{[agent.version]}-%{+yyyy.MM.dd}"
when.contains:
tags: "access"
- index: "error-%{[agent.version]}-%{+yyyy.MM.dd}"
when.contains:
tags: "error"
- index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}"
when.contains:
tags: "tomcat"
- index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}"
when.contains:
tags: "es-java"

setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.overwrite: true
setup.template.enabled: true
setup.ilm.enabled: false

[root@node3 ~]# systemctl restart filebeat

5 生成错误日志

然后修改配置文件，重启产生错误日志

到kibana查看，错误日志一起显示

收集Java日志，配置完成