启动
1、logstash-sample.conf
# Sample Logstash configuration for creating a simple
input {
redis {
batch_count => 1
type => "redis-input"
data_type => "list"
key => "logstash_test_list"
host => "127.0.0.1"
port => 6379
password => "Aroot1234@A"
db => 0
threads => 5
codec => "json"
}
beats{
host => "127.0.0.1"
port => 5044
}
}
filter {
}
output {
if [fields][document_type]=="api" {
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "apinadiyi-%{+YYYY.MM.dd}"
# template_name => "apinadiyi"
}
stdout {
codec => rubydebug
}
}
if \[type\]=="redis-input" {
elasticsearch {
hosts => \["127.0.0.1:9200"\]
index => "logstash-%{+YYYY.MM.dd}"
# document\_type => "logs" # 7之后不支持了
}
stdout {
codec => rubydebug
}
}
}
2、filebeat.yml
###################### Filebeat Configuration Example #########################
#=========================== Filebeat inputs =============================
filebeat.inputs:
type: log
enabled: true
paths:
#- /var/log/*.log
#- c:\programdata\elasticsearch\logs\*
fields:
document_type: "api"
#exclude_lines: ['^DBG']
#include_lines: ['^ERR', '^WARN']
#exclude_files: ['.gz$']
#fields:
#multiline.pattern: ^\[
#multiline.negate: false
#multiline.match: after
#============================= Filebeat modules ===============================
filebeat.config.modules:
# Glob pattern for configuration loading
path: ${path.config}/modules.d/*.yml
# Set to true to enable config reloading
reload.enabled: true
# Period on which files under path should be checked for changes
#reload.period: 10s
#==================== Elasticsearch template setting ==========================
setup.template.settings:
index.number_of_shards: 1
#index.codec: best_compression
#_source.enabled: false
#================================ General =====================================
#name:
#tags: ["service-X", "web-tier"]
#fields:
#============================== Dashboards =====================================
#setup.dashboards.enabled: false
#setup.dashboards.url:
#============================== Kibana =====================================
setup.kibana:
# Kibana Host
# Scheme and port can be left out and will be set to the default (http and 5601)
# In case you specify and additional path, the scheme is required: http://localhost:5601/path
# IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
#host: "localhost:5601"
# Kibana Space ID
# ID of the Kibana Space into which the dashboards should be loaded. By default,
# the Default Space will be used.
#space.id:
#============================= Elastic Cloud ==================================
#cloud.id:
#cloud.auth:
#================================ Outputs =====================================
#-------------------------- Elasticsearch output ------------------------------
#output.elasticsearch:
# Array of hosts to connect to.
#hosts: ["localhost:9200"]
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
#----------------------------- Logstash output --------------------------------
output.logstash:
# The Logstash hosts
hosts: ["127.0.0.1:5044"]
# Optional SSL. By default is off.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
#================================ Processors =====================================
processors:
#================================ Logging =====================================
#logging.level: debug
#logging.selectors: ["*"]
#============================== X-Pack Monitoring ===============================
#monitoring.enabled: false
#monitoring.cluster_uuid:
#monitoring.elasticsearch:
#================================= Migration ==================================
#migration.6_to_7.enabled: true
1、filebeat.yml
###################### Filebeat Configuration Example #########################
#=========================== Filebeat prospectors =============================
filebeat.prospectors:
input_type: log
paths:
#排除更改时间超过定义的文件,时间字符串可以用2h表示2小时,5m表示5分钟,默认0
ignore_older: 0
#prospector扫描新文件的时间间隔,默认10秒
scan_frequency: 5s
#harvester_buffer_size: 16384
#单文件最大收集的字节数,单文件超过此字节数后的字节将被丢弃,默认10MB,需要增大,保持与日志输出配置的单文件最大值一致即可
max_bytes: 1048576000
#exclude_lines: ["^DBG"]
#include_lines: ["^ERR", "^WARN"]
#exclude_files: [".gz$"]
#fields:
#multiline.pattern: ^\[
#multiline.negate: false
#multiline.match: after
#========================= Filebeat global options ============================
#filebeat.spool_size: 2048
#filebeat.publish_async: false
#filebeat.idle_timeout: 5s
#filebeat.registry_file: ${path.data}/registry
#filebeat.config_dir:
#filebeat.shutdown_timeout: 0
#================================ General =====================================
#name:
#tags: ["service-X", "web-tier"]
#fields:
#处理管道中单个事件内的队列大小,默认1000
queue_size: 2000
#bulk_queue_size: 0
#================================ Outputs =====================================
#-------------------------- Elasticsearch output ------------------------------
#output.elasticsearch:
# Array of hosts to connect to.
#hosts: ["localhost:9200"]
# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
#password: "changeme"
#----------------------------- Logstash output --------------------------------
output.logstash:
# The Logstash hosts
hosts: ["192.168.6.204:4501"]
# Optional SSL. By default is off.
# List of root certificates for HTTPS server verifications
#ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
# Certificate for SSL client authentication
#ssl.certificate: "/etc/pki/client/cert.pem"
# Client Certificate Key
#ssl.key: "/etc/pki/client/cert.key"
#================================ Logging =====================================
#logging.level: debug
#logging.selectors: ["*"]
2、nginx_log.conf
input {
file {
path => "/opt/logstash-6.7.0/data/*.test.json"
type => "test"
start_position => "beginning"
sincedb_path => "/opt/logstash-6.7.0/data/test-sincedb"
}
}
input {
beats {
port => 4501
ssl => false
}
}
filter {
if [fields][logtype] in ["test", "nginx"] or [type]=="test" {
grok {
#设置自定义正则路径
patterns_dir => ["/opt/logstash-6.7.0/config/patterns/nginx"]
match => {
"message" => "%{NGINXACCESS}"
}
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
target => "@timestamp"
locale => "cn"
}
useragent{
source => "agent"
prefix => "agent\_"
remove\_field => "agent"
}
#定义客户端的IP是哪个字段(上面定义的数据格式)
if \[clientip\] {
geoip {
source => "clientip" # 取自nginx中的客户端ip
}
}
if !\[request\_time\] {
mutate {
add\_field => {
"request\_time" => "0.0"
}
}
}
if !\[upstream\_response\_time\] {
mutate {
add\_field => {
"upstream\_response\_time" => "0.0"
}
}
}
#需要进行转换的字段,这里是将访问的时间转成int,再传给Elasticsearch
mutate {
convert => \["bytes", "integer"\]
convert => \["\[geoip\]\[coordinates\]", "float" \]
convert => \["request\_time", "float"\]
convert => \["upstream\_response\_time", "float"\]
}
}
# 对模板点击参数进行分解
# /Mould/GetLmsgBoard/?t=1554737610678&id=1&m\_no=F2018\_11\_21\_00100
# func=Mould
# module=GetLmsgBoard
# para=?t=...
if \[type\]=="api" {
mutate{
add\_field => {
"tempmessage" => "%{\[request\]}"
}
}
mutate{
split => \["tempmessage","/"\]
add\_field => {
"module" => "%{\[tempmessage\]\[1\]}"
}
add\_field => {
"func" => "%{\[tempmessage\]\[2\]}"
}
add\_field => {
"para" => "%{\[tempmessage\]\[3\]}"
}
}
if \[func\] == "GetLmsgBoard" {
kv {
source => "para"
include\_keys => \["id","m\_no"\]
prefix => "msg\_"
field\_split => "&? "
add\_field => {
"type" => "api"
}
}
mutate{
convert => \[ "meg\_id", "integer"\]
replace => { "type" => "message"}
}
}
mutate{
remove\_field => "tempmessage"
remove\_field => "para"
}
}
#####################################################################
# Baidu
#####################################################################
if \[type\]=="baidu" {
date {
match => \["datetime", "yyyy/MM/dd HH:mm:ss Z"\]
target => "@timestamp"
locale => "cn"
timezone => "Asia/Shanghai"
}
#定义客户端的IP是哪个字段(上面定义的数据格式)
geoip {
source => "ip"
}
mutate {
convert => { "datetime" => "string" }
}
}
#####################################################################
# ERP User
#####################################################################
if \[type\]=="erpuser" {
date {
match => \["time", "yyyy-MM-dd HH:mm:ss Z"\]
target => "@timestamp"
locale => "cn"
timezone => "Asia/Shanghai"
}
#定义客户端的IP是哪个字段(上面定义的数据格式)
geoip {
source => "ip"
}
mutate {
convert => { "time" => "string" }
}
}
}
output {
if [type]=="fanyi" {
elasticsearch {
hosts => ["http://192.168.6.204:9200"]
index => "nginx-fanyi-%{+YYYY.MM}"
template_name => "nginx"
}
}
if \[type\]=="api" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "apinadiyi-%{+YYYY.MM.dd}"
template\_name => "apinadiyi"
}
}
if \[type\]=="message" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "message-%{+YYYY.MM.dd}"
template\_name => "message"
}
}
if \[type\]=="beimu" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "nginx-beimu-%{+YYYY.MM}"
template\_name => "nginx"
}
}
if \[type\]=="syd" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "nginx-syd-%{+YYYY.MM}"
template\_name => "nginx"
}
}
if \[type\]=="nadiyi" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "nginx-nadiyi-%{+YYYY.MM}"
template\_name => "nginx"
}
}
if \[type\]=="jiajiao" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "nginx-jiajiao-%{+YYYY.MM}"
template\_name => "nginx"
}
}
if \[type\]=="tingclass" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "nginx-tingclass-%{+YYYY.MM}"
template\_name => "nginx"
}
}
if \[type\]=="baidu" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "baidu-%{+YYYY.MM.dd}"
template\_name => "baidu"
}
#stdout {
# codec => rubydebug
#}
}
if \[type\]=="erpuser" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "erpuser-new-%{+YYYY.MM}"
template\_name => "erpuser"
}
}
if \[type\] == "test" {
elasticsearch {
hosts => \["http://192.168.6.204:9200"\]
index => "nginx-access-test-%{+YYYY.MM}"
}
stdout {
codec => rubydebug
}
}
}
3、nginx_template.json
PUT _template/nginx
{
"order" : 0,
"version" : 190407,
"index_patterns" : [
"nginx-*"
],
"settings" : {
"number_of_shards": 1,
"number_of_replicas": 1,
"index" : {
"refresh_interval" : "30s"
}
},
"mappings" : {
"doc" : {
"dynamic_templates" : [
{
"message_field" : {
"path_match" : "message",
"mapping" : {
"index": "false",
"norms" : false,
"type" : "text"
},
"match_mapping_type" : "string"
}
},
{
"string_fields" : {
"mapping" : {
"index": "true",
"analyzer": "ik_max_word",
"search_analyzer": "ik_smart",
"norms" : false,
"type" : "text",
"fields" : {
"keyword" : {
"ignore_above" : 512,
"type" : "keyword"
}
}
},
"match_mapping_type" : "string",
"match" : "*"
}
}
],
"properties" : {
"@timestamp" : {
"type" : "date"
},
"geoip" : {
"dynamic" : true,
"properties" : {
"ip" : {
"type" : "ip"
},
"latitude" : {
"type" : "half_float"
},
"location" : {
"type" : "geo_point"
},
"longitude" : {
"type" : "half_float"
}
}
},
"@version" : {
"type" : "keyword"
}
}
}
},
"aliases" : {
"nginx_this_week":{}
}
}
4、nginx
URIPATH1 (?:/[\\A-Za-z0-9$.+!*'(){},~:;=@#% \[\]_<>^\-&?]*)+
URI1 %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATH1 })?
NGINXACCESS %{IPORHOST:clientip} \[%{HTTPDATE:timestamp}\] %{NUMBER:response} (?:%{WORD:catch}|-) %{WORD:verb} %{URIPATH1:request} HTTP/%{NUMBER:httpversion} (?:%{NUMBER:bytes}|-) (?:%{NUMBER:request_time}|-) (?:%{NUMBER:upstream_response_time}|-) (?:%{URI1:referrer}|-) (?:%{QS:agent}|-) (?:(%{IPORHOST:upsteam_server}:%{POSINT:up_port})|-) (?:%{NUMBER:up_request}|-)
手机扫一扫
移动阅读更方便
你可能感兴趣的文章