logstash indexer和shipper的配置

来源：互联网发布：centos linux 删除软件编辑：程序博客网时间：2024/06/05 09:42
[elk@zjtest7-frontend config]$ cat logstash_agent.conf input {        file {                type => "zj_nginx_access"                path => ["/rsyslog/data/nginx/zjzc/nginx_access0*_log.*"]                ignore_older => 87400        }           file {                 type => "uat_nginx_access"                 path => ["/rsyslog/data/nginx/uat/nginx_access0*_log.*"]                 ignore_older => 87400         }  }filter {    grok {        match => {            "message" => "%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?<http_referer>\S+)\" \"(?<http_user_agent>\S+)\" \"(?<http_x_forwarded_for>\S+)\""        }    }   }output {     if [type] == "zj_nginx_access" {         redis {                host => "192.168.32.67"                data_type => "list"                key => "zj_nginx:redis"                port=>"6379"                password => "1234567"        }}      else if [type] == "uat_nginx_access"{       redis {                 host => "192.168.32.67"                 data_type => "list"                 key => "uat_nginx:redis"                 port=>"6379"                 password => "1234567"         } }} indexer.conf:input {        redis {                host => "192.168.32.67"                data_type => "list"                key => "zj_nginx:redis"                password => "1234567"                port =>"6379"        }        redis {                host => "192.168.32.67"                data_type => "list"                key => "uat_nginx:redis"                password => "1234567"                port =>"6379"        }}output {      if   [type] == "zj_nginx_access"{         elasticsearch {                hosts => "192.168.32.80:9200"                index => "logstash-zjzc-nginx-%{+YYYY.MM.dd}"        }stdout {codec => rubydebug}      }        else if  [type] == "uat_nginx_access"{      elasticsearch {                hosts => "192.168.32.81:9200"                index => "logstash-uat-nginx-%{+YYYY.MM.dd}"        }                stdout {                        codec => rubydebug                }     }} redis消息里有type字段；127.0.0.1:6379> LPOP "zj_nginx:redis""{\"message\":\" 120.26.44.206:8001 120.26.44.206 120.26.44.206 [22/Aug/2016:22:12:58 +0800] \\\"GET / HTTP/1.1\\\" - 200 30626 \\\"-\\\" \\\"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.15.3 zlib/1.2.3 libidn/1.18 libssh2/1.4.2\\\" 0.000 -\",\"@version\":\"1\",\"@timestamp\":\"2016-08-22T14:10:55.846Z\",\"path\":\"/rsyslog/data/nginx/zjzc/nginx_access01_log.2016-08-22\",\"host\":\"0.0.0.0\",\"type\":\"zj_nginx_access\",\"tags\":[\"_grokparsefailure\"]}"{       "message" => " 120.26.44.206:8001 120.26.44.206 120.26.44.206 [22/Aug/2016:22:18:58 +0800] \"GET / HTTP/1.1\" - 200 30626 \"-\" \"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.15.3 zlib/1.2.3 libidn/1.18 libssh2/1.4.2\" 0.000 -",      "@version" => "1",    "@timestamp" => "2016-08-22T14:16:55.738Z",          "path" => "/rsyslog/data/nginx/zjzc/nginx_access01_log.2016-08-22",          "host" => "0.0.0.0",          "type" => "zj_nginx_access",          "tags" => [        [0] "_grokparsefailure"    ]}{       "message" => " 121.40.189.90:8001 121.40.189.90 120.26.44.206 [22/Aug/2016:22:14:13 +0800] \"GET / HTTP/1.1\" - 200 30338 \"-\" \"curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.15.3 zlib/1.2.3 libidn/1.18 libssh2/1.4.2\" 0.001 -",      "@version" => "1",    "@timestamp" => "2016-08-22T14:17:04.110Z",          "path" => "/rsyslog/data/nginx/uat/nginx_access01_log.2016-08-22",          "host" => "0.0.0.0",          "type" => "uat_nginx_access",          "tags" => [        [0] "_grokparsefailure"    ]}
0 0