logstash 处理各种时间格式

来源：互联网发布：剑网南风捏脸米苏数据编辑：程序博客网时间：2024/06/01 10:25
tomcat access日志:{            "@version" => "1",          "@timestamp" => "2016-10-22T12:58:07.000Z",                "path" => "/data01/applog_backup/zjzc_log/zj-api-access01.2016-10-22",                "host" => "dr-mysql01.zjcap.com",                "type" => "zj_api_access",            "clientip" => "10.252.142.174",                "time" => "22/Oct/2016:20:58:07 +0800",                "verb" => "GET",                 "api" => "/api/validate/code/send",         "httpversion" => "1.1",    "http_status_code" => "200",               "bytes" => "52",            "remoteip" => "115.51.148.47",       "response_time" => 0.015,            "messager" => "zj_api_access- 10.252.142.174 - - [22/Oct/2016:20:58:07 +0800] \"GET /api/validate/code/send?mobilePhone=15090308333&messageType=1&_=1454297673274 HTTP/1.1\" 200 52 0.015 115.51.148.47"}"message" , "\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?<api>(\S+))\?.*\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+(%{BASE16FLOAT:request_time})\s+%{IPORHOST:remoteip}","message" ,"\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?<api>(\S+))\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+(%{BASE16FLOAT:request_time})\s+%{IPORHOST:remoteip}","message" ,"\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?<api>(\S+))\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+\-\s+(%{BASE16FLOAT:request_time})\s+%{IPORHOST:remoteip}","message","\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?<api>(\S+))\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+\-\s+(%{BASE16FLOAT:request_time})\s+(%{IPORHOST:remoteip}|-)"tomcat catalina日志；{    "@timestamp" => "2016-10-22T12:59:22.877Z",      "@version" => "1",          "path" => "/data01/applog_backup/zjzc_log/zj-api02-catalina.out.2016-10-22",          "host" => "dr-mysql01.zjcap.com",          "type" => "zj_api",      "messager" => "zj_api- 2016-10-22 20:59:22,877 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",          "time" => "2016-10-22 20:59:22,877",         "Level" => "INFO"}filter {    grok {        match => [ "message","\s*%{TIMESTAMP_ISO8601:time}\s+(?<Level>(\S+)).*"]     }     date {        match => ["time", "yyyy-MM-dd HH:mm:ss,SSS"]    }     mutate {       remove_field =>["message"]        }}nginx access 日志；{                 "message" => " 10.171.246.184 [22/Oct/2016:21:00:40 +0800] \"GET /resources/images/icon/icon_phone_gray.273e583f.png HTTP/1.1\" - 200 352 \"https://www.zjcap.cn/resources/css/base.css?06212016\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36\" 0.000 115.236.160.82",                "@version" => "1",              "@timestamp" => "2016-10-22T13:00:40.000Z",                    "path" => "/data01/applog_backup/zjzc_log/zj-frontend01-access.2016-10-22",                    "host" => "dr-mysql01.zjcap.com",                    "type" => "zj_frontend_access",                "clientip" => "10.171.246.184",                    "time" => "22/Oct/2016:21:00:40 +0800",                    "verb" => "GET",                 "request" => "/resources/images/icon/icon_phone_gray.273e583f.png",             "httpversion" => "1.1",        "http_status_code" => "200",                   "bytes" => "352",            "http_referer" => "https://www.zjcap.cn/resources/css/base.css?06212016",         "http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",    "http_x_forwarded_for" => "115.236.160.82",                   "geoip" => {                      "ip" => "115.236.160.82",           "country_code2" => "CN",           "country_code3" => "CHN",            "country_name" => "China",          "continent_code" => "AS",             "region_name" => "02",               "city_name" => "Hangzhou",                "latitude" => 30.293599999999998,               "longitude" => 120.16140000000001,                "timezone" => "Asia/Shanghai",        "real_region_name" => "Zhejiang",                "location" => [            [0] 120.16140000000001,            [1] 30.293599999999998        ],             "coordinates" => [            [0] 120.16140000000001,            [1] 30.293599999999998        ]    },           "response_time" => 0.0,                "messager" => "zj_frontend_access 10.171.246.184 [22/Oct/2016:21:00:40 +0800] \"GET /resources/images/icon/icon_phone_gray.273e583f.png HTTP/1.1\" - 200 352 \"https://www.zjcap.cn/resources/css/base.css?06212016\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36\" 0.000 115.236.160.82"filter {    grok {        match =>[              "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request}\?.* HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?<http_referer>\S+)\" \"(?<http_user_agent>(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",              "message" , "%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?<http_referer>\S+)\" \"(?<http_user_agent>(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} (?<http_url>\S+)\s+HTTP/%{NUMBER:httpversion}\"\s+\-\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+\"\-\"\s+\"(?<http_user_agent>(\S+))\"\s+(%{BASE16FLOAT:request_time})\s+(%{IPORHOST:http_x_forwarded_for}|-)",             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"\" \"(?<http_user_agent>(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"                     ]    }   nginx error 日志；         "message" => " 2016/10/22 21:00:32 [error] 12890#0: *98081 open() \"/var/www/zjzc-web-frontEnd/favicon.ico\" failed (2: No such file or directory), client: 10.171.246.184, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"www.zjcap.cn\"",        "@version" => "1",      "@timestamp" => "2016-10-22T13:00:32.000Z",            "path" => "/data01/applog_backup/zjzc_log/zj-frontend01-error.2016-10-22",            "host" => "dr-mysql01.zjcap.com",            "type" => "zj_frontend_error",            "time" => "2016/10/22 21:00:32",        "severity" => "error",             "pid" => "12890",    "errormessage" => "*98081 open() \"/var/www/zjzc-web-frontEnd/favicon.ico\" failed (2: No such file or directory)",     "remote_addr" => "10.171.246.184",          "server" => "localhost",         "request" => "\"GET /favicon.ico HTTP/1.1\"",    "request_host" => "\"www.zjcap.cn\""}filter {        grok {            match => [ "message" , "(?<time>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<remote_addr>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)(?:, request: %{QS:request})?(?:, upstream: (?<upstream>\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:referrer}\")?"]        }         date {        match => ["time", "yyyy/MM/dd HH:mm:ss"]    }}
0 0