logstash 处理各种时间格式

tomcat access日志:

{

            "@version" => "1",

          "@timestamp" => "2016-10-22T12:58:07.000Z",

                "path" => "/data01/applog_backup/zjzc_log/zj-api-access01.2016-10-22",

                "host" => "dr-mysql01.zjcap.com",

                "type" => "zj_api_access",

            "clientip" => "10.252.142.174",

                "time" => "22/Oct/2016:20:58:07 +0800",

                "verb" => "GET",

                 "api" => "/api/validate/code/send",

         "httpversion" => "1.1",

    "http_status_code" => "200",

               "bytes" => "52",

            "remoteip" => "115.51.148.47",

       "response_time" => 0.015,

            "messager" => "zj_api_access- 10.252.142.174 - - [22/Oct/2016:20:58:07 +0800] \"GET /api/validate/code/send?mobilePhone=15090308333&messageType=1&_=1454297673274 HTTP/1.1\" 200 52 0.015 115.51.148.47"

}

"message" , "\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?<api>(\S+))\?.*\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+(%{BASE16FLOAT:request_time})\s+%{IPORHOST:remoteip}",

"message" ,"\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?<api>(\S+))\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+(%{BASE16FLOAT:request_time})\s+%{IPORHOST:remoteip}",

"message" ,"\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?<api>(\S+))\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+\-\s+(%{BASE16FLOAT:request_time})\s+%{IPORHOST:remoteip}",

"message","\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?<api>(\S+))\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+\-\s+(%{BASE16FLOAT:request_time})\s+(%{IPORHOST:remoteip}|-)"

tomcat catalina日志；

{

    "@timestamp" => "2016-10-22T12:59:22.877Z",

      "@version" => "1",

          "path" => "/data01/applog_backup/zjzc_log/zj-api02-catalina.out.2016-10-22",

          "host" => "dr-mysql01.zjcap.com",

          "type" => "zj_api",

      "messager" => "zj_api- 2016-10-22 20:59:22,877 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",

          "time" => "2016-10-22 20:59:22,877",

         "Level" => "INFO"

}

filter {

    grok {

        match => [ "message","\s*%{TIMESTAMP_ISO8601:time}\s+(?<Level>(\S+)).*"]

     }

     date {

        match => ["time", "yyyy-MM-dd HH:mm:ss,SSS"]

    }

     mutate {

       remove_field =>["message"]

        }

}

nginx access 日志；

{

                 "message" => " 10.171.246.184 [22/Oct/2016:21:00:40 +0800] \"GET /resources/images/icon/icon_phone_gray.273e583f.png HTTP/1.1\" - 200 352 \"https://www.zjcap.cn/resources/css/base.css?06212016\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36\" 0.000 115.236.160.82",

                "@version" => "1",

              "@timestamp" => "2016-10-22T13:00:40.000Z",

                    "path" => "/data01/applog_backup/zjzc_log/zj-frontend01-access.2016-10-22",

                    "host" => "dr-mysql01.zjcap.com",

                    "type" => "zj_frontend_access",

                "clientip" => "10.171.246.184",

                    "time" => "22/Oct/2016:21:00:40 +0800",

                    "verb" => "GET",

                 "request" => "/resources/images/icon/icon_phone_gray.273e583f.png",

             "httpversion" => "1.1",

        "http_status_code" => "200",

                   "bytes" => "352",

            "http_referer" => "https://www.zjcap.cn/resources/css/base.css?06212016",

         "http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",

    "http_x_forwarded_for" => "115.236.160.82",

                   "geoip" => {

                      "ip" => "115.236.160.82",

           "country_code2" => "CN",

           "country_code3" => "CHN",

            "country_name" => "China",

          "continent_code" => "AS",

             "region_name" => "02",

               "city_name" => "Hangzhou",

                "latitude" => 30.293599999999998,

               "longitude" => 120.16140000000001,

                "timezone" => "Asia/Shanghai",

        "real_region_name" => "Zhejiang",

                "location" => [

            [0] 120.16140000000001,

            [1] 30.293599999999998

        ],

             "coordinates" => [

            [0] 120.16140000000001,

            [1] 30.293599999999998

        ]

    },

           "response_time" => 0.0,

                "messager" => "zj_frontend_access 10.171.246.184 [22/Oct/2016:21:00:40 +0800] \"GET /resources/images/icon/icon_phone_gray.273e583f.png HTTP/1.1\" - 200 352 \"https://www.zjcap.cn/resources/css/base.css?06212016\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36\" 0.000 115.236.160.82"

filter {

    grok {

        match =>[

             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request}\?.* HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?<http_referer>\S+)\" \"(?<http_user_agent>(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",

             "message" , "%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?<http_referer>\S+)\" \"(?<http_user_agent>(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",

             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} (?<http_url>\S+)\s+HTTP/%{NUMBER:httpversion}\"\s+\-\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+\"\-\"\s+\"(?<http_user_agent>(\S+))\"\s+(%{BASE16FLOAT:request_time})\s+(%{IPORHOST:http_x_forwarded_for}|-)",

             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"\" \"(?<http_user_agent>(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"

        ]

    }   

nginx error 日志；

         "message" => " 2016/10/22 21:00:32 [error] 12890#0: *98081 open() \"/var/www/zjzc-web-frontEnd/favicon.ico\" failed (2: No such file or directory), client: 10.171.246.184, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"www.zjcap.cn\"",

        "@version" => "1",

      "@timestamp" => "2016-10-22T13:00:32.000Z",

            "path" => "/data01/applog_backup/zjzc_log/zj-frontend01-error.2016-10-22",

            "host" => "dr-mysql01.zjcap.com",

            "type" => "zj_frontend_error",

            "time" => "2016/10/22 21:00:32",

        "severity" => "error",

             "pid" => "12890",

    "errormessage" => "*98081 open() \"/var/www/zjzc-web-frontEnd/favicon.ico\" failed (2: No such file or directory)",

     "remote_addr" => "10.171.246.184",

          "server" => "localhost",

         "request" => "\"GET /favicon.ico HTTP/1.1\"",

    "request_host" => "\"www.zjcap.cn\""

}

filter {

        grok {

            match => [ "message" , "(?<time>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<remote_addr>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)(?:, request: %{QS:request})?(?:, upstream: (?<upstream>\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:referrer}\")?"]

        }

         date {

        match => ["time", "yyyy/MM/dd HH:mm:ss"]

    }

}
巴特西

logstash 处理各种时间格式

最新文章

热门文章
