Logstash处理数据用法示例---待完善

filter {

  mutate {

    rename => [ "message", "blog_html" ]

    copy => { "blog_html" => "blog_text" }

    copy => { "published" => "@timestamp" }

  }

  mutate {

    gsub => [

      "blog_text", "<.*?>", "",

      "blog_text", "[\n\t]", " "

    ]

    remove_field => [ "published", "author" ]

  }

}

apache日志

日志格式

83.149.9.216 - - [17/May/2015:10:05:03 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-search.png HTTP/1.1" 200 203023 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"

83.149.9.216 - - [17/May/2015:10:05:43 +0000] "GET /presentations/logstash-monitorama-2013/images/kibana-dashboard3.png HTTP/1.1" 200 171717 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"

filter {

  grok { # 通过正则表达式来匹配我们的每一条log信息,把相应的值赋值给相应的变量

    match => {

      "message" => '%{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response:int} (?:-|%{NUMBER:bytes:int}) %{QS:referrer} %{QS:agent}'

    }

  }

  date { # @timestamp表示的在运行时的当前timestamp。把@timestamp转换成来自于log里的时间信息，也就是timestamp所表述的时间。

    match => [ "timestamp", "dd/MMM/YYYY:HH:mm:ss Z" ]

    locale => en

  }

  geoip { # 根据IP地址解析是来自哪一个地方的以及它的经纬度等信息

    source => "clientip"

  }

  useragent { # 添加有关useragent（如系列，操作系统，版本和设备）的信息

    source => "agent"

    target => "useragent"

  }

}

csv文件

input {

  file {

    path => "/Users/liuxg/data/cars.csv"

    start_position => "beginning"

    sincedb_path => "null"

  }

}

filter {

  csv {

    # 分隔符

    separator => ","

    # 表格的列名

    columns => [ "maker", "model", "mileage", "manufacture_year", "engine_displacement",

    "engine_power", "body_type", "color_slug", "stk_year", "transmission", "door_count",

    "seat_count", "fuel_type", "date_created", "date_last_seen", "price_eur" ]

  }

  # 转换数据类型

  mutate { convert => ["mileage", "integer"] }

  mutate { convert => ["price_eur", "float"] }

  mutate { convert => ["engine_power", "integer"] }

  mutate { convert => ["door_count", "integer"] }

  mutate { convert => ["seat_count", "integer"] }

}

# 在input中，path指向csv文件的位置。start_position指向beginning。对于一个实时的数据源，它通常是ending，表示它每次都是从最后拿到那个数据。sincedb_path通常指向一个文件。这个文件保存上次操作的位置。设置为/dev/null表明我们不存储这个数据

# 在filter中，column都来自于csv表格。通常Logstash会把每个数据都当做是字符串。mileage是一个整型数，price_eur是一个浮点数，可以通过mutate来完成

巴特西

Logstash处理数据用法示例---待完善

最新文章

热门文章