logstash split插件的使用(将一个事件拆分成多个事件）

kafka中的原始数据格式(1条数据)

{
    "body": {
        "cwd": "/home/test/",
        "monitor": {
            "proc_num": 2,
            "procs": [{
                "cmd": "",
                "cpu_usage_rate": 2.0,
                "mem_usage_rate": 3.0,
                "pid": 4976,
                "procname": "test-name"
            }, {
                "cmd": "/home/test2",
                "cpu_usage_rate": 5.0,
                "mem_usage_rate": 6.0,
                "pid": 4977,
                "procname": "test-name2"
            }],
            "timestamp": 1547124214814
        },
        "os_tag": "Linux",
        "system": {
            "connection": {
                "haddr": "00:50:56:B3:7E:7A",
                "ip": "192.168.21.80",
                "name": "ens160"
            },
            "cpu": ["Intel Xeon", "Intel Xeon", "Intel Xeon", "Intel Xeon"],
            "memory": {
                "swap_total": "7918841856",
                "total": "15600787456"
            },
            "uname": "Linux Linux 3.10.0-862.el7.x86_64 x86_64 x86_64",
            "vendor": "CentOS 7.5.1804"
        }
    },
    "meta": {
        "request_id": "3-14865"
    }
}

logstash处理后的数据格式(2条数据)

{
    "hostname": "test",
    "procs": {
      "mem_usage_rate": 2.0,
      "cpu_usage_rate": 3.0,
      "pid": 4976,
      "cmd": "",
      "procname": "test-name"
    },
    "@timestamp": "2019-01-11T02:08:57.225Z",
    "memory": {
      "total": "3975188480",
      "swap_total": "4177522688"
    },
    "connection": {
      "ip": "192.168.31.182",
      "name": "ens160",
      "haddr": "00:50:56:B3:7E:35"
    },
    "proc_num": 4
  }


{
    "hostname": "test",
    "procs": {
      "mem_usage_rate": 5.0,
      "cpu_usage_rate": 6.0,
      "pid": 4976,
      "cmd": "test",
      "procname": "test-name"
    },
    "connection": {
      "ip": "192.168.31.182",
      "name": "ens160",
      "haddr": "00:50:56:B3:7E:35"
    },
    "proc_num": 4
  }

logstash的配置

input {
    kafka {
        bootstrap_servers=> "192.168.31.92:9092,192.168.31.93:9092,192.168.31.94:9092"
        group_id => "test_group"
        topics =>"test_topic"
        auto_offset_reset => "earliest"
        type => "test_type"
        consumer_threads => 1
        codec => "json"
    }
}

filter{

    if !([body][monitor][procs]) {
        drop { }
    }

    mutate {
    remove_field => ["body[cwd]","body[os_tag]","body[system][filesystem]","body[system][cpu]","body[system][disk]",
    "body[system][has_docker]","body[system][if]","body[system][uname]","body[system][vendor]","meta","url","body[configuration]"]
  }

    date {
        match => ["body[monitor][timestamp]","UNIX_MS"]

        remove_field => ["body[monitor][timestamp]"]
    }

    mutate {
        add_field => {
            "client_id" => "%{params[client_id]}"
            "system" => "%{body[system]}"
            "monitor" => "%{body[monitor]}"
        }
        remove_field => ["body","params"]
    }

    json {
        source => "system"
        remove_field => ["system"]
    }

    json {
        source => "monitor"
        remove_field => ["monitor"]
    }

    if ([procs]) {
        split {
            field => "procs"
        }
    }
}

output {
    elasticsearch {
        hosts => ["192.168.21.80:9200"]
        index => "test_index"
        codec => "json"
    }
}

巴特西

logstash split插件的使用(将一个事件拆分成多个事件）

kafka中的原始数据格式(1条数据)

logstash处理后的数据格式(2条数据)

logstash的配置

最新文章

热门文章