logback输出json格式日志（包括mdc）发送到kafka

1，pom.xml

               <!-- kafka -->

        <dependency>

            <groupId>com.github.danielwegener</groupId>

            <artifactId>logback-kafka-appender</artifactId>

            <version>0.2.0-RC1</version>

            <scope>runtime</scope>

        </dependency>

 <!-- logback-->

        <dependency>

            <groupId>ch.qos.logback</groupId>

            <artifactId>logback-core</artifactId>

        </dependency>

        <dependency>

            <groupId>ch.qos.logback</groupId>

            <artifactId>logback-classic</artifactId>

            <version>1.2.3</version>

            <scope>runtime</scope>

        </dependency>

        <dependency>

            <groupId>net.logstash.logback</groupId>

            <artifactId>logstash-logback-encoder</artifactId>

            <version>5.0</version>

        </dependency>

 <!-- other-->

     <dependency>

        <groupId>org.slf4j</groupId>

        <artifactId>slf4j-api</artifactId>

        <version>1.7.25</version>

        <scope>compile</scope>

    </dependency>

            <dependency>

                <groupId>ch.qos.logback</groupId>

                <artifactId>logback-access</artifactId>

            </dependency>

2, spring-logback.xml

<?xml version="1.0" encoding="UTF-8"?>

<configuration debug="false" scan="true" scanPeriod="600000">

    <!--定义日志文件的存储地址 勿在 LogBack 的配置中使用相对路径 -->

    <property name="LOG_HOME" value="/var/log" />

    <contextName>${HOSTNAME}</contextName>

    <springProperty scope="context" name="appName"

        source="spring.application.name" />

    <springProperty scope="context" name="ip"

        source="spring.cloud.client.ipAddress" />

    <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符 -->

    <property name="CONSOLE_LOG_PATTERN"

        value="[%d{yyyy-MM-dd HH:mm:ss.SSS} ${ip} ${appName} %highlight(%-5level) %yellow(%X{X-B3-TraceId}),%green(%X{X-B3-SpanId}),%blue(%X{X-B3-ParentSpanId}) %yellow(%thread) %green(%logger) %msg%n" />

    <!-- <logger name="org.springframework.web" level="DEBUG" /> -->

    <!-- show parameters for hibernate sql 专为 Hibernate 定制 -->

    <!--<logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="TRACE"

        /> -->

    <!--<logger name="org.hibernate.type.descriptor.sql.BasicExtractor" level="DEBUG"

        /> -->

    <!--<logger name="org.hibernate.engine.QueryParameters" level="DEBUG" /> -->

    <!--<logger name="org.hibernate.engine.query.HQLQueryPlan" level="DEBUG"

        /> -->

    <!-- <logger name="org.hibernate.SQL" level="DEBUG" /> -->

    <logger name="logging.level.com.italktv.platform" level="info" />

    <!-- 控制台输出 -->

    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">

        <encoder>

            <pattern>${CONSOLE_LOG_PATTERN}</pattern>

            <charset>utf-8</charset>

        </encoder>

        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">

            <level>debug</level>

        </filter>

    </appender>

    <!-- 按照每天生成日志文件 -->

    <appender name="FILE"

        class="ch.qos.logback.core.rolling.RollingFileAppender">

        <!-- 正在记录的日志文件的路径及文件名 -->

        <file>${LOG_HOME}/bigdata/data-api.log</file>

        <rollingPolicy

            class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">

            <!--日志文件输出的文件名 -->

            <FileNamePattern>${LOG_HOME}/bigdata/data-api.%d{yyyy-MM-dd}.%i.log

            </FileNamePattern>

            <!--日志文件保留天数 -->

            <MaxHistory>30</MaxHistory>

            <maxFileSize>1MB</maxFileSize>

            <totalSizeCap>10MB</totalSizeCap>

        </rollingPolicy>

        <encoder

            class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">

            <providers>

                <timestamp>

                    <timeZone>UTC</timeZone>

                </timestamp>

                <pattern>

                    <pattern>

                        {

                        "level": "%level",

                        "trace": "%X{X-B3-TraceId:-}",

                        "requestId": "%X{requestId}",

                        "remoteIp": "%X{remoteIp}",

                        "span": "%X{X-B3-SpanId:-}",

                        "parent":

                        "%X{X-B3-ParentSpanId:-}",

                        "thread": "%thread",

                        "class":

                        "%logger{40}",

                        "message": "%message",

                        "stack_trace":

                        "%exception{10}"

                        }

                    </pattern>

                </pattern>

            </providers>

        </encoder>

        <!--日志文件最大的大小 <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">

            <MaxFileSize>10KB</MaxFileSize> </triggeringPolicy> -->

    </appender>

        <!-- pom dependency

    <dependency>

        <groupId>com.github.danielwegener</groupId>

        <artifactId>logback-kafka-appender</artifactId>

        <version>0.1.0</version>

        <scope>runtime</scope>

    </dependency>

    <dependency>

        <groupId>net.logstash.logback</groupId>

        <artifactId>logstash-logback-encoder</artifactId>

        <version>4.11</version>

    </dependency>

    <dependency>

        <groupId>org.slf4j</groupId>

        <artifactId>slf4j-api</artifactId>

        <version>1.7.25</version>

        <scope>compile</scope>

    </dependency>

    <dependency>

        <groupId>ch.qos.logback</groupId>

        <artifactId>logback-core</artifactId>

        <version>1.1.11</version>

    </dependency>

    <dependency>

        <groupId>ch.qos.logback</groupId>

        <artifactId>logback-classic</artifactId>

        <version>1.1.11</version>

    </dependency>

     -->

    <!--  other appender

    <appender name="kafkaAppenderAnotherEncode"

        class="com.github.danielwegener.logback.kafka.KafkaAppender">

        <encoder

            class="com.github.danielwegener.logback.kafka.encoding.PatternLayoutKafkaMessageEncoder">

            <layout class="net.logstash.logback.layout.LogstashLayout">

                <includeMdc>true</includeMdc>

                <includeContext>true</includeContext>

                <includeCallerData>true</includeCallerData>

                <customFields>{"system":"test"}</customFields>

                <fieldNames class="net.logstash.logback.fieldnames.ShortenedFieldNames" />

            </layout>

        </encoder>

        <topic>tv_server_logstash_log</topic>

        <keyingStrategy

            class="com.github.danielwegener.logback.kafka.keying.HostNameKeyingStrategy" />

        <deliveryStrategy

            class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" />

        <producerConfig>bootstrap.servers=211.100.75.227:9092</producerConfig>

        <producerConfig>acks=0</producerConfig>

        <producerConfig>linger.ms=1000</producerConfig>

        <producerConfig>block.on.buffer.full=false</producerConfig>

        <appender-ref ref="STDOUT" />

    </appender>

-->

    <!-- https://www.cnblogs.com/maxzhang1985/p/9522507.html

    https://logback.qos.ch/manual/layouts.html

    -->

    <appender name="kafkaAppender"

        class="com.github.danielwegener.logback.kafka.KafkaAppender">

        <encoder charset="UTF-8"

            class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">

            <providers>

                <mdc />

                <context />

                <timestamp>

                    <timeZone>UTC</timeZone>

                </timestamp>

                <pattern>

                    <pattern>

                        {

                        "level": "%level",

                        "trace": "%X{X-B3-TraceId:-}",

                        "span":

                        "%X{X-B3-SpanId:-}",

                        "parent": "%X{X-B3-ParentSpanId:-}",

                        "thread":

                        "%thread",

                        "class": "%logger{40}",

                        "message": "%message",

                        "stack_trace": "%exception{10}"

                        }

                    </pattern>

                </pattern>

            </providers>

        </encoder>

        <topic>tv_server_logstash_log</topic>

        <keyingStrategy

            class="com.github.danielwegener.logback.kafka.keying.HostNameKeyingStrategy" />

        <deliveryStrategy

            class="com.github.danielwegener.logback.kafka.delivery.AsynchronousDeliveryStrategy" />

        <producerConfig>bootstrap.servers=127.0.0.1:9092</producerConfig>

        <!-- don't wait for a broker to ack the reception of a batch. -->

        <producerConfig>acks=0</producerConfig>

        <!-- wait up to 1000ms and collect log messages before sending them as

            a batch -->

        <producerConfig>linger.ms=1000</producerConfig>

        <!-- even if the producer buffer runs full, do not block the application

            but start to drop messages -->

        <!--<producerConfig>max.block.ms=0</producerConfig> -->

        <producerConfig>block.on.buffer.full=false</producerConfig>

        <!-- kafka连接失败后，使用下面配置进行日志输出 -->

        <appender-ref ref="STDOUT" />

    </appender>

    <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender">

        <appender-ref ref="kafkaAppender" />

    </appender>

    <!-- 日志输出级别 -->

    <root level="INFO">

        <!-- 生产上不输出stdout log -->

        <appender-ref ref="STDOUT" />

        <!-- <appender-ref ref="FILE" /> -->

        <!-- <appender-ref ref="kafkaAppender" /> -->

        <appender-ref ref="ASYNC" />

    </root>

</configuration>

3, 添加一个mdc在logback

import java.util.UUID;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import org.slf4j.MDC;

import org.springframework.stereotype.Component;

import org.springframework.web.servlet.HandlerInterceptor;

import org.springframework.web.servlet.ModelAndView;

@Component

public class LogInterceptor implements HandlerInterceptor {

    private final static String REQUEST_ID = "requestId";

    private static final Logger LOGGER  = LoggerFactory.getLogger(LogInterceptor.class);

    @Override

    public boolean preHandle(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o) throws Exception {

        String xForwardedForHeader = httpServletRequest.getHeader("X-Forwarded-For");

        String remoteIp = httpServletRequest.getRemoteAddr();

        String uuid = UUID.randomUUID().toString();

        LOGGER.info("put requestId ({}) to logger", uuid);

        LOGGER.info("request id:{}, client ip:{}, X-Forwarded-For:{}", uuid, remoteIp, xForwardedForHeader);

        MDC.put(REQUEST_ID, uuid);

        MDC.put("remoteIp", remoteIp);

        return true;

    }

    @Override

    public void postHandle(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o,

            ModelAndView modelAndView) throws Exception {

        String uuid = MDC.get(REQUEST_ID);

        LOGGER.info("remove requestId ({}) from logger", uuid);

        MDC.remove(REQUEST_ID);

    }

    @Override

    public void afterCompletion(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, Object o, Exception e)

            throws Exception {

    }

}

4，添加切面 intercept

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.context.annotation.Configuration;

import org.springframework.web.servlet.config.annotation.InterceptorRegistry;

import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;

@Configuration

public class WebMvcConfigurer extends WebMvcConfigurerAdapter {

    @Autowired

    private LogInterceptor logInterceptor;

    @Override

    public void addInterceptors(InterceptorRegistry registry) {

        registry.addInterceptor(logInterceptor);

        super.addInterceptors(registry);

    }

}

参考：

logback 手册：https://logback.qos.ch/manual/layouts.html

http://www.importnew.com/28541.html

https://www.jianshu.com/p/a26da0c55255

https://blog.csdn.net/Soinice/article/details/84033382

https://examples.javacodegeeks.com/enterprise-java/logback/logback-kafka-appender-example/

http://stevetarver.github.io/2016/04/20/whole-product-logging.html 讲解详细

巴特西

logback输出json格式日志（包括mdc）发送到kafka

最新文章

热门文章