Spring Boot 整合Logstash

前言

    ELK是目前主流的日志收集、存储和可视化的工具。

• ElasticSearch:用于存储日志信息;

• Logstash:用于收集、处理和转发日志信息;

• Kibana:提供可视化的Web界面.

 这里主要实现的是通过logbook和logstash完成日志的收集功能。

1、添加依赖包

<!--json转换-->
<dependency>
    <groupId>com.alibaba</groupId>
    <artifactId>fastjson</artifactId>
</dependency>

<!--调用远程接口-->
<dependency>
    <groupId>io.github.openfeign</groupId>
    <artifactId>feign-core</artifactId>
    <version>11.9.1</version>
</dependency>
<dependency>
    <groupId>io.github.openfeign</groupId>
    <artifactId>feign-gson</artifactId>
    <version>11.9.1</version>
</dependency>

<!--集成logstash-->
<dependency>
    <groupId>net.logstash.logback</groupId>
    <artifactId>logstash-logback-encoder</artifactId>
    <version>7.2</version>
</dependency>

2、创建logback-spring.xml文件

在main\resources路径下创建logback-spring.xml文件，用于配置logback。

因为我的logstash需要Authorization验证，所以就自己实现了httpClient，小伙伴们需要将 Logstash的<appender>和<httpClient>中的class指向自己的项目路径。

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>

    <springProperty scope="context" name="projectName" source="tec.project-name"
                    defaultValue="tec"/>
    <springProperty scope="context" name="stage" source="stage" defaultValue="local"/>

    <appender name="LOCAL-CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>%d{YYYY-MM-dd HH:mm:ss.SSS} %5level %logger{0} [%t] [%mdc] - %msg%n</pattern>
        </encoder>
    </appender>

    <!--指向自己的项目文件-->
    <appender name="LOGSTASH"
              class="net.logstash.logback.appender.LoggingEventAsyncDisruptorAppender">
        <ringBufferSize>8192</ringBufferSize>
        <appender class="com.unicorn.tec.logstash.HttpAppender">
            <!--指向自己的项目文件-->
            <httpClient class="com.unicorn.tec.logstash.HttpClient">
                <!--logstash地址和端口号-->
                <destination>http://47.**.**.**:****</destination>
                <!--Authorization的Username和Password-->
                <username>username</username>
                <password>password</password>
                <topic>my-topic</topic>
                <app>my-app</app>
                <!--日志收集级别-->
                <level>ERROR</level>
            </httpClient>
            <encoder class="net.logstash.logback.encoder.LogstashEncoder">
                <fieldNames>
                    <timestamp>timestamp</timestamp>
                </fieldNames>
            </encoder>
        </appender>
    </appender>

    <root level="ERROR">
        <appender-ref ref="LOGSTASH"/>
        <appender-ref ref="LOCAL-CONSOLE"/>
    </root>

</configuration>

3、实现HttpAppender和HttpClient

在实现HttpAppender和HttpClient之前，我们需要先声明logstash的远程调用接口，即创建ILogstashService文件

package com.unicorn.tec.logstash;

import feign.Headers;
import feign.RequestLine;
import feign.Response;
import org.springframework.web.bind.annotation.RequestBody;

public interface ILogstashService
{
    @Headers("Content-Type: application/json")
    @RequestLine("PUT /")
    Response addLog(@RequestBody String body);
}

HttpClient代码：

package com.unicorn.tec.logstash;

import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.spi.ContextAwareBase;
import ch.qos.logback.core.spi.LifeCycle;
import ch.qos.logback.core.status.Status;
import com.alibaba.fastjson.JSON;
import feign.Feign;
import feign.Logger;
import feign.Response;
import feign.auth.BasicAuthRequestInterceptor;
import lombok.var;
import org.springframework.beans.factory.annotation.Value;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.util.*;

public class HttpClient extends ContextAwareBase implements LifeCycle
{
    private String destination;
    private String username;
    private String password;
    private String topic;
    private String app;
    private String level;
    @Value("${spring.profiles.active}")
    private String env;
    private ILogstashService logstash;
    private volatile boolean isStarted = false;
    @Override
    public synchronized void start(){
        if(isStarted){
            return;
        }

        if (destination == null || destination.isEmpty()) {
            addError("No encoder was configured. Use <destination> URL");
        }

        if (hasAnyErrors()) {
            return;
        }

        logstash = Feign.builder()
                .decoder((response, type) -> response)
                .logLevel(Logger.Level.NONE)
                .requestInterceptor(new BasicAuthRequestInterceptor(username, password))
                .target(ILogstashService.class, destination);

        isStarted = true;
        addInfo("HttpClient started: '" + destination + "'");
    }

    public void addLog(ILoggingEvent event)
    {

        Level minLevel = Level.toLevel(level);
        if(!event.getLevel().isGreaterOrEqual(minLevel)){
            return;
        }

        //自定义自己想要的内容
        Map<String,Object> request = new HashMap<>();
        request.put("Topic", topic);
        request.put("Message",event.getMessage());
        request.put("Level",toLevelString(event.getLevel()));
        request.put("Logger",event.getLoggerName());
        request.put("Time",event.getTimeStamp());
        request.put("App", app);
        request.put("Env","dev");
        request.put("Source","");
        request.put("Scope","");

        var dic = event.getMDCPropertyMap();
        if(dic.containsKey("url")){
            request.put("Url",dic.get("url"));
        }
        if(dic.containsKey("parameters")){
            request.put("Parameters",dic.get("parameters"));
        }
        if(dic.containsKey("cookie")){
            request.put("Session",dic.get("cookie"));
        }

        var throwable = event.getThrowableProxy();
        if(throwable != null){
            request.put("Exception", throwable.getMessage());

            StringBuffer stackTraceString = new StringBuffer();
            stackTraceString.append(throwable.getClassName())
                            .append(": ")
                            .append(throwable.getMessage());

            for (var element : throwable.getStackTraceElementProxyArray()){
                stackTraceString.append("\n ")
                        .append(element.getSTEAsString());
            }
            request.put("StackTrace",stackTraceString);
        }

        String json = JSON.toJSONString(request);

        sendLogstash(json);
    }

    private String toLevelString(Level level)
    {
        if(level == Level.INFO)
        {
            return "Information";
        }else if(level == Level.DEBUG)
        {
            return "Debug";
        } else if (level == Level.ERROR)
        {
            return "Error";
        } else if (level == Level.WARN)
        {
            return "Warning";
        }

        return "";
    }

    private void sendLogstash(String json){

        if (!isStarted()) {
            return;
        }

        try {

            verifyResponse(logstash.addLog(json));

        } catch (Exception ex) {
            addWarn("Can't execute PUT request. URL: '" + destination + "'", ex);
        }
    }

    @Override
        public synchronized void stop() {
            if (isStarted) {
                isStarted = false;
            }
        }

        @Override
        public boolean isStarted() {
            return isStarted;
        }
    
    @Override
    public String toString() {
        return destination == null ? "HttpClient" : "HttpClient --> " + destination;
    }

    public void setDestination(String destination) {
        this.destination = destination;
    }

    public void setUsername(String username){
        this.username = username;
    }

    public void setPassword(String password){
        this.password = password;
    }

    public void setTopic(String topic){
        this.topic = topic;
    }

    public void setApp(String app){
        this.app = app;
    }

    public void setLevel(String level){
        this.level = level;
    }

    public List<Status> getStatusList() {
        return getStatusManager().getCopyOfStatusList();
    }

    private void verifyResponse(Response response) throws IOException {
        if (response.status() != HttpURLConnection.HTTP_OK) {
            String msg = "" +
                    "ResponseCode: " + response.status() + "; " +
                    "Reason: " + response.reason() + "; " +
                    "URL: " + response.request().url();
            throw new IOException(msg);
        }
    }

    private boolean hasAnyErrors() {
        return getStatusList().stream().anyMatch(x -> x.getLevel() > 1);
    }
}

HttpAppender代码：

package com.unicorn.tec.logstash;

import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.AppenderBase;
import net.logstash.logback.encoder.LogstashEncoder;
import org.springframework.stereotype.Component;

@Component
public class HttpAppender extends AppenderBase<ILoggingEvent>
{
    private LogstashEncoder encoder;
    private HttpClient httpClient;

    @Override
    public synchronized void start(){
        if(isStarted()){
            return;
        }

        verifyConfigurationParameters();
        addInfo("HttpAppender started");

        super.start();
    }

    @Override
    public void append(ILoggingEvent event) {
        if (!isStarted()) {
            return;
        }

        httpClient.addLog(event);

    }

    public void setEncoder(LogstashEncoder encoder) {
        this.encoder = encoder;
    }

    public void setHttpClient(HttpClient httpClient) {
        this.httpClient = httpClient;
    }

    private void verifyConfigurationParameters(){
        if(encoder == null){
            addError("No encoder was configured. Use <encoder> to specify the class name");
        }

        if(httpClient == null){
            addError("No destination was configured. Use <httpClient> to specify HttpClient");
        }
    }
}

至此，项目中所有error级别的log都会被推送到logstash中。但如果想要记录程序运行时的异常的话，则需要通过Spring框架截获异常并记录到logback中。

4、截获异常并记录

为了方便定位问题，我将request的一些信息也记录在日志里了。

@ControllerAdvice
@Slf4j
public class TecExceptionHandler extends ResponseEntityExceptionHandler {
    @ExceptionHandler(value = {Exception.class})
    protected Object handleExceptionInternal(Exceptionex, WebRequest request) {

        MDC.put("url", ((ServletWebRequest) request).getRequest().getRequestURI());
        MDC.put("parameters",JSON.toJSONString(request.getParameterMap()));
        MDC.put("cookie",request.getHeader("cookie"));

        log.error(ex.getMessage(), ex);

        MDC.clear();
        
        return .....
    }
}

5、总结

    核心思路就是通过监听logback的ILoggingEvent事件，然后封装成自己想要的内容，发送到自己的logstash中。因为logback无法直接记录exception，所以需要截获exception并记录到log中(可能是自己没找到合适的方法，希望大神纠正)。

参考：

idealo/ logstash-logback-http