Best way to write custom json messages using log4j2

Question

I have been using log4j for different kind of projects and have some experience with log4j2. All implementations used the default appender and layout. Currently i need to wr

Answer 1

I found a solution which works for me; slf4j-json-logger. It is a slf4j framework, so should be included in the pom.xml. Sample project files;

pom.xml

  <?xml version="1.0" encoding="UTF-8"?>
  <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.reddipped</groupId>
     <artifactId>JSONLogger_2</artifactId>
     <version>1.0-SNAPSHOT</version>
     <packaging>jar</packaging>
     <properties>

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.6</maven.compiler.source>
        <maven.compiler.target>1.6</maven.compiler.target>

        <mainClass>com.reddipped.jsonlogger_2.Test</mainClass>

        <slf4j.version>1.7.21</slf4j.version>
        <!-- current log4j 2 release -->
        <log4j.version>2.6.2</log4j.version> 

     </properties>

     <dependencies>
        <dependency>
           <groupId>org.slf4j</groupId>
           <artifactId>slf4j-api</artifactId>
           <version>${slf4j.version}</version>
        </dependency>
        <!-- Binding for Log4J -->
        <dependency>
           <groupId>org.apache.logging.log4j</groupId>
           <artifactId>log4j-slf4j-impl</artifactId>
           <version>${log4j.version}</version>
        </dependency>
        <!-- Log4j API and Core implementation required for binding -->
        <dependency>
           <groupId>org.apache.logging.log4j</groupId>
           <artifactId>log4j-api</artifactId>
           <version>${log4j.version}</version>
        </dependency>
        <dependency>
           <groupId>org.apache.logging.log4j</groupId>
           <artifactId>log4j-core</artifactId>
           <version>${log4j.version}</version>
        </dependency>
        <!-- Logger slf4j-json-logger -->
        <dependency>
           <groupId>com.savoirtech.logging</groupId>
           <artifactId>slf4j-json-logger</artifactId>
           <version>2.0.2</version>
        </dependency>
     </dependencies> 
  </project>

log4j2.xml

  <?xml version="1.0" encoding="UTF-8"?>

  <!--

  Use java property log4j.configurationFile to specify log4j2.xml location
  if not available in classpath

  -    Dlog4j.configurationFile="/Users/petervannes/NetBeansProjects/JSONLogger_2/src/mann/java/resources/log4j2.xml"

  -->
  <configuration status="trace">
     <Properties>
        <Property name="log-path">/Users/petervannes/NetBeansProjects/JSONLogger_2/logfiles</Property>
     </Properties>
     <appenders>
        <RollingFile name="RollingFile" fileName="${log-path}/jsonlogger.json"
                  filePattern="${log-path}/%d{yyyyMMdd}_jsonlogger-%i.json" >
           <PatternLayout>
              <pattern>%m%n</pattern>
           </PatternLayout> 
           <Policies>
              <TimeBasedTriggeringPolicy />
              <SizeBasedTriggeringPolicy size="1 KB" />
           </Policies>
           <DefaultRolloverStrategy max="4"/>
        </RollingFile>
     </appenders>
     <Loggers>
        <Logger name="JSONLogger" level="debug" additivity="false">
           <AppenderRef ref="RollingFile" />
        </Logger>
        <Root level="debug">
           <AppenderRef ref="RollingFile" />
        </Root>
     </Loggers>
  </configuration>

Example Java code

  package com.reddipped.jsonlogger_2;

  import com.savoirtech.logging.slf4j.json.LoggerFactory;
  import java.util.HashMap;
  import java.util.Map;

  /**
   *
   * @author petervannes
   */
  public class Test {
      public static void main(String[] args) {   
        Test t = new Test() ;
     }


     public Test() {

        LoggerFactory.setIncludeLoggerName(false);
        LoggerFactory.setDateFormatString("yyyy-MM-dd HH:mm:ss.SSS");

         com.savoirtech.logging.slf4j.json.logger.Logger LOGGER =  LoggerFactory.getLogger("JSONLogger");

     Map<String, String> optionalFields = new HashMap();
     optionalFields.put("CaseNumber", "C12.12343");
     optionalFields.put("Step","Assignment") ;
     optionalFields.put("Department","BPM") ;

     String LOB = "Business Administration" ;
     String Service = "DocumentService" ;
     String Process = "AddAttachements" ;
     String Reason = "Technical" ; 

     LOGGER.error().message("Conversion error 'incompatible PDF document'")
           .field("LOB",LOB)
           .field("Service", Service)
           .field("Process",Process)
           .field("Reason", Reason)
           .map("OptionalFields", optionalFields).log() ;
     }

  }

JSON Log Entry

  {
    "message": "Conversion error  'incompatible PDF document'",
    "LOB": "Business Administration",
    "Service": "DocumentService",
    "Process": "AddAttachements",
    "Reason": "Technical",
    "OptionalFields": {
     "Step": "Assignment",
     "Department": "BPM",
     "CaseNumber": "C12.12343"
    },
    "level": "ERROR",
    "thread_name": "main",
    "class": "com.reddipped.jsonlogger_2.Test",
    "@timestamp": "2016-09-23 10:18:06.623"
  }

Answer 2

The question is about writing a custom json messages using log4j2.

This is possible since version 2.11 of log4j2 version:

https://issues.apache.org/jira/browse/LOG4J2-2190

The new parameter of for JSONLayout is called

objectMessageAsJsonObject

. Sample project files;

log4j2.properties

status = error


appender.ana_whitespace.type = RollingFile
appender.ana_whitespace.name = ana_whitespace
appender.ana_whitespace.fileName = ${sys:es.logs.base_path:-target}${sys:file.separator}ana_whitespace.log
appender.ana_whitespace.layout.type = JsonLayout
appender.ana_whitespace.layout.propertiesAsList = false
appender.ana_whitespace.layout.compact = false
appender.ana_whitespace.layout.eventEol = true
appender.ana_whitespace.layout.objectMessageAsJsonObject = true
appender.ana_whitespace.layout.complete= true
appender.ana_whitespace.layout.properties= true
appender.ana_whitespace.filePattern = ${sys:es.logs.base_path:-target}${sys:file.separator}ana_whitespace-%d{yyyy-MM-dd}.log
appender.ana_whitespace.filter.1.type = MarkerFilter
appender.ana_whitespace.filter.1.onMismatch=DENY
appender.ana_whitespace.filter.1.onMatch=ACCEPT
appender.ana_whitespace.filter.1.marker=ANA_WHITESPACE
appender.ana_whitespace.policies.type = Policies
appender.ana_whitespace.policies.time.type = TimeBasedTriggeringPolicy
appender.ana_whitespace.policies.time.interval = 1
appender.ana_whitespace.policies.time.modulate = true
appender.ana_whitespace.policies.size.type = SizeBasedTriggeringPolicy
appender.ana_whitespace.policies.size.size = 10 MB

rootLogger.level = info
rootLogger.appenderRef.ana_whitespace.ref = ana_whitespace

Example Java code

package de.es.stemmer;

import java.io.IOException;
import java.util.Map;
import java.util.TreeMap;

import org.apache.http.client.ClientProtocolException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.Marker;
import org.apache.logging.log4j.MarkerManager;
import org.apache.logging.log4j.ThreadContext;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.message.ObjectMessage;

public class JsonLoggerTest {
    final static Logger log = LogManager.getLogger(JsonLoggerTest.class);
    final static Marker MARKER_WHITESPACE = MarkerManager.getMarker("ANA_WHITESPACE");

    public static void main(String[] args) throws ClientProtocolException, IOException {
        System.setProperty("es.logs.base_path", "target");
        System.setProperty("es.logs.cluster_name", "_cluster");
        LoggerContext.getContext().reconfigure();
        ThreadContext.put("orig", "MDC_origValue");
        ThreadContext.put("source", "MDC_sourceSnippet");
        Map<String, String> map = new TreeMap<>();
        map.put("orig", "msg_origValue");
        map.put("source", "msg_sourceSnippet");
        ObjectMessage msg = new ObjectMessage(map);
        log.info(MARKER_WHITESPACE, msg);
        ThreadContext.remove("orig");
        ThreadContext.remove("source");
    }

}

JSON Log Entry

[
{
  "thread" : "main",
  "level" : "INFO",
  "loggerName" : "de.es.stemmer.JsonLoggerTest",
  "marker" : {
    "name" : "ANA_WHITESPACE"
  },
  "message" : {
    "orig" : "msg_origValue",
    "source" : "msg_sourceSnippet"
  },
  "endOfBatch" : false,
  "loggerFqcn" : "org.apache.logging.log4j.spi.AbstractLogger",
  "instant" : {
    "epochSecond" : 1526576578,
    "nanoOfSecond" : 184000000
  },
  "contextMap" : {
    "orig" : "MDC_origValue",
    "source" : "MDC_sourceSnippet"
  },
  "threadId" : 1,
  "threadPriority" : 5
}

]

Answer 3

If you're looking for a way to generate customized JSON log messages without any of the "noise" added by log4j2 you could create an implementation of the Message interface and use a different layout - for example PatternLayout. Below is an example of this approach:

First a class to implement Message inferface

import java.util.HashMap;
import java.util.Map;

import org.apache.logging.log4j.message.Message;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;

public class JSONMessage implements Message {

    private static final long serialVersionUID = 538439258494853324L;
    private String messageString;
    private static final Gson GSON = new GsonBuilder()
            .setPrettyPrinting()
            .create();

    public JSONMessage(){
        this(null);
    }

    public JSONMessage(Object msgObj){
        parseMessageAsJson(msgObj);
    }

    public JSONMessage(String msgStr){
        Map<String,String> msgObj = new HashMap<>();
        msgObj.put("message", msgStr);
        parseMessageAsJson(msgObj);
    }

    private void parseMessageAsJson(Object msgObj){
        messageString = GSON.toJson(msgObj);
    }

    @Override
    public String getFormattedMessage() {
        return messageString;
    }

    @Override
    public String getFormat() {
        return messageString;
    }

    @Override
    public Object[] getParameters() {
        return null;
    }

    @Override
    public Throwable getThrowable() {
        return null;
    }

}

Next the log4j2.xml configuration:

<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="WARN" name="App">
    <Appenders>
        <Console name="Console" target="SYSTEM_OUT">
                <PatternLayout pattern="%m%n" />
        </Console>
    </Appenders>

    <Loggers>
        <Root level="trace">
            <AppenderRef ref="Console" />
        </Root>
    </Loggers>
</Configuration>

Now a simple application class to generate a log event

import java.util.HashMap;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;


public class App {

    private static final Logger logger = LogManager.getLogger();

    public static void main( String[] args )
    {
        HashMap<String,Object> msgMap = new HashMap<>();
        msgMap.put("someInt", 123);
        msgMap.put("note", "Maybe you put a message here");

        HashMap<String,Object> anotherMap = new HashMap<>();
        anotherMap.put("key1", "value1");
        anotherMap.put("key2", "value2");
        msgMap.put("map", anotherMap);
        logger.info(new JSONMessage(msgMap));
    }
}

Here is some sample output:

{
  "note": "Maybe you put a message here",
  "map": {
    "key1": "value1",
    "key2": "value2"
  },
  "someInt": 123
}

Note that I'm using Gson to create the JSON output, but you could use any library you want. Also note that this code does not generate "complete" JSON in that it does not add the square brackets at the start and end of the log or the comma between message objects.