Log directly to Logstash from Payara Server

In first step, we replace the backend for Payara Server’s logging with Logstash.

Project Goodees was started by me to collect useful small components for Java EE, and Payara Server in particular. The first released component is payara-logback, that includes all libraries properly packaged to use logback as the logging backend in Payara Server and also includes an installation script.

Follow the instructions there to switch your logging.

Logback and logstash configuration

<configuration scan="true">
  <jmxConfigurator/>
  <shutdownHook/>
  <!-- always a good activate OnConsoleStatusListener -->
  <statusListener class="ch.qos.logback.core.status.OnErrorConsoleStatusListener" />

  <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>${LOG_DIR}/glassfish/${instanceName}/${instanceName}.access.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>${LOG_DIR}/glassfish/${instanceName}/${instanceName}.access.%d{yyyy-MM-dd}.log.zip</fileNamePattern>
      <maxHistory>90</maxHistory>
    </rollingPolicy>

    <encoder>
      <pattern>combined</pattern>
    </encoder>
  </appender>
  <appender-ref ref="FILE" />

  <appender name="json" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>${LOG_DIR}/glassfish/${instanceName}/json/${instanceName}.access.json</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>${LOG_DIR}/glassfish/${instanceName}/json/${instanceName}.access.%d{yyyy-MM-dd}.json.zip</fileNamePattern>
      <maxHistory>10</maxHistory>
    </rollingPolicy>
    <encoder class="net.logstash.logback.encoder.AccessEventCompositeJsonEncoder">
      <providers>
         <timestamp/>
         <version/>
         <method/>
         <protocol/>
         <statusCode/>
         <requestedUri/>
         <remoteHost/>
         <hostname/>
         <remoteUser/>
         <contentLength/>
         <elapsedTime/>
         <context/>
         <pattern><pattern>
           {"instance": "${instanceName}",
            "forwarded_for": "%header{X-Forwarded-For}",
            "server_name": "%v",
            "type": "access"}
         </pattern></pattern>
      </providers>
    </encoder>
  </appender>

  <appender name="logstash-socket" class="net.logstash.logback.appender.LogstashAccessTcpSocketAppender">
     <destination>${LOGSTASH_DEST}</destination>
     <secondaryConnectionTTL>5 minutes</secondaryConnectionTTL>
     <encoder class="net.logstash.logback.encoder.AccessEventCompositeJsonEncoder">
      <providers>
         <timestamp/>
         <version/>
         <method/>
         <protocol/>
         <statusCode/>
         <requestedUri/>
         <remoteHost/>
         <hostname/>
         <remoteUser/>
         <contentLength/>
         <elapsedTime/>
         <context/>
         <pattern><pattern>
           {"instance": "${instanceName}",
            "server_name": "%v",
            "forwarded_for": "%header{X-Forwarded-For}",
            "type": "access"}
         </pattern></pattern>
      </providers>
    </encoder>
  </appender>

  <appender-ref ref="logstash-socket" />
  <appender-ref ref="json" />
</configuration>

Finally, the Logstash configuration.

It is quite a simple configuration, however access log events tend to have field names that are incompatible with recent versions of elasticsearch. To work around this problem, they need to be adapted a bit, as shown in the filter below:

input {
  tcp {
    port => 1065
    codec => json_lines
  }
}

filter {
  if [type] == "access" {
     mutate {
        rename => {
          "@fields.HOSTNAME" => "HOSTNAME"
          "@fields.content_length" => "content_length"
          "@fields.elapsed_time" => "elapsed_time"
          "@fields.method" => "method"
          "@fields.protocol" => "protocol"
          "@fields.remote_host" => "remote_host"
          "@fields.remote_user" => "remote_user"
          "@fields.requested_uri" => "requested_uri"
          "@fields.status_code" => "status_code"
        }
     }
     if [forwarded_for] != "-" {
        mutate {
          rename => {
             "forwarded_for" => "remote_host"
          }
        }
     }
     mutate {
       remove_field => ["forwarded_for"]
     }
   }
}

output {
  elasticsearch  {
    hosts => #....
  }
}