App Engine Node.js: how to link app logs and requests logs

Question

I am using Node.js on App Engine Standard and Flexible.

In the logs viewer, is it possible to display application logs nested inside request logs?

Answer 1

Yes it is possible to correlate application logs and request logs. This is the end result in the Logs Viewer:

To achieve this you can either:

Use both the @google-cloud/trace-agent and @google-cloud/logging-bunyan modules in your application. When you do so, your logs are automatically annotated with the correct Trace ID (see docs for Bunyan).

Extract the trace ID from the request header If you do not want to use the Trace module, you can extract the trace ID from the request header, use the following code to extract the traceId:

const traceHeader = req && req.headers ? req.headers['x-cloud-trace-context'] || '' : '';
const traceId = traceHeader ? traceHeader.split('/')[0] : '';

Then, you need to populate the trace attribute of your log entries. When using the Bunyan logger, use the following code:

logger.info({
  'logging.googleapis.com/trace': `projects/${project}/traces/${traceId}`
}, 'your message');

Answer 2

I also faced the same issue sometime back and did some workaround to make it. But in the above-mentioned solution might not help in some use cases where you have don't req, res object reference.

So here the solution. it will group all the logs under the request log.

File Name: correlate-logs.js

import bunyan from 'bunyan';
import { LOGGING_TRACE_KEY } from '@google-cloud/logging-bunyan';
import cls from 'cls-hooked';
import uuid from 'uuid/v1';

/**
 * CreateLogger will return loggerContextMiddleware and log.
 * Bind the loggerContextMiddleware on top to corelate other middleware logs. `app.use(loggerContextMiddleware);`
 * then you can log like this anywhere `log.info('This is helpful to see corelated logs in nodejs.)` and it will show with-in reqeust log.
 * @param {*} options
 */
export default function createLogger(projectId, bunyanLoggerOptions) {
  if (!projectId || !bunyanLoggerOptions) throw new Error('Please pass the required fields projectId and bunyanLoggerOption');

  const ns = cls.createNamespace(`logger/${uuid()}`); // To create unique namespace.
  const logger = bunyan.createLogger(bunyanLoggerOptions);

  /**
     * Express Middleware to add request context to logger for corelating the logs in GCP.
     * @param {*} req
     * @param {*} res
     * @param {*} next
     */
  const loggerContextMiddleware = (req, res, next) => {
    const traceHeader = (req && req.headers && req.headers['x-cloud-trace-context']) || '';
    if (traceHeader) {
      ns.bindEmitter(req);
      ns.bindEmitter(res);

      const traceId = traceHeader ? traceHeader.split('/')[0] : '';

      const trace = `projects/${projectId}/traces/${traceId}`;

      ns.run(() => {
        ns.set('trace', trace);
        next();
      });
    } else {
      next();
    }
  };

  /**
     * Helper method to get the trace id from CLS hook.
     */
  function getTrace() {
    if (ns && ns.active) return ns.get('trace');
    return '';
  }

  /**
     * Simple wrapper to avoid pushing dev logs to cloud.
     * @param {*} level
     * @param {*} msg
     */
  function printLog(level, ...msg) {
    const trace = getTrace();
    if (trace) { logger[level]({ [LOGGING_TRACE_KEY]: trace }, ...msg); } else { logger[level](...msg); }
  }

  /**
     * Little wrapper to abstract the log level.
     */
  const log = ['trace', 'debug', 'info', 'warn', 'error', 'fatal'].reduce((prev, curr) => ({ [curr]: (...msg) => printLog(curr, ...msg), ...prev }), {});

  return { loggerContextMiddleware, log };
}

File Name: logger.js

import { LoggingBunyan } from '@google-cloud/logging-bunyan';
import createLogger from '../lib/corelate-logs';
import { getProjectId, ifDev } from './config';

// Creates a Bunyan Stackdriver Logging client
const loggingBunyan = new LoggingBunyan();

let loggerOption;

if (ifDev()) {
  const bunyanDebugStream = require('bunyan-debug-stream'); // eslint-disable-line
  loggerOption = {
    name: 'my-service',
    streams: [{
      level: 'info',
      type: 'raw',
      stream: bunyanDebugStream({
        forceColor: true,
      }),
    }],
    serializers: bunyanDebugStream.serializers,
  };
} else {
  loggerOption = {
    name: 'my-service',
    level: 'info',
    streams: [loggingBunyan.stream('info')],
  };
}


const { loggerContextMiddleware, log } = createLogger(getProjectId() || 'dev', loggerOption);

export { loggerContextMiddleware, log };

Hope this helps somebody.