Observability and Monitoring for Production Applications

Observability lets you understand system behavior from external outputs. Metrics, logs, and traces form the three pillars of observability.

The Three Pillars#

Metrics:
- Numerical measurements over time
- CPU, memory, request counts, latencies
- Aggregated and sampled

Logs:
- Discrete events with context
- Errors, requests, business events
- Detailed but voluminous

Traces:
- Request flow across services
- Timing and dependencies
- End-to-end visibility

Metrics with Prometheus#

import { Registry, Counter, Histogram, Gauge } from 'prom-client';

const register = new Registry();

// Request counter
const httpRequestsTotal = new Counter({
  name: 'http_requests_total',
  help: 'Total HTTP requests',
  labelNames: ['method', 'path', 'status'],
  registers: [register],
});

// Request duration histogram
const httpRequestDuration = new Histogram({
  name: 'http_request_duration_seconds',
  help: 'HTTP request duration in seconds',
  labelNames: ['method', 'path', 'status'],
  buckets: [0.01, 0.05, 0.1, 0.5, 1, 5],
  registers: [register],
});

// Active connections gauge
const activeConnections = new Gauge({
  name: 'active_connections',
  help: 'Number of active connections',
  registers: [register],
});

// Middleware
function metricsMiddleware(req: Request, res: Response, next: NextFunction) {
  const start = Date.now();

  res.on('finish', () => {
    const duration = (Date.now() - start) / 1000;
    const labels = {
      method: req.method,
      path: req.route?.path || req.path,
      status: res.statusCode.toString(),
    };

    httpRequestsTotal.inc(labels);
    httpRequestDuration.observe(labels, duration);
  });

  next();
}

// Expose metrics endpoint
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', register.contentType);
  res.send(await register.metrics());
});

Business Metrics#

// Track business-specific metrics
const ordersCreated = new Counter({
  name: 'orders_created_total',
  help: 'Total orders created',
  labelNames: ['plan', 'country'],
});

const orderValue = new Histogram({
  name: 'order_value_dollars',
  help: 'Order value in dollars',
  buckets: [10, 50, 100, 500, 1000, 5000],
});

const activeUsers = new Gauge({
  name: 'active_users',
  help: 'Currently active users',
});

// Usage
async function createOrder(order: Order) {
  const result = await db.order.create({ data: order });

  ordersCreated.inc({
    plan: order.plan,
    country: order.country,
  });
  orderValue.observe(order.total);

  return result;
}

Structured Logging#

import pino from 'pino';

const logger = pino({
  level: process.env.LOG_LEVEL || 'info',
  formatters: {
    level: (label) => ({ level: label }),
  },
  base: {
    service: 'api-server',
    version: process.env.APP_VERSION,
    environment: process.env.NODE_ENV,
  },
});

// Create child logger with context
function createRequestLogger(req: Request) {
  return logger.child({
    requestId: req.id,
    userId: req.user?.id,
    path: req.path,
    method: req.method,
  });
}

// Usage
app.use((req, res, next) => {
  req.log = createRequestLogger(req);
  req.log.info('Request started');

  res.on('finish', () => {
    req.log.info({
      statusCode: res.statusCode,
      duration: Date.now() - req.startTime,
    }, 'Request completed');
  });

  next();
});

// Log levels
logger.trace('Detailed debugging');
logger.debug('Debugging information');
logger.info('Normal operations');
logger.warn('Warning conditions');
logger.error({ err: error }, 'Error occurred');
logger.fatal('System unusable');

Distributed Tracing#

import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';

// Initialize OpenTelemetry
const sdk = new NodeSDK({
  traceExporter: new OTLPTraceExporter({
    url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
  }),
  instrumentations: [getNodeAutoInstrumentations()],
  serviceName: 'api-server',
});

sdk.start();

// Manual spans
import { trace, SpanStatusCode } from '@opentelemetry/api';

const tracer = trace.getTracer('api-server');

async function processOrder(orderId: string) {
  return tracer.startActiveSpan('process-order', async (span) => {
    try {
      span.setAttribute('order.id', orderId);

      // Nested span
      await tracer.startActiveSpan('validate-order', async (validateSpan) => {
        await validateOrder(orderId);
        validateSpan.end();
      });

      await tracer.startActiveSpan('charge-payment', async (paymentSpan) => {
        const result = await chargePayment(orderId);
        paymentSpan.setAttribute('payment.amount', result.amount);
        paymentSpan.end();
      });

      span.setStatus({ code: SpanStatusCode.OK });
      return { success: true };
    } catch (error) {
      span.setStatus({
        code: SpanStatusCode.ERROR,
        message: error.message,
      });
      span.recordException(error);
      throw error;
    } finally {
      span.end();
    }
  });
}

Error Tracking#

import * as Sentry from '@sentry/node';

Sentry.init({
  dsn: process.env.SENTRY_DSN,
  environment: process.env.NODE_ENV,
  release: process.env.APP_VERSION,
  tracesSampleRate: 0.1,
  integrations: [
    new Sentry.Integrations.Http({ tracing: true }),
    new Sentry.Integrations.Express({ app }),
    new Sentry.Integrations.Prisma({ client: prisma }),
  ],
});

// Add request handler
app.use(Sentry.Handlers.requestHandler());
app.use(Sentry.Handlers.tracingHandler());

// Error handler
app.use(Sentry.Handlers.errorHandler());

// Manual error capture
try {
  await riskyOperation();
} catch (error) {
  Sentry.captureException(error, {
    tags: { operation: 'risky-operation' },
    extra: { userId: user.id },
  });
  throw error;
}

// Capture message
Sentry.captureMessage('Important event occurred', {
  level: 'info',
  tags: { feature: 'checkout' },
});

Health Checks#

interface HealthCheck {
  name: string;
  check: () => Promise<{ healthy: boolean; details?: any }>;
}

const healthChecks: HealthCheck[] = [
  {
    name: 'database',
    check: async () => {
      try {
        await db.$queryRaw`SELECT 1`;
        return { healthy: true };
      } catch (error) {
        return { healthy: false, details: error.message };
      }
    },
  },
  {
    name: 'redis',
    check: async () => {
      try {
        await redis.ping();
        return { healthy: true };
      } catch (error) {
        return { healthy: false, details: error.message };
      }
    },
  },
  {
    name: 'external-api',
    check: async () => {
      try {
        const response = await fetch('https://api.external.com/health');
        return { healthy: response.ok };
      } catch (error) {
        return { healthy: false, details: error.message };
      }
    },
  },
];

app.get('/health', async (req, res) => {
  const results = await Promise.all(
    healthChecks.map(async (check) => ({
      name: check.name,
      ...(await check.check()),
    }))
  );

  const healthy = results.every((r) => r.healthy);

  res.status(healthy ? 200 : 503).json({
    status: healthy ? 'healthy' : 'unhealthy',
    checks: results,
    timestamp: new Date().toISOString(),
  });
});

Alerting Rules#

# prometheus-rules.yaml
groups:
  - name: api-alerts
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          /
          sum(rate(http_requests_total[5m])) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: High error rate detected
          description: Error rate is {{ $value | humanizePercentage }}

      - alert: HighLatency
        expr: |
          histogram_quantile(0.95,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
          ) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: High latency detected
          description: P95 latency is {{ $value }}s

      - alert: ServiceDown
        expr: up == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: Service is down

Dashboard Queries#

# Request rate
sum(rate(http_requests_total[5m])) by (path)

# Error rate
sum(rate(http_requests_total{status=~"5.."}[5m]))
/
sum(rate(http_requests_total[5m]))

# P50, P95, P99 latency
histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))
histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))

# Active connections
active_connections

# Memory usage
process_resident_memory_bytes

Best Practices#

Metrics:
✓ Use consistent naming conventions
✓ Add relevant labels
✓ Set appropriate buckets
✓ Monitor cardinality

Logging:
✓ Use structured logging
✓ Include correlation IDs
✓ Log at appropriate levels
✓ Don't log sensitive data

Tracing:
✓ Propagate context across services
✓ Add meaningful span names
✓ Include relevant attributes
✓ Sample appropriately

Alerting:
✓ Alert on symptoms, not causes
✓ Include runbooks
✓ Avoid alert fatigue
✓ Test alerts regularly

Observability requires metrics, logs, and traces working together. Start with basic health checks and metrics, add structured logging, then implement tracing for complex systems. Good observability reduces mean time to resolution and improves system reliability.