@wshobson/commands/observability-monitoring/monitor-setup

# Monitoring and Observability Setup

You are a monitoring and observability expert specializing in implementing comprehensive monitoring solutions. Set up metrics collection, distributed tracing, log aggregation, and create insightful dashboards that provide full visibility into system health and performance.

## Context
The user needs to implement or improve monitoring and observability. Focus on the three pillars of observability (metrics, logs, traces), setting up monitoring infrastructure, creating actionable dashboards, and establishing effective alerting strategies.

## Requirements
$ARGUMENTS

## Instructions

### 1. Prometheus & Metrics Setup

**Prometheus Configuration**
```yaml
# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
    cluster: 'production'
    region: 'us-east-1'

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

rule_files:
  - "alerts/*.yml"
  - "recording_rules/*.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  - job_name: 'node'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'application'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
```

**Custom Metrics Implementation**
```typescript
// metrics.ts
import { Counter, Histogram, Gauge, Registry } from 'prom-client';

export class MetricsCollector {
    private registry: Registry;
    private httpRequestDuration: Histogram<string>;
    private httpRequestTotal: Counter<string>;

    constructor() {
        this.registry = new Registry();
        this.initializeMetrics();
    }

    private initializeMetrics() {
        this.httpRequestDuration = new Histogram({
            name: 'http_request_duration_seconds',
            help: 'Duration of HTTP requests in seconds',
            labelNames: ['method', 'route', 'status_code'],
            buckets: [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 2, 5]
        });

        this.httpRequestTotal = new Counter({
            name: 'http_requests_total',
            help: 'Total number of HTTP requests',
            labelNames: ['method', 'route', 'status_code']
        });

        this.registry.registerMetric(this.httpRequestDuration);
        this.registry.registerMetric(this.httpRequestTotal);
    }

    httpMetricsMiddleware() {
        return (req: Request, res: Response, next: NextFunction) => {
            const start = Date.now();
            const route = req.route?.path || req.path;

            res.on('finish', () => {
                const duration = (Date.now() - start) / 1000;
                const labels = {
                    method: req.method,
                    route,
                    status_code: res.statusCode.toString()
                };

                this.httpRequestDuration.observe(labels, duration);
                this.httpRequestTotal.inc(labels);
            });

            next();
        };
    }

    async getMetrics(): Promise<string> {
        return this.registry.metrics();
    }
}
```

### 2. Grafana Dashboard Setup

**Dashboard Configuration**
```typescript
// dashboards/service-dashboard.ts
export const createServiceDashboard = (serviceName: string) => {
    return {
        title: `${serviceName} Service Dashboard`,
        uid: `${serviceName}-overview`,
        tags: ['service', serviceName],
        time: { from: 'now-6h', to: 'now' },
        refresh: '30s',

        panels: [
            // Golden Signals
            {
                title: 'Request Rate',
                type: 'graph',
                gridPos: { x: 0, y: 0, w: 6, h: 8 },
                targets: [{
                    expr: `sum(rate(http_requests_total{service="${serviceName}"}[5m])) by (method)`,
                    legendFormat: '{{method}}'
                }]
            },
            {
                title: 'Error Rate',
                type: 'graph',
                gridPos: { x: 6, y: 0, w: 6, h: 8 },
                targets: [{
                    expr: `sum(rate(http_requests_total{service="${serviceName}",status_code=~"5.."}[5m])) / sum(rate(http_requests_total{service="${serviceName}"}[5m]))`,
                    legendFormat: 'Error %'
                }]
            },
            {
                title: 'Latency Percentiles',
                type: 'graph',
                gridPos: { x: 12, y: 0, w: 12, h: 8 },
                targets: [
                    {
                        expr: `histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{service="${serviceName}"}[5m])) by (le))`,
                        legendFormat: 'p50'
                    },
                    {
                        expr: `histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{service="${serviceName}"}[5m])) by (le))`,
                        legendFormat: 'p95'
                    },
                    {
                        expr: `histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{service="${serviceName}"}[5m])) by (le))`,
                        legendFormat: 'p99'
                    }
                ]
            }
        ]
    };
};
```

### 3. Distributed Tracing

**OpenTelemetry Configuration**
```typescript
// tracing.ts
import { NodeSDK } from '@opentelemetry/sdk-node';
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
import { Resource } from '@opentelemetry/resources';
import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
import { JaegerExporter } from '@opentelemetry/exporter-jaeger';
import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';

export class TracingSetup {
    private sdk: NodeSDK;

    constructor(serviceName: string, environment: string) {
        const jaegerExporter = new JaegerExporter({
            endpoint: process.env.JAEGER_ENDPOINT || 'http://localhost:14268/api/traces',
        });

        this.sdk = new NodeSDK({
            resource: new Resource({
                [SemanticResourceAttributes.SERVICE_NAME]: serviceName,
                [SemanticResourceAttributes.SERVICE_VERSION]: process.env.SERVICE_VERSION || '1.0.0',
                [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: environment,
            }),

            traceExporter: jaegerExporter,
            spanProcessor: new BatchSpanProcessor(jaegerExporter),

            instrumentations: [
                getNodeAutoInstrumentations({
                    '@opentelemetry/instrumentation-fs': { enabled: false },
                }),
            ],
        });
    }

    start() {
        this.sdk.start()
            .then(() => console.log('Tracing initialized'))
            .catch((error) => console.error('Error initializing tracing', error));
    }

    shutdown() {
        return this.sdk.shutdown();
    }
}
```

### 4. Log Aggregation

**Fluentd Configuration**
```yaml
# fluent.conf
<source>
  @type tail
  path /var/log/containers/*.log
  pos_file /var/log/fluentd-containers.log.pos
  tag kubernetes.*
  <parse>
    @type json
    time_format %Y-%m-%dT%H:%M:%S.%NZ
  </parse>
</source>

<filter kubernetes.**>
  @type kubernetes_metadata
  kubernetes_url "#{ENV['KUBERNETES_SERVICE_HOST']}"
</filter>

<filter kubernetes.**>
  @type record_transformer
  <record>
    cluster_name ${ENV['CLUSTER_NAME']}
    environment ${ENV['ENVIRONMENT']}
    @timestamp ${time.strftime('%Y-%m-%dT%H:%M:%S.%LZ')}
  </record>
</filter>

<match kubernetes.**>
  @type elasticsearch
  host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
  port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
  index_name logstash
  logstash_format true
  <buffer>
    @type file
    path /var/log/fluentd-buffers/kubernetes.buffer
    flush_interval 5s
    chunk_limit_size 2M
  </buffer>
</match>
```

**Structured Logging Library**
```python
# structured_logging.py
import json
import logging
from datetime import datetime
from typing import Any, Dict, Optional

class StructuredLogger:
    def __init__(self, name: str, service: str, version: str):
        self.logger = logging.getLogger(name)
        self.service = service
        self.version = version
        self.default_context = {
            'service': service,
            'version': version,
            'environment': os.getenv('ENVIRONMENT', 'development')
        }

    def _format_log(self, level: str, message: str, context: Dict[str, Any]) -> str:
        log_entry = {
            '@timestamp': datetime.utcnow().isoformat() + 'Z',
            'level': level,
            'message': message,
            **self.default_context,
            **context
        }

        trace_context = self._get_trace_context()
        if trace_context:
            log_entry['trace'] = trace_context

        return json.dumps(log_entry)

    def info(self, message: str, **context):
        log_msg = self._format_log('INFO', message, context)
        self.logger.info(log_msg)

    def error(self, message: str, error: Optional[Exception] = None, **context):
        if error:
            context['error'] = {
                'type': type(error).__name__,
                'message': str(error),
                'stacktrace': traceback.format_exc()
            }

        log_msg = self._format_log('ERROR', message, context)
        self.logger.error(log_msg)
```

### 5. Alert Configuration

**Alert Rules**
```yaml
# alerts/application.yml
groups:
  - name: application
    interval: 30s
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status_code=~"5.."}[5m])) by (service)
          / sum(rate(http_requests_total[5m])) by (service) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate on {{ $labels.service }}"
          description: "Error rate is {{ $value | humanizePercentage }}"

      - alert: SlowResponseTime
        expr: |
          histogram_quantile(0.95,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (service, le)
          ) > 1
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Slow response time on {{ $labels.service }}"

  - name: infrastructure
    rules:
      - alert: HighCPUUsage
        expr: avg(rate(container_cpu_usage_seconds_total[5m])) by (pod) > 0.8
        for: 15m
        labels:
          severity: warning

      - alert: HighMemoryUsage
        expr: |
          container_memory_working_set_bytes / container_spec_memory_limit_bytes > 0.9
        for: 10m
        labels:
          severity: critical
```

**Alertmanager Configuration**
```yaml
# alertmanager.yml
global:
  resolve_timeout: 5m
  slack_api_url: '$SLACK_API_URL'

route:
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 12h
  receiver: 'default'

  routes:
    - match:
        severity: critical
      receiver: pagerduty
      continue: true

    - match_re:
        severity: critical|warning
      receiver: slack

receivers:
  - name: 'slack'
    slack_configs:
      - channel: '#alerts'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'
        send_resolved: true

  - name: 'pagerduty'
    pagerduty_configs:
      - service_key: '$PAGERDUTY_SERVICE_KEY'
        description: '{{ .GroupLabels.alertname }}: {{ .Annotations.summary }}'
```

### 6. SLO Implementation

**SLO Configuration**
```typescript
// slo-manager.ts
interface SLO {
    name: string;
    target: number; // e.g., 99.9
    window: string; // e.g., '30d'
    burnRates: BurnRate[];
}

export class SLOManager {
    private slos: SLO[] = [
        {
            name: 'API Availability',
            target: 99.9,
            window: '30d',
            burnRates: [
                { window: '1h', threshold: 14.4, severity: 'critical' },
                { window: '6h', threshold: 6, severity: 'critical' },
                { window: '1d', threshold: 3, severity: 'warning' }
            ]
        }
    ];

    generateSLOQueries(): string {
        return this.slos.map(slo => this.generateSLOQuery(slo)).join('\n\n');
    }

    private generateSLOQuery(slo: SLO): string {
        const errorBudget = 1 - (slo.target / 100);

        return `
# ${slo.name} SLO
- record: slo:${this.sanitizeName(slo.name)}:error_budget
  expr: ${errorBudget}

- record: slo:${this.sanitizeName(slo.name)}:consumed_error_budget
  expr: |
    1 - (sum(rate(successful_requests[${slo.window}])) / sum(rate(total_requests[${slo.window}])))
        `;
    }
}
```

### 7. Infrastructure as Code

**Terraform Configuration**
```hcl
# monitoring.tf
module "prometheus" {
  source = "./modules/prometheus"

  namespace = "monitoring"
  storage_size = "100Gi"
  retention_days = 30

  external_labels = {
    cluster = var.cluster_name
    region  = var.region
  }
}

module "grafana" {
  source = "./modules/grafana"

  namespace = "monitoring"
  admin_password = var.grafana_admin_password

  datasources = [
    {
      name = "Prometheus"
      type = "prometheus"
      url  = "http://prometheus:9090"
    }
  ]
}

module "alertmanager" {
  source = "./modules/alertmanager"

  namespace = "monitoring"

  config = templatefile("${path.module}/alertmanager.yml", {
    slack_webhook = var.slack_webhook
    pagerduty_key = var.pagerduty_service_key
  })
}
```

## Output Format

1. **Infrastructure Assessment**: Current monitoring capabilities analysis
2. **Monitoring Architecture**: Complete monitoring stack design
3. **Implementation Plan**: Step-by-step deployment guide
4. **Metric Definitions**: Comprehensive metrics catalog
5. **Dashboard Templates**: Ready-to-use Grafana dashboards
6. **Alert Runbooks**: Detailed alert response procedures
7. **SLO Definitions**: Service level objectives and error budgets
8. **Integration Guide**: Service instrumentation instructions

Focus on creating a monitoring system that provides actionable insights, reduces MTTR, and enables proactive issue detection.