クラウド監視ツール比較：Prometheus/Grafana/Datadogによるプロアクティブな運用

# Prometheusの基本設定例（prometheus.yml）
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "alert_rules.yml"

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
  
  - job_name: 'node_exporter'
    static_configs:
      - targets: ['node-exporter:9100']
  
  - job_name: 'kubernetes-apiservers'
    kubernetes_sd_configs:
      - role: endpoints
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https

// Grafanaダッシュボード設定例（一部）
{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": null,
  "graphTooltip": 0,
  "id": 1,
  "links": [],
  "panels": [
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "Prometheus",
      "fill": 1,
      "fillGradient": 0,
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "hiddenSeries": false,
      "id": 2,
      "legend": {
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "show": true,
        "total": false,
        "values": false
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": {
        "dataLinks": []
      },
      "percentage": false,
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "targets": [
        {
          "expr": "node_cpu_seconds_total{mode=\"user\"}",
          "refId": "A"
        }
      ],
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "CPU Usage",
      "tooltip": {
        "shared": true,
        "sort": 0,
        "value_type": "individual"
      },
      "type": "graph",
      "xaxis": {
        "buckets": null,
        "mode": "time",
        "name": null,
        "show": true,
        "values": []
      },
      "yaxes": [
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        },
        {
          "format": "short",
          "label": null,
          "logBase": 1,
          "max": null,
          "min": null,
          "show": true
        }
      ],
      "yaxis": {
        "align": false,
        "alignLevel": null
      }
    }
  ],
  "schemaVersion": 22,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Node Exporter Dashboard",
  "uid": "rYdddlPWk",
  "version": 1
}

# Datadog Agentの基本設定例（datadog.yaml）
api_key: YOUR_API_KEY
site: datadoghq.com
logs_enabled: true

apm_config:
  enabled: true
  
process_config:
  enabled: true

logs_config:
  container_collect_all: true

kubernetes_metadata_tag_update_freq: 60

# オートディスカバリー設定
listeners:
  - name: docker
  - name: kubelet

config_providers:
  - name: docker
    polling: true
  - name: kubelet
    polling: true

# インテグレーション設定
integrations:
  docker:
    collect_container_size: true
    collect_container_count: true
    collect_volume_count: true
    collect_images_stats: true
  kubernetes:
    collect_service_tags: true
    collect_events: true

# Docker Composeを使った簡易Prometheus + Grafana環境
version: '3'
services:
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    ports:
      - "9090:9090"
    
  grafana:
    image: grafana/grafana:latest
    depends_on:
      - prometheus
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
    volumes:
      - grafana-data:/var/lib/grafana
    
  node-exporter:
    image: prom/node-exporter:latest
    ports:
      - "9100:9100"
    
volumes:
  grafana-data:

# Prometheusの設定例（Datadog統合）
remote_write:
  - url: https://prometheus-intake.datadoghq.com/api/v1/prometheus?api_key=<DATADOG_API_KEY>
    basic_auth:
      username: <DATADOG_APP_KEY>

# Terraform例: GrafanaダッシュボードをIaCで管理
resource "grafana_dashboard" "service_overview" {
  config_json = file("dashboards/service_overview.json")
  folder = grafana_folder.monitoring.id
  overwrite = true
}

resource "grafana_folder" "monitoring" {
  title = "Service Monitoring"
}

resource "grafana_alert_notification" "slack" {
  name = "Slack Notifications"
  type = "slack"
  settings = {
    url = var.slack_webhook_url
    recipient = "#alerts"
    mention = "@oncall"
  }
}

// Datadogのシンセティックテストとアラート設定例
{
  "name": "API Health Check",
  "type": "api",
  "subtype": "http",
  "request": {
    "method": "GET",
    "url": "https://api.dx-media.example/health",
    "timeout": 30,
    "headers": {
      "Content-Type": "application/json"
    }
  },
  "assertions": [
    {
      "type": "statusCode",
      "operator": "is",
      "target": 200
    },
    {
      "type": "responseTime",
      "operator": "lessThan",
      "target": 1000
    }
  ],
  "locations": ["aws:ap-northeast-1", "aws:us-west-2"],
  "options": {
    "tick_every": 60,
    "monitor_options": {
      "notify_audit": true,
      "locked": false,
      "timeout_h": 0,
      "renotify_interval": 120
    }
  },
  "message": "@slack-oncall-team APIヘルスチェック失敗。緊急対応が必要です。\n\n{{#is_alert}}サービスがダウンしています{{/is_alert}}\n\n{{#is_recovery}}サービスが復旧しました{{/is_recovery}}",
  "tags": ["service:api", "env:production", "team:backend"],
  "priority": 1
}

# Prometheusの高度なアラートルール例（SLOベース）
groups:
- name: SLO_Alerts
  rules:
  - alert: APIErrorBudgetBurn
    expr: |
      sum(increase(http_requests_total{job="api",status=~"5.."}[1h])) by (service)
        / 
      sum(increase(http_requests_total{job="api"}[1h])) by (service)
        > 0.01
    for: 15m
    labels:
      severity: warning
      categories:slo
    annotations:
      summary: "API error budget burning too fast"
      description: "Service {{ $labels.service }} has error rate above 1% for past 15min, burning error budget too quickly."
      runbook_url: "https://wiki.dx-media.example/runbooks/api-errors"