AWS ECS Fargate の SRE 実践ガイド：ADOT分散トレース・EMFメトリクス・SLO/エラーバジェット/バーンレートアラート設計

// logger.ts — 構造化ロガーの最小実装
import { AsyncLocalStorage } from "node:async_hooks";

type LogLevel = "debug" | "info" | "warn" | "error";

// レベルを数値化して、環境変数で出力しきい値を制御する
const LEVELS: Record<LogLevel, number> = { debug: 10, info: 20, warn: 30, error: 40 };
const MIN_LEVEL = LEVELS[(process.env.LOG_LEVEL as LogLevel) ?? "info"];

// リクエストスコープの相関コンテキストを保持する（後述のミドルウェアで設定）
export const requestContext = new AsyncLocalStorage<{
  requestId: string;
  traceId?: string;
}>();

// PIIをログに残さないための墨消し。鍵名ベースで再帰的にマスクする
const PII_KEYS = new Set(["email", "password", "cardNumber", "phone", "token"]);
function redact(value: unknown): unknown {
  if (Array.isArray(value)) return value.map(redact);
  if (value && typeof value === "object") {
    return Object.fromEntries(
      Object.entries(value as Record<string, unknown>).map(([k, v]) =>
        PII_KEYS.has(k) ? [k, "[REDACTED]"] : [k, redact(v)],
      ),
    );
  }
  return value;
}

function emit(level: LogLevel, message: string, fields: Record<string, unknown> = {}): void {
  if (LEVELS[level] < MIN_LEVEL) return;
  const ctx = requestContext.getStore();
  const line = {
    level,
    message,
    timestamp: new Date().toISOString(),
    // 相関ID：これがログを横断検索可能にする要
    requestId: ctx?.requestId,
    traceId: ctx?.traceId,
    ...redact(fields),
  };
  // CloudWatch Logs は標準出力の1行=1イベントとして取り込む
  process.stdout.write(`${JSON.stringify(line)}\n`);
}

export const log = {
  debug: (m: string, f?: Record<string, unknown>) => emit("debug", m, f),
  info: (m: string, f?: Record<string, unknown>) => emit("info", m, f),
  warn: (m: string, f?: Record<string, unknown>) => emit("warn", m, f),
  error: (m: string, f?: Record<string, unknown>) => emit("error", m, f),
};

// middleware.ts — Express の相関IDミドルウェア
import { randomUUID } from "node:crypto";
import { trace } from "@opentelemetry/api";
import { requestContext } from "./logger";
import type { Request, Response, NextFunction } from "express";

export function correlationMiddleware(req: Request, res: Response, next: NextFunction): void {
  // 上流（ALB/フロント）が付けた X-Request-Id を尊重し、無ければ採番する
  const requestId = (req.header("x-request-id") ?? randomUUID()).slice(0, 64);
  // 進行中のスパンから traceId を取得（OTel計装が前提）
  const traceId = trace.getActiveSpan()?.spanContext().traceId;

  res.setHeader("x-request-id", requestId);
  // このリクエストの処理が終わるまで、ログは自動で相関IDを含む
  requestContext.run({ requestId, traceId }, () => next());
}

# 必要パッケージ（OpenTelemetry公式の手順に準拠）
npm install @opentelemetry/sdk-node \
  @opentelemetry/api \
  @opentelemetry/auto-instrumentations-node \
  @opentelemetry/exporter-trace-otlp-proto

// instrumentation.ts — アプリより前にロードされる
import { NodeSDK } from "@opentelemetry/sdk-node";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";

const sdk = new NodeSDK({
  // 同一タスク内のADOTサイドカーへ OTLP/HTTP で送る（localhost:4318）
  traceExporter: new OTLPTraceExporter({
    url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT
      ? `${process.env.OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces`
      : "http://localhost:4318/v1/traces",
  }),
  instrumentations: [getNodeAutoInstrumentations()],
});

sdk.start();

// payment.ts — 手動計装でビジネス区間を可視化する
import { trace, SpanStatusCode } from "@opentelemetry/api";

const tracer = trace.getTracer("payment-service");

export async function charge(orderId: string, amountJpy: number): Promise<void> {
  // span 名は「動作.対象」の規約に寄せると検索しやすい
  await tracer.startActiveSpan("payment.charge", async (span) => {
    // 高カーディナリティのIDは属性に。メトリクスのディメンションには使わない
    span.setAttribute("order.id", orderId);
    span.setAttribute("payment.amount_jpy", amountJpy);
    try {
      await callPaymentProvider(orderId, amountJpy);
      span.setStatus({ code: SpanStatusCode.OK });
    } catch (err) {
      // 例外をスパンに記録すると、トレース上でエラー区間が赤くなる
      span.recordException(err as Error);
      span.setStatus({ code: SpanStatusCode.ERROR, message: "charge failed" });
      throw err;
    } finally {
      span.end();
    }
  });
}

# ecs_task.tf — アプリ + ADOTサイドカー の Fargate タスク定義
resource "aws_ecs_task_definition" "app" {
  family                   = "payment-api"
  requires_compatibilities = ["FARGATE"]
  network_mode             = "awsvpc"
  cpu                      = "1024"
  memory                   = "3072"
  task_role_arn            = aws_iam_role.task.arn       # アプリ + 計装の権限
  execution_role_arn       = aws_iam_role.execution.arn  # イメージpull/ログ書込

  container_definitions = jsonencode([
    {
      name      = "payment-api"
      image     = "${aws_ecr_repository.app.repository_url}:latest"
      essential = true
      environment = [
        # awsvpc では同一タスクのコンテナは localhost で通信できる
        { name = "OTEL_EXPORTER_OTLP_ENDPOINT", value = "http://localhost:4318" },
        { name = "OTEL_SERVICE_NAME", value = "payment-api" },
        { name = "LOG_LEVEL", value = "info" }
      ]
      # Collector を先に起動してから、アプリを起動する
      dependsOn = [{ containerName = "aws-otel-collector", condition = "START" }]
      logConfiguration = {
        logDriver = "awslogs"
        options = {
          "awslogs-group"         = "/ecs/payment-api"
          "awslogs-region"        = var.region
          "awslogs-stream-prefix" = "ecs"
          "awslogs-create-group"  = "true"
        }
      }
    },
    {
      name      = "aws-otel-collector"
      image     = "public.ecr.aws/aws-observability/aws-otel-collector:v0.30.0"
      essential = true
      # ECSが同梱する設定を使う例。X-Ray + EMF を有効化する設定に差し替え可能
      command   = ["--config=/etc/ecs/ecs-default-config.yaml"]
      logConfiguration = {
        logDriver = "awslogs"
        options = {
          "awslogs-group"         = "/ecs/ecs-aws-otel-sidecar-collector"
          "awslogs-region"        = var.region
          "awslogs-stream-prefix" = "ecs"
          "awslogs-create-group"  = "true"
        }
      }
    }
  ])
}

# iam.tf — タスクロールに可観測性の最小権限を付与
resource "aws_iam_role_policy_attachment" "xray" {
  role       = aws_iam_role.task.name
  policy_arn = "arn:aws:iam::aws:policy/AWSXRayDaemonWriteAccess"
}

resource "aws_iam_role_policy_attachment" "cw_agent" {
  role       = aws_iam_role.task.name
  policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
}

# otel-config.yaml — ADOT Collector の受信/処理/送信
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch: {}   # スパン/メトリクスをまとめて送り、API呼び出しを削減する

exporters:
  awsxray: {}                 # トレース → AWS X-Ray
  awsemf:                     # メトリクス → CloudWatch（EMF経由）
    namespace: PaymentApi
    log_group_name: /metrics/payment-api

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [awsxray]
    metrics:
      receivers: [otlp]
      processors: [batch]
      exporters: [awsemf]

// emf.ts — RED の Duration/Errors を EMF で発行する
type EmfUnit = "Milliseconds" | "Count";

function putEmf(
  namespace: string,
  metrics: { name: string; unit: EmfUnit; value: number }[],
  dimensions: Record<string, string>,
): void {
  const line: Record<string, unknown> = {
    _aws: {
      Timestamp: Date.now(),
      CloudWatchMetrics: [
        {
          Namespace: namespace,
          // 注意：高カーディナリティ（requestId等）をディメンションにしない。
          // 組み合わせごとに課金対象メトリクスが生成される
          Dimensions: [Object.keys(dimensions)],
          Metrics: metrics.map((m) => ({ Name: m.name, Unit: m.unit })),
        },
      ],
    },
    ...dimensions,
  };
  for (const m of metrics) line[m.name] = m.value;
  process.stdout.write(`${JSON.stringify(line)}\n`);
}

export function recordRequest(route: string, status: number, durationMs: number): void {
  putEmf(
    "PaymentApi",
    [
      { name: "Duration", unit: "Milliseconds", value: durationMs },
      { name: "Errors", unit: "Count", value: status >= 500 ? 1 : 0 },
      { name: "Count", unit: "Count", value: 1 },
    ],
    // ディメンションは低カーディナリティに保つ（route と環境程度）
    { Route: route, Environment: process.env.NODE_ENV ?? "production" },
  );
}

{
  "_aws": {
    "Timestamp": 1574109732004,
    "CloudWatchMetrics": [
      {
        "Namespace": "PaymentApi",
        "Dimensions": [["Route", "Environment"]],
        "Metrics": [{ "Name": "Duration", "Unit": "Milliseconds" }]
      }
    ]
  },
  "Route": "/charge",
  "Environment": "production",
  "Duration": 135.5
}

# alarms.tf — 1時間/5分の2窓を AND 結合する Page アラーム
resource "aws_cloudwatch_metric_alarm" "burn_long" {
  alarm_name          = "slo-burn-1h"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  threshold           = 14.4 # 14.4倍速のバーンレート
  # metric_query でエラー率 / (1 - SLO) を計算する想定（式は省略）
  metric_name = "ErrorRate"
  namespace   = "PaymentApi"
  period      = 3600
  statistic   = "Average"
}

resource "aws_cloudwatch_metric_alarm" "burn_short" {
  alarm_name          = "slo-burn-5m"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  threshold           = 14.4
  metric_name         = "ErrorRate"
  namespace           = "PaymentApi"
  period              = 300
  statistic           = "Average"
}

resource "aws_cloudwatch_composite_alarm" "burn_page" {
  alarm_name = "slo-burn-page"
  # 両方の窓が同時に発火したときだけ、オンコールを呼び出す
  alarm_rule = join(" ", [
    "ALARM(${aws_cloudwatch_metric_alarm.burn_long.alarm_name})",
    "AND",
    "ALARM(${aws_cloudwatch_metric_alarm.burn_short.alarm_name})",
  ])
  alarm_actions = [aws_sns_topic.oncall.arn]
}

# logs.tf — ロググループに保持期間を必ず設定する
resource "aws_cloudwatch_log_group" "app" {
  name              = "/ecs/payment-api"
  retention_in_days = 30 # 無期限保持はコスト事故。用途に応じて明示する
}

// payment.test.ts — 生成スパンを検証する（vitest想定）
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
import {
  InMemorySpanExporter,
  SimpleSpanProcessor,
} from "@opentelemetry/sdk-trace-base";

const exporter = new InMemorySpanExporter();
const provider = new NodeTracerProvider({
  spanProcessors: [new SimpleSpanProcessor(exporter)],
});
provider.register();

afterEach(() => exporter.reset());

test("charge emits a single OK span", async () => {
  await charge("order-1", 1200);
  const spans = exporter.getFinishedSpans();
  expect(spans).toHaveLength(1);
  expect(spans[0].name).toBe("payment.charge");
  expect(spans[0].attributes["payment.amount_jpy"]).toBe(1200);
});