[ { "id": "01-system-context", "title": "Observability Platform — System Context", "layout": "context", "canvas": { "width": 1860 }, "colWidth": 300, "system": { "label": "Observability Platform", "sub": "metrics · logs · traces" }, "groups": [ { "side": "left", "title": "People who read it", "nodes": [ { "id": "oncall", "label": "On-call engineer", "kind": "actor", "rel": "page to cause", "dir": "in" }, { "id": "owner", "label": "Service owner", "kind": "actor", "rel": "SLOs, budgets", "dir": "in" }, { "id": "sre", "label": "SRE / platform", "kind": "actor", "rel": "estate health", "dir": "in" }, { "id": "sec", "label": "Security engineer", "kind": "actor", "rel": "compliance", "dir": "in" } ] }, { "side": "top", "title": "What it observes", "nodes": [ { "id": "svc", "label": "900 in-house services", "kind": "app", "rel": "OTLP", "dir": "in" }, { "id": "infra", "label": "EKS · EC2 · managed AWS", "kind": "platform", "rel": "infra metrics", "dir": "in" }, { "id": "clients", "label": "Web & mobile clients", "kind": "external", "rel": "untrusted path", "dir": "in", "kind2": "async" } ] }, { "side": "right", "title": "Systems it hands off to", "nodes": [ { "id": "paging", "label": "Paging platform", "kind": "external", "rel": "firing alert", "kind2": "async" }, { "id": "cmdb", "label": "Service catalogue", "kind": "external", "rel": "ownership", "dir": "in", "icon": "fa5_book" }, { "id": "idp", "label": "Identity provider", "kind": "security", "rel": "identity", "dir": "in" } ] }, { "side": "bottom", "title": "Deliberately outside the boundary", "nodes": [ { "id": "audit", "label": "Regulatory audit log", "kind": "external", "rel": "own custody" }, { "id": "analytics", "label": "Product analytics", "kind": "external", "rel": "not here" }, { "id": "ir", "label": "Incident management", "kind": "external", "rel": "handover" } ] } ], "note": "Routing, escalation and acknowledgement belong to the paging platform. This set stops at the firing alert.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "02-high-level-architecture", "title": "Observability Platform — High-Level Architecture", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "align": "middle", "stages": [ { "title": "Emit", "nodes": [ { "id": "sdk", "label": "OTel SDK", "sub": "in-process", "kind": "app" }, { "id": "agent", "label": "Node agent", "sub": "ADOT DaemonSet", "kind": "platform" } ] }, { "title": "Admit", "nodes": [ { "id": "gw", "label": "Gateway tier", "sub": "OTel Collector", "kind": "integration" }, { "id": "card", "label": "Cardinality admission", "sub": "budget per service", "kind": "decision" }, { "id": "redact", "label": "Redaction", "sub": "before write", "kind": "security" } ] }, { "title": "Reduce", "nodes": [ { "id": "agg", "label": "Aggregate", "sub": "metric path — unsampled", "kind": "app" }, { "id": "tail", "label": "Tail sampler", "sub": "45 s hold, whole traces", "kind": "app" }, { "id": "logclass", "label": "Log class reduction", "sub": "by declared class", "kind": "app" } ] }, { "title": "Buffer", "nodes": [ { "id": "msk", "label": "Ingest buffer", "sub": "Amazon MSK", "kind": "queue" } ] }, { "title": "Store", "nodes": [ { "id": "mimir", "label": "Metric store", "sub": "Mimir on S3", "kind": "store" }, { "id": "tempo", "label": "Trace store", "sub": "Tempo on S3", "kind": "store" }, { "id": "ch", "label": "Log store", "sub": "ClickHouse on S3", "kind": "store" }, { "id": "exem", "label": "Exemplar index", "sub": "DynamoDB", "kind": "store" } ] }, { "title": "Read", "nodes": [ { "id": "ruler", "label": "Alert evaluation", "sub": "independent plane", "kind": "app" }, { "id": "graf", "label": "Query & consoles", "sub": "Managed Grafana", "kind": "integration" } ] } ], "edges": [ { "from": "gw", "to": "agg", "label": "unsampled", "kind": "sync" }, { "from": "gw", "to": "tail", "label": "evidence", "kind": "async", "route": "gutter" }, { "from": "mimir", "to": "ruler", "label": "unsampled only", "kind": "sync" }, { "from": "exem", "to": "graf", "label": "aggregate to trace", "kind": "sync" } ], "note": "Two paths leave the gateway and never rejoin except through the exemplar index, which carries pointers and never quantities.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "03-actors-and-journeys", "title": "Who the Platform Is For, and What They Get To Do", "layout": "actors", "canvas": { "width": 1740 }, "cardWidth": 268, "groups": [ { "title": "The people it is built for", "kind": "boundary", "actors": [ { "id": "oncall", "label": "On-call engineer", "sub": "70 rotations", "goal": "I was paged twelve minutes ago for a service I have never opened. Tell me what is broken before my second coffee.", "journeys": [ { "id": "j-rootcause", "label": "Page to root cause", "sub": "view 04" }, { "label": "Raise my sampling mid-incident" }, { "label": "Prove the last 5 minutes exist" } ] }, { "id": "owner", "label": "Service owner", "sub": "900 services", "goal": "I want my new service observable the day it ships, without filing a ticket or learning three query languages.", "journeys": [ { "id": "j-onboard", "label": "Onboard a service", "sub": "view 05" }, { "label": "Define an SLO and burn alert" }, { "label": "Fix a cardinality rejection" } ] }, { "id": "sre", "label": "SRE / platform team", "sub": "estate-wide", "goal": "I need to know the platform is blind before anyone else discovers it is, and to keep one team's bad deploy off everyone else's dashboards.", "journeys": [ { "label": "Contain a cardinality incident" }, { "label": "Shed by class under a storm" }, { "label": "Prove self-telemetry is alive" } ] } ] }, { "title": "The people who pay for it and police it", "kind": "trust", "actors": [ { "id": "eng-dir", "label": "Engineering director", "sub": "budget holder", "goal": "Observability is 4% of infrastructure spend and rising. Show me what to cut that will not cost us an outage.", "journeys": [ { "id": "j-bill", "label": "Cut the bill, stay sighted", "sub": "view 06" }, { "label": "Read the per-team scorecard" } ] }, { "id": "secops", "label": "Security engineer", "sub": "compliance class", "goal": "Logs will contain personal data whatever the policy says. I need it redacted before it is written and erasable after it is.", "journeys": [ { "label": "Erase a named subject" }, { "label": "Audit who read production logs" } ] } ] }, { "title": "Machines in the cast", "kind": "cloud", "actors": [ { "id": "svc", "label": "Instrumented service", "kind": "app", "sub": "900 of them", "goal": "Take my telemetry without ever making me wait, retry or hold a buffer.", "journeys": [ { "label": "Push OTLP to the local agent" } ] }, { "id": "pager", "label": "Paging platform", "kind": "external", "sub": "out of scope", "goal": "Hand me a firing alert with the owner, the indicator and a scoped view already attached.", "journeys": [ { "label": "Receive a firing alert" } ] }, { "id": "deadman", "label": "Dead-man canary", "kind": "platform", "sub": "separate account", "goal": "Notice that the observability platform has stopped speaking, from somewhere it cannot take down with it.", "journeys": [ { "label": "Alert that the platform is blind" } ] } ] } ], "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "04-journey-page-to-root-cause", "title": "Journey — On-Call Engineer: Page to Root Cause", "layout": "journey", "canvas": { "width": 1740 }, "actor": { "label": "On-call engineer", "sub": "never seen this service", "goal": "Find the failing dependency without knowing the query in advance", "trigger": "A burn-rate page at 03:14 for checkout-api", "success": "Names the failing component and its blast radius inside 10 minutes" }, "phases": [ { "title": "Paged", "sub": "03:14" }, { "title": "Orient", "sub": "is it real" }, { "title": "Localise", "moment": true }, { "title": "Prove", "moment": true }, { "title": "Hand over" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Opens the alert link" } ], [ { "label": "Reads the SLO burn" }, { "label": "Checks recent deploys" } ], [ { "label": "Clicks the exemplar" }, { "label": "Groups errors by attribute" } ], [ { "label": "Reads that trace's logs" } ], [ { "label": "Posts the trace link" } ] ] }, { "title": "What the platform gives", "kind": "system", "cells": [ [ { "label": "Scoped view, owner, indicator" } ], [ { "label": "Unsampled error rate" }, { "label": "Deploy markers" } ], [ { "label": "Aggregate to trace pivot" }, { "label": "High-cardinality group-by" } ], [ { "label": "Span to log join" }, { "label": "Completeness label" } ], [ { "label": "Shareable permalink" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Working", "Lost" ], "points": [ 1, 1, 0, 2, 2 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Last 4 min still writing" } ], [ { "label": "The trace was sampled away" } ], [ { "label": "Log class was reduced" } ], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Alert carries context" } ], [ { "label": "Freshness lag shown, not hidden" } ], [ { "label": "Errors are always-keep" } ], [ { "label": "Coverage stated on the result" } ], [ { "label": "Query audit records the hunt" } ] ] } ], "chain": true, "note": "The trough is phase 3: the pivot fails when the evidence was sampled away. The always-keep rule for errors exists for exactly this moment.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "05-journey-onboard-a-service", "title": "Journey — Service Owner: Observable On The Day It Ships", "layout": "journey", "canvas": { "width": 1740 }, "actor": { "label": "Service owner", "sub": "shipping a new service", "goal": "Be observable without a platform-team ticket", "trigger": "A new service registered in the catalogue", "success": "Dashboards, SLO and budget live before the first production deploy" }, "phases": [ { "title": "Register" }, { "title": "Instrument" }, { "title": "First deploy", "moment": true }, { "title": "Add an SLO" }, { "title": "Live" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Claims service in catalogue" } ], [ { "label": "Adds the OTel SDK" }, { "label": "Names its log classes" } ], [ { "label": "Ships to staging" } ], [ { "label": "Declares indicator + target" } ], [ { "label": "Reads the scorecard" } ] ] }, { "title": "What the platform gives", "kind": "system", "cells": [ [ { "label": "Owner, budget, retention class" } ], [ { "label": "Identity from EKS Pod Identity" } ], [ { "label": "Default dashboards generated" }, { "label": "Cardinality estimate" } ], [ { "label": "Burn-rate rule compiled" } ], [ { "label": "Daily cost attribution" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Easy", "Working", "Blocked" ], "points": [ 2, 1, 0, 1, 2 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Which class is this log?" } ], [ { "label": "Rejected: user_id as a label" } ], [ { "label": "No unsampled indicator yet" } ], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Catalogue is the front door" } ], [ { "label": "Class defaults, overridable" } ], [ { "label": "Caught in staging, attribute named" } ], [ { "label": "Rules refuse sampled inputs" } ], [ { "label": "Unread data is reported" } ] ] } ], "chain": true, "note": "The cardinality rejection at first deploy is the design working. It is a staging event with a named attribute, not a 3 a.m. discovery from the store.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "06-journey-cut-the-bill", "title": "Journey — Engineering Director: Cut The Bill Without Going Blind", "layout": "journey", "canvas": { "width": 1740 }, "actor": { "label": "Engineering director", "sub": "owns the 4% line", "goal": "Take 30% out of observability spend without losing an incident to it", "trigger": "Quarterly spend review: $22 per host per month, rising", "success": "A cut with a named owner per change and a stated risk per change" }, "phases": [ { "title": "See the bill" }, { "title": "Find the waste" }, { "title": "Decide", "moment": true }, { "title": "Change" }, { "title": "Verify" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Opens per-team scorecard" } ], [ { "label": "Sorts by unread data" }, { "label": "Reads reduction ratios" } ], [ { "label": "Picks tier, rate or budget" } ], [ { "label": "Teams apply the change" } ], [ { "label": "Reviews next month" } ] ] }, { "title": "What the platform gives", "kind": "system", "cells": [ [ { "label": "Cost per signal, per team" } ], [ { "label": "Last-queried per service" }, { "label": "Active series per team" } ], [ { "label": "Cost preview at the change" } ], [ { "label": "Retention + sampling as config" } ], [ { "label": "Spend against MTTD/MTTL" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "In control", "Working", "Guessing" ], "points": [ 1, 2, 0, 1, 2 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [], [ { "label": "Which cut loses an incident?" } ], [ { "label": "Quota drops data mid-incident" } ], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Attribution within 24 h" } ], [ { "label": "Unread data is the waste class" } ], [ { "label": "Metric path is not a lever" } ], [ { "label": "Incident raise with expiry" } ], [ { "label": "Cost per host, published" } ] ] } ], "chain": true, "note": "The decision phase is the trough because the platform can price every lever but cannot price the incident a cut will lose. That is the question ADR-19 leaves open.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "07-layered-architecture", "title": "Layered Architecture", "layout": "bands", "canvas": { "width": 1780 }, "layerHeaderWidth": 170, "bands": [ { "name": "Consumption", "nodes": [ { "id": "graf", "label": "Consoles & dashboards", "sub": "Amazon Managed Grafana", "kind": "integration" }, { "id": "pager", "label": "Paging platform", "sub": "out of scope", "kind": "external" }, { "id": "score", "label": "Team scorecards", "sub": "cost, cardinality, unread", "kind": "app" }, { "id": "qapi", "label": "Query API", "sub": "PromQL · TraceQL · SQL", "kind": "integration" } ] }, { "name": "Query & evaluation", "nodes": [ { "id": "qf", "label": "Query frontend", "sub": "per-tenant admission", "kind": "app" }, { "id": "hc", "label": "High-cardinality engine", "sub": "group-by on raw attrs", "kind": "app" }, { "id": "ruler", "label": "Alert evaluation", "sub": "isolated plane", "kind": "app" }, { "id": "rehyd", "label": "Cold rehydration", "sub": "Athena over Glacier", "kind": "app" } ] }, { "name": "Storage", "nodes": [ { "id": "mimir", "label": "Metric store", "sub": "Mimir blocks on S3", "kind": "store" }, { "id": "tempo", "label": "Trace store", "sub": "Tempo blocks on S3", "kind": "store" }, { "id": "ch", "label": "Log store", "sub": "ClickHouse on S3", "kind": "store" }, { "id": "exem", "label": "Exemplar index", "sub": "DynamoDB", "kind": "store" } ] }, { "name": "Buffer", "nodes": [ { "id": "msk", "label": "Ingest buffer", "sub": "MSK — the replay point", "kind": "queue" } ] }, { "name": "Reduction", "nodes": [ { "id": "agg", "label": "Aggregation", "sub": "complete, unsampled", "kind": "app" }, { "id": "tail", "label": "Tail sampler", "sub": "45 s hold", "kind": "app" }, { "id": "logred", "label": "Log class reducer", "sub": "per declared class", "kind": "app" } ] }, { "name": "Admission", "nodes": [ { "id": "gw", "label": "Gateway collectors", "sub": "OTel Collector on EKS", "kind": "integration" }, { "id": "budget", "label": "Cardinality admission", "sub": "250k series / service", "kind": "decision" }, { "id": "redact", "label": "Redaction", "sub": "before any write", "kind": "security" }, { "id": "norm", "label": "Normalisation", "sub": "units, names, clocks", "kind": "app" } ] }, { "name": "Collection", "nodes": [ { "id": "sdk", "label": "OTel SDK", "sub": "in-process exporter", "kind": "app" }, { "id": "agent", "label": "Node agent", "sub": "ADOT DaemonSet", "kind": "platform" }, { "id": "scrape", "label": "Infrastructure scrape", "sub": "CloudWatch streams", "kind": "platform" }, { "id": "rum", "label": "Untrusted client edge", "sub": "web & mobile", "kind": "external" } ] }, { "name": "Control & governance", "accent": "#e1d5e7", "nodes": [ { "id": "cat", "label": "Service catalogue sync", "sub": "owner, class, budget", "kind": "platform", "icon": "fa5_book" }, { "id": "policy", "label": "Policy & rules store", "sub": "Aurora PostgreSQL", "kind": "store" }, { "id": "gitops", "label": "Config as code", "sub": "Git + Argo CD", "kind": "platform" }, { "id": "ledger", "label": "Cost ledger", "sub": "per team, daily", "kind": "app" } ] } ], "edges": [ { "from": "gw", "to": "msk", "label": "after admission", "kind": "sync" }, { "from": "msk", "to": "mimir", "label": "replayable", "kind": "async" }, { "from": "mimir", "to": "ruler", "label": "unsampled only", "kind": "sync" } ], "note": "Control & governance is drawn as a layer because it is deployed as one; it is read by every layer above it and is the only store here that is not derived.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "08-completeness-boundary", "title": "The Completeness Boundary — Two Paths That Never Rejoin", "layout": "nested", "canvas": { "width": 1720 }, "boxes": [ { "title": "COMPLETE — every observation contributes. Counting happens here.", "kind": "trust", "dir": "row", "children": [ { "title": "Reduce by aggregation", "kind": "boundary", "nodes": [ { "id": "agg", "label": "Aggregation", "sub": "at the gateway", "kind": "app" }, { "id": "mimir", "label": "Metric store", "sub": "60 M active series", "kind": "store" } ] }, { "title": "Everything that decides", "kind": "boundary", "nodes": [ { "id": "ruler", "label": "Alert evaluation", "sub": "30 s period", "kind": "app" }, { "id": "slo", "label": "SLO burn rate", "sub": "error budget", "kind": "app" }, { "id": "absence", "label": "Absence detection", "sub": "stopped exporter", "kind": "app" } ] } ] }, { "title": "SAMPLED — an evidence corpus. Reading happens here, counting never does.", "kind": "boundary", "dir": "row", "children": [ { "title": "Reduce by selection", "kind": "boundary", "nodes": [ { "id": "tail", "label": "Tail sampler", "sub": "keep rules + rate stamp", "kind": "app" }, { "id": "logred", "label": "Log class reducer", "sub": "per declared class", "kind": "app" } ] }, { "title": "The corpus", "kind": "boundary", "nodes": [ { "id": "tempo", "label": "Trace store", "sub": "≈6% retained", "kind": "store" }, { "id": "ch", "label": "Log store", "sub": "6 TB/day after reduction", "kind": "store" } ] } ] }, { "title": "THE ONLY JOIN — pointers, never quantities", "kind": "plain", "nodes": [ { "id": "exem", "label": "Exemplar index", "sub": "aggregate ↔ trace id", "kind": "store" }, { "id": "query", "label": "Query surface", "sub": "labels every result", "kind": "integration" } ] } ], "outside": [ { "id": "gw", "label": "Gateway tier — where the two paths separate", "kind": "integration" } ], "edges": [ { "from": "gw", "to": "agg", "label": "unsampled", "kind": "sync" }, { "from": "gw", "to": "tail", "label": "selected", "kind": "async" }, { "from": "mimir", "to": "exem", "label": "exemplar id", "kind": "sync" }, { "from": "tempo", "to": "exem", "label": "trace id", "kind": "sync" }, { "from": "exem", "to": "query", "kind": "sync" }, { "from": "tempo", "to": "ruler", "label": "refused", "kind": "error" } ], "note": "The red edge is the rule: an alert rule whose condition reads the sampled corpus is rejected at submission, not warned about at runtime.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "09-collection-tier", "title": "Collection Tier — Agent, Gateway and the Untrusted Edge", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "AWS eu-west-1 — EKS data plane", "kind": "cloud", "dir": "col", "children": [ { "title": "Workload node (× 12,000)", "kind": "boundary", "nodes": [ { "id": "app", "label": "Application pod", "sub": "OTel SDK, no buffer", "kind": "app" }, { "id": "agent", "label": "ADOT node agent", "sub": "≤ 2% core, ≤ 350 MB", "kind": "platform" }, { "id": "spool", "label": "Bounded local spool", "sub": "sheds by class", "kind": "queue" }, { "id": "host", "label": "Host & container metrics", "sub": "no app cooperation", "kind": "platform" } ] }, { "title": "Gateway namespace — the admission point", "kind": "trust", "nodes": [ { "id": "nlb", "label": "Network Load Balancer", "sub": "OTLP/gRPC", "kind": "integration" }, { "id": "norm", "label": "Normalise", "sub": "units, names, clock source", "kind": "app" }, { "id": "redact", "label": "Redact", "sub": "before any write", "kind": "security" }, { "id": "budget", "label": "Cardinality admission", "sub": "drop label, then reject", "kind": "decision" }, { "id": "split", "label": "Path split", "sub": "aggregate ǀ select", "kind": "decision" } ] }, { "title": "Trace affinity tier", "kind": "boundary", "nodes": [ { "id": "lbexp", "label": "Load-balancing exporter", "sub": "shard by trace id", "kind": "integration" }, { "id": "tail", "label": "Tail sampler shard", "sub": "holds whole traces", "kind": "app" } ] } ] }, { "title": "Everything not on EKS", "kind": "boundary", "dir": "row", "children": [ { "title": "EC2 and managed services", "kind": "boundary", "nodes": [ { "id": "ec2", "label": "EC2 agent", "sub": "systemd unit", "kind": "platform" }, { "id": "cwstream", "label": "CloudWatch metric streams", "sub": "RDS, MSK, NLB", "kind": "platform" } ] }, { "title": "Untrusted ingest — its own trust zone", "kind": "trust", "nodes": [ { "id": "waf", "label": "WAF + API Gateway", "sub": "rate limited per key", "kind": "security" }, { "id": "rum", "label": "Client telemetry receiver", "sub": "attribute allow-list", "kind": "integration" } ] } ] } ], "outside": [ { "id": "idp", "label": "EKS Pod Identity / IAM Roles Anywhere", "kind": "security" }, { "id": "msk", "label": "Ingest buffer (MSK)", "kind": "queue" } ], "edges": [ { "from": "app", "to": "agent", "kind": "sync" }, { "from": "agent", "to": "nlb", "label": "OTLP/gRPC", "kind": "sync" }, { "from": "ec2", "to": "nlb", "kind": "sync" }, { "from": "budget", "to": "split", "kind": "sync" }, { "from": "split", "to": "lbexp", "label": "evidence", "kind": "async", "route": "gutter" }, { "from": "split", "to": "msk", "label": "aggregated", "kind": "sync" }, { "from": "tail", "to": "msk", "label": "kept traces", "kind": "async" }, { "from": "idp", "to": "nlb", "label": "workload identity", "kind": "sync" } ], "note": "The application never holds a durable buffer and never retries to a remote endpoint. The untrusted client receiver reaches the same gateway as everything else, but through its own keyed, rate-limited path and never as a service identity.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "10-platform-services", "title": "Platform Services — Query, Evaluation and Control", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Read plane — scales with readers, not with ingest", "kind": "boundary", "dir": "row", "children": [ { "title": "Query services", "kind": "boundary", "nodes": [ { "id": "qf", "label": "Query frontend", "sub": "admission + result cache", "kind": "app" }, { "id": "mq", "label": "Metric querier", "sub": "Mimir", "kind": "app" }, { "id": "tq", "label": "Trace querier", "sub": "Tempo", "kind": "app" }, { "id": "lq", "label": "Log & group-by engine", "sub": "ClickHouse", "kind": "app" }, { "id": "athena", "label": "Cold rehydration", "sub": "Athena, ≤ 15 min", "kind": "app" } ] }, { "title": "Consoles", "kind": "boundary", "nodes": [ { "id": "graf", "label": "Amazon Managed Grafana", "sub": "SAML from the IdP", "kind": "integration" }, { "id": "svcview", "label": "Generated service views", "sub": "from the catalogue", "kind": "app" } ] } ] }, { "title": "Alert evaluation plane — separate node group, separate scaling, separate blast radius", "kind": "trust", "nodes": [ { "id": "ruler", "label": "Rule evaluator", "sub": "30 s period", "kind": "app" }, { "id": "burn", "label": "SLO burn evaluator", "sub": "multi-window", "kind": "app" }, { "id": "absence", "label": "Absence detector", "sub": "expected emission", "kind": "app" }, { "id": "dispatch", "label": "Alert dispatcher", "sub": "context attached", "kind": "integration" } ] }, { "title": "Control plane — the only non-derived state in the platform", "kind": "boundary", "dir": "row", "children": [ { "title": "Authoring and policy", "kind": "boundary", "nodes": [ { "id": "gitops", "label": "Config as code", "sub": "Git + Argo CD", "kind": "platform" }, { "id": "policy", "label": "Policy & rule store", "sub": "Aurora PostgreSQL", "kind": "store" }, { "id": "catsync", "label": "Catalogue sync", "sub": "owner, class, budget", "kind": "platform" } ] }, { "title": "Economics", "kind": "boundary", "nodes": [ { "id": "meter", "label": "Usage meter", "sub": "bytes, series, queries", "kind": "app" }, { "id": "ledger", "label": "Cost ledger", "sub": "per team, daily", "kind": "store" } ] } ] } ], "outside": [ { "id": "pager", "label": "Paging platform (out of scope)", "kind": "external" }, { "id": "cat", "label": "Service catalogue", "kind": "external", "icon": "fa5_book" } ], "edges": [ { "from": "qf", "to": "mq", "kind": "sync" }, { "from": "qf", "to": "lq", "kind": "sync" }, { "from": "graf", "to": "qf", "label": "all reads", "kind": "sync" }, { "from": "ruler", "to": "dispatch", "kind": "sync" }, { "from": "dispatch", "to": "pager", "label": "firing alert", "kind": "async" }, { "from": "cat", "to": "catsync", "label": "ownership", "kind": "batch" }, { "from": "policy", "to": "ruler", "label": "last-known rules", "kind": "sync" } ], "note": "The ruler reads the metric store directly, never the query frontend, so a query-tier outage cannot stop a page.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "11-integration-surface", "title": "Integration Surface", "layout": "hub", "canvas": { "width": 1760 }, "left": { "title": "Producers", "nodes": [ { "id": "sdk", "label": "Services & node agents", "sub": "900 services, 12,000 hosts", "kind": "app", "rel": "OTLP/gRPC" }, { "id": "cw", "label": "CloudWatch metric streams", "sub": "managed services", "kind": "platform", "rel": "Firehose" }, { "id": "rum", "label": "Web & mobile clients", "sub": "untrusted zone", "kind": "external", "rel": "HTTPS keyed" } ] }, "centre": { "title": "Observability Platform", "nodes": [ { "id": "plat", "label": "Observability Platform", "sub": "ingest · query · rules", "kind": "integration" } ] }, "right": { "title": "Consumers and dependencies", "nodes": [ { "id": "pager", "label": "Paging platform", "sub": "firing alerts only", "kind": "external", "rel": "alerts", "dir": "out", "kind2": "async" }, { "id": "graf", "label": "Consoles & notebooks", "sub": "Managed Grafana", "kind": "integration", "rel": "reads", "dir": "out" }, { "id": "cat", "label": "Service catalogue", "sub": "owners, classes", "kind": "external", "rel": "owners", "dir": "out", "kind2": "batch", "icon": "fa5_book" } ] }, "note": "Four inbound protocols, one of them untrusted. Outbound is a firing alert, a query result or a ledger — the platform pushes telemetry nowhere. The compliance archive and the cost feed are shown in views 12 and 06.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "12-storage-zones", "title": "Storage Zones — By Who Owns It and Whether It Can Be Rebuilt", "layout": "nested", "canvas": { "width": 1740 }, "boxes": [ { "title": "NOT DERIVED — losing this loses the platform. RPO ≤ 1 min, RTO ≤ 15 min.", "kind": "trust", "nodes": [ { "id": "policy", "label": "Catalogue, budgets, rules", "sub": "Aurora PostgreSQL, Multi-AZ", "kind": "store" }, { "id": "gitops", "label": "Config as code", "sub": "Git — the second copy", "kind": "platform" }, { "id": "audit", "label": "Query audit", "sub": "13 months", "kind": "store" }, { "id": "ledger", "label": "Cost ledger", "sub": "per team, daily", "kind": "store" } ] }, { "title": "DERIVED — replayable from the buffer while it still holds the window. No RPO.", "kind": "boundary", "dir": "row", "children": [ { "title": "Hot — interactive query", "kind": "boundary", "nodes": [ { "id": "ingester", "label": "Metric ingesters", "sub": "recent window in memory", "kind": "app" }, { "id": "chhot", "label": "Log hot parts", "sub": "ClickHouse, 7 days", "kind": "store" }, { "id": "exem", "label": "Exemplar index", "sub": "DynamoDB, 90 days", "kind": "store" } ] }, { "title": "Warm and cold — object storage is the substrate", "kind": "boundary", "nodes": [ { "id": "s3std", "label": "S3 Standard", "sub": "blocks, parts, traces", "kind": "store" }, { "id": "s3ia", "label": "S3 Glacier IR", "sub": "logs to 400 days", "kind": "store" }, { "id": "lock", "label": "S3 Object Lock", "sub": "compliance class, 7 years", "kind": "store" } ] } ] }, { "title": "VOLATILE — its loss is a telemetry gap, shown as a gap and never interpolated", "kind": "plain", "nodes": [ { "id": "spool", "label": "Node agent spool", "sub": "bounded, sheds by class", "kind": "queue" }, { "id": "batch", "label": "Gateway batch", "sub": "in memory", "kind": "queue" }, { "id": "hold", "label": "Tail sampler hold", "sub": "45 s of in-flight traces", "kind": "queue" } ] } ], "outside": [ { "id": "msk", "label": "Ingest buffer — MSK, 72 h retention, RPO ≤ 5 min", "kind": "queue" }, { "id": "kms", "label": "KMS — keys held outside the platform", "kind": "security" } ], "edges": [ { "from": "msk", "to": "ingester", "label": "replay", "kind": "async" }, { "from": "msk", "to": "chhot", "label": "replay", "kind": "async" }, { "from": "ingester", "to": "s3std", "label": "flush", "kind": "batch" }, { "from": "chhot", "to": "s3ia", "label": "age out", "kind": "batch" }, { "from": "kms", "to": "lock", "label": "crypto-shred", "kind": "sync" } ], "note": "The zones are drawn by recovery obligation, not by technology: four stores carry a strict RPO, six carry none, and three are allowed to vanish.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "13-data-model", "title": "Data Model — Telemetry, Its Owners and Its Enforcement Record", "layout": "er", "canvas": { "width": 1740, "cols": 4 }, "rowGap": 250, "entities": [ { "id": "team", "name": "team", "kind": "store", "row": 0, "col": 0, "attrs": [ "team_id PK", "name", "series_budget", "retention_class", "cost_centre" ] }, { "id": "service", "name": "service", "kind": "store", "row": 0, "col": 1, "attrs": [ "service_id PK", "team_id FK -> team", "environment", "region", "log_classes", "sampling_policy" ] }, { "id": "series", "name": "metric_series", "kind": "store", "row": 0, "col": 2, "attrs": [ "series_id PK", "service_id FK -> service", "metric_name", "label_set_hash", "first_seen", "last_seen" ] }, { "id": "exemplar", "name": "exemplar", "kind": "store", "row": 0, "col": 3, "attrs": [ "series_id FK -> metric_series", "ts", "value", "trace_id FK -> trace" ] }, { "id": "ledger", "name": "usage_ledger", "kind": "store", "row": 1, "col": 0, "attrs": [ "day PK", "team_id FK -> team", "signal", "bytes_ingested", "active_series", "queries", "cost" ] }, { "id": "enforce", "name": "enforcement_event", "kind": "risk", "row": 1, "col": 1, "attrs": [ "event_id PK", "service_id FK -> service", "kind", "attribute", "records_affected", "ts" ] }, { "id": "trace", "name": "trace", "kind": "store", "row": 1, "col": 2, "attrs": [ "trace_id PK", "root_service_id FK", "duration_ms", "status", "keep_reason", "effective_rate" ] }, { "id": "span", "name": "span", "kind": "store", "row": 1, "col": 3, "attrs": [ "span_id PK", "trace_id FK -> trace", "parent_span_id", "service_id FK", "name", "start_ts", "duration_ms" ] }, { "id": "log", "name": "log_record", "kind": "store", "row": 2, "col": 2, "attrs": [ "log_id PK", "service_id FK", "class", "ts", "trace_id FK -> trace", "body_redacted", "subject_key_id" ] } ], "relations": [ { "from": "team", "to": "service", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "service", "to": "series", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "series", "to": "exemplar", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "exemplar", "to": "trace", "label": "N : 1", "from_side": "s", "to_side": "n3" }, { "from": "trace", "to": "span", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "trace", "to": "log", "label": "1 : N", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "team", "to": "ledger", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "service", "to": "enforce", "label": "1 : N", "from_side": "s", "to_side": "n" } ], "note": "trace_id on log_record is optional by design: a log from an unsampled request still exists, and the join simply returns nothing rather than the record being discarded. Alert rules and SLO definitions are control-plane config and live in the store shown in view 12.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "14-retention-and-tiers", "title": "Retention Tiers — What Survives, At What Resolution, For How Long", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "top", "stages": [ { "title": "Written", "nodes": [ { "id": "m0", "label": "Metrics", "sub": "10 s resolution", "kind": "store" }, { "id": "t0", "label": "Traces", "sub": "kept whole", "kind": "store" }, { "id": "l0", "label": "Logs", "sub": "by declared class", "kind": "store" } ] }, { "title": "Hot — interactive", "nodes": [ { "id": "m1", "label": "10 s · 15 days", "sub": "p95 ≤ 2 s panels", "kind": "store" }, { "id": "t1", "label": "Traces · 14 days", "sub": "lookup p95 ≤ 1.5 s", "kind": "store" }, { "id": "l1", "label": "Logs · 7 days", "sub": "search p95 ≤ 15 s", "kind": "store" } ] }, { "title": "Warm — slower, cheaper", "nodes": [ { "id": "m2", "label": "1 min · 90 days", "sub": "mergeable sketches", "kind": "store" }, { "id": "t2", "label": "Exemplar-linked · 90 days", "sub": "pivot targets only", "kind": "store" }, { "id": "l2", "label": "Logs · 30 days", "sub": "S3 Standard", "kind": "store" } ] }, { "title": "Cold — rehydrate to read", "nodes": [ { "id": "m3", "label": "5 min · 13 months", "sub": "capacity planning", "kind": "store" }, { "id": "l3", "label": "Logs · 400 days", "sub": "Glacier IR, ≤ 15 min", "kind": "store" } ] }, { "title": "Long and locked", "nodes": [ { "id": "m4", "label": "1 hour · 3 years", "sub": "trend only", "kind": "store" }, { "id": "c4", "label": "Compliance class · 7 years", "sub": "S3 Object Lock", "kind": "security" } ] }, { "title": "Gone", "nodes": [ { "id": "exp", "label": "Lifecycle expiry", "sub": "per class, automatic", "kind": "app" }, { "id": "erase", "label": "Subject erasure", "sub": "all tiers, ≤ 30 days", "kind": "security" }, { "id": "shred", "label": "Crypto-shred", "sub": "last resort", "kind": "security" } ] } ], "edges": [ { "from": "m1", "to": "m2", "label": "downsample", "kind": "batch" }, { "from": "m2", "to": "m3", "label": "downsample", "kind": "batch" }, { "from": "l1", "to": "l2", "label": "age out", "kind": "batch" }, { "from": "l3", "to": "erase", "label": "reachable", "kind": "error" } ], "note": "Percentile-bearing metrics are stored as mergeable sketches, never as pre-computed percentiles, so a tier change never silently changes what a chart means.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "15-metric-path-to-page", "title": "Critical Flow — A Sample Arrives and a Human Is Paged", "layout": "sequence", "canvas": { "width": 1720 }, "lifelines": [ { "id": "app", "label": "Service + OTel SDK", "kind": "app" }, { "id": "agent", "label": "Node agent", "kind": "platform" }, { "id": "gw", "label": "Gateway collector", "kind": "integration" }, { "id": "msk", "label": "Ingest buffer", "kind": "queue" }, { "id": "mimir", "label": "Metric store", "kind": "store" }, { "id": "ruler", "label": "Alert evaluation", "kind": "app" }, { "id": "pager", "label": "Paging platform", "kind": "external" } ], "messages": [ { "from": "app", "to": "agent", "label": "OTLP over localhost, fire and forget", "kind": "call" }, { "from": "agent", "to": "gw", "label": "batched OTLP/gRPC, resource identity attached", "kind": "call" }, { "from": "gw", "to": "gw", "label": "normalise units, names and clock source", "kind": "self" }, { "from": "gw", "to": "gw", "label": "redact before anything is written", "kind": "self" }, { "from": "gw", "to": "gw", "label": "cardinality admission against the service budget", "kind": "self" }, { "from": "gw", "to": "agent", "label": "label dropped — enforcement event counted", "kind": "error" }, { "from": "gw", "to": "gw", "label": "aggregate — every observation contributes", "kind": "self" }, { "from": "gw", "to": "msk", "label": "produce, partitioned by service", "kind": "async" }, { "from": "msk", "to": "mimir", "label": "consume, then commit the offset", "kind": "call" }, { "from": "mimir", "to": "mimir", "label": "WAL, then block flush to S3", "kind": "self" }, { "from": "ruler", "to": "mimir", "label": "evaluate, 30 s period", "kind": "call" }, { "from": "mimir", "to": "ruler", "label": "unsampled series only", "kind": "return" }, { "from": "ruler", "to": "ruler", "label": "burn rate, two windows", "kind": "self" }, { "from": "ruler", "to": "pager", "label": "firing alert with context", "kind": "async" } ], "note": "Budget: sample arrival to alert emission p95 ≤ 45 s, p99 ≤ 90 s. The ruler reads the store, never the query frontend — step 11 has no dependency on anything a human is also using.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "16-tail-sampling", "title": "Tail Sampling — Selecting the Evidence Worth Keeping", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "top", "stages": [ { "title": "Arrive", "nodes": [ { "id": "spans", "label": "Spans", "sub": "2.5 M/s offered", "kind": "app" }, { "id": "head", "label": "Head sample floor", "sub": "volume control only", "kind": "decision" } ] }, { "title": "Shard", "nodes": [ { "id": "lb", "label": "Load-balancing exporter", "sub": "consistent hash on trace id", "kind": "integration" }, { "id": "affin", "label": "Trace affinity", "sub": "all spans, one shard", "kind": "app" } ] }, { "title": "Hold", "nodes": [ { "id": "buf", "label": "In-memory trace buffer", "sub": "45 s window", "kind": "queue" }, { "id": "ceil", "label": "Memory ceiling", "sub": "sized on 5× burst", "kind": "risk" } ] }, { "title": "Decide", "nodes": [ { "id": "err", "label": "Error status", "sub": "always keep", "kind": "decision" }, { "id": "slow", "label": "Above service p99", "sub": "always keep", "kind": "decision" }, { "id": "dbg", "label": "Debug flag set", "sub": "always keep", "kind": "decision" }, { "id": "rare", "label": "Rare key combination", "sub": "dynamic rate", "kind": "decision" }, { "id": "cap", "label": "Per-service keep ceiling", "sub": "a keep rule is not a free path", "kind": "risk" } ] }, { "title": "Stamp", "nodes": [ { "id": "rate", "label": "Effective rate", "sub": "written on the trace", "kind": "app" }, { "id": "reason", "label": "Keep reason", "sub": "why this one survived", "kind": "app" } ] }, { "title": "Emit", "nodes": [ { "id": "keep", "label": "Whole trace to the buffer", "sub": "≈150 k spans/s retained", "kind": "queue" }, { "id": "drop", "label": "Dropped, counted, attributed", "sub": "reduction ratio published", "kind": "app" } ] } ], "edges": [ { "from": "buf", "to": "drop", "label": "hold expired", "kind": "error" }, { "from": "cap", "to": "drop", "label": "over ceiling", "kind": "error" }, { "from": "rate", "to": "keep", "kind": "sync" } ], "note": "The hold window is the whole cost of tail sampling: it buys the error trace and pays for it in memory at the worst burst, and it is why trace freshness carries 45 s the metric path does not.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "17-page-to-root-cause", "title": "Critical Flow — The Pivot From Aggregate To Instance", "layout": "sequence", "canvas": { "width": 1720 }, "lifelines": [ { "id": "eng", "label": "On-call engineer", "kind": "actor" }, { "id": "console", "label": "Console", "kind": "integration" }, { "id": "qf", "label": "Query frontend", "kind": "app" }, { "id": "mimir", "label": "Metric store", "kind": "store" }, { "id": "exem", "label": "Exemplar index", "kind": "store" }, { "id": "tempo", "label": "Trace store", "kind": "store" }, { "id": "ch", "label": "Log store", "kind": "store" } ], "messages": [ { "from": "eng", "to": "console", "label": "opens the scoped view from the alert", "kind": "call" }, { "from": "console", "to": "qf", "label": "error rate and burn, last 6 h", "kind": "call" }, { "from": "qf", "to": "mimir", "label": "unsampled aggregate", "kind": "call" }, { "from": "mimir", "to": "qf", "label": "series + freshness lag", "kind": "return" }, { "from": "qf", "to": "console", "label": "result, labelled complete", "kind": "return" }, { "from": "eng", "to": "console", "label": "clicks the spike", "kind": "call" }, { "from": "console", "to": "exem", "label": "exemplar at that bucket", "kind": "call" }, { "from": "exem", "to": "tempo", "label": "trace id — a pointer, not a count", "kind": "call" }, { "from": "tempo", "to": "console", "label": "whole trace, effective rate 1 : 16", "kind": "return" }, { "from": "tempo", "to": "console", "label": "no exemplar — sampled away, and it says so", "kind": "error" }, { "from": "console", "to": "ch", "label": "logs for that trace id and span", "kind": "call" }, { "from": "ch", "to": "console", "label": "records + class reduction applied", "kind": "return" }, { "from": "eng", "to": "qf", "label": "group errors by any attribute, 24 h", "kind": "call" }, { "from": "qf", "to": "eng", "label": "partial result, coverage 92% stated", "kind": "return" } ], "note": "Step 10 is the failure the design accepts: when the evidence was not kept, the platform says so rather than returning an empty panel that reads like health.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "18-telemetry-storm", "title": "Telemetry Storm — Shedding By Class, Not By Luck", "layout": "swimlane", "canvas": { "width": 1780 }, "laneHeaderWidth": 180, "stages": [ "Steady state", "Onset — 2×", "Saturation — 5×", "Beyond the budget", "Recovery" ], "lanes": [ { "title": "Metric path", "cells": [ [ { "label": "25 M samples/s", "kind": "app" } ], [ { "label": "New series from error labels", "kind": "app" } ], [ { "label": "Cardinality budget bites", "kind": "decision" } ], [ { "label": "Never shed", "kind": "opportunity" } ], [ { "label": "Complete throughout", "kind": "opportunity" } ] ] }, { "title": "Trace path", "cells": [ [ { "label": "6% retained", "kind": "app" } ], [ { "label": "Error keep rules fire", "kind": "app" } ], [ { "label": "Keep ceiling reached", "kind": "risk" } ], [ { "label": "Rate cut, stamped", "kind": "decision" } ], [ { "label": "Rate restored", "kind": "app" } ] ] }, { "title": "Log path", "cells": [ [ { "label": "1.2 M lines/s", "kind": "app" } ], [ { "label": "Debug class surges", "kind": "app" } ], [ { "label": "Debug class shed first", "kind": "decision" } ], [ { "label": "Access class shed next", "kind": "risk" } ], [ { "label": "Backlog drained from buffer", "kind": "queue" } ] ] }, { "title": "Platform response", "cells": [ [ { "label": "Headroom 1.3×", "kind": "platform" } ], [ { "label": "Gateway scales out", "kind": "platform" } ], [ { "label": "Buffer absorbs 10 min", "kind": "queue" } ], [ { "label": "Shed volume counted per team", "kind": "app" } ], [ { "label": "Replay from retained offset", "kind": "async" } ] ] }, { "title": "What the reader sees", "cells": [ [ { "label": "Nothing unusual", "kind": "journey" } ], [ { "label": "Freshness lag shown", "kind": "journey" } ], [ { "label": "Coverage below 100%", "kind": "journey" } ], [ { "label": "Named gap, named class", "kind": "risk" } ], [ { "label": "Gap stays on the record", "kind": "journey" } ] ] } ], "note": "The order of shedding is declared in advance and is the same every time: debug logs, then access logs, then trace rate. The metric path is not in the order, which is what keeps every count true through the incident.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "19-deployment-architecture", "title": "Deployment Architecture — Three Regions and One Place That Is Not", "layout": "nested", "canvas": { "width": 1780 }, "boxes": [ { "title": "AWS eu-west-1 — primary region, observability account", "kind": "cloud", "dir": "col", "children": [ { "title": "Ingest VPC — spread across az-a, az-b, az-c", "kind": "boundary", "nodes": [ { "id": "nlb", "label": "Network Load Balancer", "sub": "cross-zone", "kind": "integration" }, { "id": "gwng", "label": "Gateway node group", "sub": "EKS, compute optimised", "kind": "platform" }, { "id": "tailng", "label": "Tail sampler node group", "sub": "memory optimised, 5× headroom", "kind": "platform" }, { "id": "msk", "label": "Amazon MSK", "sub": "3 AZ, 72 h retention", "kind": "queue" } ] }, { "title": "Storage & query VPC", "kind": "boundary", "nodes": [ { "id": "mimir", "label": "Mimir node group", "sub": "EKS", "kind": "platform" }, { "id": "ch", "label": "ClickHouse cluster", "sub": "EC2 i4i, S3 tiering", "kind": "store" }, { "id": "tempo", "label": "Tempo node group", "sub": "EKS", "kind": "platform" }, { "id": "ddb", "label": "DynamoDB", "sub": "exemplar index", "kind": "store" }, { "id": "s3", "label": "S3 buckets", "sub": "hot · warm · Glacier IR · Object Lock", "kind": "store" } ] }, { "title": "Evaluation & control — isolated node groups, separate scaling", "kind": "trust", "nodes": [ { "id": "ruler", "label": "Ruler node group", "sub": "no shared capacity with query", "kind": "platform" }, { "id": "aurora", "label": "Aurora PostgreSQL", "sub": "Multi-AZ, RPO ≤ 1 min", "kind": "store" }, { "id": "argo", "label": "Argo CD", "sub": "config as code", "kind": "platform" } ] } ] }, { "title": "eu-central-1 and us-east-1 — collect and store locally, replicate a reduced set", "kind": "cloud", "nodes": [ { "id": "r2gw", "label": "Regional gateway + buffer", "sub": "same stack, own state", "kind": "platform" }, { "id": "r2store", "label": "Regional stores", "sub": "queryable from any region", "kind": "store" }, { "id": "r2rep", "label": "Reduced cross-region replica", "sub": "aggregates + SLO series only", "kind": "store" }, { "id": "graf", "label": "Amazon Managed Grafana", "sub": "workspace per region", "kind": "integration" } ] }, { "title": "Separate AWS account, separate region — deliberately outside the failure domain it watches", "kind": "trust", "nodes": [ { "id": "amp", "label": "Amazon Managed Prometheus", "sub": "platform self-telemetry", "kind": "platform" }, { "id": "amg", "label": "Amazon Managed Grafana", "sub": "the console of last resort", "kind": "integration" }, { "id": "canary", "label": "Heartbeat canary", "sub": "writes and reads a probe", "kind": "app" }, { "id": "deadman", "label": "Dead-man switch", "sub": "alerts on our silence", "kind": "security" } ] } ], "outside": [ { "id": "pager", "label": "Paging platform", "kind": "external" }, { "id": "pl", "label": "PrivateLink from every workload account", "kind": "security" } ], "edges": [ { "from": "pl", "to": "nlb", "label": "no public ingest", "kind": "sync" }, { "from": "r2rep", "to": "mimir", "label": "reduced set", "kind": "async" }, { "from": "canary", "to": "amp", "kind": "sync" }, { "from": "deadman", "to": "pager", "label": "platform is blind", "kind": "error" } ], "note": "The self-telemetry stack is managed rather than self-hosted on purpose: it must not share an EKS control plane, an account limit or an on-call engineer with the thing it is watching.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "20-release-pipeline", "title": "Release Pipeline — Config, Rules and the Agent Fleet", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "top", "stages": [ { "title": "Source", "nodes": [ { "id": "cfg", "label": "Collector config", "sub": "pipelines, processors", "kind": "platform" }, { "id": "rules", "label": "Alert & SLO rules", "sub": "per service", "kind": "platform" }, { "id": "budg", "label": "Budgets & classes", "sub": "from the catalogue", "kind": "platform" } ] }, { "title": "Build", "nodes": [ { "id": "render", "label": "Render manifests", "sub": "one artefact per region", "kind": "app" }, { "id": "unit", "label": "Rule unit tests", "sub": "fixture series in, alert out", "kind": "app" } ] }, { "title": "Gates", "nodes": [ { "id": "g1", "label": "Sampled input refused", "sub": "rules must read the complete path", "kind": "decision" }, { "id": "g2", "label": "Cardinality estimate", "sub": "before it reaches production", "kind": "decision" }, { "id": "g3", "label": "Config schema + replay", "sub": "against a recorded corpus", "kind": "decision" }, { "id": "g4", "label": "Cost delta preview", "sub": "on the pull request", "kind": "decision" } ] }, { "title": "Environments", "nodes": [ { "id": "stg", "label": "Staging", "sub": "full pipeline, 1% traffic mirror", "kind": "app" }, { "id": "can", "label": "Canary region", "sub": "one region, 30 min soak", "kind": "app" }, { "id": "all", "label": "All regions", "sub": "wave by wave", "kind": "app" } ] }, { "title": "Fleet rollout", "nodes": [ { "id": "ring", "label": "Agent rollout rings", "sub": "1% · 10% · 100% of hosts", "kind": "platform" }, { "id": "conv", "label": "Two conventions in flight", "sub": "no synchronised deploy", "kind": "app" }, { "id": "back", "label": "Rollback to last-known config", "sub": "one action", "kind": "opportunity" } ] } ], "edges": [ { "from": "can", "to": "back", "kind": "error", "route": "gutter" } ], "note": "An agent rollout is an estate-wide deploy that no service team asked for. It rides rings and is reversible in one action, because the blast radius is every host at once. The red path is a failed canary soak.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "21-self-observability", "title": "Observing the Observability Platform", "layout": "grid", "canvas": { "width": 1780 }, "laneHeaderWidth": 190, "columns": [ "Collect", "Admit", "Buffer", "Store", "Query", "Alert" ], "rows": [ { "title": "Health", "cells": [ [ { "label": "Agent up, CPU, RSS", "kind": "platform" } ], [ { "label": "Gateway error rate", "kind": "platform" } ], [ { "label": "Broker in-sync replicas", "kind": "platform" } ], [ { "label": "Writer restarts", "kind": "platform" } ], [ { "label": "Frontend 5xx", "kind": "platform" } ], [ { "label": "Missed evaluations", "kind": "risk" } ] ] }, { "title": "Freshness & lag", "cells": [ [ { "label": "Agent spool depth", "kind": "app" } ], [ { "label": "Batch age", "kind": "app" } ], [ { "label": "Consumer lag", "kind": "app" } ], [ { "label": "Ingest to queryable", "kind": "app" } ], [ { "label": "Query p95 by class", "kind": "app" } ], [ { "label": "Sample to page", "kind": "app" } ] ] }, { "title": "Completeness", "cells": [ [ { "label": "Hosts not reporting", "kind": "risk" } ], [ { "label": "Shed by class, per team", "kind": "risk" } ], [ { "label": "Records dropped", "kind": "risk" } ], [ { "label": "Gap markers written", "kind": "risk" } ], [ { "label": "Coverage on results", "kind": "app" } ], [ { "label": "Rules on stale data", "kind": "risk" } ] ] }, { "title": "Cardinality & cost", "cells": [ [], [ { "label": "Series admitted / rejected", "kind": "decision" } ], [ { "label": "Bytes per team", "kind": "app" } ], [ { "label": "Active series, tier bytes", "kind": "app" } ], [ { "label": "Query cost per team", "kind": "app" } ], [ { "label": "Rule evaluation cost", "kind": "app" } ] ] }, { "title": "From outside the estate", "cells": [ [ { "label": "Canary writes a probe", "kind": "security" } ], [], [], [], [ { "label": "Canary reads it back", "kind": "security" } ], [ { "label": "Dead-man switch", "kind": "security" } ] ] } ], "note": "The bottom row is the only one that does not depend on the platform being alive. Four blank cells in it are deliberate: an outside prober should test the round trip, not the internals.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "22-cardinality-incident-loop", "title": "The Cardinality Loop — Admitted, Attributed, Fixed", "layout": "cycle", "canvas": { "width": 1600 }, "centre": { "label": "Cardinality control", "id": "hub" }, "nodes": [ { "label": "A deploy adds a label", "sub": "user_id on an error metric", "kind": "app", "id": "c0" }, { "label": "Gateway admission sees it", "sub": "before the store does", "kind": "decision", "id": "c1" }, { "label": "Label dropped, series kept", "sub": "the aggregate survives", "kind": "opportunity", "id": "c2" }, { "label": "Enforcement event written", "sub": "attribute named, team attached", "kind": "risk", "id": "c3" }, { "label": "Owning team notified", "sub": "same day, not month end", "kind": "actor", "id": "c4" }, { "label": "Fix, or an exception with an expiry", "sub": "never a permanent waiver", "kind": "decision", "id": "c5" }, { "label": "Pre-production estimate updated", "sub": "caught at review next time", "kind": "platform", "id": "c6" } ], "ringLabels": [ "admitted", "degraded", "recorded", "attributed", "decided", "learned", "next deploy" ], "rx": 470, "ry": 215, "note": "The loop closes at the estimate: every incident that reaches admission is a question the pre-production check should have asked.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "23-security-trust-zones", "title": "Trust Zones — Telemetry Is Production Data", "layout": "zones", "canvas": { "width": 1760 }, "zones": [ { "title": "Internet — untrusted, and assumed hostile", "kind": "trust", "nodes": [ { "id": "web", "label": "Web & mobile clients", "sub": "identity is a claim", "kind": "external" }, { "id": "saas", "label": "Third-party SaaS", "sub": "webhook or pull", "kind": "external" } ] }, { "title": "Edge — its own trust zone, its own limits", "kind": "trust", "nodes": [ { "id": "waf", "label": "WAF", "sub": "per-key rate limits", "kind": "security" }, { "id": "apigw", "label": "API Gateway", "sub": "client ingest only", "kind": "integration" }, { "id": "allow", "label": "Attribute allow-list", "sub": "no service identity", "kind": "decision" } ] }, { "title": "Workload accounts — where telemetry is produced", "kind": "trust", "nodes": [ { "id": "pod", "label": "Service pods", "sub": "OTel SDK", "kind": "app" }, { "id": "agent", "label": "Node agents", "sub": "identity per workload", "kind": "platform" }, { "id": "pl", "label": "PrivateLink endpoint", "sub": "no public ingest path", "kind": "security" } ] }, { "title": "Observability account — the platform", "kind": "trust", "nodes": [ { "id": "gw", "label": "Gateway + redaction", "sub": "before any write", "kind": "security" }, { "id": "buf", "label": "Buffer & stores", "sub": "encrypted with KMS", "kind": "store" }, { "id": "qf", "label": "Query frontend", "sub": "authorises by class", "kind": "app" }, { "id": "ruler", "label": "Alert evaluation", "sub": "reads, never writes", "kind": "app" } ] }, { "title": "Restricted — narrower audience than the rest of telemetry", "kind": "trust", "nodes": [ { "id": "lock", "label": "Compliance class", "sub": "S3 Object Lock, 7 years", "kind": "store" }, { "id": "kms", "label": "KMS keys", "sub": "held outside the platform", "kind": "security" }, { "id": "audit", "label": "Query audit", "sub": "reading is an audited act", "kind": "store" } ] }, { "title": "Outside the failure domain — separate account, separate region", "kind": "trust", "nodes": [ { "id": "self", "label": "Self-telemetry stack", "sub": "managed services", "kind": "platform" }, { "id": "dead", "label": "Dead-man switch", "sub": "break-glass path", "kind": "security" } ] } ], "edges": [ { "from": "web", "to": "waf", "label": "HTTPS, keyed", "kind": "async" }, { "from": "apigw", "to": "gw", "label": "keyed", "kind": "async" }, { "from": "pod", "to": "agent", "kind": "sync" }, { "from": "agent", "to": "pl", "label": "SigV4", "kind": "sync" }, { "from": "pl", "to": "gw", "label": "private", "kind": "sync" }, { "from": "gw", "to": "buf", "label": "redacted", "kind": "sync" }, { "from": "qf", "to": "lock", "label": "narrow role", "kind": "sync" }, { "from": "qf", "to": "audit", "label": "logged first", "kind": "sync" }, { "from": "kms", "to": "lock", "label": "crypto-shred", "kind": "batch" }, { "from": "gw", "to": "self", "label": "heartbeat", "kind": "async" } ], "note": "Six zones, and the one that matters most is the fifth: a debugging interface over production logs is a production data interface, so reading it is authorised narrowly and written to the audit before it executes. Client telemetry crosses the edge keyed and rate limited, and never carries a service identity.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "24-identity-and-access", "title": "Identity — Who Proves What, To Whom, In What Order", "layout": "sequence", "canvas": { "width": 1720 }, "lifelines": [ { "id": "pod", "label": "Workload pod", "kind": "app" }, { "id": "pi", "label": "EKS Pod Identity", "kind": "security" }, { "id": "sts", "label": "AWS STS", "kind": "security" }, { "id": "gw", "label": "Gateway", "kind": "integration" }, { "id": "eng", "label": "Engineer", "kind": "actor" }, { "id": "idc", "label": "IAM Identity Center", "kind": "security" }, { "id": "qf", "label": "Query frontend", "kind": "app" }, { "id": "audit", "label": "Query audit", "kind": "store" } ], "messages": [ { "from": "pod", "to": "pi", "label": "credentials for this service account", "kind": "call" }, { "from": "pi", "to": "sts", "label": "assume the workload role", "kind": "call" }, { "from": "sts", "to": "pod", "label": "short-lived credentials", "kind": "return" }, { "from": "pod", "to": "gw", "label": "OTLP, SigV4 signed", "kind": "call" }, { "from": "gw", "to": "gw", "label": "identity from the role, not the payload", "kind": "self" }, { "from": "gw", "to": "pod", "label": "rejected: a team credential is not a workload", "kind": "error" }, { "from": "eng", "to": "idc", "label": "SSO with MFA", "kind": "call" }, { "from": "idc", "to": "qf", "label": "assertion with group claims", "kind": "call" }, { "from": "qf", "to": "qf", "label": "authorise by team and data class", "kind": "self" }, { "from": "qf", "to": "audit", "label": "write the read before executing it", "kind": "call" }, { "from": "qf", "to": "eng", "label": "results, compliance class withheld", "kind": "return" } ], "note": "Step 5 is the rule the platform cannot compromise on: resource identity comes from the credential the workload proved, never from a service name the payload asserts.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "25-failure-modes", "title": "Failure Modes — What Is Assumed To Fail, and What Happens", "layout": "grid", "canvas": { "width": 1780 }, "laneHeaderWidth": 210, "columns": [ "Detected by", "Contained by", "Degrades to", "Recovers when" ], "rows": [ { "title": "Telemetry storm", "cells": [ [ { "label": "Ingest rate vs 5× budget", "kind": "app" } ], [ { "label": "Gateway admission + buffer depth", "kind": "decision" } ], [ { "label": "Shed by declared class", "kind": "risk" } ], [ { "label": "Backlog replays from offset", "kind": "opportunity" } ] ] }, { "title": "Cardinality explosion", "cells": [ [ { "label": "Series admitted vs budget", "kind": "app" } ], [ { "label": "Per-service budget at the gateway", "kind": "decision" } ], [ { "label": "Label dropped, aggregate kept", "kind": "risk" } ], [ { "label": "Team fixes or takes an expiring exception", "kind": "opportunity" } ] ] }, { "title": "Query stampede", "cells": [ [ { "label": "Frontend queue depth", "kind": "app" } ], [ { "label": "Per-tenant query admission", "kind": "decision" } ], [ { "label": "Partial results, coverage stated", "kind": "risk" } ], [ { "label": "Cache warms, incident ends", "kind": "opportunity" } ] ] }, { "title": "Buffer unavailable", "cells": [ [ { "label": "Produce error rate", "kind": "app" } ], [ { "label": "Bounded gateway spool", "kind": "queue" } ], [ { "label": "Shed by class, gap marked", "kind": "risk" } ], [ { "label": "Brokers return; no catch-up flood", "kind": "opportunity" } ] ] }, { "title": "Storage-writer lag", "cells": [ [ { "label": "Consumer lag, freshness p95", "kind": "app" } ], [ { "label": "Buffer retention 72 h", "kind": "queue" } ], [ { "label": "Freshness incident, lag shown in UI", "kind": "risk" } ], [ { "label": "Writers scale, lag drains", "kind": "opportunity" } ] ] }, { "title": "Query-tier outage", "cells": [ [ { "label": "Frontend 5xx, canary read fails", "kind": "app" } ], [ { "label": "Ruler on separate node groups", "kind": "security" } ], [ { "label": "No dashboards; paging unaffected", "kind": "risk" } ], [ { "label": "Frontend restored; no data lost", "kind": "opportunity" } ] ] }, { "title": "Control-plane outage", "cells": [ [ { "label": "Aurora health, Argo sync state", "kind": "app" } ], [ { "label": "Last-known config on every tier", "kind": "decision" } ], [ { "label": "Fail static — no rule or budget changes", "kind": "risk" } ], [ { "label": "Restored from Git, then Aurora", "kind": "opportunity" } ] ] }, { "title": "Region loss", "cells": [ [ { "label": "Regional ingest to zero", "kind": "app" } ], [ { "label": "Regions collect and store locally", "kind": "decision" } ], [ { "label": "Reduced replica only for that region", "kind": "risk" } ], [ { "label": "Region returns; local data intact", "kind": "opportunity" } ] ] }, { "title": "Platform is blind", "cells": [ [ { "label": "Dead-man switch, separate account", "kind": "security" } ], [ { "label": "Self-telemetry outside the domain", "kind": "security" } ], [ { "label": "Page the platform team directly", "kind": "risk" } ], [ { "label": "Heartbeat resumes", "kind": "opportunity" } ] ] }, { "title": "Silent gap", "cells": [ [ { "label": "Absence detection per producer", "kind": "security" } ], [ { "label": "Expected emission from the catalogue", "kind": "decision" } ], [ { "label": "Gap marker written, never interpolated", "kind": "risk" } ], [ { "label": "Producer resumes; gap stays on record", "kind": "opportunity" } ] ] } ], "note": "The last row is the worst class because it looks like health. Every other row is detected by something going up; this one is detected by something that should be there and is not.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } } ]