[ { "id": "01-system-context", "title": "Incident Management Platform — System Context", "layout": "context", "colWidth": 280, "system": { "label": "Incident Management Platform", "sub": "400 responders · 25 rotations", "icon": "fa5_bell" }, "groups": [ { "side": "top", "title": "Signals it receives", "nodes": [ { "id": "c-mon", "label": "Monitoring and alerting", "sub": "metric · log · synthetic", "kind": "external", "rel": "signed webhooks", "dir": "in", "icon": "prometheus" }, { "id": "c-mail", "label": "Email and ticket sources", "sub": "vendors · helpdesk", "kind": "external", "rel": "SMTP · webhook", "dir": "in", "icon": "fa5_envelope" } ] }, { "side": "left", "title": "People it serves and informs", "nodes": [ { "id": "c-oncall", "label": "On-call responder", "kind": "actor", "rel": "acknowledges", "dir": "in", "icon": "fa5_user_clock" }, { "id": "c-ic", "label": "Incident commander", "kind": "actor", "rel": "coordinates", "dir": "in", "icon": "fa5_user_tie" }, { "id": "c-rot", "label": "Rotation owner", "kind": "actor", "rel": "schedules · overrides", "dir": "in", "icon": "fa5_calendar_alt" }, { "id": "c-svc", "label": "Service owner", "kind": "actor", "rel": "reviews · noise", "dir": "in", "icon": "fa5_user_cog" }, { "id": "c-stake", "label": "Stakeholders", "kind": "actor", "rel": "updates · status page", "dir": "out", "icon": "fa5_user_friends" } ] }, { "side": "right", "title": "What it reaches", "nodes": [ { "id": "c-carrier", "label": "Voice and SMS carriers", "sub": "two, independent", "kind": "external", "rel": "SIP · SMPP", "icon": "fa5_broadcast_tower" }, { "id": "c-push", "label": "Apple and Google push", "sub": "APNs · FCM", "kind": "external", "rel": "critical alerts", "icon": "fa5_mobile_alt" }, { "id": "c-chat", "label": "Mattermost · tracker", "sub": "channels · action items", "kind": "external", "rel": "channels · actions", "icon": "fa5_comments" } ] }, { "side": "bottom", "title": "What it borrows, and must survive losing", "nodes": [ { "id": "c-idp", "label": "Corporate identity provider", "sub": "federated only", "kind": "external", "rel": "SSO", "dir": "in", "icon": "fa5_id_card" }, { "id": "c-chg", "label": "Delivery pipelines", "sub": "deploy · change events", "kind": "external", "rel": "change events", "dir": "in", "kind2": "async", "icon": "fa5_shipping_fast" } ] } ], "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "02-high-level-architecture", "title": "High-Level Architecture — Two Planes, One Loop", "layout": "flow", "chain": true, "stages": [ { "title": "Receive", "nodes": [ { "id": "h-edge", "label": "HAProxy edge", "sub": "anycast · quotas", "kind": "integration", "icon": "haproxy" }, { "id": "h-gw", "label": "Ingest gateway", "sub": "verify · normalise", "kind": "app", "icon": "fa5_door_open" } ] }, { "title": "Decide", "nodes": [ { "id": "h-eng", "label": "Incident engine", "sub": "dedup · group · suppress", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "h-cov", "label": "Coverage snapshot", "sub": "JetStream KV", "kind": "store", "icon": "fa5_user_clock" } ] }, { "title": "Page", "nodes": [ { "id": "h-tim", "label": "Escalation timers", "sub": "durable · ± 5 s", "kind": "app", "icon": "fa5_stopwatch" }, { "id": "h-disp", "label": "Dispatcher", "sub": "channel policy", "kind": "app", "icon": "fa5_satellite_dish" } ] }, { "title": "Reach", "nodes": [ { "id": "h-voice", "label": "Asterisk · Jasmin", "sub": "two carrier routes", "kind": "integration", "icon": "fa5_phone_volume" }, { "id": "h-push", "label": "Push relay", "sub": "APNs · FCM", "kind": "integration", "icon": "fa5_mobile_alt" } ] }, { "title": "Confirm", "nodes": [ { "id": "h-ack", "label": "Ack receiver", "sub": "every channel", "kind": "app", "icon": "fa5_check_double" }, { "id": "h-led", "label": "Notification ledger", "sub": "per attempt", "kind": "store", "icon": "fa5_receipt" } ] }, { "title": "Record · control plane", "nodes": [ { "id": "h-log", "label": "Incident event log", "sub": "PostgreSQL · append-only", "kind": "store", "icon": "postgresql" }, { "id": "h-res", "label": "Schedule resolver", "sub": "72 h ahead", "kind": "app", "icon": "fa5_calendar_alt" } ] } ], "edges": [ { "from": "h-res", "to": "h-cov", "label": "snapshot ≤ 60 s", "kind": "async", "route": "gutter" } ], "note": "The first five stages are the paging path and run with the sixth stage entirely unavailable. The only arrow into it from the control plane is a replicated snapshot.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "03-actors-and-journeys", "title": "Actors — Who the Platform Is For and What They Get to Do", "layout": "actors", "cardWidth": 280, "groups": [ { "title": "People woken by it", "kind": "boundary", "actors": [ { "id": "a-oncall", "label": "On-call responder", "sub": "≈ 400 across 25 rotations", "goal": "Wake me only for something real, and let one keypress stop the pager.", "journeys": [ { "id": "j-woken", "label": "Acknowledge a SEV1 at 03:12", "icon": "fa5_bell" }, { "label": "Hand over at the end of a shift", "icon": "fa5_exchange_alt" }, { "label": "Pull a colleague in by role", "icon": "fa5_comments" } ], "icon": "fa5_user_clock" }, { "id": "a-ic", "label": "Incident commander", "sub": "every SEV1 and SEV2", "goal": "Get the right people in one place, and keep stakeholders informed without letting them into the channel.", "journeys": [ { "id": "j-ic", "label": "Run a SEV1 to mitigation", "icon": "fa5_bell" }, { "label": "Split a second failure out", "icon": "fa5_code_branch" }, { "label": "Post a stakeholder update", "icon": "fa5_bullhorn" } ], "icon": "fa5_user_tie" } ] }, { "title": "People who own what it pages for", "kind": "onprem", "actors": [ { "id": "a-rot", "label": "Rotation owner", "sub": "25 rotations · 3 timezones", "goal": "Hear about the hole in my rotation a week before it matters, not when a page fails.", "journeys": [ { "id": "j-dst", "label": "Close a coverage gap before DST", "icon": "fa5_user_clock" }, { "label": "Approve a holiday override", "icon": "fa5_calendar_alt" }, { "label": "Chase an unverified phone number", "icon": "fa5_mobile_alt" } ], "icon": "fa5_calendar_alt" }, { "id": "a-svc", "label": "Service owner", "sub": "40 services", "goal": "Pay less for noise, and make the review the least effort of anything I do that week.", "journeys": [ { "id": "j-review", "label": "Close a review and its actions", "icon": "fa5_clipboard_list" }, { "label": "Read the weekly noise and cost report", "icon": "fa5_coins" }, { "label": "Declare a maintenance window", "icon": "fa5_calendar_minus" } ], "icon": "fa5_user_cog" } ] }, { "title": "Machines and readers", "kind": "plain", "actors": [ { "id": "a-mon", "label": "Monitoring system", "kind": "platform", "sub": "Alertmanager · synthetics", "goal": "Hand an alert over once, retry safely, and be throttled rather than dropped.", "journeys": [ { "label": "Retry a webhook after a timeout", "icon": "fa5_plug" }, { "label": "Survive its own quota", "icon": "fa5_tachometer_alt" } ], "icon": "prometheus" }, { "id": "a-stake", "label": "Stakeholder", "kind": "external", "sub": "support · leadership", "goal": "Know what customers see and when I will hear next, without joining the incident.", "journeys": [ { "label": "Read a templated update", "icon": "fa5_clipboard_list" }, { "label": "Check the status page", "icon": "fa5_globe" } ], "icon": "fa5_user_friends" } ] } ], "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "04-journey-acknowledge-a-sev1-at-night", "title": "Journey — An On-Call Responder Is Woken for a SEV1 at 03:12", "layout": "journey", "actor": { "label": "On-call responder", "sub": "payments primary", "goal": "Be reached, understand why, act", "trigger": "checkout error rate at 18%, SEV1 proposed", "success": "Acknowledged in 3 min, rollback started", "icon": "fa5_user_clock" }, "phases": [ { "title": "Alert fires" }, { "title": "Push" }, { "title": "Call rings", "moment": true }, { "title": "Acknowledge" }, { "title": "Orient" }, { "title": "Mitigate" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Asleep, phone on DND", "icon": "fa5_mobile_alt" } ], [ { "label": "Sleeps through it", "icon": "fa5_moon" } ], [ { "label": "Answers the call", "icon": "fa5_phone_volume" } ], [ { "label": "Presses 1", "icon": "fa5_phone_volume" } ], [ { "label": "Opens the incident link", "icon": "fa5_link" } ], [ { "label": "Rolls back the deploy", "icon": "fa5_history" } ] ] }, { "title": "System behind it", "kind": "system", "cells": [ [ { "label": "Engine · snapshot", "icon": "fa5_user_clock" } ], [ { "label": "Push relay", "icon": "fa5_mobile_alt" } ], [ { "label": "Asterisk · carrier A", "icon": "fa5_phone_volume" } ], [ { "label": "Ack receiver", "icon": "fa5_check_double" } ], [ { "label": "Timeline · catalogue", "icon": "fa5_sitemap" } ], [ { "label": "Change events", "icon": "fa5_history" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Calm", "Groggy", "Stressed" ], "points": [ "Calm", "Groggy", "Stressed", "Stressed", "Groggy", "Calm" ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Push swallowed?", "icon": "fa5_mobile_alt" } ], [ { "label": "Why me, what broke?", "icon": "fa5_exclamation_circle" } ], [], [ { "label": "Where is the runbook?", "icon": "fa5_sitemap" } ], [] ] }, { "title": "What the platform gives", "kind": "gain", "cells": [ [ { "label": "Right human, one lookup", "icon": "fa5_bell" } ], [ { "label": "Critical alert for SEV1", "icon": "fa5_bell" } ], [ { "label": "Service named in speech", "icon": "fa5_phone_volume" } ], [ { "label": "Escalation off ≤ 5 s", "icon": "fa5_bell" } ], [ { "label": "Runbook from catalogue", "icon": "fa5_sitemap" } ], [ { "label": "Deploy on the timeline", "icon": "fa5_history" } ] ] } ], "chain": true, "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "05-journey-run-a-sev1-to-mitigation", "title": "Journey — An Incident Commander Runs a SEV1 to Mitigation", "layout": "journey", "actor": { "label": "Incident commander", "sub": "checkout outage", "goal": "Stop user impact, keep everyone informed", "trigger": "Responder raises the incident to SEV1", "success": "Mitigated in 41 min, updates on time", "icon": "fa5_user_tie" }, "phases": [ { "title": "Declare" }, { "title": "Assemble" }, { "title": "Organise" }, { "title": "Split", "moment": true }, { "title": "Update" }, { "title": "Mitigated" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Takes the IC role", "icon": "fa5_comments" } ], [ { "label": "Pages database team", "icon": "fa5_bell" } ], [ { "label": "Names scribe and comms", "icon": "fa5_comments" } ], [ { "label": "Splits the DNS alerts out", "icon": "fa5_code_branch" } ], [ { "label": "Posts the 30-min update", "icon": "fa5_bullhorn" } ], [ { "label": "Marks mitigated", "icon": "fa5_history" } ] ] }, { "title": "System behind it", "kind": "system", "cells": [ [ { "label": "Severity obligations", "icon": "fa5_bell" } ], [ { "label": "Service catalogue", "icon": "fa5_sitemap" } ], [ { "label": "Role events", "icon": "fa5_comments" } ], [ { "label": "Split event", "icon": "fa5_code_branch" } ], [ { "label": "Update template", "icon": "fa5_clipboard_list" } ], [ { "label": "Three clocks", "icon": "fa5_history" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Calm", "Tense", "Stressed" ], "points": [ "Tense", "Tense", "Tense", "Stressed", "Tense", "Calm" ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Who owns payments-db?", "icon": "fa5_sitemap" } ], [ { "label": "Twenty people typing", "icon": "fa5_comments" } ], [ { "label": "Second failure hidden", "icon": "fa5_code_branch" } ], [ { "label": "Leaders asking in DMs", "icon": "fa5_bullhorn" } ], [] ] }, { "title": "What the platform gives", "kind": "gain", "cells": [ [ { "label": "Channel opened for SEV1", "icon": "fa5_comments" } ], [ { "label": "Page by service owner", "icon": "fa5_sitemap" } ], [ { "label": "One holder per role", "icon": "fa5_comments" } ], [ { "label": "New service re-pages", "icon": "fa5_code_branch" } ], [ { "label": "Next-update-by timer", "icon": "fa5_bullhorn" } ], [ { "label": "Mitigated is not resolved", "icon": "fa5_history" } ] ] } ], "chain": true, "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "06-journey-close-a-coverage-gap-before-dst", "title": "Journey — A Rotation Owner Closes a Gap Nine Days Before DST", "layout": "journey", "actor": { "label": "Rotation owner", "sub": "platform-db · US/Eastern", "goal": "No hour without a responder", "trigger": "Gap warning for the spring-forward night", "success": "Covered, verified, provable later", "icon": "fa5_calendar_alt" }, "phases": [ { "title": "Warned" }, { "title": "Inspect" }, { "title": "Override" }, { "title": "Verify" }, { "title": "The night", "moment": true }, { "title": "Prove" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Reads the gap warning", "icon": "fa5_calendar_alt" } ], [ { "label": "Opens coverage preview", "icon": "fa5_user_clock" } ], [ { "label": "Adds a 01:00–04:00 override", "icon": "fa5_moon" } ], [ { "label": "Reads the snapshot diff", "icon": "fa5_user_clock" } ], [ { "label": "Sleeps", "icon": "fa5_moon" } ], [ { "label": "Answers who was on call", "icon": "fa5_phone_volume" } ] ] }, { "title": "System behind it", "kind": "system", "cells": [ [ { "label": "Gap detector · 14 d", "icon": "fa5_calendar_alt" } ], [ { "label": "Resolved timeline", "icon": "fa5_history" } ], [ { "label": "Override layer", "icon": "fa5_calendar_alt" } ], [ { "label": "Second resolver", "icon": "fa5_balance_scale" } ], [ { "label": "Snapshot · 72 h ahead", "icon": "fa5_user_clock" } ], [ { "label": "Schedule audit", "icon": "fa5_history" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Unsure", "Anxious" ], "points": [ "Anxious", "Unsure", "Unsure", "Confident", "Anxious", "Confident" ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "02:30 does not exist", "icon": "fa5_moon" } ], [ { "label": "Four layers deep", "icon": "fa5_exclamation_circle" } ], [], [], [ { "label": "Will anyone be paged?", "icon": "fa5_exclamation_circle" } ], [] ] }, { "title": "What the platform gives", "kind": "gain", "cells": [ [ { "label": "Warned 9 days early" } ], [ { "label": "Shown in local time", "icon": "fa5_moon" } ], [ { "label": "Override beats all layers", "icon": "fa5_calendar_alt" } ], [ { "label": "Two resolvers agree", "icon": "fa5_balance_scale" } ], [ { "label": "Nothing computed live", "icon": "fa5_check_circle" } ], [ { "label": "Answer from the record", "icon": "fa5_exclamation_circle" } ] ] } ], "chain": true, "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "07-journey-close-a-review-and-its-actions", "title": "Journey — A Service Owner Turns a SEV2 Into Fixed Conditions", "layout": "journey", "actor": { "label": "Service owner", "sub": "search-api", "goal": "A review that changes something", "trigger": "SEV2 resolved, review due in 5 business days", "success": "Reviewed on day 3, actions closed with change", "icon": "fa5_user_cog" }, "phases": [ { "title": "Draft" }, { "title": "Timeline" }, { "title": "Factors" }, { "title": "Actions", "moment": true }, { "title": "Follow up" }, { "title": "Recur" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Opens the draft", "icon": "fa5_history" } ], [ { "label": "Checks the intervals", "icon": "fa5_history" } ], [ { "label": "Writes conditions" } ], [ { "label": "Files three actions", "icon": "fa5_tasks" } ], [ { "label": "Chases the overdue one", "icon": "fa5_tasks" } ], [ { "label": "Searches past reviews", "icon": "fa5_clipboard_list" } ] ] }, { "title": "System behind it", "kind": "system", "cells": [ [ { "label": "Review generator", "icon": "fa5_clipboard_list" } ], [ { "label": "Event log", "icon": "postgresql" } ], [ { "label": "Blameless template", "icon": "fa5_clipboard_list" } ], [ { "label": "Tracker sync", "icon": "fa5_sync_alt" } ], [ { "label": "Overdue report", "icon": "fa5_chart_bar" } ], [ { "label": "Factor search", "icon": "fa5_clipboard_list" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Unsure", "Reluctant" ], "points": [ "Reluctant", "Confident", "Unsure", "Reluctant", "Unsure", "Confident" ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "Another form to fill", "icon": "fa5_clipboard_list" } ], [], [ { "label": "Urge to name a person", "icon": "fa5_clipboard_list" } ], [ { "label": "Items die in trackers", "icon": "fa5_tasks" } ], [ { "label": "Closed, nothing changed", "icon": "fa5_tasks" } ], [] ] }, { "title": "What the platform gives", "kind": "gain", "cells": [ [ { "label": "Pre-filled on resolve", "icon": "fa5_history" } ], [ { "label": "Intervals computed", "icon": "fa5_history" } ], [ { "label": "No field for a name", "icon": "fa5_clipboard_list" } ], [ { "label": "Two-way sync" } ], [ { "label": "No-change closures flagged", "icon": "fa5_tasks" } ], [ { "label": "Same factor, third time", "icon": "fa5_clipboard_list" } ] ] } ], "chain": true, "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "08-layered-architecture", "title": "Layered Architecture", "layout": "bands", "layerHeaderWidth": 170, "bands": [ { "name": "Signal edge", "nodes": [ { "id": "l-edge", "label": "HAProxy edge", "sub": "BGP anycast · quotas", "kind": "integration", "icon": "haproxy" }, { "id": "l-mx", "label": "Postfix inbound", "sub": "email ingest · replies", "kind": "integration", "icon": "fa5_envelope" }, { "id": "l-gw", "label": "Ingest gateway", "sub": "Go · CEL mappings", "kind": "app", "icon": "fa5_door_open" }, { "id": "l-quar", "label": "Quarantine", "sub": "per integration", "kind": "risk", "icon": "fa5_ban" } ] }, { "name": "Paging path", "nodes": [ { "id": "l-eng", "label": "Incident engine", "sub": "dedup · group · suppress", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "l-tim", "label": "Escalation timers", "sub": "sharded wheel", "kind": "app", "icon": "fa5_stopwatch" }, { "id": "l-disp", "label": "Dispatcher", "sub": "channel policy", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "l-ack", "label": "Ack receiver", "sub": "DTMF · SMS · push", "kind": "app", "icon": "fa5_check_double" } ] }, { "name": "Paging state", "nodes": [ { "id": "l-js", "label": "NATS JetStream", "sub": "stretched · 3 sites", "kind": "queue", "icon": "fa5_stream" }, { "id": "l-cov", "label": "Coverage snapshot", "sub": "KV · 72 h ahead", "kind": "store", "icon": "fa5_user_clock" }, { "id": "l-led", "label": "Notification ledger", "sub": "stream · 30 d", "kind": "store", "icon": "fa5_receipt" } ] }, { "name": "Delivery", "nodes": [ { "id": "l-push", "label": "Push relay", "sub": "APNs · FCM", "kind": "integration", "icon": "fa5_mobile_alt" }, { "id": "l-jas", "label": "Jasmin SMS", "sub": "SMPP · two routes", "kind": "integration", "icon": "fa5_sms" }, { "id": "l-ast", "label": "Asterisk", "sub": "SIP · two carriers", "kind": "integration", "icon": "fa5_phone_volume" }, { "id": "l-smtp", "label": "Postfix outbound", "sub": "two smarthosts", "kind": "integration", "icon": "fa5_mail_bulk" } ] }, { "name": "Control plane", "nodes": [ { "id": "l-res", "label": "Schedule resolver", "sub": "IANA zones", "kind": "app", "icon": "fa5_calendar_alt" }, { "id": "l-inc", "label": "Incident service", "sub": "roles · updates", "kind": "app", "icon": "fa5_fire_alt" }, { "id": "l-rev", "label": "Review module", "sub": "drafts · actions", "kind": "app", "icon": "fa5_clipboard_list" }, { "id": "l-con", "label": "Console", "sub": "web · break-glass", "kind": "app", "icon": "fa5_desktop" } ] }, { "name": "Record and insight", "nodes": [ { "id": "l-pg", "label": "PostgreSQL", "sub": "event log · projections", "kind": "store", "icon": "postgresql" }, { "id": "l-ch", "label": "ClickHouse", "sub": "ledger · raw · noise", "kind": "store", "icon": "clickhouse" }, { "id": "l-ceph", "label": "Ceph RGW", "sub": "Object Lock archive", "kind": "store", "icon": "fa5_archive" } ] }, { "name": "Identity, secrets and reporting", "nodes": [ { "id": "l-kc", "label": "Keycloak", "sub": "OIDC broker", "kind": "security", "icon": "keycloak" }, { "id": "l-bao", "label": "OpenBao", "sub": "secrets · transit keys", "kind": "security", "icon": "fa5_key" }, { "id": "l-graf", "label": "Grafana", "sub": "reports · dashboards", "kind": "platform", "icon": "grafana" } ] } ], "edges": [ { "from": "l-gw", "to": "l-eng", "label": "normalised", "kind": "async" }, { "from": "l-disp", "to": "l-ast", "label": "call · SMS" }, { "from": "l-inc", "to": "l-pg", "label": "append" } ], "note": "The top four layers are the paging path and depend on nothing below them at run time. The control plane reaches up only by publishing a snapshot into the paging state.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "09-paging-cell", "title": "Container Architecture — One Paging Cell", "layout": "nested", "boxes": [ { "title": "Paging cell · site A · three dedicated hosts · systemd, no orchestrator", "kind": "onprem", "dir": "row", "children": [ { "title": "Receive", "kind": "boundary", "dir": "col", "nodes": [ { "id": "p-edge", "label": "HAProxy edge", "sub": "anycast VIP · BFD", "kind": "integration", "icon": "haproxy" }, { "id": "p-gw", "label": "Ingest gateway", "sub": "ingest.slice · capped", "kind": "app", "icon": "fa5_door_open" } ] }, { "title": "State", "kind": "trust", "dir": "col", "nodes": [ { "id": "p-jsi", "label": "Ingest domain", "sub": "JetStream · site R3", "kind": "queue", "icon": "fa5_stream" }, { "id": "p-jsp", "label": "Paging domain", "sub": "JetStream · 3-site R3", "kind": "queue", "icon": "fa5_stream" } ] }, { "title": "Decide and page · paging.slice, reserved CPU", "kind": "boundary", "dir": "col", "nodes": [ { "id": "p-eng", "label": "Incident engine", "sub": "grouping key CAS", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "p-tim", "label": "Escalation timers", "sub": "shard lease", "kind": "app", "icon": "fa5_stopwatch" }, { "id": "p-disp", "label": "Dispatcher", "sub": "human dedup 60 s", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "p-ack", "label": "Ack receiver", "sub": "signed ack tokens", "kind": "app", "icon": "fa5_check_double" } ] }, { "title": "Delivery", "kind": "boundary", "dir": "col", "nodes": [ { "id": "p-ast", "label": "Asterisk", "sub": "IVR · DTMF", "kind": "integration", "icon": "fa5_phone_volume" }, { "id": "p-jas", "label": "Jasmin SMS", "sub": "SMPP · MO replies", "kind": "integration", "icon": "fa5_sms" }, { "id": "p-push", "label": "Push relay", "sub": "APNs · FCM", "kind": "integration", "icon": "fa5_mobile_alt" } ] } ] } ], "outside": [ { "id": "p-mon", "label": "Monitoring sources", "sub": "signed webhooks", "kind": "external", "icon": "prometheus" }, { "id": "p-cells", "label": "Cells B and C", "sub": "Raft peers", "kind": "external", "icon": "fa5_server" }, { "id": "p-carr", "label": "Carrier A", "sub": "SIP trunk · SMPP", "kind": "external", "icon": "fa5_broadcast_tower" } ], "edges": [ { "from": "p-mon", "to": "p-edge", "label": "HTTPS" }, { "from": "p-gw", "to": "p-jsi", "label": "publish", "kind": "async" }, { "from": "p-eng", "to": "p-jsp", "label": "CAS · outbox" }, { "from": "p-ast", "to": "p-carr", "label": "SIP · TLS" }, { "from": "p-jsp", "to": "p-cells", "label": "Raft", "kind": "async" } ], "note": "Omitted for legibility: the engine consuming the ingest domain, timers and dispatcher reading the paging domain, and every ack path (view 19).", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "10-control-plane", "title": "Container Architecture — The Control Plane", "layout": "nested", "boxes": [ { "title": "Control plane · site A · platform-owned Kubernetes, warm rebuild at site B", "kind": "onprem", "dir": "row", "children": [ { "title": "Response", "kind": "boundary", "dir": "col", "nodes": [ { "id": "k-inc", "label": "Incident service", "sub": "state · roles · updates", "kind": "app", "icon": "fa5_fire_alt" }, { "id": "k-con", "label": "Console", "sub": "TypeScript web UI", "kind": "app", "icon": "fa5_desktop" }, { "id": "k-rev", "label": "Review module", "sub": "drafts · actions", "kind": "app", "icon": "fa5_clipboard_list" } ] }, { "title": "Scheduling", "kind": "boundary", "dir": "col", "nodes": [ { "id": "k-cat", "label": "Service catalogue", "sub": "YAML in Git", "kind": "app", "icon": "fa5_sitemap" }, { "id": "k-res", "label": "Schedule resolver", "sub": "72 h · 14 d gaps", "kind": "app", "icon": "fa5_calendar_alt" }, { "id": "k-ver", "label": "DST verifier", "sub": "second implementation", "kind": "decision", "icon": "fa5_balance_scale" } ] }, { "title": "Data", "kind": "trust", "dir": "col", "nodes": [ { "id": "k-pg", "label": "PostgreSQL", "sub": "CloudNativePG · sync", "kind": "store", "icon": "postgresql" }, { "id": "k-proj", "label": "Projector", "sub": "rebuildable views", "kind": "app", "icon": "fa5_redo" }, { "id": "k-ch", "label": "ClickHouse", "sub": "ledger · noise · cost", "kind": "store", "icon": "clickhouse" } ] }, { "title": "Integration and identity", "kind": "boundary", "dir": "col", "nodes": [ { "id": "k-kc", "label": "Keycloak", "sub": "broker · break-glass", "kind": "security", "icon": "keycloak" }, { "id": "k-bot", "label": "Chat bridge", "sub": "channel per incident", "kind": "integration", "icon": "fa5_comments" }, { "id": "k-sync", "label": "Tracker sync", "sub": "two-way · reconciled", "kind": "integration", "icon": "fa5_sync_alt" } ] } ] } ], "outside": [ { "id": "k-cells", "label": "Paging cells", "sub": "coverage KV · outbox", "kind": "external", "icon": "fa5_server" }, { "id": "k-idp", "label": "Corporate IdP", "sub": "OIDC · SAML", "kind": "external", "icon": "fa5_id_card" } ], "edges": [ { "from": "k-res", "to": "k-cells", "label": "publish snapshot", "kind": "async" }, { "from": "k-inc", "to": "k-pg", "label": "append" }, { "from": "k-kc", "to": "k-idp", "label": "federate" } ], "note": "Omitted for legibility: the reconciler reading the paging outbox (view 18), the chat bridge and tracker sync reaching Mattermost and the tracker (view 11), and OpenBao, which every service reads at start.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "11-integration-surface", "title": "Integration Surface — Every Interface", "layout": "hub", "left": { "title": "What sends to it", "nodes": [ { "id": "i-mon", "label": "Monitoring sources", "kind": "external", "rel": "HMAC webhooks", "icon": "prometheus" }, { "id": "i-mail", "label": "Email · change feed", "kind": "external", "rel": "SMTP · events", "icon": "fa5_envelope" }, { "id": "i-api", "label": "Automation clients", "kind": "app", "rel": "REST · on-call lookup", "icon": "fa5_robot" } ] }, "centre": { "title": "Platform", "nodes": [ { "id": "i-core", "label": "Incident Management Platform", "sub": "decide · page · record", "icon": "fa5_bell" } ] }, "right": { "title": "What it reaches", "nodes": [ { "id": "i-carr", "label": "Carriers · APNs · FCM", "kind": "external", "rel": "SIP · SMPP · push", "dir": "out", "icon": "fa5_broadcast_tower" }, { "id": "i-chat", "label": "Mattermost · tracker", "kind": "external", "rel": "channels · actions", "dir": "out", "icon": "fa5_comments" }, { "id": "i-out", "label": "Outbound event stream", "kind": "queue", "rel": "lifecycle events", "dir": "out", "kind2": "async", "icon": "fa5_stream" } ] }, "note": "Only the left column and the carriers sit on the paging path. Everything else on the right is allowed to be late.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "12-anatomy-of-a-page", "title": "Anatomy of a Page — Every Step Local to One Cell", "layout": "flow", "chain": true, "stages": [ { "title": "Admit", "nodes": [ { "id": "n-quota", "label": "Integration quota", "sub": "HAProxy stick table", "kind": "decision", "icon": "fa5_tachometer_alt" }, { "id": "n-sig", "label": "Signature check", "sub": "HMAC · two secrets", "kind": "security", "icon": "fa5_fingerprint" } ] }, { "title": "Normalise", "nodes": [ { "id": "n-cel", "label": "CEL mapping", "sub": "per integration", "kind": "app", "icon": "fa5_code" }, { "id": "n-fp", "label": "Fingerprint", "sub": "source · key · fp", "kind": "app", "icon": "fa5_fingerprint" } ] }, { "title": "Decide", "nodes": [ { "id": "n-dedup", "label": "Dedup window", "sub": "24 h · site KV", "kind": "decision", "icon": "fa5_clone" }, { "id": "n-group", "label": "Grouping key CAS", "sub": "one open incident", "kind": "decision", "icon": "fa5_layer_group" } ] }, { "title": "Resolve", "nodes": [ { "id": "n-cov", "label": "Coverage bucket", "sub": "rotation · 15 min", "kind": "store", "icon": "fa5_user_clock" }, { "id": "n-pol", "label": "Escalation policy", "sub": "versioned KV", "kind": "store", "icon": "fa5_sort_amount_down" } ] }, { "title": "Dispatch", "nodes": [ { "id": "n-hdd", "label": "Human dedup", "sub": "60 s per channel", "kind": "decision", "icon": "fa5_user_check" }, { "id": "n-chan", "label": "Channel policy", "sub": "per severity", "kind": "app", "icon": "fa5_random" } ] }, { "title": "Confirm", "nodes": [ { "id": "n-ack", "label": "Acknowledgement", "sub": "cancels step ≤ 5 s", "kind": "app", "icon": "fa5_check_double" }, { "id": "n-led", "label": "Ledger entry", "sub": "per attempt", "kind": "store", "icon": "fa5_receipt" } ] } ], "note": "Budget for alert receipt to first dispatch: p95 15 s. Nothing on this page makes a network call outside the cell except the carrier submit.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "13-page-or-not", "title": "May the Platform Decide Not to Page? — The Order of the Checks", "layout": "flow", "chain": true, "stages": [ { "title": "Seen before?", "nodes": [ { "id": "s-dup", "label": "Exact duplicate", "sub": "source key · 24 h", "kind": "decision", "icon": "fa5_clone" }, { "id": "s-dupo", "label": "Recorded, not paged", "sub": "duplicate count", "kind": "store", "icon": "fa5_list_alt" } ] }, { "title": "Exempt?", "nodes": [ { "id": "s-ex", "label": "SEV1 or declared", "sub": "by a human", "kind": "decision", "icon": "fa5_bell" }, { "id": "s-exo", "label": "Bypasses suppression", "sub": "still grouped", "kind": "opportunity", "icon": "fa5_bolt" } ] }, { "title": "Maintenance?", "nodes": [ { "id": "s-mw", "label": "Declared window", "sub": "owner · expiry", "kind": "decision", "icon": "fa5_calendar_minus" }, { "id": "s-mwo", "label": "Recorded, visible", "sub": "rule id logged", "kind": "store", "icon": "fa5_eye" } ] }, { "title": "Already open?", "nodes": [ { "id": "s-grp", "label": "Open incident", "sub": "same grouping key", "kind": "decision" }, { "id": "s-grpo", "label": "Attach, no re-page", "sub": "unless severity rises", "kind": "store", "icon": "fa5_link" } ] }, { "title": "Flooding?", "nodes": [ { "id": "s-fl", "label": "Rate per key", "sub": "over threshold", "kind": "decision", "icon": "fa5_tachometer_alt" }, { "id": "s-flo", "label": "Attach, count", "sub": "never dropped", "kind": "store", "icon": "fa5_link" } ] }, { "title": "Page", "nodes": [ { "id": "s-page", "label": "Escalation starts", "sub": "policy step 1", "kind": "app", "icon": "fa5_bell" }, { "id": "s-log", "label": "Decision event", "sub": "rule that decided", "kind": "store", "icon": "fa5_list_alt" } ] } ], "note": "Every box in the lower row writes an event naming the rule that decided. Dependency-aware suppression is absent on purpose; it annotates, it never withholds.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "14-four-stores", "title": "Four Stores, Separated by What May Be Lost and What Must Be Fast", "layout": "nested", "boxes": [ { "title": "Platform state, by recovery obligation", "kind": "plain", "dir": "row", "children": [ { "title": "Coverage · RPO 0", "kind": "trust", "dir": "col", "nodes": [ { "id": "d-cov", "label": "Coverage snapshot", "sub": "JetStream KV · R3", "kind": "store", "icon": "fa5_user_clock" }, { "id": "d-pol", "label": "Escalation policies", "sub": "KV · versioned", "kind": "store", "icon": "fa5_sort_amount_down" }, { "id": "d-sched", "label": "Schedule store", "sub": "PostgreSQL · source", "kind": "store", "icon": "postgresql" } ] }, { "title": "Event log · system of record · 7 y", "kind": "boundary", "dir": "col", "nodes": [ { "id": "d-out", "label": "Paging outbox", "sub": "JetStream · 30 d", "kind": "queue", "icon": "fa5_stream" }, { "id": "d-log", "label": "Incident event log", "sub": "PostgreSQL · append-only", "kind": "store", "icon": "postgresql" }, { "id": "d-anch", "label": "Chain anchors", "sub": "Ceph · Object Lock", "kind": "store", "icon": "fa5_archive" } ] }, { "title": "Notification ledger · 13 mo", "kind": "boundary", "dir": "col", "nodes": [ { "id": "d-led", "label": "Ledger stream", "sub": "JetStream · 30 d", "kind": "queue", "icon": "fa5_stream" }, { "id": "d-ledch", "label": "Ledger archive", "sub": "ClickHouse · 13 mo", "kind": "store", "icon": "clickhouse" } ] }, { "title": "Operational · rebuildable", "kind": "boundary", "dir": "col", "nodes": [ { "id": "d-proj", "label": "Projections", "sub": "PostgreSQL schema", "kind": "store", "icon": "postgresql" }, { "id": "d-search", "label": "Review search", "sub": "full-text index", "kind": "store", "icon": "fa5_search" } ] }, { "title": "Evidence · loss-tolerant", "kind": "boundary", "dir": "col", "nodes": [ { "id": "d-raw", "label": "Raw payloads", "sub": "ClickHouse · 90 d", "kind": "store", "icon": "clickhouse" }, { "id": "d-marts", "label": "Noise and cost marts", "sub": "ClickHouse · 13 mo", "kind": "store", "icon": "clickhouse" } ] } ] } ], "note": "The first column is small and must be current. The second must never change. The third must survive the second. The last two may be rebuilt or lost.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "15-data-model", "title": "Incidents, Coverage and Evidence — Data Model", "layout": "er", "canvas": { "width": 1640, "cols": 4 }, "rowGap": 170, "entities": [ { "id": "e-int", "name": "integration", "kind": "store", "row": 0, "col": 0, "attrs": [ "integration_id PK", "owner_team", "quota_per_s", "secret_ref two active", "mapping_version" ] }, { "id": "e-alert", "name": "alert", "kind": "store", "row": 0, "col": 1, "attrs": [ "alert_id PK", "integration_id FK", "source_key, fingerprint", "proposed_severity", "source_ts, received_ts", "raw_ref 90 d" ] }, { "id": "e-rev", "name": "review", "kind": "store", "row": 0, "col": 2, "attrs": [ "review_id PK", "incident_id FK", "contributing_factors", "system_conditions", "due_at 5 business days", "state" ] }, { "id": "e-act", "name": "action_item", "kind": "store", "row": 0, "col": 3, "attrs": [ "action_id PK", "review_id FK", "owner_team, due_at", "tracker_ref", "state", "closed_with_change" ] }, { "id": "e-svc", "name": "service", "kind": "store", "row": 1, "col": 0, "attrs": [ "service_id PK", "owner_team", "policy_id FK", "runbook_url", "depends_on" ] }, { "id": "e-inc", "name": "incident", "kind": "store", "row": 1, "col": 1, "attrs": [ "incident_id PK", "service_id FK", "grouping_key", "state, severity", "acked_at, mitigated_at", "resolved_at", "restricted bool" ] }, { "id": "e-ev", "name": "incident_event", "kind": "store", "row": 1, "col": 2, "attrs": [ "event_id PK idempotent", "incident_id FK", "seq, received_ts", "type, actor", "payload", "prev_hash, hash" ] }, { "id": "e-pol", "name": "escalation_policy", "kind": "store", "row": 2, "col": 0, "attrs": [ "policy_id, version PK", "steps targets, wait_s", "final_step NOT NULL", "channel_policy" ] }, { "id": "e-att", "name": "notification_attempt", "kind": "store", "row": 2, "col": 1, "attrs": [ "attempt_id PK", "incident_id, step", "channel, provider", "provider_msg_id", "dispatched_at, outcome", "acked_at, cost" ] }, { "id": "e-cm", "name": "contact_method", "kind": "store", "row": 2, "col": 2, "attrs": [ "method_id PK", "responder_id", "type push|sms|voice|email", "address encrypted", "verified_at" ] }, { "id": "e-rot", "name": "rotation", "kind": "store", "row": 3, "col": 0, "attrs": [ "rotation_id PK", "iana_zone", "owner_team", "handoff_local_time" ] }, { "id": "e-cb", "name": "coverage_bucket", "kind": "store", "row": 3, "col": 2, "attrs": [ "rotation_id, bucket_utc PK", "layers ordered", "method_ids verified only", "snapshot_version", "verified_by both resolvers" ] }, { "id": "e-lay", "name": "schedule_layer", "kind": "store", "row": 4, "col": 0, "attrs": [ "layer_id PK", "rotation_id FK", "kind base|secondary|mgmt|override", "rule or window", "changed_by, changed_at" ] } ], "relations": [ { "from": "e-int", "to": "e-alert", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-alert", "to": "e-inc", "label": "N : 1", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "e-svc", "to": "e-inc", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-inc", "to": "e-ev", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-rev", "to": "e-ev", "label": "cites", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "e-rev", "to": "e-act", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-inc", "to": "e-att", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "e-cm", "to": "e-att", "label": "1 : N", "from_side": "w", "to_side": "e" }, { "from": "e-svc", "to": "e-pol", "label": "N : 1", "from_side": "s", "to_side": "n" }, { "from": "e-pol", "to": "e-rot", "label": "N : M", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "e-rot", "to": "e-cb", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-cm", "to": "e-cb", "label": "N : M", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "e-rot", "to": "e-lay", "label": "1 : N", "from_side": "s", "to_side": "n" } ], "note": "incident is a projection of incident_event. contact_method.address is encrypted under a per-responder key; the event log holds only responder ids.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "16-ingest-pipeline", "title": "Alert Ingest — Untrusted In, Normalised Out, Nothing Lost", "layout": "flow", "stages": [ { "title": "Sources", "nodes": [ { "id": "g-am", "label": "Alertmanager", "sub": "grouped upstream", "kind": "external", "icon": "prometheus" }, { "id": "g-mail", "label": "Vendor email", "sub": "SMTP", "kind": "external", "icon": "fa5_envelope" } ] }, { "title": "Edge", "nodes": [ { "id": "g-hap", "label": "HAProxy edge", "sub": "quota · 429", "kind": "integration", "icon": "haproxy" }, { "id": "g-pf", "label": "Postfix inbound", "sub": "per-integration address", "kind": "integration", "icon": "fa5_envelope" } ] }, { "title": "Gateway", "nodes": [ { "id": "g-ver", "label": "Verify", "sub": "HMAC · replay window", "kind": "security", "icon": "fa5_fingerprint" }, { "id": "g-cel", "label": "Normalise", "sub": "CEL · schema check", "kind": "app", "icon": "fa5_code" } ] }, { "title": "Buffer", "nodes": [ { "id": "g-raw", "label": "Raw and normalised", "sub": "one message, both", "kind": "queue", "icon": "fa5_stream" }, { "id": "g-quar", "label": "Quarantine", "sub": "schema failures", "kind": "risk", "icon": "fa5_ban" } ] }, { "title": "Consume", "nodes": [ { "id": "g-eng", "label": "Incident engine", "sub": "per-integration consumer", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "g-qmon", "label": "Quarantine monitor", "sub": "threshold · SEV4", "kind": "decision", "icon": "fa5_eye" } ] }, { "title": "Keep", "nodes": [ { "id": "g-ch", "label": "Raw payloads", "sub": "ClickHouse · 90 d", "kind": "store", "icon": "clickhouse" }, { "id": "g-own", "label": "Integration owner", "sub": "low-severity incident", "kind": "actor", "icon": "fa5_user_cog" } ] } ], "edges": [ { "from": "g-am", "to": "g-hap", "label": "HTTPS" }, { "from": "g-mail", "to": "g-pf", "label": "SMTP" }, { "from": "g-hap", "to": "g-ver" }, { "from": "g-pf", "to": "g-cel", "label": "parsed" }, { "from": "g-ver", "to": "g-cel" }, { "from": "g-cel", "to": "g-raw", "label": "valid", "kind": "async" }, { "from": "g-cel", "to": "g-quar", "label": "invalid", "kind": "error" }, { "from": "g-raw", "to": "g-eng", "kind": "async" }, { "from": "g-quar", "to": "g-qmon", "kind": "async" }, { "from": "g-eng", "to": "g-ch", "label": "sink", "kind": "batch" }, { "from": "g-qmon", "to": "g-own", "label": "over threshold", "kind": "error" } ], "note": "Source and receive timestamps are both kept and never reconciled. A malformed payload is kept and counted, never rejected silently.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "17-coverage-materialisation", "title": "Coverage Snapshot — Resolved Ahead, Verified Twice, Read in One Lookup", "layout": "flow", "chain": true, "stages": [ { "title": "Edit", "nodes": [ { "id": "m-ed", "label": "Schedule editor", "sub": "rotation owners only", "kind": "app", "icon": "fa5_calendar_alt" }, { "id": "m-ov", "label": "Override", "sub": "wins every layer", "kind": "app", "icon": "fa5_calendar_plus" } ] }, { "title": "Store", "nodes": [ { "id": "m-pg", "label": "Schedule store", "sub": "PostgreSQL", "kind": "store", "icon": "postgresql" }, { "id": "m-aud", "label": "Schedule audit", "sub": "actor · window", "kind": "store", "icon": "fa5_history" } ] }, { "title": "Resolve", "nodes": [ { "id": "m-res", "label": "Schedule resolver", "sub": "IANA zones · 72 h", "kind": "app", "icon": "fa5_calendar_alt" }, { "id": "m-gap", "label": "Gap detector", "sub": "14 d horizon", "kind": "decision", "icon": "fa5_search" } ] }, { "title": "Verify", "nodes": [ { "id": "m-ver", "label": "DST verifier", "sub": "independent code", "kind": "decision", "icon": "fa5_balance_scale" }, { "id": "m-diff", "label": "Diff gate", "sub": "hold on disagreement", "kind": "security", "icon": "fa5_traffic_light" } ] }, { "title": "Publish", "nodes": [ { "id": "m-kv", "label": "Coverage buckets", "sub": "15 min · R3 quorum", "kind": "store", "icon": "fa5_user_clock" }, { "id": "m-ver2", "label": "Snapshot version", "sub": "stamped on pages", "kind": "store", "icon": "fa5_tags" } ] }, { "title": "Page", "nodes": [ { "id": "m-disp", "label": "Dispatcher", "sub": "one KV get", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "m-stale", "label": "Staleness alarm", "sub": "p99 ≤ 60 s", "kind": "platform", "icon": "fa5_hourglass_half" } ] } ], "note": "A change is visible to paging within 60 s at p99. If the two resolvers disagree, the previous verified buckets keep serving and the owner is told why.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "18-event-log-and-projections", "title": "The Log Is the Record — Events In, Projections Out", "layout": "flow", "chain": true, "stages": [ { "title": "Emit", "nodes": [ { "id": "r-cell", "label": "Paging outbox", "sub": "cells A · B · C", "kind": "queue", "icon": "fa5_stream" }, { "id": "r-cmd", "label": "Console commands", "sub": "notes · severity · roles", "kind": "app", "icon": "fa5_desktop" } ] }, { "title": "Reconcile", "nodes": [ { "id": "r-rec", "label": "Reconciler", "sub": "idempotent on event_id", "kind": "app", "icon": "fa5_equals" }, { "id": "r-min", "label": "Minority replay", "sub": "after partition", "kind": "decision", "icon": "fa5_history" } ] }, { "title": "Append", "nodes": [ { "id": "r-log", "label": "Incident event log", "sub": "no UPDATE · no DELETE", "kind": "store", "icon": "postgresql" }, { "id": "r-hash", "label": "Hash chain", "sub": "per incident", "kind": "security", "icon": "fa5_link" } ] }, { "title": "Project", "nodes": [ { "id": "r-proj", "label": "Projector", "sub": "replayable", "kind": "app", "icon": "fa5_redo" }, { "id": "r-ops", "label": "Operational store", "sub": "open incidents · views", "kind": "store", "icon": "postgresql" } ] }, { "title": "Serve", "nodes": [ { "id": "r-con", "label": "Console", "sub": "p95 ≤ 1.5 s", "kind": "app", "icon": "fa5_desktop" }, { "id": "r-outs", "label": "Outbound stream", "sub": "idempotency keys", "kind": "queue", "icon": "fa5_stream" } ] }, { "title": "Preserve", "nodes": [ { "id": "r-anch", "label": "Daily anchors", "sub": "Ceph · Object Lock", "kind": "store", "icon": "fa5_archive" }, { "id": "r-drill", "label": "Rebuild drill", "sub": "quarterly · ≤ 60 min", "kind": "platform", "icon": "fa5_redo" } ] } ], "note": "A correction is a new event that supersedes an old one. Merge and split are events too, so both sides of either keep their original timelines.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "19-alert-to-acknowledgement", "title": "Alert to Acknowledgement — The Core Loop", "layout": "sequence", "lifelines": [ { "id": "q-mon", "label": "Monitoring source", "kind": "external", "icon": "prometheus" }, { "id": "q-gw", "label": "Ingest gateway", "kind": "app", "icon": "fa5_door_open" }, { "id": "q-eng", "label": "Incident engine", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "q-cov", "label": "Coverage snapshot", "kind": "store", "icon": "fa5_user_clock" }, { "id": "q-tim", "label": "Escalation timers", "kind": "app", "icon": "fa5_stopwatch" }, { "id": "q-disp", "label": "Dispatcher", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "q-phone", "label": "Responder phone", "kind": "actor", "icon": "fa5_mobile_alt" }, { "id": "q-ack", "label": "Ack receiver", "kind": "app", "icon": "fa5_check_double" } ], "messages": [ { "from": "q-mon", "to": "q-gw", "label": "POST alert · HMAC", "kind": "call" }, { "from": "q-gw", "to": "q-mon", "label": "202 · alert id", "kind": "return" }, { "from": "q-gw", "to": "q-eng", "label": "normalised + raw", "kind": "async" }, { "from": "q-eng", "to": "q-eng", "label": "dedup · CAS grouping key · rules", "kind": "self" }, { "from": "q-eng", "to": "q-cov", "label": "get rotation, current bucket", "kind": "call" }, { "from": "q-cov", "to": "q-eng", "label": "layers · verified methods", "kind": "return" }, { "from": "q-eng", "to": "q-tim", "label": "arm step 1 · write timer", "kind": "call" }, { "from": "q-tim", "to": "q-disp", "label": "fire step 1", "kind": "async" }, { "from": "q-disp", "to": "q-phone", "label": "push · critical alert", "kind": "call" }, { "from": "q-disp", "to": "q-phone", "label": "SMS at 30 s · voice at 2 min", "kind": "call" }, { "from": "q-phone", "to": "q-ack", "label": "DTMF 1", "kind": "call" }, { "from": "q-ack", "to": "q-tim", "label": "CAS state acked · cancel", "kind": "call" }, { "from": "q-ack", "to": "q-disp", "label": "withdraw pending sends", "kind": "async" } ], "note": "Delivery receipts are written to the ledger and change nothing. Only the keypress, the SMS reply or the app button stops escalation.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "20-provider-failover-within-a-step", "title": "Provider Failover Within a Step — Same Channel on Failure, Next Channel on Silence", "layout": "sequence", "lifelines": [ { "id": "f-disp", "label": "Dispatcher", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "f-a", "label": "SMS route A", "kind": "integration", "icon": "fa5_sms" }, { "id": "f-b", "label": "SMS route B", "kind": "integration", "icon": "fa5_sms" }, { "id": "f-voice", "label": "Asterisk voice", "kind": "integration", "icon": "fa5_phone_volume" }, { "id": "f-resp", "label": "Responder", "kind": "actor", "icon": "fa5_user_clock" }, { "id": "f-led", "label": "Notification ledger", "kind": "store", "icon": "fa5_receipt" } ], "messages": [ { "from": "f-disp", "to": "f-a", "label": "submit SMS", "kind": "call" }, { "from": "f-a", "to": "f-disp", "label": "NACK · throttled", "kind": "return" }, { "from": "f-disp", "to": "f-led", "label": "attempt 1 · failed", "kind": "async" }, { "from": "f-disp", "to": "f-b", "label": "same SMS · route B", "kind": "call" }, { "from": "f-b", "to": "f-disp", "label": "accepted · msg id", "kind": "return" }, { "from": "f-disp", "to": "f-led", "label": "attempt 2 · accepted", "kind": "async" }, { "from": "f-disp", "to": "f-disp", "label": "no receipt in 20 s", "kind": "self" }, { "from": "f-disp", "to": "f-voice", "label": "call now, not at 2 min", "kind": "call" }, { "from": "f-voice", "to": "f-resp", "label": "rings · reads summary", "kind": "call" }, { "from": "f-resp", "to": "f-voice", "label": "presses 1", "kind": "return" }, { "from": "f-voice", "to": "f-disp", "label": "acknowledged", "kind": "return" }, { "from": "f-disp", "to": "f-led", "label": "ack · step closed", "kind": "async" } ], "note": "A hard failure retries the same channel on the other route. Silence moves to a different channel, so a slow route never produces a second SMS inside 60 s.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "21-alert-storm", "title": "Alert Storm — 20,000 Alerts a Second for 120 Seconds", "layout": "swimlane", "laneHeaderWidth": 160, "stages": [ "T0 · burst", "T + 5 s", "T + 60 s", "T + 120 s", "After" ], "lanes": [ { "title": "Edge", "cells": [ [ { "label": "Noisy source over quota", "kind": "decision", "icon": "fa5_tachometer_alt" } ], [ { "label": "429 to that source only", "kind": "security", "icon": "fa5_tachometer_alt" } ], [ { "label": "Others admitted", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Source retries drain", "kind": "integration", "icon": "fa5_tachometer_alt" } ], [ { "label": "Shed count reported", "kind": "store", "icon": "fa5_chart_bar" } ] ] }, { "title": "Ingest domain", "cells": [ [ { "label": "Burst buffered, R3", "kind": "queue", "icon": "fa5_tachometer_alt" } ], [ { "label": "Consumer lag rises", "kind": "decision", "icon": "fa5_tachometer_alt" } ], [ { "label": "Lag bounded, alarmed", "kind": "platform", "icon": "fa5_tachometer_alt" } ], [ { "label": "Drained", "kind": "queue", "icon": "fa5_tachometer_alt" } ], [] ] }, { "title": "Incident engine", "cells": [ [ { "label": "One grouping key", "kind": "app", "icon": "fa5_layer_group" } ], [ { "label": "One incident opened", "kind": "app", "icon": "fa5_layer_group" } ], [ { "label": "Flood control attaches", "kind": "app", "icon": "fa5_tachometer_alt" } ], [ { "label": "2.4 M counted, 0 dropped", "kind": "store", "icon": "fa5_layer_group" } ], [ { "label": "Noise report entry", "kind": "store", "icon": "fa5_chart_bar" } ] ] }, { "title": "Paging slice", "cells": [ [ { "label": "Reserved CPU untouched", "kind": "security", "icon": "fa5_microchip" } ], [ { "label": "One page dispatched", "kind": "app", "icon": "fa5_check_circle" } ], [ { "label": "Open incidents on time", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "No re-page", "kind": "opportunity", "icon": "fa5_check_circle" } ], [] ] }, { "title": "Responder", "cells": [ [], [ { "label": "Woken once", "kind": "app", "icon": "fa5_check_circle" } ], [ { "label": "Sees live alert count", "kind": "app" } ], [], [ { "label": "Tunes the rule", "kind": "app", "icon": "fa5_filter" } ] ] } ], "note": "Ingest and paging share hosts but not CPU: systemd slices cap the gateway and reserve the dispatcher, so a storm raises lag, never time to first notification.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "22-dependency-outages", "title": "Dependency Outages — What Still Works When Something Else Is Down", "layout": "grid", "laneHeaderWidth": 190, "columns": [ "Paging", "Acknowledgement", "Console", "Record" ], "rows": [ { "title": "Event log down", "cells": [ [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Read-only, stale", "kind": "decision", "icon": "fa5_database" } ], [ { "label": "Outbox holds 30 d", "kind": "security", "icon": "fa5_database" } ] ] }, { "title": "Control-plane cluster down", "cells": [ [ { "label": "Last snapshot · 72 h", "kind": "opportunity", "icon": "fa5_user_clock" } ], [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Cell status page only", "kind": "risk", "icon": "fa5_globe" } ], [ { "label": "Outbox holds 30 d", "kind": "security", "icon": "fa5_database" } ] ] }, { "title": "Corporate IdP down", "cells": [ [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Unaffected · no login", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Break-glass keys", "kind": "decision", "icon": "fa5_key" } ], [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ] ] }, { "title": "OpenBao down", "cells": [ [ { "label": "Keys held in memory", "kind": "decision", "icon": "fa5_key" } ], [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "No secret rotation", "kind": "decision", "icon": "fa5_key" } ], [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ] ] }, { "title": "Mattermost down", "cells": [ [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Channel creation queued", "kind": "decision", "icon": "fa5_comments" } ], [ { "label": "Unaffected", "kind": "opportunity", "icon": "fa5_check_circle" } ] ] }, { "title": "Site A lost", "cells": [ [ { "label": "Cells B and C · quorum", "kind": "opportunity", "icon": "fa5_network_wired" } ], [ { "label": "Anycast moves ≤ 2 min", "kind": "decision", "icon": "fa5_network_wired" } ], [ { "label": "Rebuilt at B ≤ 15 min", "kind": "risk", "icon": "fa5_database" } ], [ { "label": "Sync standby · RPO 0", "kind": "security", "icon": "fa5_database" } ] ] } ], "note": "The degradation order is the requirement's: console first, then reviews, then analytics. No row reaches the paging column in red.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "23-minority-cell-still-pages", "title": "Partition — A Cell Without Quorum Still Pages", "layout": "sequence", "lifelines": [ { "id": "m-probe", "label": "Outpost synthetic probe", "kind": "platform", "icon": "fa5_heartbeat" }, { "id": "m-eng", "label": "Cell C engine", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "m-js", "label": "Paging domain", "kind": "queue", "icon": "fa5_stream" }, { "id": "m-disp", "label": "Cell C dispatcher", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "m-carr", "label": "Carrier B", "kind": "external", "icon": "fa5_broadcast_tower" }, { "id": "m-resp", "label": "Responder", "kind": "actor", "icon": "fa5_user_clock" }, { "id": "m-rec", "label": "Reconciler", "kind": "app", "icon": "fa5_equals" } ], "messages": [ { "from": "m-probe", "to": "m-eng", "label": "estate checks failing", "kind": "call" }, { "from": "m-eng", "to": "m-js", "label": "CAS open incident", "kind": "call" }, { "from": "m-js", "to": "m-eng", "label": "no quorum · 1 of 3", "kind": "return" }, { "from": "m-eng", "to": "m-eng", "label": "minority mode · local log", "kind": "self" }, { "from": "m-eng", "to": "m-disp", "label": "page from last snapshot", "kind": "call" }, { "from": "m-disp", "to": "m-carr", "label": "SMS and voice · route B", "kind": "call" }, { "from": "m-carr", "to": "m-resp", "label": "rings", "kind": "call" }, { "from": "m-resp", "to": "m-disp", "label": "ack by SMS reply", "kind": "return" }, { "from": "m-js", "to": "m-eng", "label": "quorum restored", "kind": "async" }, { "from": "m-eng", "to": "m-rec", "label": "replay minority log", "kind": "async" }, { "from": "m-rec", "to": "m-rec", "label": "mark duplicates · keep both", "kind": "self" } ], "note": "Quorum decides what the record says, never whether a human is paged. The cost is a possible duplicate page, which the reconciler labels rather than hides.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "24-dst-boundary", "title": "A DST Boundary — Found Nine Days Early, Not at 02:30", "layout": "swimlane", "laneHeaderWidth": 160, "stages": [ "T − 14 d", "T − 9 d", "T − 72 h", "02:00 local", "After" ], "lanes": [ { "title": "Resolver", "cells": [ [ { "label": "Handoff at 02:30 local", "kind": "app", "icon": "fa5_moon" } ], [ { "label": "Instant does not exist", "kind": "risk", "icon": "fa5_moon" } ], [ { "label": "Buckets materialised", "kind": "app", "icon": "fa5_user_clock" } ], [], [] ] }, { "title": "Verifier", "cells": [ [], [ { "label": "Second resolver disagrees", "kind": "decision", "icon": "fa5_balance_scale" } ], [ { "label": "Both agree after fix", "kind": "decision", "icon": "fa5_balance_scale" } ], [], [] ] }, { "title": "Rotation owner", "cells": [ [], [ { "label": "Gap warning received", "kind": "actor", "icon": "fa5_calendar_alt" } ], [ { "label": "Override 01:00–04:00", "kind": "actor", "icon": "fa5_moon" } ], [ { "label": "Asleep", "kind": "actor", "icon": "fa5_moon" } ], [] ] }, { "title": "Snapshot", "cells": [ [], [ { "label": "Old buckets held", "kind": "store", "icon": "fa5_user_clock" } ], [ { "label": "Verified buckets live", "kind": "store", "icon": "fa5_user_clock" } ], [ { "label": "Served unchanged", "kind": "store" } ], [] ] }, { "title": "Paging", "cells": [ [], [], [], [ { "label": "Page lands on override", "kind": "opportunity", "icon": "fa5_calendar_alt" } ], [ { "label": "Audit names who", "kind": "store" } ] ] } ], "note": "Nothing about the clock change is computed on the night. The resolver's rule for a missing local time is to hand over at the next valid instant, and the verifier checks it did.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "25-escalation-exhausted", "title": "Escalation Exhausted — The Worst Outcome Is Loud", "layout": "sequence", "lifelines": [ { "id": "x-tim", "label": "Escalation timers", "kind": "app", "icon": "fa5_stopwatch" }, { "id": "x-disp", "label": "Dispatcher", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "x-p", "label": "Primary", "kind": "actor", "icon": "fa5_user_clock" }, { "id": "x-s", "label": "Secondary", "kind": "actor", "icon": "fa5_user_clock" }, { "id": "x-m", "label": "Management layer", "kind": "actor", "icon": "fa5_user_tie" }, { "id": "x-team", "label": "Owning team", "kind": "actor", "icon": "fa5_users" }, { "id": "x-plat", "label": "Platform rotation", "kind": "actor", "icon": "fa5_users_cog" } ], "messages": [ { "from": "x-tim", "to": "x-disp", "label": "step 1 · 5 min", "kind": "async" }, { "from": "x-disp", "to": "x-p", "label": "push · SMS · voice", "kind": "call" }, { "from": "x-tim", "to": "x-disp", "label": "no ack · step 2", "kind": "async" }, { "from": "x-disp", "to": "x-s", "label": "push · SMS · voice", "kind": "call" }, { "from": "x-disp", "to": "x-p", "label": "primary keeps ringing", "kind": "call" }, { "from": "x-tim", "to": "x-disp", "label": "no ack · final step", "kind": "async" }, { "from": "x-disp", "to": "x-m", "label": "voice · every channel", "kind": "call" }, { "from": "x-disp", "to": "x-team", "label": "broadcast to whole team", "kind": "call" }, { "from": "x-tim", "to": "x-tim", "label": "state: exhausted", "kind": "self" }, { "from": "x-disp", "to": "x-plat", "label": "platform SEV2 · route B", "kind": "call" }, { "from": "x-tim", "to": "x-disp", "label": "repeat broadcast every 5 min", "kind": "async" } ], "note": "Exhausted is a state, not an ending. It stays open, keeps broadcasting, and opens an incident against the platform, because either the policy or the rota is wrong.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "26-deployment-architecture", "title": "Deployment — Two Estate Sites and an Outpost That Shares Neither", "layout": "nested", "boxes": [ { "title": "Three locations, three power feeds, two upstream carriers", "kind": "plain", "dir": "row", "children": [ { "title": "Site A · data centre", "kind": "onprem", "dir": "col", "children": [ { "title": "Paging cell A · 3 hosts", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-na", "label": "NATS JetStream", "sub": "ingest R3 · paging voter", "kind": "queue", "icon": "fa5_stream" }, { "id": "y-pa", "label": "Paging services", "sub": "engine · timers · dispatch", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "y-da", "label": "Asterisk · Jasmin", "sub": "carrier A", "kind": "integration", "icon": "fa5_phone_volume" } ] }, { "title": "Control plane · platform Kubernetes", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-ca", "label": "Control services", "sub": "incident · schedule · review", "kind": "app", "icon": "fa5_fire_alt" }, { "id": "y-pga", "label": "PostgreSQL primary", "sub": "CloudNativePG", "kind": "store", "icon": "postgresql" } ] } ] }, { "title": "Site B · data centre", "kind": "onprem", "dir": "col", "children": [ { "title": "Paging cell B · 3 hosts", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-nb", "label": "NATS JetStream", "sub": "ingest R3 · paging voter", "kind": "queue", "icon": "fa5_stream" }, { "id": "y-pb", "label": "Paging services", "sub": "active-active with A", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "y-db", "label": "Asterisk · Jasmin", "sub": "carrier A", "kind": "integration", "icon": "fa5_phone_volume" } ] }, { "title": "Warm control plane", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-gb", "label": "Argo CD manifests", "sub": "scaled to zero", "kind": "app", "icon": "argocd" }, { "id": "y-pgb", "label": "PostgreSQL standby", "sub": "synchronous", "kind": "store", "icon": "postgresql" } ] } ] }, { "title": "Site C · colocation outpost", "kind": "onprem", "dir": "col", "children": [ { "title": "Paging cell C · 3 hosts", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-nc", "label": "NATS JetStream", "sub": "paging voter", "kind": "queue", "icon": "fa5_stream" }, { "id": "y-pc", "label": "Paging services", "sub": "minority-capable", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "y-dc", "label": "Asterisk · Jasmin", "sub": "carrier B · GSM bank", "kind": "integration", "icon": "fa5_phone_volume" } ] }, { "title": "Outpost-only duties", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-syn", "label": "Synthetic and watchdog", "sub": "dead-man switch", "kind": "platform", "icon": "fa5_heartbeat" }, { "id": "y-stat", "label": "Status page", "sub": "nginx · static", "kind": "app", "icon": "fa5_globe" } ] } ] } ] } ], "note": "The estate runs in A and B. C runs nothing the estate needs and needs nothing the estate runs: own ISP, own DNS resolvers, own time source.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "27-release-pipeline", "title": "Changing the Paging Path — One Cell at a Time, Never During a SEV1", "layout": "flow", "chain": true, "stages": [ { "title": "Source", "nodes": [ { "id": "w-git", "label": "Forgejo", "sub": "signed commits · 2 reviewers", "kind": "store", "icon": "git" }, { "id": "w-ci", "label": "Forgejo Actions", "sub": "static Go binaries", "kind": "app", "icon": "fa5_cogs" } ] }, { "title": "Prove", "nodes": [ { "id": "w-dst", "label": "Schedule tests", "sub": "every DST to 2030", "kind": "decision", "icon": "fa5_calendar_check" }, { "id": "w-replay", "label": "Storm replay", "sub": "recorded 20k/s burst", "kind": "decision", "icon": "fa5_history" } ] }, { "title": "Sign", "nodes": [ { "id": "w-cos", "label": "cosign · SBOM", "sub": "Harbor OCI artefact", "kind": "security", "icon": "fa5_file_signature" }, { "id": "w-stage", "label": "Staging cells", "sub": "synthetic pages", "kind": "app", "icon": "fa5_server" } ] }, { "title": "Gate", "nodes": [ { "id": "w-freeze", "label": "Change freeze check", "sub": "no open SEV1 or SEV2", "kind": "decision", "icon": "fa5_ban" }, { "id": "w-time", "label": "Business hours", "sub": "both regions awake", "kind": "decision", "icon": "fa5_business_time" } ] }, { "title": "Cell C", "nodes": [ { "id": "w-c", "label": "Ansible · cell C", "sub": "host by host", "kind": "app", "icon": "ansible" }, { "id": "w-cs", "label": "Synthetic gate", "sub": "3 pages acked", "kind": "decision", "icon": "fa5_traffic_light" } ] }, { "title": "Cells B then A", "nodes": [ { "id": "w-b", "label": "Ansible · cell B", "sub": "next day", "kind": "app", "icon": "ansible" }, { "id": "w-a", "label": "Ansible · cell A", "sub": "day after", "kind": "app", "icon": "ansible" } ] } ], "edges": [ { "from": "w-cs", "to": "w-c", "label": "fail", "kind": "error" } ], "note": "Binaries and configuration are staged on every host before a switch, so a deploy needs no registry, DNS or Git reachable at the moment it happens.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "28-observability", "title": "Observability — Watching the Watcher From Outside", "layout": "grid", "laneHeaderWidth": 170, "columns": [ "Ingest", "Decide", "Dispatch", "Acknowledge", "Record" ], "rows": [ { "title": "Metrics", "cells": [ [ { "label": "Accepted · shed", "kind": "platform", "icon": "fa5_tachometer_alt" } ], [ { "label": "Suppression rate", "kind": "platform", "icon": "fa5_stopwatch" } ], [ { "label": "Receipt latency", "kind": "platform", "icon": "fa5_stopwatch" } ], [ { "label": "Time to ack", "kind": "platform", "icon": "fa5_bell" } ], [ { "label": "Reconcile lag", "kind": "platform", "icon": "fa5_tachometer_alt" } ] ] }, { "title": "Freshness", "cells": [ [ { "label": "Consumer lag", "kind": "decision", "icon": "fa5_tachometer_alt" } ], [ { "label": "Snapshot age", "kind": "decision", "icon": "fa5_user_clock" } ], [ { "label": "Timer drift p99", "kind": "decision", "icon": "fa5_stopwatch" } ], [ { "label": "Cancel latency p99", "kind": "decision", "icon": "fa5_stopwatch" } ], [ { "label": "Projection age", "kind": "decision", "icon": "fa5_database" } ] ] }, { "title": "Synthetic", "cells": [ [ { "label": "Test alert · 60 s", "kind": "app", "icon": "fa5_heartbeat" } ], [ { "label": "Test incident opened", "kind": "app", "icon": "fa5_layer_group" } ], [ { "label": "SMS and call to own SIM", "kind": "app", "icon": "fa5_phone_volume" } ], [ { "label": "Robot presses 1", "kind": "app", "icon": "fa5_phone_volume" } ], [ { "label": "Event appears in log", "kind": "app", "icon": "fa5_database" } ] ] }, { "title": "Outside view", "cells": [ [], [], [ { "label": "Carrier B only", "kind": "security", "icon": "fa5_network_wired" } ], [ { "label": "Dead-man switch", "kind": "security", "icon": "fa5_heartbeat" } ], [] ] } ], "note": "The platform's own alerts are paged through itself and, independently, by the outpost watchdog on carrier B. A silent platform is the alert.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "29-cost-and-human-load", "title": "Cost and Human Load — What Gets Measured, Charged and Reported", "layout": "grid", "laneHeaderWidth": 180, "columns": [ "Measured from", "Attributed to", "Reported", "Lever" ], "rows": [ { "title": "Voice minutes", "cells": [ [ { "label": "Ledger · carrier rate card", "kind": "store", "icon": "fa5_receipt" } ], [ { "label": "Service · rotation", "kind": "app", "icon": "fa5_calendar_alt" } ], [ { "label": "Weekly · with noise", "kind": "platform", "icon": "fa5_chart_bar" } ], [ { "label": "Push first below SEV2", "kind": "opportunity", "icon": "fa5_mobile_alt" } ] ] }, { "title": "SMS", "cells": [ [ { "label": "Ledger · per segment", "kind": "store", "icon": "fa5_receipt" } ], [ { "label": "Service · rotation", "kind": "app", "icon": "fa5_calendar_alt" } ], [ { "label": "Weekly · with noise", "kind": "platform", "icon": "fa5_chart_bar" } ], [ { "label": "Fix the noisiest rule", "kind": "opportunity", "icon": "fa5_filter" } ] ] }, { "title": "Outpost standby", "cells": [ [ { "label": "Colocation · ISP · trunks", "kind": "store", "icon": "fa5_coins" } ], [ { "label": "Platform budget", "kind": "app", "icon": "fa5_coins" } ], [ { "label": "Yearly · stated up front", "kind": "platform", "icon": "fa5_chart_bar" } ], [ { "label": "Insurance, not waste", "kind": "decision", "icon": "fa5_coins" } ] ] }, { "title": "Synthetic traffic", "cells": [ [ { "label": "Ledger · test target", "kind": "store", "icon": "fa5_receipt" } ], [ { "label": "Platform budget", "kind": "app", "icon": "fa5_coins" } ], [ { "label": "Monthly", "kind": "platform", "icon": "fa5_chart_bar" } ], [ { "label": "Interval per channel", "kind": "decision", "icon": "fa5_comments" } ] ] }, { "title": "Out-of-hours pages", "cells": [ [ { "label": "Ledger · responder zone", "kind": "store", "icon": "fa5_receipt" } ], [ { "label": "Responder · rotation", "kind": "app", "icon": "fa5_calendar_alt" } ], [ { "label": "Weekly · target ≤ 2", "kind": "platform", "icon": "fa5_chart_bar" } ], [ { "label": "Rotation size · rules", "kind": "opportunity", "icon": "fa5_calendar_alt" } ] ] }, { "title": "Actionable rate", "cells": [ [ { "label": "Ack then change or note", "kind": "store", "icon": "fa5_bell" } ], [ { "label": "Service · alert rule", "kind": "app" } ], [ { "label": "Weekly · target ≥ 70%", "kind": "platform", "icon": "fa5_chart_bar" } ], [ { "label": "Demote or delete rule", "kind": "opportunity", "icon": "fa5_filter" } ] ] } ], "note": "Human cost is measured and reported, never enforced by suppression. A rotation over budget gets a report, not fewer pages.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "30-roadmap", "title": "Roadmap — What Ships First, and Why in That Order", "layout": "flow", "chain": true, "stages": [ { "title": "Prove · 8 weeks", "nodes": [ { "id": "rm-p1", "label": "Stretched JetStream", "sub": "3-site latency", "kind": "decision", "icon": "fa5_stream" }, { "id": "rm-p2", "label": "Carrier routes", "sub": "receipts · DTMF", "kind": "decision", "icon": "fa5_broadcast_tower" }, { "id": "rm-p3", "label": "Critical alerts", "sub": "iOS entitlement", "kind": "risk", "icon": "fa5_bell" } ] }, { "title": "MVP", "nodes": [ { "id": "rm-m1", "label": "Ingest · dedup · raw", "sub": "webhooks", "kind": "app", "icon": "fa5_door_open" }, { "id": "rm-m2", "label": "Schedules · snapshot", "sub": "gaps · DST verifier", "kind": "app", "icon": "fa5_calendar_alt" }, { "id": "rm-m3", "label": "Escalation · 4 channels", "sub": "two providers · acks", "kind": "app", "icon": "fa5_sort_amount_down" }, { "id": "rm-m4", "label": "Event log · synthetic", "sub": "break-glass · APIs", "kind": "app", "icon": "postgresql" } ] }, { "title": "Phase 2", "nodes": [ { "id": "rm-2a", "label": "Grouping · merge · split", "sub": "maintenance · flood", "kind": "app", "icon": "fa5_layer_group" }, { "id": "rm-2b", "label": "Channels · roles", "sub": "stakeholder updates", "kind": "app", "icon": "fa5_comments" }, { "id": "rm-2c", "label": "Reviews · tracker sync", "sub": "computed intervals", "kind": "app", "icon": "fa5_clipboard_list" }, { "id": "rm-2d", "label": "Outbound stream · cost", "sub": "status page", "kind": "app", "icon": "fa5_stream" } ] }, { "title": "Phase 3", "nodes": [ { "id": "rm-3a", "label": "Dependency hints", "sub": "annotate, never withhold", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "rm-3b", "label": "Interrupt analytics", "sub": "rotation fairness", "kind": "app", "icon": "fa5_chart_bar" }, { "id": "rm-3c", "label": "Regional failover drills", "sub": "automated", "kind": "platform", "icon": "fa5_redo" }, { "id": "rm-3d", "label": "Sensitive incidents", "sub": "factor search · fallback kit", "kind": "security", "icon": "fa5_user_lock" } ] } ], "note": "The MVP already runs across three sites. Independence is not a Phase 3 feature, because a paging path that shares the estate's failure is not an MVP of this platform.", "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "31-incident-learning-loop", "title": "The Loop the Platform Exists to Close", "layout": "cycle", "centre": { "label": "Incident", "icon": "fa5_bell" }, "nodes": [ { "id": "o1", "label": "Signal", "sub": "alert or a person", "kind": "external", "icon": "fa5_signal" }, { "id": "o2", "label": "Page", "sub": "right human", "kind": "app", "icon": "fa5_satellite_dish" }, { "id": "o3", "label": "Acknowledge", "sub": "one keypress", "kind": "app", "icon": "fa5_check_double" }, { "id": "o4", "label": "Mitigate", "sub": "impact stops", "kind": "app", "icon": "fa5_band_aid" }, { "id": "o5", "label": "Review", "sub": "conditions, not people", "kind": "decision", "icon": "fa5_clipboard_list" }, { "id": "o6", "label": "Action items", "sub": "closed with change", "kind": "store", "icon": "fa5_tasks" }, { "id": "o7", "label": "Alert hygiene", "sub": "noise · cost report", "kind": "platform", "icon": "fa5_filter" } ], "ringLabels": [ "decided", "reached", "coordinated", "resolved", "tracked", "tuned", "fewer, truer" ], "rx": 430, "ry": 215, "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } }, { "id": "32-security-trust-zones", "title": "Security — Trust Zones and What Crosses Them", "layout": "zones", "zones": [ { "title": "Untrusted senders", "kind": "trust", "nodes": [ { "id": "z-int", "label": "Integrations", "sub": "one credential each", "kind": "external", "icon": "fa5_plug" }, { "id": "z-phone", "label": "Responder devices", "sub": "app · SMS · voice", "kind": "external", "icon": "fa5_mobile_alt" } ] }, { "title": "Edge", "kind": "trust", "nodes": [ { "id": "z-gw", "label": "Ingest gateway", "sub": "verify before parse", "kind": "security", "icon": "fa5_door_open" }, { "id": "z-ack", "label": "Ack receiver", "sub": "signed, single-use", "kind": "security", "icon": "fa5_check_double" } ] }, { "title": "Paging cells", "kind": "trust", "nodes": [ { "id": "z-eng", "label": "Incident engine", "sub": "mTLS between cells", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "z-disp", "label": "Dispatcher", "sub": "decrypts at send", "kind": "app", "icon": "fa5_satellite_dish" } ] }, { "title": "Key custody", "kind": "trust", "nodes": [ { "id": "z-bao", "label": "OpenBao", "sub": "secrets · transit", "kind": "security", "icon": "fa5_key" }, { "id": "z-keys", "label": "Responder keys", "sub": "deleted on offboarding", "kind": "security", "icon": "fa5_key" } ] }, { "title": "Control plane", "kind": "trust", "nodes": [ { "id": "z-con", "label": "Console", "sub": "team-scoped RBAC", "kind": "app", "icon": "fa5_desktop" }, { "id": "z-pg", "label": "Event log", "sub": "row-level security", "kind": "store", "icon": "postgresql" } ] } ], "edges": [ { "from": "z-int", "to": "z-gw", "label": "HMAC · TLS" }, { "from": "z-phone", "to": "z-ack", "label": "ack token" }, { "from": "z-gw", "to": "z-eng", "label": "normalised", "kind": "async" }, { "from": "z-disp", "to": "z-keys", "label": "unwrap data key" }, { "from": "z-bao", "to": "z-con", "label": "short-lived creds", "kind": "async" } ], "note": "An integration's credential can only submit alerts for itself. A leaked ack token stops one escalation once; it cannot read an incident.", "meta": { "v": "1.0", "owner": "Security Architecture", "date": "2026-09" } }, { "id": "33-break-glass-login", "title": "Identity — Signing In When the Identity Provider Is the Outage", "layout": "sequence", "lifelines": [ { "id": "b-ic", "label": "Incident commander", "kind": "actor", "icon": "fa5_user_tie" }, { "id": "b-con", "label": "Console", "kind": "app", "icon": "fa5_desktop" }, { "id": "b-kc", "label": "Keycloak", "kind": "security", "icon": "keycloak" }, { "id": "b-idp", "label": "Corporate IdP", "kind": "external", "icon": "fa5_id_card" }, { "id": "b-key", "label": "Hardware key", "kind": "security", "icon": "fa5_fingerprint" }, { "id": "b-aud", "label": "Audit events", "kind": "store", "icon": "fa5_history" }, { "id": "b-sec", "label": "Security rotation", "kind": "actor", "icon": "fa5_user_shield" } ], "messages": [ { "from": "b-ic", "to": "b-con", "label": "open console", "kind": "call" }, { "from": "b-con", "to": "b-kc", "label": "OIDC redirect", "kind": "call" }, { "from": "b-kc", "to": "b-idp", "label": "federated login", "kind": "call" }, { "from": "b-idp", "to": "b-kc", "label": "timeout", "kind": "return" }, { "from": "b-kc", "to": "b-ic", "label": "offer break-glass", "kind": "return" }, { "from": "b-ic", "to": "b-kc", "label": "local account", "kind": "call" }, { "from": "b-kc", "to": "b-key", "label": "WebAuthn challenge", "kind": "call" }, { "from": "b-key", "to": "b-kc", "label": "signed assertion", "kind": "return" }, { "from": "b-kc", "to": "b-con", "label": "4 h session · responder role", "kind": "return" }, { "from": "b-kc", "to": "b-aud", "label": "break-glass used", "kind": "async" }, { "from": "b-aud", "to": "b-sec", "label": "notify, never page", "kind": "async" } ], "note": "Break-glass accounts can acknowledge, resolve and edit incidents. They cannot edit schedules, policies or suppression rules, which can wait for the IdP.", "meta": { "v": "1.0", "owner": "Security Architecture", "date": "2026-09" } }, { "id": "34-failure-modes", "title": "Failure Modes — Detection, Containment, What Responders See", "layout": "grid", "laneHeaderWidth": 190, "columns": [ "Detected by", "Contained by", "Responders see", "Back to normal" ], "rows": [ { "title": "Carrier degraded", "cells": [ [ { "label": "Loopback SMS · call", "kind": "platform", "icon": "fa5_phone_volume" } ], [ { "label": "Route B within step", "kind": "security", "icon": "fa5_network_wired" } ], [ { "label": "Page on time", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Route health recovers", "kind": "app", "icon": "fa5_network_wired" } ] ] }, { "title": "Alert storm", "cells": [ [ { "label": "Shed and lag metrics", "kind": "platform", "icon": "fa5_tachometer_alt" } ], [ { "label": "Quota · flood control", "kind": "security", "icon": "fa5_tachometer_alt" } ], [ { "label": "One page", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Noise report", "kind": "app", "icon": "fa5_chart_bar" } ] ] }, { "title": "Coverage gap", "cells": [ [ { "label": "14-day gap detector", "kind": "platform", "icon": "fa5_calendar_alt" } ], [ { "label": "Fall to final step", "kind": "security", "icon": "fa5_bell" } ], [ { "label": "Manager paged at once", "kind": "decision", "icon": "fa5_bell" } ], [ { "label": "Override added", "kind": "app", "icon": "fa5_calendar_alt" } ] ] }, { "title": "Unreachable human", "cells": [ [ { "label": "No ack in step", "kind": "platform", "icon": "fa5_bell" } ], [ { "label": "Channels, then steps", "kind": "security", "icon": "fa5_comments" } ], [ { "label": "Secondary paged", "kind": "decision", "icon": "fa5_bell" } ], [ { "label": "Ack from anyone", "kind": "app", "icon": "fa5_bell" } ] ] }, { "title": "Control plane down", "cells": [ [ { "label": "Synthetic · reconcile lag", "kind": "platform", "icon": "fa5_tachometer_alt" } ], [ { "label": "Snapshot · outbox", "kind": "security", "icon": "fa5_user_clock" } ], [ { "label": "Degraded banner", "kind": "decision", "icon": "fa5_exclamation_circle" } ], [ { "label": "Rebuild ≤ 15 min", "kind": "app", "icon": "fa5_database" } ] ] }, { "title": "Estate sites lost", "cells": [ [ { "label": "Outpost probes", "kind": "platform", "icon": "fa5_heartbeat" } ], [ { "label": "Minority cell pages", "kind": "security", "icon": "fa5_network_wired" } ], [ { "label": "Possible duplicate", "kind": "decision", "icon": "fa5_network_wired" } ], [ { "label": "Replay · labelled", "kind": "app", "icon": "fa5_network_wired" } ] ] }, { "title": "Poison alert", "cells": [ [ { "label": "Schema check", "kind": "platform", "icon": "fa5_ban" } ], [ { "label": "Quarantine", "kind": "security", "icon": "fa5_ban" } ], [ { "label": "Nothing", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Mapping fixed", "kind": "app", "icon": "fa5_ban" } ] ] }, { "title": "Replayed webhook", "cells": [ [ { "label": "Source key · fingerprint", "kind": "platform", "icon": "fa5_layer_group" } ], [ { "label": "Dedup · grouping CAS", "kind": "security", "icon": "fa5_layer_group" } ], [ { "label": "Nothing", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Counted", "kind": "app", "icon": "fa5_layer_group" } ] ] }, { "title": "DST or zone error", "cells": [ [ { "label": "Second resolver", "kind": "platform", "icon": "fa5_balance_scale" } ], [ { "label": "Old buckets held", "kind": "security", "icon": "fa5_user_clock" } ], [ { "label": "Nothing", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Owner fixes rule", "kind": "app", "icon": "fa5_sitemap" } ] ] }, { "title": "Escalation exhausted", "cells": [ [ { "label": "Terminal state", "kind": "platform", "icon": "fa5_bell" } ], [ { "label": "Broadcast every 5 min", "kind": "security", "icon": "fa5_bell" } ], [ { "label": "Whole team paged", "kind": "risk", "icon": "fa5_bell" } ], [ { "label": "Platform review", "kind": "app", "icon": "fa5_clipboard_list" } ] ] } ], "meta": { "v": "1.0", "owner": "Reliability Architecture", "date": "2026-09" } } ]