[ { "id": "01-system-context", "title": "Change Data Capture Pipeline — System Context", "layout": "context", "colWidth": 260, "system": { "label": "CDC Pipeline", "sub": "commit to sink ≤ 5 s p95" }, "groups": [ { "side": "top", "title": "Systems of record (out of scope)", "nodes": [ { "id": "pg", "label": "Operational PostgreSQL", "sub": "12 DBs · 180 tables", "kind": "external", "rel": "WAL decode", "dir": "in" }, { "id": "replica", "label": "Read replica", "sub": "snapshot source", "kind": "external", "rel": "snapshot read", "dir": "in" } ] }, { "side": "left", "title": "People", "nodes": [ { "id": "de", "label": "Data engineer", "kind": "actor", "rel": "registers sinks", "dir": "in" }, { "id": "pe", "label": "Product engineer", "kind": "actor", "rel": "ships migrations", "dir": "in" }, { "id": "sre", "label": "Platform SRE", "kind": "actor", "rel": "pauses, replays", "dir": "in" }, { "id": "analyst", "label": "Analyst", "kind": "actor", "rel": "queries", "dir": "in" } ] }, { "side": "right", "title": "Sinks the pipeline writes", "nodes": [ { "id": "bq", "label": "BigQuery", "sub": "mirror + changelog", "kind": "store", "rel": "upsert" }, { "id": "search", "label": "Search index", "sub": "product search", "kind": "external", "rel": "index upsert" }, { "id": "cache", "label": "Cache and webhooks", "sub": "fan-out", "kind": "external", "rel": "invalidations", "kind2": "async" } ] }, { "side": "bottom", "title": "Platform dependencies", "nodes": [ { "id": "iam", "label": "Cloud IAM", "sub": "workload identity", "kind": "security", "rel": "authorises" }, { "id": "sm", "label": "Secret Manager", "sub": "90-day rotation", "kind": "security", "rel": "credentials" }, { "id": "gcs", "label": "Cloud Storage", "sub": "13-month archive", "kind": "store", "rel": "archive", "kind2": "batch" }, { "id": "mon", "label": "Cloud Monitoring", "sub": "lag and slot SLOs", "kind": "platform", "rel": "lag, alarms" } ] } ], "note": "The pipeline owns no business logic: it owns the seam between one writer and many readers.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "02-high-level-architecture", "title": "Change Data Capture Pipeline — High-Level Architecture", "layout": "flow", "chain": true, "align": "middle", "stages": [ { "title": "Sources", "nodes": [ { "id": "pg", "label": "PostgreSQL primary", "sub": "WAL · 1 slot per DB", "kind": "external" }, { "id": "hb", "label": "Heartbeat writer", "sub": "every 10 s", "kind": "platform" } ] }, { "title": "Capture plane", "nodes": [ { "id": "cap", "label": "Datastream capture", "sub": "logical decoding", "kind": "integration" }, { "id": "snap", "label": "Snapshot chunker", "sub": "PK ranges", "kind": "app" }, { "id": "mask", "label": "Column masker", "sub": "at capture", "kind": "security" } ] }, { "title": "Change log", "nodes": [ { "id": "log", "label": "Pub/Sub change log", "sub": "7 days · key-hash", "kind": "queue" }, { "id": "dlq", "label": "Dead-letter topic", "sub": "poison events", "kind": "queue" } ] }, { "title": "Projection plane", "nodes": [ { "id": "apply", "label": "Dataflow appliers", "sub": "one job per sink", "kind": "app" }, { "id": "reg", "label": "Schema registry", "sub": "Spanner", "kind": "store" } ] }, { "title": "Sinks", "nodes": [ { "id": "bq", "label": "BigQuery", "sub": "mirror + changelog", "kind": "store" }, { "id": "idx", "label": "Search index", "kind": "external" }, { "id": "inval", "label": "Cache invalidation", "kind": "queue" } ] }, { "title": "Consumers", "nodes": [ { "id": "dash", "label": "Dashboards", "kind": "external" }, { "id": "prod", "label": "Product search", "kind": "external" }, { "id": "hooks", "label": "Partner webhooks", "kind": "external" } ] } ], "edges": [ { "from": "log", "to": "dlq", "label": "retries spent", "kind": "error" }, { "from": "reg", "to": "apply", "label": "schema version", "kind": "sync" } ], "note": "Every sink reads the same log at its own offset; no sink can slow another, or the source.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "03-actors-and-journeys", "title": "Who the Pipeline Is For, and What They Get to Do", "layout": "actors", "cardWidth": 270, "groups": [ { "title": "Product engineering", "kind": "boundary", "actors": [ { "id": "pe", "label": "Product engineer", "sub": "40 engineers", "goal": "Ship my migration on Tuesday without being the person who broke analytics.", "journeys": [ { "id": "j-migration", "label": "Ship a schema migration", "sub": "see view 05" }, { "label": "Add a table to replication" } ] }, { "id": "po", "label": "Service owner", "sub": "9 services", "goal": "Know that nothing downstream can make my database slow down.", "journeys": [ { "label": "Read the source-impact budget" } ] } ] }, { "title": "Data platform", "kind": "boundary", "actors": [ { "id": "de", "label": "Data engineer", "sub": "6 engineers", "goal": "Point a new consumer at data that is already correct, in a day, not a quarter.", "journeys": [ { "id": "j-sink", "label": "Wire up a new sink", "sub": "see view 04" }, { "label": "Rebuild a sink from the log" } ] }, { "id": "sre", "label": "Platform SRE", "sub": "on call, 24×7", "goal": "Know within a minute whether lag is a stall, a burst or a wrong number.", "journeys": [ { "id": "j-wrong", "label": "Chase a wrong number", "sub": "see view 06" }, { "label": "Pause a table safely" } ] } ] }, { "title": "Consumers of the data", "kind": "cloud", "actors": [ { "id": "analyst", "label": "Revenue analyst", "sub": "120 people", "goal": "Trust today's number enough to put it in front of the board.", "journeys": [ { "label": "Read the mirror with an as-of" } ] }, { "id": "support", "label": "Support agent", "sub": "300 seats", "goal": "See the change the customer just made while they are still on the call.", "journeys": [ { "label": "Open a freshly updated record" } ] } ] }, { "title": "Machines in the cast", "kind": "cloud", "actors": [ { "id": "recon", "label": "Reconciliation job", "kind": "platform", "sub": "daily, tier-1 tables", "goal": "Find divergence before a human does.", "journeys": [ { "label": "Publish a divergence verdict" } ] }, { "id": "partner", "label": "Partner system", "kind": "external", "sub": "14 integrated", "goal": "Receive a change event once, in order, per record.", "journeys": [ { "label": "Consume a webhook fan-out" } ] } ] } ], "note": "Four different questions arrive at this platform; the set answers all four before it draws a box.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "04-journey-wire-up-a-new-sink", "title": "Journey — Data Engineer Wires Up a New Sink", "layout": "journey", "actor": { "label": "Data engineer", "sub": "6 on the team", "goal": "Give the new fraud service a current-state mirror of 9 tables", "trigger": "A product team asks for data that already exists somewhere", "success": "Sink registered, backfilled and inside its freshness target, with no source ticket" }, "phases": [ { "title": "Ask", "sub": "before any code" }, { "title": "Register" }, { "title": "Backfill", "moment": true }, { "title": "Catch up" }, { "title": "Operate" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Names the tables" }, { "label": "Declares freshness" } ], [ { "label": "Registers the sink" }, { "label": "Picks mirror or log" } ], [ { "label": "Starts the snapshot" } ], [ { "label": "Watches lag fall" } ], [ { "label": "Subscribes to alarms" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "Shows cost per table" } ], [ { "label": "Allocates an offset" }, { "label": "Creates sink schema" } ], [ { "label": "Chunks by PK range" }, { "label": "Throttles to source budget" } ], [ { "label": "Replays log over snapshot" } ], [ { "label": "Publishes freshness" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Fine", "Anxious" ], "points": [ 1, 1, 0, 2, 2 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [], [ { "label": "Backfill ETA unknown" }, { "label": "Fear of hurting the source" } ], [], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Per-table cost report" } ], [ { "label": "Self-service registry" } ], [ { "label": "Chunk progress and ETA" }, { "label": "Declared source budget" } ], [ { "label": "Lag graph per sink" } ], [ { "label": "Freshness as a contract" } ] ] } ], "chain": true, "note": "The trough is the backfill: an unbounded snapshot against a live primary is the moment this job is feared.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "05-journey-ship-a-schema-migration", "title": "Journey — Product Engineer Ships a Schema Migration", "layout": "journey", "actor": { "label": "Product engineer", "sub": "ships weekly", "goal": "Add a column and drop a dead one, on Tuesday, as usual", "trigger": "A feature needs a field; a deprecated field is finally unused", "success": "Migration applied, analytics unbroken, nobody paged" }, "phases": [ { "title": "Write", "sub": "in the repo" }, { "title": "Apply" }, { "title": "Propagate", "moment": true }, { "title": "Find out", "moment": true }, { "title": "Resolve" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Writes the migration" } ], [ { "label": "Applies to the primary" } ], [], [ { "label": "Reads the pause alert" } ], [ { "label": "Confirms the drop" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "Nothing — no notice yet" } ], [ { "label": "Decodes the DDL event" } ], [ { "label": "Adds nullable column" }, { "label": "Evolves sink schema" } ], [ { "label": "Pauses that one table" } ], [ { "label": "Resumes from position" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Fine", "Uneasy", "Blamed" ], "points": [ 0, 0, 1, 2, 1 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "No idea what replicates" } ], [], [], [ { "label": "Alert arrives after apply" } ], [ { "label": "Is a pause my outage?" } ] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Replication list in CI" } ], [ { "label": "Detection from the stream" } ], [ { "label": "Compatible: automatic" } ], [ { "label": "Blast radius: one table" } ], [ { "label": "Resume, not re-snapshot" } ] ] } ], "chain": true, "note": "Nobody tells the pipeline a migration is coming, so the design assumes it finds out from the stream.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "06-journey-chase-a-wrong-number", "title": "Journey — Chasing a Number That Looks Wrong", "layout": "journey", "actor": { "label": "Platform SRE", "sub": "with a revenue analyst", "goal": "Say within minutes whether the dashboard is stale, wrong or right", "trigger": "\"Yesterday's revenue dropped 8% and nothing happened\"", "success": "A named cause, a repair by replay, and a check that would have caught it" }, "phases": [ { "title": "Report", "sub": "a human notices" }, { "title": "Triage" }, { "title": "Diagnose", "moment": true }, { "title": "Repair", "moment": true }, { "title": "Prevent" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Analyst raises it" } ], [ { "label": "Checks lag and as-of" } ], [ { "label": "Reads reconciliation" }, { "label": "Compares to source" } ], [ { "label": "Replays to a shadow" } ], [ { "label": "Adds the table to tier 1" } ] ] }, { "title": "What the platform shows", "kind": "system", "cells": [ [ { "label": "Freshness per sink" } ], [ { "label": "Lag within target" } ], [ { "label": "Checksum divergence" }, { "label": "Lineage to a transform" } ], [ { "label": "Replay from position" } ], [ { "label": "Daily verdict" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Calm", "Tense", "Exposed" ], "points": [ 1, 1, 2, 1, 0 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "Stale or wrong?" } ], [], [ { "label": "No check on this table" } ], [], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "As-of on every read" } ], [ { "label": "Four distinct alarms" } ], [ { "label": "Sampled checksums" }, { "label": "Per-event lineage" } ], [ { "label": "Rebuild, not repair SQL" } ], [ { "label": "Coverage as the output" } ] ] } ], "chain": true, "note": "The trough is a table nobody reconciled: the fix is coverage, not a cleverer query.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "07-layered-architecture", "title": "Layered Architecture", "layout": "bands", "layerHeaderWidth": 160, "bands": [ { "name": "Consumers", "nodes": [ { "id": "dash", "label": "Dashboards", "kind": "external" }, { "id": "psearch", "label": "Product search", "kind": "external" }, { "id": "fraud", "label": "Fraud service", "kind": "external" }, { "id": "hooks", "label": "Partner webhooks", "kind": "external" } ] }, { "name": "Serving", "nodes": [ { "id": "mirror", "label": "Current-state mirror", "sub": "BigQuery", "kind": "store" }, { "id": "clog", "label": "Changelog tables", "sub": "append-only", "kind": "store" }, { "id": "idx", "label": "Search index", "kind": "store" }, { "id": "inval", "label": "Invalidation topic", "kind": "queue" } ] }, { "name": "Projection", "nodes": [ { "id": "bqapply", "label": "Warehouse applier", "sub": "micro-batch", "kind": "app" }, { "id": "idxapply", "label": "Search indexer", "kind": "app" }, { "id": "fanout", "label": "Fan-out worker", "kind": "app" }, { "id": "xform", "label": "Stateless transform", "sub": "per event", "kind": "app" } ] }, { "name": "Transport", "nodes": [ { "id": "log", "label": "Change log", "sub": "7-day · key-hash", "kind": "queue" }, { "id": "dlq", "label": "Dead-letter topic", "kind": "queue" }, { "id": "offs", "label": "Per-sink offsets", "kind": "store" } ] }, { "name": "Capture", "nodes": [ { "id": "reader", "label": "Log reader", "sub": "logical decoding", "kind": "integration" }, { "id": "chunk", "label": "Snapshot chunker", "kind": "app" }, { "id": "ddl", "label": "Relation detector", "sub": "schema drift", "kind": "app" }, { "id": "mask", "label": "Column masker", "kind": "security" } ] }, { "name": "Sources", "nodes": [ { "id": "pg", "label": "PostgreSQL primary", "sub": "WAL", "kind": "external" }, { "id": "rep", "label": "Read replica", "kind": "external" }, { "id": "hb", "label": "Heartbeat table", "sub": "10 s", "kind": "external" } ] }, { "name": "Control and ops", "accent": "purple", "nodes": [ { "id": "reg", "label": "Stream and sink registry", "kind": "platform" }, { "id": "sreg", "label": "Schema registry", "kind": "platform" }, { "id": "obs", "label": "Lag and reconciliation", "kind": "platform" }, { "id": "audit", "label": "Operator API and audit", "kind": "security" } ] } ], "edges": [ { "from": "pg", "to": "reader", "label": "WAL", "kind": "sync" }, { "from": "rep", "to": "chunk", "label": "chunked read", "kind": "batch" }, { "from": "reader", "to": "log", "label": "change events", "kind": "async" }, { "from": "log", "to": "xform", "label": "pull by offset", "kind": "async" }, { "from": "xform", "to": "bqapply", "kind": "sync" }, { "from": "bqapply", "to": "mirror", "label": "upsert", "kind": "sync" }, { "from": "idxapply", "to": "idx", "kind": "sync" }, { "from": "mirror", "to": "dash", "label": "SQL", "kind": "sync" }, { "from": "idx", "to": "psearch", "kind": "sync" } ], "note": "The only layer that may read the source is Capture, and it only ever reads.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "08-platform-components", "title": "Platform Components — Container View", "layout": "nested", "boxes": [ { "title": "Google Cloud · europe-west1", "kind": "cloud", "dir": "col", "children": [ { "title": "Capture plane — reads the source, never writes it", "kind": "boundary", "nodes": [ { "id": "reader", "label": "Log reader", "sub": "Datastream", "kind": "integration" }, { "id": "chunk", "label": "Snapshot chunker", "sub": "PK ranges", "kind": "app" }, { "id": "ddl", "label": "Relation detector", "sub": "schema drift", "kind": "app" }, { "id": "mask", "label": "Column masker", "sub": "before the log", "kind": "security" }, { "id": "pub", "label": "Event publisher", "sub": "normalise + key", "kind": "app" } ] }, { "title": "Transport — the system of record for change", "kind": "boundary", "nodes": [ { "id": "log", "label": "Change log", "sub": "Pub/Sub · 7 days", "kind": "queue" }, { "id": "dlq", "label": "Dead-letter topic", "kind": "queue" }, { "id": "arch", "label": "Changelog archive", "sub": "Cloud Storage", "kind": "store" } ] }, { "title": "Projection plane — one job per sink, one offset per sink", "kind": "boundary", "nodes": [ { "id": "bqapply", "label": "Warehouse applier", "sub": "Dataflow", "kind": "app" }, { "id": "idxapply", "label": "Search indexer", "sub": "Dataflow", "kind": "app" }, { "id": "fanout", "label": "Fan-out worker", "sub": "Cloud Run", "kind": "app" }, { "id": "rebuild", "label": "Rebuild runner", "sub": "shadow then swap", "kind": "app" } ] }, { "title": "Control plane — small, exact, authoritative", "kind": "trust", "nodes": [ { "id": "reg", "label": "Stream and sink registry", "sub": "Spanner", "kind": "platform" }, { "id": "sreg", "label": "Schema registry", "sub": "versioned", "kind": "platform" }, { "id": "ops", "label": "Operator API", "sub": "Cloud Run", "kind": "security" }, { "id": "recon", "label": "Reconciliation runner", "sub": "daily", "kind": "platform" } ] } ] } ], "outside": [ { "id": "pg", "label": "Operational PostgreSQL", "sub": "12 databases", "kind": "external" }, { "id": "bq", "label": "BigQuery", "kind": "store" }, { "id": "idx", "label": "Search index", "kind": "external" } ], "edges": [ { "from": "pg", "to": "reader", "label": "WAL", "kind": "sync" }, { "from": "pub", "to": "log", "label": "publish", "kind": "async" }, { "from": "log", "to": "bqapply", "label": "by offset", "kind": "async" }, { "from": "bqapply", "to": "bq", "label": "MERGE", "kind": "sync" }, { "from": "log", "to": "arch", "kind": "batch" }, { "from": "sreg", "to": "pub", "label": "schema version", "kind": "sync" } ], "note": "Omitted for clarity: the search and fan-out sink edges, Secret Manager and IAM into capture, and every metrics edge.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "09-integration-surface", "title": "Integration Surface — Sources, Platform, Destinations", "layout": "hub", "left": { "title": "Sources (read-only)", "nodes": [ { "id": "pg1", "label": "PostgreSQL · orders", "sub": "62 tables", "kind": "external", "rel": "WAL", "dir": "in" }, { "id": "pg2", "label": "PostgreSQL · billing", "sub": "44 tables", "kind": "external", "rel": "WAL decode", "dir": "in" }, { "id": "pg3", "label": "PostgreSQL · identity", "sub": "31 tables", "kind": "external", "rel": "WAL · masked", "dir": "in" } ] }, "centre": { "title": "CDC Platform", "nodes": [ { "id": "cap", "label": "Capture plane", "kind": "integration" }, { "id": "log", "label": "Change log", "sub": "7-day retention", "kind": "queue" }, { "id": "arch", "label": "Changelog archive", "sub": "13 months", "kind": "store" }, { "id": "proj", "label": "Projection plane", "kind": "app" }, { "id": "ctl", "label": "Control plane", "kind": "platform" } ] }, "right": { "title": "Destinations and dependencies", "nodes": [ { "id": "bq", "label": "BigQuery", "sub": "mirror + changelog", "kind": "store", "rel": "MERGE micro-batch", "dir": "out" }, { "id": "idx", "label": "Search index", "kind": "external", "rel": "upsert", "dir": "out" }, { "id": "hooks", "label": "Cache and webhook fan-out", "sub": "14 partners", "kind": "integration", "rel": "async events", "dir": "out", "kind2": "async" } ] }, "note": "Three source interfaces in, three sinks out, one contract each way. The snapshot read path and heartbeat probe are internal to capture (view 14).", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "10-change-event-flow", "title": "Data Flow — One Row Change, End to End", "layout": "flow", "chain": true, "align": "middle", "stages": [ { "title": "Committed change", "nodes": [ { "id": "txn", "label": "Row commit", "sub": "INSERT · UPDATE · DELETE", "kind": "external" }, { "id": "wal", "label": "WAL record", "sub": "LSN + txid", "kind": "external" } ] }, { "title": "Decode and enrich", "nodes": [ { "id": "dec", "label": "Decoded change", "sub": "pre + post image", "kind": "app" }, { "id": "mask", "label": "Mask or exclude", "sub": "declared columns", "kind": "security" }, { "id": "stamp", "label": "Stamp metadata", "sub": "tenant · schema v", "kind": "app" } ] }, { "title": "Log", "nodes": [ { "id": "log", "label": "Change log", "sub": "partition = hash(PK)", "kind": "queue" }, { "id": "arch", "label": "Archive writer", "sub": "13 months", "kind": "store" } ] }, { "title": "Fan-out", "nodes": [ { "id": "s1", "label": "Warehouse offset", "kind": "app" }, { "id": "s2", "label": "Search offset", "kind": "app" }, { "id": "s3", "label": "Fan-out offset", "kind": "app" } ] }, { "title": "Materialise", "nodes": [ { "id": "mb", "label": "Micro-batch", "sub": "5 s or 10k rows", "kind": "app" }, { "id": "idem", "label": "Idempotent apply", "sub": "key = PK + LSN", "kind": "decision" }, { "id": "stale", "label": "Discard if stale", "sub": "LSN ≤ current", "kind": "decision" } ] }, { "title": "Sinks", "nodes": [ { "id": "mirror", "label": "Current-state mirror", "kind": "store" }, { "id": "clog", "label": "Changelog table", "kind": "store" }, { "id": "idx", "label": "Search index", "kind": "store" } ] } ], "edges": [ { "from": "log", "to": "arch", "label": "continuous", "kind": "batch" } ], "note": "The post-image, the primary key and the log position are the three fields everything downstream relies on.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "11-storage-zones", "title": "Storage Zones — Ownership and Rebuildability", "layout": "nested", "boxes": [ { "title": "Zones ordered by what happens if the data is lost", "kind": "plain", "dir": "col", "children": [ { "title": "Not ours — operational truth, read-only to this platform", "kind": "onprem", "nodes": [ { "id": "pg", "label": "PostgreSQL primaries", "sub": "application-owned", "kind": "external" }, { "id": "wal", "label": "WAL retention", "sub": "the hard deadline", "kind": "risk" } ] }, { "title": "System of record for change — losing this loses history", "kind": "trust", "nodes": [ { "id": "log", "label": "Change log", "sub": "7 days · replayable", "kind": "queue" }, { "id": "arch", "label": "Changelog archive", "sub": "13 months · tiered", "kind": "store" } ] }, { "title": "Control state — small, exact, must survive", "kind": "trust", "nodes": [ { "id": "reg", "label": "Stream and sink registry", "sub": "Spanner", "kind": "store" }, { "id": "sv", "label": "Schema versions", "sub": "effective LSN", "kind": "store" }, { "id": "off", "label": "Sink offsets", "kind": "store" }, { "id": "ck", "label": "Snapshot chunk state", "kind": "store" }, { "id": "aud", "label": "Operator audit log", "sub": "13 months", "kind": "store" } ] }, { "title": "Derived — rebuildable from the log, no retention claim", "kind": "boundary", "nodes": [ { "id": "mirror", "label": "Current-state mirror", "sub": "BigQuery", "kind": "store" }, { "id": "clog", "label": "Changelog tables", "sub": "BigQuery", "kind": "store" }, { "id": "idx", "label": "Search index", "kind": "store" }, { "id": "cache", "label": "Read caches", "kind": "store" } ] } ] } ], "edges": [ { "from": "arch", "to": "log", "label": "re-publish past 7 days", "kind": "batch" }, { "from": "log", "to": "mirror", "label": "replay", "kind": "async" } ], "note": "Only two zones have a durability requirement of their own; the bottom zone is deliberately disposable.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "12-data-model", "title": "Control-Plane Data Model", "layout": "er", "canvas": { "width": 1680, "cols": 4 }, "rowGap": 230, "entities": [ { "id": "src", "name": "source_database", "kind": "store", "row": 0, "col": 0, "attrs": [ "source_id PK", "engine", "replication_slot", "log_retention_hours", "source_budget_pct" ] }, { "id": "tbl", "name": "replicated_table", "kind": "store", "row": 0, "col": 1, "attrs": [ "table_id PK", "source_id FK -> source_database", "tier (1 | 2)", "pre_image_available", "state (live | paused | snapshotting)" ] }, { "id": "sv", "name": "schema_version", "kind": "store", "row": 0, "col": 2, "attrs": [ "schema_version_id PK", "table_id FK -> replicated_table", "effective_lsn", "columns JSON", "compatibility (compatible | breaking)" ] }, { "id": "sink", "name": "sink", "kind": "store", "row": 1, "col": 2, "attrs": [ "sink_id PK", "shape (mirror | changelog)", "freshness_target_s", "delete_mode (soft | hard)" ] }, { "id": "sub", "name": "sink_subscription", "kind": "store", "row": 1, "col": 1, "attrs": [ "subscription_id PK", "sink_id FK -> sink", "table_id FK -> replicated_table", "offset_lsn", "lag_seconds" ] }, { "id": "chunk", "name": "snapshot_chunk", "kind": "store", "row": 1, "col": 0, "attrs": [ "chunk_id PK", "table_id FK -> replicated_table", "pk_range_low", "pk_range_high", "state (pending | done)" ] }, { "id": "evt", "name": "change_event", "kind": "queue", "row": 0, "col": 3, "attrs": [ "table_id + pk + lsn PK", "op (c | u | d | r)", "schema_version_id FK", "commit_ts", "post_image JSON" ] }, { "id": "dl", "name": "dead_letter_event", "kind": "store", "row": 1, "col": 3, "attrs": [ "dlq_id PK", "table_id FK -> replicated_table", "lsn", "error_class", "payload raw" ] }, { "id": "rec", "name": "reconciliation_run", "kind": "store", "row": 2, "col": 1, "attrs": [ "run_id PK", "subscription_id FK -> sink_subscription", "verdict (match | diverged)", "rows_source / rows_sink", "sampled_key_ranges" ] }, { "id": "act", "name": "operator_action", "kind": "store", "row": 2, "col": 2, "attrs": [ "action_id PK", "actor", "action (pause | resume | resnapshot | reset)", "target_sink_id FK -> sink", "target_table_id FK -> replicated_table", "at immutable" ] } ], "relations": [ { "from": "src", "to": "tbl", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "tbl", "to": "sv", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "sv", "to": "evt", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "tbl", "to": "chunk", "label": "1 : N", "from_side": "s1", "to_side": "n" }, { "from": "tbl", "to": "sub", "label": "1 : N", "from_side": "s3", "to_side": "n" }, { "from": "sink", "to": "sub", "label": "1 : N", "from_side": "w", "to_side": "e" }, { "from": "evt", "to": "dl", "label": "1 : N", "kind": "optional", "from_side": "s", "to_side": "n" }, { "from": "sub", "to": "rec", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "sink", "to": "act", "label": "1 : N", "kind": "optional", "from_side": "s", "to_side": "n" } ], "note": "change_event is the log's shape, not a table: it is keyed by (table, primary key, log position) so an apply is idempotent.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "13-commit-to-sink-sequence", "title": "Critical Flow — Commit to Sink in Five Seconds", "layout": "sequence", "lifelines": [ { "id": "app", "label": "Product service", "kind": "external" }, { "id": "pg", "label": "PostgreSQL", "kind": "external" }, { "id": "cap", "label": "Capture plane", "kind": "integration" }, { "id": "log", "label": "Change log", "kind": "queue" }, { "id": "apply", "label": "Warehouse applier", "kind": "app" }, { "id": "bq", "label": "BigQuery mirror", "kind": "store" }, { "id": "mon", "label": "Monitoring", "kind": "platform" } ], "messages": [ { "from": "app", "to": "pg", "label": "UPDATE order SET status", "kind": "call" }, { "from": "pg", "to": "app", "label": "commit acknowledged", "kind": "return" }, { "from": "pg", "to": "cap", "label": "WAL record at LSN", "kind": "async" }, { "from": "cap", "to": "cap", "label": "decode, mask, stamp schema v", "kind": "self" }, { "from": "cap", "to": "log", "label": "publish, key = hash(PK)", "kind": "async" }, { "from": "log", "to": "cap", "label": "durable ack", "kind": "return" }, { "from": "log", "to": "apply", "label": "pull batch from offset", "kind": "async" }, { "from": "apply", "to": "apply", "label": "transform, group by key, keep max LSN", "kind": "self" }, { "from": "apply", "to": "bq", "label": "MERGE micro-batch", "kind": "call" }, { "from": "bq", "to": "apply", "label": "rows affected", "kind": "return" }, { "from": "apply", "to": "log", "label": "commit offset", "kind": "call" }, { "from": "apply", "to": "mon", "label": "end-to-end lag from commit_ts", "kind": "async" }, { "from": "apply", "to": "log", "label": "non-retryable: to dead letter", "kind": "error" } ], "note": "The offset is committed after the sink write, never before: that is what makes a crash a duplicate rather than a gap.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "14-snapshot-to-stream-handoff", "title": "Snapshot to Stream — The Handoff With No Gap", "layout": "flow", "chain": true, "align": "top", "stages": [ { "title": "Register", "nodes": [ { "id": "reg", "label": "Table registered", "sub": "allow-list applied", "kind": "app" }, { "id": "slot", "label": "Slot position recorded", "sub": "LSN₀", "kind": "decision" } ] }, { "title": "Stream first", "nodes": [ { "id": "buf", "label": "Stream from LSN₀", "sub": "into the log", "kind": "queue" }, { "id": "noapply", "label": "Sink stays empty", "sub": "offset parked", "kind": "app" } ] }, { "title": "Chunked snapshot", "nodes": [ { "id": "chunk", "label": "Next PK range", "sub": "resumable", "kind": "app" }, { "id": "read", "label": "Read from replica", "sub": "op = READ", "kind": "integration" }, { "id": "thr", "label": "Yield to live traffic", "sub": "≤ 5% source CPU", "kind": "decision" } ] }, { "title": "Overlap resolved", "nodes": [ { "id": "merge", "label": "Replay log over snapshot", "sub": "from LSN₀", "kind": "app" }, { "id": "mono", "label": "Keep the higher LSN", "sub": "per primary key", "kind": "decision" } ] }, { "title": "Steady state", "nodes": [ { "id": "live", "label": "Table marked live", "sub": "one apply path", "kind": "app" }, { "id": "lag", "label": "Lag inside target", "sub": "p95 ≤ 5 s", "kind": "platform" } ] }, { "title": "If it breaks", "nodes": [ { "id": "resume", "label": "Resume at last chunk", "sub": "never restart", "kind": "opportunity" }, { "id": "trunc", "label": "Log truncated: re-snapshot", "sub": "fail loudly", "kind": "risk" } ] } ], "edges": [ { "from": "slot", "to": "merge", "label": "LSN₀", "kind": "sync", "route": "gutter" }, { "from": "chunk", "to": "resume", "label": "chunk state", "kind": "sync" } ], "note": "Snapshot rows and stream rows are the same event type with a different op, so one apply path serves both.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "15-schema-change-paths", "title": "Schema Drift — Four Changes, Four Paths", "layout": "swimlane", "laneHeaderWidth": 180, "stages": [ "Detect", "Classify", "Pipeline action", "Sink action", "Outcome" ], "lanes": [ { "title": "Added nullable column", "cells": [ [ { "label": "Relation metadata", "sub": "from the stream", "kind": "app" } ], [ { "label": "Compatible", "kind": "decision" } ], [ { "label": "Register new version", "sub": "effective LSN", "kind": "app" } ], [ { "label": "Additive DDL", "sub": "before first event", "kind": "store" } ], [ { "label": "No pause", "sub": "≤ 60 s", "kind": "opportunity" } ] ] }, { "title": "New table in allow-list", "cells": [ [ { "label": "Registry change", "kind": "app" } ], [ { "label": "Compatible", "kind": "decision" } ], [ { "label": "Chunked snapshot", "sub": "view 14", "kind": "app" } ], [ { "label": "Create sink table", "kind": "store" } ], [ { "label": "Live after backfill", "kind": "opportunity" } ] ] }, { "title": "Dropped or renamed column", "cells": [ [ { "label": "DDL event", "kind": "app" } ], [ { "label": "Breaking", "kind": "decision" } ], [ { "label": "Pause this table only", "sub": "≤ 10 s", "kind": "risk" } ], [ { "label": "Sink untouched", "sub": "no destructive DDL", "kind": "store" } ], [ { "label": "Operator decision", "sub": "resume from position", "kind": "actor" } ] ] }, { "title": "Primary-key change", "cells": [ [ { "label": "DDL event", "kind": "app" } ], [ { "label": "Breaking", "kind": "decision" } ], [ { "label": "Re-snapshot required", "sub": "keys no longer match", "kind": "risk" } ], [ { "label": "Shadow then swap", "kind": "store" } ], [ { "label": "Rebuild, not migrate", "kind": "opportunity" } ] ] }, { "title": "Unregistered column seen", "cells": [ [ { "label": "Decode mismatch", "kind": "app" } ], [ { "label": "Not a third option", "kind": "decision" } ], [ { "label": "Propagate or exclude", "sub": "never silently drop", "kind": "security" } ], [ { "label": "Evolve or ignore", "kind": "store" } ], [ { "label": "Explicit either way", "kind": "opportunity" } ] ] } ], "note": "Only the breaking rows stop anything, and they stop exactly one table.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "16-deployment-architecture", "title": "Deployment — Regional, With a Capture Standby", "layout": "nested", "boxes": [ { "title": "Google Cloud", "kind": "cloud", "dir": "col", "children": [ { "title": "europe-west1 — active", "kind": "boundary", "dir": "row", "children": [ { "title": "Capture (regional managed)", "kind": "boundary", "nodes": [ { "id": "ds", "label": "Datastream", "sub": "one stream per DB", "kind": "integration" }, { "id": "snapjob", "label": "Snapshot workers", "sub": "Dataflow · 3 zones", "kind": "app" } ] }, { "title": "Transport", "kind": "boundary", "nodes": [ { "id": "ps", "label": "Pub/Sub", "sub": "regional · 7 days", "kind": "queue" }, { "id": "dlq", "label": "Dead-letter topic", "kind": "queue" } ] }, { "title": "Projection and control", "kind": "boundary", "nodes": [ { "id": "df", "label": "Dataflow appliers", "sub": "one job per sink", "kind": "app" }, { "id": "cr", "label": "Cloud Run", "sub": "operator API", "kind": "app" }, { "id": "sp", "label": "Spanner", "sub": "regional · control state", "kind": "store" } ] } ] }, { "title": "Data residency and durability", "kind": "boundary", "dir": "row", "children": [ { "title": "Multi-region EU", "kind": "boundary", "nodes": [ { "id": "bq", "label": "BigQuery", "sub": "EU multi-region", "kind": "store" } ] }, { "title": "Dual-region archive", "kind": "boundary", "nodes": [ { "id": "gcs", "label": "Cloud Storage", "sub": "eur4 · 13 months", "kind": "store" } ] } ] }, { "title": "europe-west4 — capture standby (RTO 30 min)", "kind": "onprem", "nodes": [ { "id": "ds2", "label": "Datastream standby", "sub": "resumes at last LSN", "kind": "integration" }, { "id": "df2", "label": "Applier templates", "sub": "deployed, scaled to zero", "kind": "app" } ] } ] } ], "outside": [ { "id": "sql", "label": "Cloud SQL for PostgreSQL", "sub": "HA, zonal failover", "kind": "external" }, { "id": "idx", "label": "Search index", "sub": "managed, regional", "kind": "external" } ], "edges": [ { "from": "sql", "to": "ds", "label": "replication slot", "kind": "sync" }, { "from": "ds", "to": "ps", "kind": "async" }, { "from": "ps", "to": "df", "label": "by offset", "kind": "async" }, { "from": "df", "to": "bq", "label": "MERGE", "kind": "sync" }, { "from": "sp", "to": "ds2", "label": "last committed LSN", "kind": "sync", "route": "gutter" } ], "note": "One write region. The standby holds no state of its own: it resumes from the position in Spanner. Archive and metrics edges omitted.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "17-observability", "title": "Observability — Signal by Pipeline Stage", "layout": "grid", "laneHeaderWidth": 190, "columns": [ "Source", "Capture", "Change log", "Projection", "Sink" ], "rows": [ { "title": "Lag", "cells": [ [ { "label": "Commit timestamp", "sub": "heartbeat 10 s", "kind": "platform" } ], [ { "label": "Capture lag", "sub": "p95 ≤ 1 s", "kind": "platform" } ], [ { "label": "Transport lag", "kind": "platform" } ], [ { "label": "Apply lag", "kind": "platform" } ], [ { "label": "End-to-end lag", "sub": "p95 ≤ 5 s · SLO", "kind": "platform" } ] ] }, { "title": "Throughput", "cells": [ [ { "label": "Row changes/s", "sub": "9k steady", "kind": "platform" } ], [ { "label": "Decode rate", "kind": "platform" } ], [ { "label": "Publish rate", "kind": "platform" } ], [ { "label": "Batches/min", "kind": "platform" } ], [ { "label": "Rows applied/s", "kind": "platform" } ] ] }, { "title": "Errors", "cells": [ [], [ { "label": "Decode failures", "kind": "risk" } ], [ { "label": "Dead-letter depth", "sub": "alarm on growth", "kind": "risk" } ], [ { "label": "Transform exceptions", "kind": "risk" } ], [ { "label": "Apply rejections", "kind": "risk" } ] ] }, { "title": "Saturation", "cells": [ [ { "label": "Slot retention headroom", "sub": "the disk-full alarm", "kind": "risk" } ], [ { "label": "Source CPU added", "sub": "≤ 5% budget", "kind": "platform" } ], [ { "label": "Unacked backlog", "kind": "platform" } ], [ { "label": "Worker autoscale", "kind": "platform" } ], [ { "label": "Sink quota", "kind": "platform" } ] ] }, { "title": "Correctness", "cells": [ [ { "label": "Row counts", "sub": "tier-1 daily", "kind": "platform" } ], [ { "label": "Snapshot chunk state", "kind": "platform" } ], [ { "label": "Position continuity", "sub": "gap detection", "kind": "risk" } ], [ { "label": "Schema version in use", "kind": "platform" } ], [ { "label": "Sampled checksum", "sub": "divergence alarm", "kind": "platform" } ] ] }, { "title": "Cost", "cells": [ [], [], [ { "label": "Bytes logged per table", "kind": "platform" } ], [ { "label": "Worker hours per sink", "kind": "platform" } ], [ { "label": "£ per million events", "sub": "target ≤ 0.60", "kind": "platform" } ] ] } ], "note": "Four alarms, four actions: lag against target, slot headroom, dead-letter growth, reconciliation divergence.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "18-sink-lifecycle", "title": "Sink Lifecycle — Rebuild as a Routine, Not a Disaster", "layout": "cycle", "centre": { "label": "Sink lifecycle" }, "nodes": [ { "id": "register", "label": "Register", "sub": "shape and freshness", "kind": "app" }, { "id": "backfill", "label": "Backfill", "sub": "chunked snapshot", "kind": "app" }, { "id": "serve", "label": "Serve", "sub": "inside freshness target", "kind": "store" }, { "id": "verify", "label": "Verify", "sub": "daily reconciliation", "kind": "platform" }, { "id": "shadow", "label": "Rebuild to shadow", "sub": "log or archive", "kind": "app" }, { "id": "swap", "label": "Swap in", "sub": "atomic, no partial reads", "kind": "decision" }, { "id": "retire", "label": "Retire", "sub": "no consumer, no cost", "kind": "opportunity" } ], "ringLabels": [ "offset parked", "replay over snapshot", "as-of published", "divergence or bad transform", "diff against live", "offsets realigned", "registry entry removed" ], "rx": 440, "ry": 215, "note": "The loop closes because a rebuild is the same code path as a backfill, run against a shadow table.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "19-security-trust-zones", "title": "Security — Trust Zones and What Crosses Them", "layout": "zones", "zones": [ { "title": "Source estate — application-owned, highest sensitivity", "kind": "trust", "nodes": [ { "id": "pg", "label": "PostgreSQL primaries", "sub": "no platform write path", "kind": "external" }, { "id": "rep", "label": "Read replicas", "kind": "external" }, { "id": "role", "label": "Replication role", "sub": "replication + SELECT only", "kind": "security" } ] }, { "title": "Capture zone — the only reader of the source", "kind": "trust", "nodes": [ { "id": "cap", "label": "Capture plane", "sub": "workload identity", "kind": "integration" }, { "id": "mask", "label": "Column masker", "sub": "regulated values never logged", "kind": "security" }, { "id": "sm", "label": "Secret Manager", "sub": "90-day rotation", "kind": "security" } ] }, { "title": "Platform zone — change log and projection", "kind": "trust", "nodes": [ { "id": "ops", "label": "Operator API", "sub": "privileged actions", "kind": "security" }, { "id": "log", "label": "Change log", "sub": "CMEK at rest", "kind": "queue" }, { "id": "proj", "label": "Appliers", "sub": "per-sink identity", "kind": "app" }, { "id": "arch", "label": "Changelog archive", "sub": "crypto-shreddable", "kind": "store" } ] }, { "title": "Sink and consumer zone — tenant-scoped", "kind": "trust", "nodes": [ { "id": "bq", "label": "BigQuery", "sub": "per-tenant datasets", "kind": "store" }, { "id": "idx", "label": "Search index", "kind": "store" }, { "id": "cons", "label": "Consumers", "sub": "never see the raw log", "kind": "external" } ] } ], "edges": [ { "from": "pg", "to": "cap", "label": "WAL, read-only", "kind": "sync" }, { "from": "cap", "to": "log", "label": "masked", "kind": "async" }, { "from": "log", "to": "proj", "label": "by offset", "kind": "async" }, { "from": "proj", "to": "bq", "label": "per tenant", "kind": "sync" }, { "from": "bq", "to": "cons", "label": "scoped SQL", "kind": "sync" }, { "from": "ops", "to": "cap", "label": "pause", "kind": "error" } ], "note": "The masker is inside the capture zone deliberately: a regulated value that reaches the log can only be deleted, never un-logged.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "20-operator-authorisation-flow", "title": "Identity and Access — Authorising a Re-Snapshot", "layout": "sequence", "lifelines": [ { "id": "op", "label": "Platform SRE", "kind": "actor" }, { "id": "api", "label": "Operator API", "kind": "security" }, { "id": "iam", "label": "Cloud IAM", "kind": "security" }, { "id": "ctl", "label": "Control plane", "kind": "platform" }, { "id": "cap", "label": "Capture plane", "kind": "integration" }, { "id": "aud", "label": "Audit log", "kind": "store" } ], "messages": [ { "from": "op", "to": "api", "label": "POST /tables/orders:resnapshot", "kind": "call" }, { "from": "api", "to": "iam", "label": "verify identity and role", "kind": "call" }, { "from": "iam", "to": "api", "label": "cdc.tableOperator granted", "kind": "return" }, { "from": "api", "to": "ctl", "label": "check preconditions", "kind": "call" }, { "from": "ctl", "to": "ctl", "label": "retention headroom, no snapshot in flight", "kind": "self" }, { "from": "ctl", "to": "api", "label": "refused: would exhaust WAL retention", "kind": "error" }, { "from": "api", "to": "aud", "label": "record attempt and refusal", "kind": "async" }, { "from": "op", "to": "api", "label": "retry with source budget raised", "kind": "call" }, { "from": "ctl", "to": "cap", "label": "start chunked snapshot at LSN", "kind": "call" }, { "from": "cap", "to": "ctl", "label": "chunk progress", "kind": "async" }, { "from": "ctl", "to": "aud", "label": "record action, actor, target", "kind": "async" } ], "note": "A privileged action the platform can refuse is the point: the capture plane never takes an operator's word for retention headroom.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } }, { "id": "21-failure-classes", "title": "Assurance — Failure Classes and Their Handling", "layout": "grid", "laneHeaderWidth": 210, "columns": [ "Detected by", "Contained to", "Recovery", "Correctness risk" ], "rows": [ { "title": "Transient (blip, throttle)", "cells": [ [ { "label": "Retry counters", "kind": "platform" } ], [ { "label": "One batch", "kind": "app" } ], [ { "label": "Backoff with jitter", "kind": "opportunity" } ], [ { "label": "None — lag absorbs it", "kind": "opportunity" } ] ] }, { "title": "Source unavailable", "cells": [ [ { "label": "Connection loss", "kind": "platform" } ], [ { "label": "One source", "kind": "app" } ], [ { "label": "Resume at last LSN", "kind": "opportunity" } ], [ { "label": "None", "kind": "opportunity" } ] ] }, { "title": "Source log truncated", "cells": [ [ { "label": "Position gap", "sub": "slot headroom alarm", "kind": "risk" } ], [ { "label": "One table", "kind": "app" } ], [ { "label": "Re-snapshot", "sub": "fail loudly first", "kind": "risk" } ], [ { "label": "Silent gap if unchecked", "kind": "risk" } ] ] }, { "title": "Source failover / new timeline", "cells": [ [ { "label": "LSN discontinuity", "kind": "risk" } ], [ { "label": "One source", "kind": "app" } ], [ { "label": "Reconcile or re-snapshot", "kind": "decision" } ], [ { "label": "Bounded duplicate or gap", "kind": "risk" } ] ] }, { "title": "Breaking schema change", "cells": [ [ { "label": "DDL event", "kind": "platform" } ], [ { "label": "One table", "sub": "slot keeps running", "kind": "app" } ], [ { "label": "Operator decision", "kind": "actor" } ], [ { "label": "None if paused", "kind": "opportunity" } ] ] }, { "title": "Poison event", "cells": [ [ { "label": "Retry budget spent", "kind": "platform" } ], [ { "label": "One event", "kind": "app" } ], [ { "label": "Dead-letter with context", "kind": "opportunity" } ], [ { "label": "One row stale", "kind": "risk" } ] ] }, { "title": "Sink unavailable or slow", "cells": [ [ { "label": "Apply lag per sink", "kind": "platform" } ], [ { "label": "That sink only", "kind": "app" } ], [ { "label": "Buffer in the log", "sub": "to 7 days", "kind": "opportunity" } ], [ { "label": "Past retention: re-snapshot", "kind": "risk" } ] ] }, { "title": "Bad transform deployed", "cells": [ [ { "label": "Reconciliation verdict", "kind": "platform" } ], [ { "label": "That sink only", "kind": "app" } ], [ { "label": "Replay to shadow, swap", "kind": "opportunity" } ], [ { "label": "Wrong values until found", "kind": "risk" } ] ] }, { "title": "Regional outage", "cells": [ [ { "label": "Health checks", "kind": "platform" } ], [ { "label": "Whole pipeline", "kind": "risk" } ], [ { "label": "Standby from Spanner LSN", "sub": "RTO 30 min", "kind": "decision" } ], [ { "label": "None — RPO 0 vs source", "kind": "opportunity" } ] ] } ], "note": "Only one row has a whole-pipeline blast radius, and no row has a recovery that is hand-written SQL.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-10" } } ]