[ { "id": "01-system-context", "title": "Distributed Job Scheduler — System Context", "layout": "context", "canvas": { "width": 1700 }, "colWidth": 265, "system": { "label": "Scheduled Trigger Service", "sub": "decides when; never runs the work" }, "groups": [ { "side": "left", "title": "People", "nodes": [ { "id": "dev", "label": "Tenant developer", "kind": "actor", "rel": "declares", "dir": "in" }, { "id": "oncall", "label": "Tenant on-call", "kind": "actor", "rel": "asks \"did it run?\"", "dir": "in" }, { "id": "sre", "label": "Platform SRE", "kind": "actor", "rel": "owns punctuality", "dir": "in" }, { "id": "sec", "label": "Security reviewer", "kind": "actor", "rel": "audits privilege", "dir": "in" } ] }, { "side": "right", "title": "Where the work runs", "nodes": [ { "id": "http", "label": "Tenant HTTP target", "kind": "external", "rel": "signed dispatch", "dir": "out" }, { "id": "queue", "label": "Tenant queue", "kind": "queue", "rel": "enqueue", "dir": "out", "kind2": "async" }, { "id": "exec", "label": "Platform executor", "kind": "external", "rel": "fire handle", "dir": "out" }, { "id": "wf", "label": "Workflow engine", "kind": "external", "rel": "out of scope", "dir": "out", "kind2": "batch" } ] }, { "side": "top", "title": "Platform dependencies", "nodes": [ { "id": "time", "label": "Time authority", "kind": "platform", "rel": "ε bound", "dir": "in", "kind2": "bidirectional" }, { "id": "iam", "label": "Workload identity", "kind": "security", "rel": "tokens", "dir": "in" } ] }, { "side": "bottom", "title": "Where the evidence goes", "nodes": [ { "id": "obs", "label": "Monitoring", "kind": "platform", "rel": "signals", "dir": "out" }, { "id": "audit", "label": "Audit sink", "kind": "store", "rel": "privileged acts", "dir": "out" }, { "id": "bill", "label": "Billing", "kind": "external", "rel": "fire counts", "dir": "out", "kind2": "batch" } ] } ], "note": "In scope: deciding an instant is due and handing over a durable fire record. Out of scope: running the work, and orchestrating more than one step of it. Key management is omitted here and appears in views 10 and 18.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "02-high-level-architecture", "title": "Distributed Job Scheduler — High-Level Architecture", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "middle", "nodeWidth": 185, "stages": [ { "title": "Declare", "nodes": [ { "id": "api", "label": "Management API", "sub": "Cloud Run", "kind": "integration" }, { "id": "valid", "label": "Validator", "sub": "next 3 instants", "kind": "app" } ] }, { "title": "Remember", "nodes": [ { "id": "reg", "label": "Trigger registry", "sub": "Spanner", "kind": "store" }, { "id": "due", "label": "Due index", "sub": "time-ordered", "kind": "store" } ] }, { "title": "Decide", "nodes": [ { "id": "own", "label": "Partition owner", "sub": "lease held", "kind": "app" }, { "id": "gate", "label": "Clock gate", "sub": "waits out ε", "kind": "decision" } ] }, { "title": "Commit", "nodes": [ { "id": "ledger", "label": "Fire ledger", "sub": "append-only, keyed", "kind": "store" } ] }, { "title": "Shape", "nodes": [ { "id": "lanes", "label": "Dispatch lanes", "sub": "Cloud Tasks", "kind": "queue" }, { "id": "smear", "label": "Jitter smear", "sub": "peak second", "kind": "app" } ] }, { "title": "Deliver", "nodes": [ { "id": "disp", "label": "Dispatcher", "sub": "signs + mints", "kind": "app" }, { "id": "tgt", "label": "Tenant target", "kind": "external" } ] }, { "title": "Account", "nodes": [ { "id": "hist", "label": "Fire history", "sub": "Bigtable 90 d", "kind": "store" }, { "id": "rep", "label": "Lateness facts", "sub": "BigQuery", "kind": "store" } ] } ], "edges": [ { "from": "gate", "to": "ledger", "label": "commit first" }, { "from": "ledger", "to": "hist", "label": "state changes", "kind": "async", "route": "gutter" } ], "note": "The spine is the fire path. Nothing is dispatched that is not already committed to the ledger.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" }, "stageGap": 44 }, { "id": "03-actors-and-journeys", "title": "Who the Scheduler Is For, and What They Get to Do", "layout": "actors", "canvas": { "width": 1740 }, "cardWidth": 268, "groups": [ { "title": "Tenant engineering", "kind": "boundary", "actors": [ { "id": "dev", "label": "Tenant developer", "kind": "actor", "sub": "4,200 across 50 k tenants", "goal": "I want a digest to go out every weekday at 09:00 in my customers' time zone, and I do not want to learn anything about distributed timers to get it.", "journeys": [ { "id": "j-ship", "label": "Ship a scheduled job", "sub": "see view 04" }, { "label": "Amend a schedule safely" }, { "label": "Dry-run before committing" } ] }, { "id": "oncall", "label": "Tenant on-call", "kind": "actor", "sub": "pages at 07:40", "goal": "Something did not land overnight. Tell me whether it fired, whether you decided not to, or whether my endpoint refused it — before I start reading my own logs.", "journeys": [ { "id": "j-after", "label": "The morning after an outage", "sub": "see view 05" }, { "label": "Pause a misbehaving trigger" } ] } ] }, { "title": "Platform", "kind": "cloud", "actors": [ { "id": "sre", "label": "Platform SRE", "kind": "actor", "sub": "6 on rotation", "goal": "I need one number that tells me the scheduler is not firing, because a scheduler that is up and silent looks perfectly healthy on every other dashboard.", "journeys": [ { "label": "Drain a zone without losing a fire" }, { "label": "Throttle a tenant's catch-up" }, { "label": "Rebuild the due index in shadow" } ] }, { "id": "owner", "label": "Product owner", "kind": "actor", "sub": "sets defaults", "goal": "I want the default behaviour after an outage to be the one that is least likely to bill a customer twice.", "journeys": [ { "label": "Set a tenant's quota" }, { "label": "Approve a backfill" } ] } ] }, { "title": "Assurance", "kind": "trust", "actors": [ { "id": "sec", "label": "Security reviewer", "kind": "actor", "sub": "quarterly", "goal": "Show me that a schedule cannot be pointed at a host the tenant does not own, and that the person who can backfill is not simply anyone who can edit a trigger.", "journeys": [ { "label": "Trace a dispatch credential" }, { "label": "Audit a horizon extension" } ] } ] }, { "title": "Machines in the cast", "kind": "plain", "actors": [ { "id": "target", "label": "Tenant target", "kind": "external", "sub": "HTTP, queue, executor", "goal": "Hand me a fire with a key I can deduplicate on, and do not assume my answer means the work finished.", "journeys": [ { "label": "Accept a signed dispatch" }, { "label": "Report an outcome" } ] }, { "id": "clock", "label": "Time authority", "kind": "platform", "sub": "ε ≤ 10 ms", "goal": "Tell the truth about how wrong I might be, and be believed rather than averaged.", "journeys": [ { "label": "Serve a bounded interval" } ] } ] } ], "note": "Goals are in each actor's own voice. The two journeys with their own view are the ones where the architecture is visible to the person living it.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "04-journey-ship-a-scheduled-job", "title": "Journey — A Developer Ships a Scheduled Job", "layout": "journey", "canvas": { "width": 1740 }, "cellWidth": 215, "actor": { "label": "Tenant developer", "sub": "first scheduled job", "goal": "A weekday 09:00 digest, in my customers' zone, live today", "trigger": "A product ask: \"can this email go out every morning?\"", "success": "It fires at 09:00 local, and I can prove it fired" }, "phases": [ { "title": "Express", "sub": "cron string + zone" }, { "title": "Check", "sub": "dry-run" }, { "title": "Choose policy", "moment": true }, { "title": "First fire", "moment": true }, { "title": "Live", "sub": "day two onwards" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Writes 0 9 * * 1-5" }, { "label": "Names a zone" } ], [ { "label": "Reads next 3 instants" } ], [ { "label": "Picks missed-fire policy" }, { "label": "Picks overlap policy" } ], [ { "label": "Watches the 09:00" } ], [ { "label": "Checks the history tab" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "Validates expression" }, { "label": "Resolves the zone" } ], [ { "label": "Same code path as prod" } ], [ { "label": "Explains each option" } ], [ { "label": "Commits, then dispatches" } ], [ { "label": "Publishes lateness" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Fine", "Uneasy" ], "points": [ 1, 0, 2, 1, 0 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [], [ { "label": "No idea what fire-all costs" } ], [ { "label": "Is 3 s late a problem?" } ], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Zone picker, not UTC maths" } ], [ { "label": "Dry-run before commit" } ], [ { "label": "Default is fire-once-now" } ], [ { "label": "Published p99 budget" } ], [ { "label": "Per-fire record" } ] ] } ], "chain": true, "note": "The trough is the policy screen: this is the one moment a developer is asked a distributed-systems question, and the default has to be the safe one.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "05-journey-the-morning-after", "title": "Journey — The Morning After an Outage", "layout": "journey", "canvas": { "width": 1740 }, "cellWidth": 215, "actor": { "label": "Tenant on-call", "sub": "paged at 07:40", "goal": "Know what ran, what did not, and what is about to", "trigger": "A customer asks why last night's statement never arrived", "success": "An answer good enough to tell the customer, without reading platform logs" }, "phases": [ { "title": "Notice" }, { "title": "Look" }, { "title": "Understand", "moment": true }, { "title": "Decide", "moment": true }, { "title": "Recover" }, { "title": "Prevent" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Gets the complaint" } ], [ { "label": "Opens the trigger" } ], [ { "label": "Reads the skip cause" } ], [ { "label": "Backfill, or let it go" } ], [ { "label": "Requests a backfill" } ], [ { "label": "Changes the policy" } ] ] }, { "title": "What the platform shows", "kind": "system", "cells": [ [], [ { "label": "Every instant, every state" } ], [ { "label": "\"expired: past horizon\"" } ], [ { "label": "Cost of the backfill" } ], [ { "label": "Own lane, own rate cap" } ], [ { "label": "Dry-run of the new policy" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Calm", "Tense", "Alarmed" ], "points": [ 2, 1, 0, 1, 0, 0 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "Found out from a customer" } ], [], [ { "label": "Horizon was a default" } ], [ { "label": "Backfill needs approval" } ], [], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Lateness alert to tenant" } ], [ { "label": "Scheduled vs dispatched" } ], [ { "label": "Cause on every skip" } ], [ { "label": "Privilege is the guardrail" } ], [ { "label": "Original instant preserved" } ], [ { "label": "Policy is declarable" } ] ] } ], "chain": true, "note": "The trough is \"expired: past horizon\" — a correct decision the tenant never agreed to. Everything in acts 3 to 7 exists to make that sentence explainable rather than surprising.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "06-layered-architecture", "title": "Distributed Job Scheduler — Layered Architecture", "layout": "bands", "canvas": { "width": 1720 }, "layerHeaderWidth": 158, "bands": [ { "name": "Tenant surface", "nodes": [ { "id": "ui", "label": "Schedules console", "kind": "app" }, { "id": "mapi", "label": "Management API", "sub": "Cloud Run", "kind": "integration" }, { "id": "dry", "label": "Dry-run / next fires", "kind": "app" }, { "id": "hapi", "label": "History API", "kind": "integration" } ] }, { "name": "Control plane", "nodes": [ { "id": "vald", "label": "Definition validator", "kind": "app" }, { "id": "comp", "label": "Recurrence compiler", "sub": "zone + DST rules", "kind": "app" }, { "id": "adm", "label": "Policy and quota admission", "kind": "app" }, { "id": "aud", "label": "Audit writer", "kind": "security" } ] }, { "name": "State of record", "nodes": [ { "id": "reg", "label": "Trigger registry", "sub": "Spanner", "kind": "store" }, { "id": "duei", "label": "Due index", "sub": "partitioned", "kind": "store" }, { "id": "fl", "label": "Fire ledger", "sub": "append-only", "kind": "store" } ] }, { "name": "Timing plane", "nodes": [ { "id": "lease", "label": "Lease manager", "kind": "app" }, { "id": "scan", "label": "Due scanner", "kind": "app" }, { "id": "cgate", "label": "Clock gate", "sub": "ε ≤ 10 ms", "kind": "decision" }, { "id": "claim", "label": "Instant claimer", "kind": "app" } ] }, { "name": "Dispatch plane", "nodes": [ { "id": "shape", "label": "Rate shaper", "kind": "app" }, { "id": "lane", "label": "Lane queues", "sub": "Cloud Tasks", "kind": "queue" }, { "id": "sign", "label": "Signer", "kind": "security" }, { "id": "att", "label": "Attempt runner", "kind": "app" } ] }, { "name": "Evidence", "nodes": [ { "id": "hist", "label": "Fire history", "sub": "Bigtable", "kind": "store" }, { "id": "arch", "label": "History archive", "sub": "Cloud Storage", "kind": "store" }, { "id": "met", "label": "Lateness metrics", "kind": "platform" }, { "id": "bq", "label": "Reporting", "sub": "BigQuery", "kind": "store" } ] }, { "name": "Platform services", "nodes": [ { "id": "time", "label": "Time authority", "kind": "platform" }, { "id": "wid", "label": "Workload identity", "kind": "security" }, { "id": "kms", "label": "Key management", "kind": "security" }, { "id": "vpcsc", "label": "Perimeter controls", "kind": "security" } ] } ], "edges": [ { "from": "mapi", "to": "vald", "label": "define" }, { "from": "adm", "to": "reg", "label": "version write" }, { "from": "scan", "to": "duei", "label": "range scan", "kind": "bidirectional" }, { "from": "cgate", "to": "time", "label": "bounded ε", "route": "gutter" }, { "from": "claim", "to": "fl", "label": "commit" }, { "from": "shape", "to": "lane", "label": "by lane" } ], "note": "The control plane and the timing plane share only the state of record. A control-plane outage stops new definitions, not scheduled fires.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "07-platform-components", "title": "Platform Components — Container View", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Google Cloud — scheduler region", "kind": "cloud", "dir": "col", "children": [ { "title": "Control plane (Cloud Run)", "kind": "boundary", "nodes": [ { "id": "mapi", "label": "Management API", "kind": "integration" }, { "id": "vald", "label": "Validator", "sub": "3-instant probe", "kind": "app" }, { "id": "comp", "label": "Recurrence compiler", "sub": "tzdata pinned", "kind": "app" }, { "id": "hapi", "label": "History API", "kind": "integration" } ] }, { "title": "Timing plane (Cloud Run, leased partitions)", "kind": "boundary", "nodes": [ { "id": "lease", "label": "Lease manager", "kind": "app" }, { "id": "scan", "label": "Due scanner", "sub": "1 s tick", "kind": "app" }, { "id": "cgate", "label": "Clock gate", "kind": "decision" }, { "id": "claim", "label": "Instant claimer", "kind": "app" }, { "id": "shadow", "label": "Shadow rebuilder", "sub": "divergence check", "kind": "app" } ] }, { "title": "Dispatch plane", "kind": "boundary", "nodes": [ { "id": "shape", "label": "Rate shaper", "kind": "app" }, { "id": "ontime", "label": "On-time lane", "sub": "Cloud Tasks", "kind": "queue" }, { "id": "catch", "label": "Catch-up lane", "sub": "capped 10%", "kind": "queue" }, { "id": "bf", "label": "Backfill lane", "kind": "queue" }, { "id": "att", "label": "Attempt runner", "sub": "signs, mints", "kind": "app" } ] }, { "title": "State and evidence", "kind": "boundary", "nodes": [ { "id": "reg", "label": "Trigger registry", "sub": "Spanner", "kind": "store" }, { "id": "duei", "label": "Due index", "sub": "Spanner", "kind": "store" }, { "id": "fl", "label": "Fire ledger", "sub": "Spanner", "kind": "store" }, { "id": "hist", "label": "Fire history", "sub": "Bigtable", "kind": "store" }, { "id": "bq", "label": "Reporting", "sub": "BigQuery", "kind": "store" } ] }, { "title": "Platform services", "kind": "trust", "nodes": [ { "id": "time", "label": "Time authority", "kind": "platform" }, { "id": "wid", "label": "Workload identity", "kind": "security" }, { "id": "kms", "label": "Key management", "kind": "security" }, { "id": "mon", "label": "Monitoring", "kind": "platform" } ] } ] } ], "outside": [ { "id": "tgt", "label": "Tenant HTTP target", "kind": "external" }, { "id": "tq", "label": "Tenant queue", "kind": "queue" }, { "id": "exec", "label": "Platform executor", "kind": "external" } ], "edges": [ { "from": "mapi", "to": "vald", "label": "validate" }, { "from": "scan", "to": "duei", "label": "scan" }, { "from": "claim", "to": "fl", "label": "commit" }, { "from": "claim", "to": "shape", "route": "gutter" }, { "from": "att", "to": "tgt", "label": "signed POST" } ], "note": "Only the attempt runner leaves the perimeter. Five edges shown; the registry writes, the history fan-out and the identity and monitoring dependencies of every tier are omitted deliberately.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "08-integration-surface", "title": "Integration Surface — Who Calls, Who Is Called", "layout": "hub", "canvas": { "width": 1700 }, "left": { "title": "Inbound", "nodes": [ { "id": "console", "label": "Schedules console", "kind": "app", "rel": "HTTPS" }, { "id": "cli", "label": "Tenant CLI / IaC", "kind": "external", "rel": "apply" }, { "id": "sdk", "label": "Tenant SDK", "kind": "external", "rel": "REST" }, { "id": "cb", "label": "Outcome callback", "kind": "integration", "rel": "signed", "kind2": "async" } ] }, "centre": { "title": "Scheduled Trigger Service", "nodes": [ { "id": "mapi", "label": "Management API", "sub": "CRUD, pause, dry-run", "kind": "integration" }, { "id": "hapi", "label": "History API", "sub": "per-fire record", "kind": "integration" }, { "id": "padm", "label": "Privileged API", "sub": "backfill, horizon, quota", "kind": "security" } ] }, "right": { "title": "Outbound", "nodes": [ { "id": "http", "label": "Tenant HTTP target", "kind": "external", "rel": "attempt", "dir": "out" }, { "id": "tq", "label": "Tenant queue", "kind": "queue", "rel": "enqueue", "dir": "out", "kind2": "async" }, { "id": "exec", "label": "Platform executor", "kind": "external", "rel": "handle", "dir": "out" }, { "id": "mon", "label": "Monitoring", "kind": "platform", "rel": "signals", "dir": "out", "kind2": "async" } ] }, "note": "Three inbound surfaces, deliberately separated: authorship, reading history, and the privileges that multiply work. The audit and billing exports are omitted here; they appear in views 10 and 16.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "09-fire-path-data-flow", "title": "Data Flow — Definition to Evidence", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "middle", "nodeWidth": 183, "stages": [ { "title": "Declared", "nodes": [ { "id": "defn", "label": "Trigger definition", "sub": "versioned", "kind": "store" }, { "id": "pol", "label": "Policies", "sub": "missed, overlap", "kind": "store" } ] }, { "title": "Projected", "nodes": [ { "id": "inst", "label": "Next instant", "sub": "recomputed", "kind": "app" }, { "id": "duei", "label": "Due index row", "kind": "store" } ] }, { "title": "Claimed", "nodes": [ { "id": "claim", "label": "Instant claim", "sub": "conditional write", "kind": "app" } ] }, { "title": "Committed", "nodes": [ { "id": "rec", "label": "Fire record", "sub": "idempotency key", "kind": "store" } ] }, { "title": "Queued", "nodes": [ { "id": "lane", "label": "Lane queue entry", "kind": "queue" } ] }, { "title": "Attempted", "nodes": [ { "id": "att", "label": "Attempt row", "sub": "n of budget", "kind": "store" }, { "id": "tok", "label": "Minted token", "sub": "attempt-scoped", "kind": "security" } ] }, { "title": "Accounted", "nodes": [ { "id": "hist", "label": "History row", "sub": "Bigtable", "kind": "store" }, { "id": "arch", "label": "Archive", "sub": "13 months", "kind": "store" }, { "id": "bq", "label": "Lateness facts", "sub": "BigQuery", "kind": "store" } ] } ], "edges": [ { "from": "rec", "to": "hist", "label": "state changes", "kind": "async", "route": "gutter" }, { "from": "defn", "to": "inst", "label": "recompute each tick", "kind": "batch" }, { "from": "att", "to": "rec", "label": "outcome", "kind": "async" } ], "note": "Only the first and fourth columns are authoritative. Everything between them is recomputable from the definition.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" }, "stageGap": 44 }, { "id": "10-storage-zones", "title": "Storage Zones — What Is Truth, What Is Rebuildable", "layout": "nested", "canvas": { "width": 1740 }, "boxes": [ { "title": "Systems of record — backed up, never rebuilt", "kind": "trust", "dir": "row", "children": [ { "title": "Trigger registry", "kind": "boundary", "nodes": [ { "id": "defs", "label": "Definitions", "sub": "versioned, immutable", "kind": "store" }, { "id": "pols", "label": "Policies and quotas", "kind": "store" }, { "id": "tz", "label": "tzdata version pin", "kind": "store" } ] }, { "title": "Fire ledger", "kind": "boundary", "nodes": [ { "id": "fires", "label": "Fire records", "sub": "PK = idempotency key", "kind": "store" }, { "id": "atts", "label": "Attempts", "kind": "store" } ] }, { "title": "Audit", "kind": "boundary", "nodes": [ { "id": "audit", "label": "Privileged acts", "sub": "7 years, immutable", "kind": "store" } ] } ] }, { "title": "Rebuildable projections — dropped and recomputed, not restored", "kind": "boundary", "dir": "row", "children": [ { "title": "Due index", "kind": "plain", "nodes": [ { "id": "duei", "label": "Next instants", "sub": "per partition", "kind": "store" }, { "id": "leases", "label": "Partition leases", "sub": "short TTL", "kind": "store" } ] }, { "title": "Serving caches", "kind": "plain", "nodes": [ { "id": "nextc", "label": "Next-fires cache", "kind": "store" }, { "id": "quotac", "label": "Quota counters", "kind": "store" } ] }, { "title": "Evidence stores", "kind": "plain", "nodes": [ { "id": "hist", "label": "Fire history", "sub": "90 d hot", "kind": "store" }, { "id": "bq", "label": "Lateness facts", "sub": "derived", "kind": "store" } ] } ] }, { "title": "Cold — retained, not queried", "kind": "onprem", "dir": "row", "children": [ { "title": "Archive", "kind": "plain", "nodes": [ { "id": "arch", "label": "History archive", "sub": "13 months, Cloud Storage", "kind": "store" }, { "id": "bc", "label": "Billing counts", "sub": "non-identifying", "kind": "store" } ] } ] } ], "edges": [ { "from": "defs", "to": "duei", "label": "recompute", "kind": "batch" }, { "from": "fires", "to": "hist", "label": "project", "kind": "async" }, { "from": "hist", "to": "arch", "label": "tier at 90 d", "kind": "batch" }, { "from": "hist", "to": "bq", "label": "load", "kind": "batch" } ], "note": "Tenant payloads live only in the registry and the ledger, encrypted under a per-tenant key, and are never copied into the evidence stores.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "11-data-model", "title": "Data Model — Trigger, Instant, Fire, Attempt", "layout": "er", "canvas": { "width": 1700, "cols": 4 }, "rowGap": 252, "entities": [ { "id": "tenant", "name": "tenant", "kind": "store", "row": 0, "col": 0, "attrs": [ "tenant_id PK", "quota_triggers", "quota_dispatch_per_s", "quota_inflight", "catchup_rate_pct" ] }, { "id": "trigger", "name": "trigger", "kind": "store", "row": 0, "col": 1, "attrs": [ "trigger_id PK", "tenant_id FK", "name UQ(tenant,name)", "state live|paused|deleted", "current_version FK" ] }, { "id": "version", "name": "trigger_version", "kind": "store", "row": 0, "col": 2, "attrs": [ "version_id PK", "trigger_id FK", "recurrence", "zone IANA", "target_id FK", "payload_ref encrypted", "missed_policy", "overlap_policy", "catchup_horizon_s", "attempt_budget" ] }, { "id": "target", "name": "target", "kind": "store", "row": 0, "col": 3, "attrs": [ "target_id PK", "tenant_id FK", "kind http|queue|exec", "endpoint", "verified_at", "verification_state" ] }, { "id": "audit", "name": "audit_entry", "kind": "store", "row": 1, "col": 0, "attrs": [ "audit_id PK", "tenant_id FK", "actor", "action", "prior_value", "at" ] }, { "id": "due", "name": "due_index_row", "kind": "store", "row": 1, "col": 1, "attrs": [ "partition_id PK1", "next_instant PK2", "trigger_id FK", "version_id FK", "claimed_by nullable" ] }, { "id": "fire", "name": "fire", "kind": "store", "row": 1, "col": 2, "attrs": [ "tenant_id PK1", "trigger_id PK2", "scheduled_instant PK3", "sequence PK4", "version_id FK", "origin sched|manual|backfill", "state", "non_dispatch_cause", "decided_at", "tzdata_version" ] }, { "id": "attempt", "name": "attempt", "kind": "store", "row": 1, "col": 3, "attrs": [ "attempt_id PK", "fire_key FK", "n", "dispatched_at", "transport_result", "http_status", "token_jti" ] }, { "id": "lease", "name": "partition_lease", "kind": "store", "row": 2, "col": 1, "attrs": [ "partition_id PK", "owner_id", "expires_at", "epsilon_ms_at_grant" ] }, { "id": "outcome", "name": "work_outcome", "kind": "store", "row": 2, "col": 2, "attrs": [ "fire_key PK/FK", "terminal_state", "reported_at", "reported_by", "cause" ] }, { "id": "histr", "name": "history_row", "kind": "store", "row": 2, "col": 3, "attrs": [ "row_key tenant#trigger#instant", "fire + attempts, projected", "ttl 90 d" ] } ], "relations": [ { "from": "tenant", "to": "trigger", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "trigger", "to": "version", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "version", "to": "target", "label": "N : 1", "from_side": "e", "to_side": "w" }, { "from": "tenant", "to": "audit", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "trigger", "to": "due", "label": "1 : 1", "from_side": "s", "to_side": "n" }, { "from": "version", "to": "fire", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "fire", "to": "attempt", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "due", "to": "lease", "label": "N : 1", "from_side": "s", "to_side": "n" }, { "from": "fire", "to": "outcome", "label": "1 : 0..1", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "attempt", "to": "histr", "label": "N : 1", "from_side": "s", "to_side": "n" } ], "note": "The fire's four-part primary key is the idempotency key: a duplicate decision is a constraint violation, not a detection problem. Non-dispatch is a field on the fire, not a separate record.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "12-due-to-dispatch-sequence", "title": "Critical Flow — One Instant, Decided and Delivered", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "clock", "label": "Time authority", "kind": "platform" }, { "id": "own", "label": "Partition owner", "kind": "app" }, { "id": "duei", "label": "Due index", "kind": "store" }, { "id": "fl", "label": "Fire ledger", "kind": "store" }, { "id": "lane", "label": "Lane queue", "kind": "queue" }, { "id": "att", "label": "Attempt runner", "kind": "app" }, { "id": "tgt", "label": "Tenant target", "kind": "external" } ], "messages": [ { "from": "own", "to": "clock", "label": "now, with ε", "kind": "call" }, { "from": "clock", "to": "own", "label": "[t-ε, t+ε]", "kind": "return" }, { "from": "own", "to": "own", "label": "ε > 100 ms? shed leases", "kind": "self" }, { "from": "own", "to": "duei", "label": "scan due ≤ t-ε", "kind": "call" }, { "from": "duei", "to": "own", "label": "instants + versions", "kind": "return" }, { "from": "own", "to": "own", "label": "apply overlap policy", "kind": "self" }, { "from": "own", "to": "fl", "label": "insert fire (key)", "kind": "call" }, { "from": "fl", "to": "own", "label": "committed", "kind": "return" }, { "from": "fl", "to": "own", "label": "duplicate key: already decided", "kind": "error" }, { "from": "own", "to": "duei", "label": "advance next_instant", "kind": "call" }, { "from": "own", "to": "lane", "label": "enqueue on-time", "kind": "async" }, { "from": "lane", "to": "att", "label": "lease task", "kind": "call" }, { "from": "att", "to": "att", "label": "mint token, sign record", "kind": "self" }, { "from": "att", "to": "tgt", "label": "POST fire + key", "kind": "call" }, { "from": "tgt", "to": "att", "label": "202 accepted", "kind": "return" }, { "from": "tgt", "to": "att", "label": "timeout: state unknown", "kind": "error" }, { "from": "att", "to": "fl", "label": "record attempt + state", "kind": "async" }, { "from": "tgt", "to": "fl", "label": "outcome callback", "kind": "async" } ], "note": "The ledger insert is the decision. Everything after it is retryable; nothing before it is dispatched.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "13-catch-up-after-an-outage", "title": "Catch-up — Draining a Backlog Without Causing the Next Outage", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "middle", "nodeWidth": 185, "stages": [ { "title": "Gap detected", "nodes": [ { "id": "res", "label": "Resume or recovery", "kind": "app" }, { "id": "miss", "label": "Missed instant set", "sub": "computed", "kind": "app" } ] }, { "title": "Horizon filter", "nodes": [ { "id": "hz", "label": "Older than horizon?", "kind": "decision" }, { "id": "exp", "label": "Record expired", "sub": "with cause", "kind": "risk" } ] }, { "title": "Policy", "nodes": [ { "id": "all", "label": "fire-all", "sub": "every instant", "kind": "app" }, { "id": "once", "label": "fire-once-now", "sub": "default", "kind": "app" }, { "id": "skip", "label": "skip", "sub": "record only", "kind": "app" } ] }, { "title": "Commit", "nodes": [ { "id": "fl", "label": "Fire records", "sub": "original instants", "kind": "store" } ] }, { "title": "Rate shape", "nodes": [ { "id": "cap", "label": "Tenant catch-up cap", "sub": "≤ 10% of quota", "kind": "decision" }, { "id": "clane", "label": "Catch-up lane", "sub": "separate queue", "kind": "queue" } ] }, { "title": "Interleave", "nodes": [ { "id": "fair", "label": "Fair across triggers", "sub": "oldest-first", "kind": "app" } ] }, { "title": "Deliver", "nodes": [ { "id": "att", "label": "Attempt runner", "kind": "app" }, { "id": "tgt", "label": "Tenant target", "kind": "external" } ] } ], "edges": [ { "from": "hz", "to": "exp", "label": "yes", "kind": "error" }, { "from": "clane", "to": "fair", "label": "drain rate" }, { "from": "fl", "to": "cap", "label": "backlog depth" } ], "note": "The on-time lane is untouched by this path. A backlog drains over hours by design — recovery is deliberately slower than the failure.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" }, "stageGap": 44 }, { "id": "14-dispatch-lanes", "title": "Dispatch Lanes — Fire Classes Through the Same Machinery", "layout": "swimlane", "canvas": { "width": 1760 }, "laneHeaderWidth": 152, "stages": [ "Admit", "Shape", "Queue", "Attempt", "Account" ], "lanes": [ { "title": "On-time", "cells": [ [ { "id": "a1", "label": "Due now", "kind": "app" } ], [ { "id": "s1", "label": "Jitter smear", "sub": "opt-out allowed", "kind": "app" } ], [ { "id": "q1", "label": "On-time lane", "sub": "drains first", "kind": "queue" } ], [ { "id": "t1", "label": "Full attempt budget", "kind": "app" } ], [ { "id": "h1", "label": "Lateness p99", "kind": "store" } ] ] }, { "title": "Catch-up", "cells": [ [ { "id": "a2", "label": "Missed instant", "kind": "app" } ], [ { "id": "s2", "label": "Cap at 10%", "kind": "decision" } ], [ { "id": "q2", "label": "Catch-up lane", "kind": "queue" } ], [ { "id": "t2", "label": "Original instant kept", "kind": "app" } ], [ { "id": "h2", "label": "Backlog depth", "kind": "store" } ] ] }, { "title": "Backfill", "cells": [ [ { "id": "a3", "label": "Authorised window", "kind": "security" } ], [ { "id": "s3", "label": "Volume ceiling", "kind": "decision" } ], [ { "id": "q3", "label": "Backfill lane", "sub": "shed first", "kind": "queue" } ], [ { "id": "t3", "label": "Labelled backfill", "kind": "app" } ], [ { "id": "h3", "label": "Audit entry", "kind": "store" } ] ] }, { "title": "Manual run", "cells": [ [ { "id": "a4", "label": "Operator action", "kind": "actor" } ], [ { "id": "s4", "label": "No displacement", "kind": "app" } ], [ { "id": "q4", "label": "On-time lane", "kind": "queue" } ], [ { "id": "t4", "label": "Own fire key", "kind": "app" } ], [ { "id": "h4", "label": "Labelled manual", "kind": "store" } ] ] }, { "title": "Retry", "cells": [ [ { "id": "a5", "label": "Transport failure", "kind": "risk" } ], [ { "id": "s5", "label": "Backoff + jitter", "kind": "app" } ], [ { "id": "q5", "label": "Origin lane", "kind": "queue" } ], [ { "id": "t5", "label": "Truncated at next instant", "kind": "decision" } ], [ { "id": "h5", "label": "Attempt rows", "kind": "store" } ] ] } ], "note": "One pipeline, five classes, one published shedding order: backfill, then catch-up, then over-quota, then on-time.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "15-deployment-architecture", "title": "Deployment — Region, Zones and the Standby", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Primary region — europe-west4", "kind": "cloud", "dir": "col", "children": [ { "title": "Zone a", "kind": "boundary", "nodes": [ { "id": "cr1", "label": "Control plane", "sub": "Cloud Run", "kind": "app" }, { "id": "tp1", "label": "Timing plane", "sub": "partitions 0-85", "kind": "app" }, { "id": "dp1", "label": "Dispatch plane", "kind": "app" } ] }, { "title": "Zone b", "kind": "boundary", "nodes": [ { "id": "cr2", "label": "Control plane", "kind": "app" }, { "id": "tp2", "label": "Timing plane", "sub": "partitions 86-170", "kind": "app" }, { "id": "dp2", "label": "Dispatch plane", "kind": "app" } ] }, { "title": "Zone c", "kind": "boundary", "nodes": [ { "id": "cr3", "label": "Control plane", "kind": "app" }, { "id": "tp3", "label": "Timing plane", "sub": "partitions 171-255", "kind": "app" }, { "id": "dp3", "label": "Dispatch plane", "kind": "app" } ] }, { "title": "Regional state — synchronous across all three zones", "kind": "trust", "nodes": [ { "id": "sp", "label": "Spanner regional", "sub": "registry, due index, ledger", "kind": "store" }, { "id": "bt", "label": "Bigtable", "sub": "fire history", "kind": "store" }, { "id": "ct", "label": "Cloud Tasks", "sub": "lane queues", "kind": "queue" } ] } ] }, { "title": "Standby region — europe-west1 (read replica, promoted by declaration)", "kind": "cloud", "dir": "row", "children": [ { "title": "Cold tiers", "kind": "plain", "nodes": [ { "id": "crs", "label": "Control plane", "sub": "scaled to zero", "kind": "app" }, { "id": "tps", "label": "Timing plane", "sub": "no leases held", "kind": "app" } ] }, { "title": "Replicated state", "kind": "plain", "nodes": [ { "id": "sps", "label": "Spanner replica", "sub": "RPO ≤ 15 s", "kind": "store" }, { "id": "gcs", "label": "History archive", "sub": "dual-region", "kind": "store" } ] } ] } ], "outside": [ { "id": "time", "label": "Time authority", "kind": "platform" }, { "id": "tgt", "label": "Tenant targets", "kind": "external" } ], "edges": [ { "from": "tp1", "to": "sp", "label": "lease + claim" }, { "from": "dp2", "to": "ct", "label": "lane queues" }, { "from": "sp", "to": "sps", "label": "async replicate", "kind": "async" }, { "from": "dp3", "to": "tgt", "label": "attempts", "route": "gutter" }, { "from": "tp2", "to": "time", "label": "bounded ε" } ], "note": "One write region. The standby holds no leases, because two regions deciding the same instant is a merge problem the fire key only partly solves.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "16-observability", "title": "Observability — Signal Type by Fire-Path Stage", "layout": "grid", "canvas": { "width": 1780 }, "laneHeaderWidth": 150, "columns": [ "Declare", "Decide", "Commit", "Dispatch", "Outcome" ], "rows": [ { "title": "The alert", "cells": [ [ { "label": "Validation failure rate", "kind": "platform" } ], [ { "label": "Oldest undispatched due age", "sub": "> 60 s pages", "kind": "risk" } ], [ { "label": "Ledger write errors", "kind": "risk" } ], [ { "label": "Lateness p99 breach", "kind": "risk" } ], [ { "label": "Unknown-state fraction", "kind": "risk" } ] ] }, { "title": "Metrics", "cells": [ [ { "label": "Definitions / s", "kind": "platform" }, { "label": "Propagation p99", "kind": "platform" } ], [ { "label": "Clock ε per node", "kind": "platform" }, { "label": "Leases vs partitions", "kind": "platform" } ], [ { "label": "Fires committed / s", "kind": "platform" }, { "label": "Duplicate-key rate", "kind": "platform" } ], [ { "label": "Lateness histogram", "kind": "platform" }, { "label": "Queue depth per lane", "kind": "platform" } ], [ { "label": "Success by target class", "kind": "platform" } ] ] }, { "title": "Traces", "cells": [ [ { "label": "Validate span", "kind": "app" } ], [ { "label": "Scan → claim span", "kind": "app" } ], [ { "label": "Commit span", "kind": "app" } ], [ { "label": "Attempt span per n", "kind": "app" } ], [ { "label": "Callback correlation", "kind": "app" } ] ] }, { "title": "Tenant-facing", "cells": [ [ { "label": "Next 3 instants", "kind": "app" } ], [ { "label": "Skip cause", "kind": "app" } ], [ { "label": "Fire record", "kind": "app" } ], [ { "label": "Per-trigger lateness", "kind": "app" } ], [ { "label": "Terminal state", "kind": "app" } ] ] }, { "title": "Audit", "cells": [ [ { "label": "Definition change", "kind": "security" } ], [ { "label": "Lease transfer", "kind": "security" } ], [ { "label": "Manual run", "kind": "security" } ], [ { "label": "Backfill grant", "kind": "security" } ], [ { "label": "Quota change", "kind": "security" } ] ] }, { "title": "Reporting", "cells": [ [ { "label": "Trigger population", "kind": "store" } ], [ { "label": "Shadow divergence", "kind": "store" } ], [ { "label": "Fire counts for billing", "kind": "store" } ], [ { "label": "Peak avoided by jitter", "kind": "store" } ], [ { "label": "Cost per million fires", "kind": "store" } ] ] } ], "note": "Oldest undispatched due age is the only signal that catches the characteristic failure: a scheduler that is up, healthy on every other dashboard, and not firing.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "17-trigger-lifecycle", "title": "Trigger Lifecycle — The Loop That Has to Close", "layout": "cycle", "canvas": { "width": 1600 }, "centre": { "label": "Trigger", "sub": "one version at a time" }, "nodes": [ { "id": "declared", "label": "Declared", "sub": "validated, versioned", "kind": "app" }, { "id": "indexed", "label": "Indexed", "sub": "next instant computed", "kind": "store" }, { "id": "decided", "label": "Decided", "sub": "instant claimed", "kind": "decision" }, { "id": "dispatched", "label": "Dispatched", "sub": "at-least-once", "kind": "app" }, { "id": "accounted", "label": "Accounted", "sub": "outcome or unknown", "kind": "store" }, { "id": "observed", "label": "Observed", "sub": "lateness published", "kind": "platform" }, { "id": "amended", "label": "Amended or paused", "sub": "new version", "kind": "app" } ], "ringLabels": [ "recompute", "clock gate", "commit then send", "callback window", "per-trigger view", "tenant acts", "revalidate" ], "rx": 440, "ry": 215, "note": "The loop closes because the tenant can see lateness. Without the observed step, a trigger that drifted would only be found by a customer.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "18-security-trust-zones", "title": "Security — Trust Zones and What Crosses Them", "layout": "zones", "canvas": { "width": 1740 }, "zones": [ { "title": "Internet — untrusted", "kind": "trust", "nodes": [ { "id": "dev", "label": "Tenant engineer", "kind": "actor" }, { "id": "tgt", "label": "Tenant HTTP target", "kind": "external" }, { "id": "atk", "label": "Forged callback", "kind": "risk" } ] }, { "title": "Perimeter", "kind": "trust", "nodes": [ { "id": "lb", "label": "Load balancer", "sub": "TLS termination", "kind": "integration" }, { "id": "armor", "label": "Edge protection", "kind": "security" }, { "id": "oidc", "label": "OIDC verification", "kind": "security" }, { "id": "cbv", "label": "Callback verifier", "sub": "signature + window", "kind": "security" } ] }, { "title": "Control zone — authorship and privilege", "kind": "trust", "nodes": [ { "id": "mapi", "label": "Management API", "kind": "integration" }, { "id": "rbac", "label": "Role check", "sub": "author ǀ policy ǀ read", "kind": "security" }, { "id": "tgtv", "label": "Target ownership check", "sub": "SSRF guard", "kind": "security" }, { "id": "aud", "label": "Audit writer", "kind": "security" } ] }, { "title": "Fire zone — no tenant request reaches here", "kind": "trust", "nodes": [ { "id": "tp", "label": "Timing plane", "kind": "app" }, { "id": "dp", "label": "Dispatch plane", "kind": "app" }, { "id": "mint", "label": "Per-attempt token", "sub": "attempt lifetime", "kind": "security" }, { "id": "sign", "label": "Dispatch signer", "kind": "security" } ] }, { "title": "Data zone", "kind": "trust", "nodes": [ { "id": "sp", "label": "Registry + ledger", "sub": "payload encrypted", "kind": "store" }, { "id": "kms", "label": "Per-tenant keys", "kind": "security" }, { "id": "al", "label": "Audit store", "sub": "immutable 7 y", "kind": "store" } ] } ], "edges": [ { "from": "dev", "to": "lb", "label": "HTTPS" }, { "from": "lb", "to": "oidc", "label": "token" }, { "from": "oidc", "to": "mapi", "label": "authenticated" }, { "from": "mapi", "to": "rbac", "label": "authorise" }, { "from": "rbac", "to": "tgtv", "label": "verify target" }, { "from": "tgtv", "to": "sp", "label": "write version" }, { "from": "aud", "to": "al", "label": "append", "kind": "async" }, { "from": "tp", "to": "sp", "label": "claim, commit" }, { "from": "sp", "to": "kms", "label": "decrypt" }, { "from": "atk", "to": "cbv", "label": "rejected", "kind": "error" }, { "from": "sign", "to": "tgt", "label": "signed dispatch", "route": "gutter" } ], "note": "Three guards carry most of the risk: the target ownership check that stops a schedule becoming an SSRF primitive, the privilege split that keeps backfill away from authorship, and the callback verifier. The monitoring and audit fan-out from every zone is omitted.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "19-privilege-and-dispatch-identity", "title": "Identity — Who Proves What, in What Order", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "actor", "label": "Tenant engineer", "kind": "actor" }, { "id": "idp", "label": "Identity provider", "kind": "security" }, { "id": "mapi", "label": "Management API", "kind": "integration" }, { "id": "rbac", "label": "Authorisation", "kind": "security" }, { "id": "wid", "label": "Workload identity", "kind": "security" }, { "id": "att", "label": "Attempt runner", "kind": "app" }, { "id": "tgt", "label": "Tenant target", "kind": "external" } ], "messages": [ { "from": "actor", "to": "idp", "label": "authenticate", "kind": "call" }, { "from": "idp", "to": "actor", "label": "OIDC token", "kind": "return" }, { "from": "actor", "to": "mapi", "label": "create trigger", "kind": "call" }, { "from": "mapi", "to": "rbac", "label": "role: author?", "kind": "call" }, { "from": "rbac", "to": "mapi", "label": "granted", "kind": "return" }, { "from": "mapi", "to": "mapi", "label": "verify target ownership", "kind": "self" }, { "from": "actor", "to": "mapi", "label": "extend horizon to 24 h", "kind": "call" }, { "from": "mapi", "to": "rbac", "label": "role: policy?", "kind": "call" }, { "from": "rbac", "to": "mapi", "label": "denied: separate grant", "kind": "error" }, { "from": "att", "to": "wid", "label": "mint for this attempt", "kind": "call" }, { "from": "wid", "to": "att", "label": "token, attempt lifetime", "kind": "return" }, { "from": "att", "to": "att", "label": "sign fire record", "kind": "self" }, { "from": "att", "to": "tgt", "label": "dispatch + signature", "kind": "call" }, { "from": "tgt", "to": "wid", "label": "verify issuer", "kind": "call" }, { "from": "wid", "to": "tgt", "label": "valid, scoped to trigger", "kind": "return" }, { "from": "tgt", "to": "att", "label": "202 accepted", "kind": "return" } ], "note": "Authorship and policy change are separate grants. The dispatch credential is minted per attempt and expires with it, so a captured dispatch buys one trigger for one timeout.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } }, { "id": "20-failure-classes", "title": "Assurance — Failure Classes and Their Answers", "layout": "grid", "canvas": { "width": 1780 }, "laneHeaderWidth": 168, "columns": [ "Assumed to fail", "How it shows", "Structural answer", "Residual risk" ], "rows": [ { "title": "Clock", "cells": [ [ { "label": "Skew, step, lost sync", "kind": "risk" } ], [ { "label": "ε above bound", "kind": "platform" } ], [ { "label": "Shed leases, wait out ε", "kind": "app" } ], [ { "label": "Fleet-wide ε drift", "kind": "risk" } ] ] }, { "title": "Owner loss", "cells": [ [ { "label": "Death between claim and send", "kind": "risk" } ], [ { "label": "Lease expiry", "kind": "platform" } ], [ { "label": "Re-derive from registry", "kind": "app" } ], [ { "label": "One duplicate attempt", "kind": "risk" } ] ] }, { "title": "Split ownership", "cells": [ [ { "label": "Two owners mid-handover", "kind": "risk" } ], [ { "label": "Duplicate-key rate", "kind": "platform" } ], [ { "label": "Fire key uniqueness", "kind": "store" } ], [ { "label": "Executor must dedupe", "kind": "risk" } ] ] }, { "title": "State unavailable", "cells": [ [ { "label": "Registry or index down", "kind": "risk" } ], [ { "label": "Rising due age", "kind": "platform" } ], [ { "label": "Fail closed on dispatch", "kind": "decision" } ], [ { "label": "Stall, then catch-up", "kind": "risk" } ] ] }, { "title": "Target", "cells": [ [ { "label": "Transient, 4xx, systematic", "kind": "risk" } ], [ { "label": "Success by target class", "kind": "platform" } ], [ { "label": "Budget, stop, isolate", "kind": "app" } ], [ { "label": "Tenant unaware of 4xx", "kind": "risk" } ] ] }, { "title": "Poison definition", "cells": [ [ { "label": "No future instant", "kind": "risk" } ], [ { "label": "Error on the trigger", "kind": "platform" } ], [ { "label": "Quarantine one trigger", "kind": "app" } ], [ { "label": "Slow compiler path", "kind": "risk" } ] ] }, { "title": "Catch-up storm", "cells": [ [ { "label": "Backlog released at once", "kind": "risk" } ], [ { "label": "Backlog depth per tenant", "kind": "platform" } ], [ { "label": "Horizon, cap, own lane", "kind": "queue" } ], [ { "label": "Alert noisy while draining", "kind": "risk" } ] ] }, { "title": "Slow executor", "cells": [ [ { "label": "Run outlives the interval", "kind": "risk" } ], [ { "label": "Unknown-state fraction", "kind": "platform" } ], [ { "label": "Overlap policy + window", "kind": "decision" } ], [ { "label": "Concurrent run, or silence", "kind": "risk" } ] ] }, { "title": "Zone or region", "cells": [ [ { "label": "Zone loss, region loss", "kind": "risk" } ], [ { "label": "Degraded lateness p99", "kind": "platform" } ], [ { "label": "Reassign; declared promotion", "kind": "app" } ], [ { "label": "15 s of decisions at RPO", "kind": "risk" } ] ] }, { "title": "tzdata change", "cells": [ [ { "label": "Future instant moves", "kind": "risk" } ], [ { "label": "Version on each fire", "kind": "platform" } ], [ { "label": "Recomputed instant governs", "kind": "decision" } ], [ { "label": "Surprise at the boundary", "kind": "risk" } ] ] } ], "note": "Every row's residual risk is accepted knowingly. The two that would change the design are fleet-wide clock drift and an executor that cannot deduplicate.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-10" } } ]