[ { "id": "01-system-context", "title": "Distributed Lock Service — System Context", "layout": "context", "colWidth": 250, "system": { "label": "Distributed Lock Service", "sub": "leases + fencing tokens", "icon": "fa5_lock" }, "groups": [ { "side": "top", "title": "Callers", "nodes": [ { "id": "c-sched", "label": "Job schedulers", "kind": "app", "rel": "run once", "dir": "in", "icon": "fa5_calendar_check" }, { "id": "c-rs", "label": "Replica sets", "kind": "app", "rel": "elect leader", "dir": "in", "icon": "fa5_crown" }, { "id": "c-work", "label": "Entity workers", "kind": "app", "rel": "lock a record", "dir": "in", "icon": "fa5_cogs" } ] }, { "side": "left", "title": "People", "nodes": [ { "id": "p-owner", "label": "Service owner", "kind": "actor", "rel": "declares classes", "dir": "in" }, { "id": "p-sre", "label": "On-call SRE", "kind": "actor", "rel": "inspects, releases", "dir": "in" }, { "id": "p-sec", "label": "Security reviewer", "kind": "actor", "rel": "reads audit", "dir": "in" } ] }, { "side": "right", "title": "Platform dependencies", "nodes": [ { "id": "d-git", "label": "Git + Argo CD", "sub": "lock class config", "kind": "platform", "rel": "classes", "dir": "in", "kind2": "batch", "icon": "argocd" }, { "id": "d-spire", "label": "SPIRE", "sub": "workload identity", "kind": "security", "rel": "SVIDs", "icon": "fa5_id_badge" }, { "id": "d-kc", "label": "Keycloak", "sub": "operator sign-in", "kind": "security", "rel": "OIDC", "icon": "keycloak" }, { "id": "d-obs", "label": "Observability", "sub": "Prometheus · Tempo", "kind": "platform", "rel": "metrics", "kind2": "async", "icon": "grafana" } ] }, { "side": "bottom", "title": "", "nodes": [ { "id": "r-db", "label": "Guarded databases", "kind": "external", "rel": "token rule", "kind2": "async", "icon": "fa5_database" }, { "id": "r-obj", "label": "Guarded object stores", "kind": "external", "rel": "token rule", "kind2": "async", "icon": "fa5_hdd" }, { "id": "r-leg", "label": "Unfenced legacy stores", "kind": "risk", "rel": "risk register", "kind2": "error", "icon": "fa5_unlock_alt" } ] } ], "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "02-high-level-architecture", "title": "Distributed Lock Service — High-Level Architecture", "layout": "flow", "chain": true, "stages": [ { "title": "Call", "nodes": [ { "id": "h-app", "label": "Caller workload", "kind": "app", "icon": "fa5_cogs" }, { "id": "h-lib", "label": "Client library", "sub": "fence callback", "kind": "app", "icon": "fa5_puzzle_piece" } ] }, { "title": "Admit", "nodes": [ { "id": "h-envoy", "label": "Envoy fan-in", "sub": "mTLS · ring hash", "kind": "integration", "icon": "envoy" }, { "id": "h-spire", "label": "SPIRE", "sub": "SPIFFE IDs", "kind": "security", "icon": "fa5_id_badge" } ] }, { "title": "Decide", "nodes": [ { "id": "h-arb", "label": "Lock arbiter", "sub": "Go · gRPC", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "h-cls", "label": "Class registry", "sub": "CRDs from Git", "kind": "platform", "icon": "fa5_clipboard_list" } ] }, { "title": "Commit", "nodes": [ { "id": "h-etcd", "label": "Lock log", "sub": "etcd · Raft", "kind": "store", "icon": "fa5_server" }, { "id": "h-tail", "label": "Audit tailer", "kind": "app", "icon": "fa5_history" }, { "id": "h-aud", "label": "Audit store", "sub": "ClickHouse", "kind": "store", "icon": "clickhouse" } ] }, { "title": "Enforce", "nodes": [ { "id": "h-res", "label": "Guarded resource", "sub": "token > high-water", "kind": "external", "icon": "fa5_database" } ] } ], "edges": [ { "from": "h-arb", "to": "h-etcd", "label": "Txn + lease" }, { "from": "h-etcd", "to": "h-tail", "label": "watch", "kind": "async" }, { "from": "h-tail", "to": "h-aud", "label": "batch", "kind": "batch" }, { "from": "h-lib", "to": "h-res", "label": "write + token", "route": "gutter" } ], "note": "The lock log decides who holds; only the resource can refuse a stale writer.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "03-actors-and-journeys", "title": "Actors and Their Core Journeys", "layout": "actors", "cardWidth": 270, "groups": [ { "title": "Resource owners", "kind": "boundary", "actors": [ { "id": "a-owner", "label": "Service owner", "sub": "~300 namespaces", "goal": "Declare a lock once, and know my store will reject a writer whose turn has passed.", "journeys": [ { "id": "j-onboard", "label": "Onboard a fenced lock class" }, { "label": "Read contention and cost" } ] } ] }, { "title": "Callers", "kind": "plain", "actors": [ { "id": "a-sched", "label": "Job scheduler", "kind": "platform", "sub": "coarse · long hold", "goal": "Run tonight's settlement once, even when two pods wake up at 02:00.", "journeys": [ { "label": "Try-acquire before running" } ], "icon": "fa5_calendar_check" }, { "id": "a-rs", "label": "Replica set", "kind": "platform", "sub": "tens of keys", "goal": "Have exactly one leader, and step down the moment I cannot prove I still am.", "journeys": [ { "label": "Hold leadership for hours" } ], "icon": "fa5_crown" }, { "id": "a-worker", "label": "Entity worker", "kind": "platform", "sub": "250k leases", "goal": "Mutate a record nobody else is mutating, without paying for it twice.", "journeys": [ { "label": "Lock, write with token, release" } ], "icon": "fa5_cogs" } ] }, { "title": "Operators and assurance", "kind": "boundary", "actors": [ { "id": "a-sre", "label": "On-call SRE", "sub": "24x7 rota", "goal": "Unstick a blocked key at 3 a.m. without turning one outage into two.", "journeys": [ { "id": "j-stuck", "label": "Clear a stuck lock" }, { "label": "Recover from quorum loss" } ] }, { "id": "a-plat", "label": "Platform team", "sub": "owns the clusters", "goal": "Grow the key space and upgrade etcd without a migration weekend.", "journeys": [ { "label": "Add an entity shard" }, { "label": "Rolling etcd upgrade" } ] }, { "id": "a-sec", "label": "Security reviewer", "sub": "quarterly", "goal": "Prove who forced a release, why, and whether it was safe to.", "journeys": [ { "label": "Review force-release trail" }, { "label": "Chase advisory waivers" } ] } ] } ], "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "04-journey-onboard-lock-class", "title": "Journey — Service Owner Onboards a Correctness Lock", "layout": "journey", "actor": { "label": "Service owner", "sub": "payments ledger team", "goal": "Stop two settlement runs from posting the same batch", "trigger": "An incident review found a duplicate posting", "success": "Stale writer rejected in a staged pause test" }, "phases": [ { "title": "Question the lock" }, { "title": "Declare the class" }, { "title": "Fence the store", "moment": true }, { "title": "Prove the failure", "moment": true }, { "title": "Go live" }, { "title": "Operate" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Checks for a CAS first" } ], [ { "label": "Opens a LockClass PR" } ], [ { "label": "Adds token column" }, { "label": "Guards the UPDATE" } ], [ { "label": "Runs pause test" } ], [ { "label": "Enables in prod" } ], [ { "label": "Watches contention" } ] ] }, { "title": "Touchpoint", "kind": "touch", "cells": [ [ { "label": "Decision guide" } ], [ { "label": "Class schema lint" } ], [ { "label": "Reference fence" } ], [ { "label": "Fault-injection kit" } ], [ { "label": "Argo CD sync", "icon": "argocd" } ], [ { "label": "Namespace dashboard" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Unsure", "Blocked" ], "points": [ "Unsure", "Confident", "Blocked", "Unsure", "Confident", "Confident" ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "No one asked if needed" } ], [], [ { "label": "Legacy store has no CAS" } ], [ { "label": "Pauses are hard to stage" } ], [], [ { "label": "Cost only seen monthly" } ] ] }, { "title": "What the platform gives", "kind": "gain", "cells": [ [ { "label": "Contention ratio up front" } ], [ { "label": "TTL cost printed in PR" } ], [ { "label": "Time-boxed advisory waiver" } ], [ { "label": "Scripted SIGSTOP holder" } ], [], [ { "label": "Per-record lock alarm" } ] ] } ], "chain": true, "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "05-journey-clear-stuck-lock", "title": "Journey — On-call SRE Clears a Stuck Lock", "layout": "journey", "actor": { "label": "On-call SRE", "sub": "night rota", "goal": "Unblock the queue behind one key without a double write", "trigger": "Longest-hold alarm on the orders namespace", "success": "Queue drains and the old holder's writes are refused" }, "phases": [ { "title": "Paged" }, { "title": "Inspect" }, { "title": "Judge safety" }, { "title": "Force-release", "moment": true }, { "title": "Verify" }, { "title": "Review" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Acknowledges page" } ], [ { "label": "Looks up the key" } ], [ { "label": "Reads fenced flag" } ], [ { "label": "Requests release", "icon": "fa5_unlock_alt" }, { "label": "Peer approves" } ], [ { "label": "Watches queue drain" } ], [ { "label": "Files holder bug" } ] ] }, { "title": "System behind it", "kind": "system", "cells": [ [ { "label": "Alertmanager" } ], [ { "label": "Inspection API" } ], [ { "label": "Class registry", "icon": "fa5_clipboard_list" } ], [ { "label": "Admin API + Keycloak", "icon": "keycloak" } ], [ { "label": "Lock log watch" } ], [ { "label": "Audit store", "icon": "clickhouse" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "In control", "Uneasy", "Stressed" ], "points": [ "Stressed", "Uneasy", "Stressed", "Uneasy", "In control", "In control" ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Holder alive, not working" } ], [ { "label": "Advisory: release unsafe" } ], [ { "label": "Approver hard to reach" } ], [], [] ] }, { "title": "What the platform gives", "kind": "gain", "cells": [ [ { "label": "Runbook in the alert" } ], [ { "label": "Holder, token, age, queue" } ], [ { "label": "Safety stated, not implied" } ], [ { "label": "Token advances on release" } ], [ { "label": "Stale write rejected" } ], [ { "label": "Trail with reason" } ] ] } ], "chain": true, "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "06-layered-architecture", "title": "Layered Architecture", "layout": "bands", "layerHeaderWidth": 150, "bands": [ { "name": "Callers", "nodes": [ { "id": "l-sched", "label": "Job schedulers", "kind": "app", "icon": "fa5_calendar_check" }, { "id": "l-rs", "label": "Replica sets", "kind": "app", "icon": "fa5_crown" }, { "id": "l-work", "label": "Entity workers", "kind": "app", "icon": "fa5_cogs" } ] }, { "name": "Client", "nodes": [ { "id": "l-lib", "label": "Client library", "sub": "Go · Java · Python", "kind": "app", "icon": "fa5_puzzle_piece" }, { "id": "l-renew", "label": "Session keepalive", "sub": "TTL / 3", "kind": "app", "icon": "fa5_heartbeat" }, { "id": "l-fence", "label": "Fence callback", "sub": "required argument", "kind": "app", "icon": "fa5_hand_paper" } ] }, { "name": "Access", "nodes": [ { "id": "l-envoy", "label": "Envoy fan-in", "sub": "mTLS · ring hash", "kind": "integration", "icon": "envoy" }, { "id": "l-spire", "label": "SPIRE agent", "sub": "X.509 SVID", "kind": "security", "icon": "fa5_id_badge" } ] }, { "name": "Coordination", "nodes": [ { "id": "l-arb", "label": "Lock arbiter", "sub": "Go · stateless truth", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "l-queue", "label": "Wait queue", "sub": "in memory", "kind": "queue", "icon": "fa5_list_ol" }, { "id": "l-admin", "label": "Admin API", "sub": "inspect · force", "kind": "app", "icon": "fa5_user_shield" } ] }, { "name": "Record", "nodes": [ { "id": "l-coord", "label": "Lock log · coord", "sub": "etcd · 5 members", "kind": "store", "icon": "fa5_server" }, { "id": "l-ent", "label": "Lock log · entity", "sub": "etcd · 3 per shard", "kind": "store", "icon": "fa5_server" }, { "id": "l-cls", "label": "Class registry", "sub": "LockClass CRDs", "kind": "platform", "icon": "fa5_clipboard_list" } ] }, { "name": "Evidence", "nodes": [ { "id": "l-tail", "label": "Audit tailer", "sub": "resumes by revision", "kind": "app", "icon": "fa5_history" }, { "id": "l-ch", "label": "Audit store", "sub": "ClickHouse", "kind": "store", "icon": "clickhouse" }, { "id": "l-minio", "label": "Audit archive", "sub": "MinIO object lock", "kind": "store", "icon": "minio" } ] }, { "name": "Platform", "nodes": [ { "id": "l-prom", "label": "Prometheus", "kind": "platform", "icon": "prometheus" }, { "id": "l-otel", "label": "OTel Collector", "kind": "platform", "icon": "opentelemetry" }, { "id": "l-argo", "label": "Argo CD", "kind": "platform", "icon": "argocd" }, { "id": "l-kc", "label": "Keycloak", "kind": "security", "icon": "keycloak" } ] } ], "edges": [ { "from": "l-work", "to": "l-lib", "label": "acquire" }, { "from": "l-lib", "to": "l-envoy", "label": "gRPC" }, { "from": "l-envoy", "to": "l-arb", "label": "mTLS" }, { "from": "l-arb", "to": "l-ent", "label": "Txn" }, { "from": "l-ent", "to": "l-tail", "label": "watch", "kind": "async" } ], "note": "Dependencies point down. Nothing below Coordination calls up, and no caller reaches etcd directly.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "07-container-architecture", "title": "Container Architecture", "layout": "nested", "boxes": [ { "title": "Kubernetes · lock-system namespace", "kind": "boundary", "dir": "row", "children": [ { "title": "Access tier", "kind": "plain", "nodes": [ { "id": "k-envoy", "label": "Envoy fan-in", "sub": "6 replicas", "kind": "integration", "icon": "envoy" } ] }, { "title": "Coordination tier", "kind": "plain", "dir": "col", "nodes": [ { "id": "k-arb", "label": "Lock arbiter", "sub": "9 replicas", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "k-admin", "label": "Admin API", "sub": "2 replicas", "kind": "app", "icon": "fa5_user_shield" } ] }, { "title": "Dedicated etcd nodes · local NVMe", "kind": "onprem", "dir": "col", "nodes": [ { "id": "k-coord", "label": "Lock log · coord", "sub": "5 members", "kind": "store", "icon": "fa5_server" }, { "id": "k-ent", "label": "Lock log · entity-01", "sub": "3 members", "kind": "store", "icon": "fa5_server" } ] }, { "title": "Evidence tier", "kind": "plain", "dir": "col", "nodes": [ { "id": "k-tail", "label": "Audit tailer", "sub": "1 per cluster", "kind": "app", "icon": "fa5_history" }, { "id": "k-ch", "label": "Audit store", "sub": "ClickHouse", "kind": "store", "icon": "clickhouse" } ] } ] } ], "outside": [ { "id": "k-lib", "label": "Client library", "sub": "in caller pod", "kind": "app", "icon": "fa5_puzzle_piece" }, { "id": "k-res", "label": "Guarded resource", "sub": "checks token", "kind": "external", "icon": "fa5_database" } ], "edges": [ { "from": "k-lib", "to": "k-envoy", "label": "gRPC mTLS" }, { "from": "k-envoy", "to": "k-arb" }, { "from": "k-arb", "to": "k-coord" }, { "from": "k-arb", "to": "k-ent", "label": "Txn" }, { "from": "k-ent", "to": "k-tail", "label": "watch", "kind": "async" }, { "from": "k-tail", "to": "k-ch", "label": "batch", "kind": "batch" }, { "from": "k-lib", "to": "k-res", "label": "write + token" } ], "note": "etcd members run on tainted, dedicated nodes (view 19). The Admin API holds a delete-only etcd role; its path is on views 24 and 25.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "08-integration-surface", "title": "Integration Surface — Every Interface", "layout": "hub", "left": { "title": "Consumers", "nodes": [ { "id": "i-lib", "label": "Client library", "kind": "app", "rel": "Lock v1 gRPC", "icon": "fa5_puzzle_piece" }, { "id": "i-cli", "label": "lockctl CLI", "kind": "app", "rel": "Admin v1 gRPC" }, { "id": "i-res", "label": "Unfencable resource", "kind": "external", "rel": "ValidateToken", "icon": "fa5_database" }, { "id": "i-prom", "label": "Prometheus", "kind": "platform", "rel": "scrape", "icon": "prometheus" } ] }, "centre": { "title": "Lock service", "nodes": [ { "id": "i-core", "label": "Lock arbiter + Admin API", "sub": "proto · buf-checked", "icon": "fa5_balance_scale" } ] }, "right": { "title": "Dependencies", "nodes": [ { "id": "i-etcd", "label": "etcd clusters", "kind": "store", "rel": "etcd v3", "dir": "out", "icon": "fa5_server" }, { "id": "i-spire", "label": "SPIRE", "kind": "security", "rel": "SVID", "dir": "out", "icon": "fa5_id_badge" }, { "id": "i-kc", "label": "Keycloak", "kind": "security", "rel": "OIDC", "dir": "out", "icon": "keycloak" }, { "id": "i-k8s", "label": "Kubernetes API", "kind": "platform", "rel": "CRD watch", "dir": "out", "kind2": "async", "icon": "kubernetes" }, { "id": "i-otel", "label": "OTel Collector", "kind": "platform", "rel": "OTLP", "dir": "out", "kind2": "async", "icon": "opentelemetry" } ] }, "note": "ValidateToken is the Phase 2 fallback. Every call puts this service on a resource write path.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "09-class-and-cluster-routing", "title": "How a Key Finds Its Class and Its Cluster", "layout": "flow", "chain": false, "stages": [ { "title": "Request", "nodes": [ { "id": "r-key", "label": "Lock key", "sub": "tenant/ns/path", "kind": "app", "icon": "fa5_key" } ] }, { "title": "Classify", "nodes": [ { "id": "r-cls", "label": "Class match", "sub": "longest prefix", "kind": "platform", "icon": "fa5_clipboard_list" }, { "id": "r-deny", "label": "No class: refuse", "kind": "risk" } ] }, { "title": "Tier", "nodes": [ { "id": "r-tier", "label": "Class tier", "kind": "decision" } ] }, { "title": "Home", "nodes": [ { "id": "r-coord", "label": "Coord tier", "sub": "leader election", "kind": "app" }, { "id": "r-map", "label": "Shard map", "sub": "rendezvous hash", "kind": "app", "icon": "fa5_sitemap" } ] }, { "title": "Commit", "nodes": [ { "id": "r-ecoord", "label": "Lock log · coord", "kind": "store", "icon": "fa5_server" }, { "id": "r-e1", "label": "Lock log · entity-01", "kind": "store", "icon": "fa5_server" }, { "id": "r-e2", "label": "Lock log · entity-02", "kind": "store", "icon": "fa5_server" } ] } ], "edges": [ { "from": "r-key", "to": "r-cls" }, { "from": "r-cls", "to": "r-deny", "label": "unmatched", "kind": "error" }, { "from": "r-cls", "to": "r-tier", "label": "class" }, { "from": "r-tier", "to": "r-coord", "label": "coarse" }, { "from": "r-tier", "to": "r-map", "label": "entity" }, { "from": "r-coord", "to": "r-ecoord" }, { "from": "r-map", "to": "r-e1" }, { "from": "r-map", "to": "r-e2" } ], "note": "The class decides the tier. A caller never names a cluster, so a caller cannot get routing wrong.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "10-key-space-and-stores", "title": "Key Space and Stores, by Authority", "layout": "nested", "boxes": [ { "title": "Authoritative · lock log (etcd, linearizable)", "kind": "trust", "dir": "row", "children": [ { "title": "/sys", "kind": "boundary", "nodes": [ { "id": "s-epoch", "label": "Epoch record", "sub": "epoch + cluster ID", "kind": "store", "icon": "fa5_sort_numeric_up" }, { "id": "s-shard", "label": "Shard map", "sub": "versioned", "kind": "store", "icon": "fa5_sitemap" } ] }, { "title": "/l · held", "kind": "boundary", "nodes": [ { "id": "s-x", "label": "Exclusive holder", "sub": "on session lease", "kind": "store", "icon": "fa5_lock" }, { "id": "s-s", "label": "Shared holders", "sub": "one key each", "kind": "store", "icon": "fa5_users" } ] }, { "title": "/q · fair classes", "kind": "boundary", "nodes": [ { "id": "s-q", "label": "Waiter keys", "sub": "create revision", "kind": "store", "icon": "fa5_list_ol" } ] } ] }, { "title": "Rebuildable · advisory", "kind": "boundary", "dir": "col", "nodes": [ { "id": "s-mq", "label": "Wait queue", "sub": "arbiter memory", "kind": "queue", "icon": "fa5_list_ol" }, { "id": "s-watch", "label": "Watch stream", "sub": "from revision", "kind": "queue", "icon": "fa5_bell" } ] }, { "title": "Evidence · asynchronous", "kind": "boundary", "dir": "col", "nodes": [ { "id": "s-ch", "label": "Audit store", "sub": "ClickHouse · 90 d", "kind": "store", "icon": "clickhouse" }, { "id": "s-arch", "label": "Audit archive", "sub": "MinIO · 1 y / 7 y", "kind": "store", "icon": "minio" } ] }, { "title": "Declared · Git", "kind": "boundary", "dir": "col", "nodes": [ { "id": "s-cls", "label": "Lock classes", "sub": "TTL · mode · fenced", "kind": "platform", "icon": "fa5_clipboard_list" }, { "id": "s-acl", "label": "Prefix grants", "sub": "identity → prefix", "kind": "security", "icon": "fa5_key" } ] } ], "edges": [ { "from": "s-x", "to": "s-watch", "label": "revisions", "kind": "async" }, { "from": "s-ch", "to": "s-arch", "label": "nightly", "kind": "batch" } ], "note": "Only the top box answers who holds what. Everything else can be lost and rebuilt without a double grant.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "11-data-model", "title": "Lock Data Model", "layout": "er", "canvas": { "width": 1560, "cols": 3 }, "rowGap": 270, "entities": [ { "id": "e-ns", "name": "namespace", "kind": "store", "row": 0, "col": 0, "attrs": [ "ns_id PK", "tenant_id", "held_lease_quota", "op_rate_quota", "cluster_tier" ] }, { "id": "e-cls", "name": "lock_class", "kind": "store", "row": 0, "col": 1, "attrs": [ "class_id PK", "ns_id FK", "key_prefix", "ttl_min / ttl_max", "mode X | S", "enforcement fenced | advisory", "on_quorum_loss closed | open" ] }, { "id": "e-epoch", "name": "epoch", "kind": "store", "row": 0, "col": 2, "attrs": [ "epoch PK 16-bit", "etcd_cluster_id", "advanced_at", "reason restore | rehome" ] }, { "id": "e-sess", "name": "session", "kind": "store", "row": 1, "col": 0, "attrs": [ "etcd_lease_id PK", "spiffe_id", "ttl_s", "opened_rev" ] }, { "id": "e-hold", "name": "hold", "kind": "store", "row": 1, "col": 1, "attrs": [ "lock_key PK", "etcd_lease_id FK", "class_id FK", "grant_rev", "token = epoch·2^48 + grant_rev", "holder_meta" ] }, { "id": "e-wait", "name": "waiter", "kind": "store", "row": 2, "col": 0, "attrs": [ "lock_key + create_rev PK", "etcd_lease_id FK", "deadline" ] }, { "id": "e-evt", "name": "audit_event", "kind": "store", "row": 2, "col": 1, "attrs": [ "cluster_id + revision PK", "type grant | release | expire | force", "lock_key", "spiffe_id", "token", "unprotected bool" ] }, { "id": "e-force", "name": "force_release", "kind": "store", "row": 2, "col": 2, "attrs": [ "request_id PK", "revision FK", "requester / approver", "reason", "class_was_fenced" ] } ], "relations": [ { "from": "e-ns", "to": "e-cls", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-cls", "to": "e-hold", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "e-sess", "to": "e-hold", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-epoch", "to": "e-hold", "label": "1 : N", "from_side": "s1", "to_side": "n3" }, { "from": "e-sess", "to": "e-wait", "label": "1 : N", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "e-hold", "to": "e-evt", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "e-force", "to": "e-evt", "label": "1 : 1", "from_side": "w", "to_side": "e" } ], "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "12-audit-data-flow", "title": "Audit Data Flow — Derived From the Log, Not Dual-Written", "layout": "flow", "chain": true, "stages": [ { "title": "Commit", "nodes": [ { "id": "a-etcd", "label": "Lock log", "sub": "every revision", "kind": "store", "icon": "fa5_server" } ] }, { "title": "Capture", "nodes": [ { "id": "a-tail", "label": "Audit tailer", "sub": "checkpoint revision", "kind": "app", "icon": "fa5_history" }, { "id": "a-gap", "label": "Compaction gap", "sub": "alarm, never skip", "kind": "risk" } ] }, { "title": "Store", "nodes": [ { "id": "a-ch", "label": "Audit store", "sub": "ClickHouse · 90 d", "kind": "store", "icon": "clickhouse" } ] }, { "title": "Retain", "nodes": [ { "id": "a-m1", "label": "Grant archive", "sub": "Parquet · 1 y", "kind": "store", "icon": "minio" }, { "id": "a-m7", "label": "Force-release archive", "sub": "object lock · 7 y", "kind": "store", "icon": "minio" } ] }, { "title": "Use", "nodes": [ { "id": "a-risk", "label": "Advisory risk report", "kind": "platform" }, { "id": "a-cost", "label": "Cost attribution", "kind": "platform" }, { "id": "a-inv", "label": "Investigation", "kind": "platform" } ] } ], "edges": [ { "from": "a-tail", "to": "a-gap", "label": "lag > window", "kind": "error" } ], "note": "Idempotent on (cluster ID, revision): a tailer restart replays, it never double counts.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "13-token-space-lifecycle", "title": "Token Space Across Restore — Why It Never Regresses", "layout": "cycle", "centre": { "label": "Token space", "sub": "(epoch, revision)", "icon": "fa5_ticket_alt" }, "nodes": [ { "id": "t1", "label": "Serving epoch E", "kind": "app", "sub": "grants compare epoch", "icon": "fa5_ticket_alt" }, { "id": "t2", "label": "Snapshot", "kind": "store", "sub": "every 30 min", "icon": "fa5_camera" }, { "id": "t3", "label": "Quorum destroyed", "kind": "risk" }, { "id": "t4", "label": "Restore", "kind": "store", "sub": "new cluster ID", "icon": "fa5_undo" }, { "id": "t5", "label": "Sealed", "kind": "decision", "sub": "ID ≠ epoch record", "icon": "fa5_ban" }, { "id": "t6", "label": "Advance to E+1", "kind": "security", "sub": "one committed Txn", "icon": "fa5_sort_numeric_up" } ], "ringLabels": [ "etcdutl snapshot", "disaster", "etcdutl restore", "grants refused", "gated job", "first grant" ], "rx": 430, "ry": 220, "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "14-acquire-sequence", "title": "Acquire, Write, Release — the Uncontended Path", "layout": "sequence", "lifelines": [ { "id": "app", "label": "Caller workload", "kind": "app", "icon": "fa5_cogs" }, { "id": "lib", "label": "Client library", "kind": "app", "icon": "fa5_puzzle_piece" }, { "id": "arb", "label": "Lock arbiter", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "led", "label": "etcd leader", "kind": "store", "icon": "fa5_server" }, { "id": "fol", "label": "etcd followers", "kind": "store", "icon": "fa5_server" }, { "id": "res", "label": "Guarded resource", "kind": "external", "icon": "fa5_database" } ], "messages": [ { "from": "app", "to": "lib", "label": "acquire(key, onFence)", "kind": "call" }, { "from": "lib", "to": "arb", "label": "Acquire · session lease L", "kind": "call" }, { "from": "arb", "to": "arb", "label": "class · prefix grant · quota", "kind": "self" }, { "from": "arb", "to": "led", "label": "Txn: epoch = E, key absent → put on L", "kind": "call" }, { "from": "led", "to": "fol", "label": "append entry", "kind": "call" }, { "from": "fol", "to": "led", "label": "fsync · majority ack", "kind": "return" }, { "from": "led", "to": "arb", "label": "committed · revision R", "kind": "return" }, { "from": "arb", "to": "lib", "label": "granted · token (E, R)", "kind": "return" }, { "from": "lib", "to": "app", "label": "token · remaining TTL", "kind": "return" }, { "from": "app", "to": "res", "label": "UPDATE … WHERE fence < token", "kind": "call" }, { "from": "res", "to": "app", "label": "1 row · high-water = token", "kind": "return" }, { "from": "app", "to": "lib", "label": "release()", "kind": "call" }, { "from": "lib", "to": "arb", "label": "Release", "kind": "call" }, { "from": "arb", "to": "led", "label": "Txn: delete if held on L", "kind": "call" } ], "note": "One consensus round trip to grant, one to release. Renewal rides the session lease, not the lock.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "15-stale-holder-fenced", "title": "The Pause Longer Than the Lease — Why the Token Exists", "layout": "sequence", "lifelines": [ { "id": "a", "label": "Holder A", "kind": "app", "icon": "fa5_pause_circle" }, { "id": "log", "label": "Lock log", "kind": "store", "icon": "fa5_server" }, { "id": "b", "label": "Holder B", "kind": "app", "icon": "fa5_cogs" }, { "id": "res", "label": "Guarded resource", "kind": "external", "icon": "fa5_database" } ], "messages": [ { "from": "a", "to": "log", "label": "acquire", "kind": "call" }, { "from": "log", "to": "a", "label": "token 41", "kind": "return" }, { "from": "a", "to": "res", "label": "write · token 41", "kind": "call" }, { "from": "res", "to": "a", "label": "ok · high-water 41", "kind": "return" }, { "from": "a", "to": "a", "label": "stop-the-world pause 22 s", "kind": "self" }, { "from": "log", "to": "log", "label": "session lapses · revoke committed", "kind": "self" }, { "from": "b", "to": "log", "label": "acquire", "kind": "call" }, { "from": "log", "to": "b", "label": "token 57", "kind": "return" }, { "from": "b", "to": "res", "label": "write · token 57", "kind": "call" }, { "from": "res", "to": "b", "label": "ok · high-water 57", "kind": "return" }, { "from": "a", "to": "res", "label": "resumes · write · token 41", "kind": "error" }, { "from": "res", "to": "a", "label": "rejected · 41 ≤ 57", "kind": "error" }, { "from": "a", "to": "a", "label": "deadline passed → onFence", "kind": "self" } ], "note": "No lock protocol prevents message 11. Only the resource can refuse it.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "16-contention-and-wait", "title": "Wait Mode Under Contention", "layout": "sequence", "lifelines": [ { "id": "w", "label": "Waiter", "kind": "app", "icon": "fa5_hourglass_half" }, { "id": "arb", "label": "Arbiter · queue owner", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "log", "label": "Lock log", "kind": "store", "icon": "fa5_server" }, { "id": "h", "label": "Current holder", "kind": "app", "icon": "fa5_lock" } ], "messages": [ { "from": "w", "to": "arb", "label": "acquire · wait ≤ 10 s", "kind": "call" }, { "from": "arb", "to": "log", "label": "Txn: key absent?", "kind": "call" }, { "from": "log", "to": "arb", "label": "held · revision R0", "kind": "return" }, { "from": "arb", "to": "arb", "label": "enqueue · position 3 of 40 max", "kind": "self" }, { "from": "arb", "to": "log", "label": "watch key from R0", "kind": "async" }, { "from": "h", "to": "log", "label": "release", "kind": "call" }, { "from": "log", "to": "arb", "label": "delete event", "kind": "async" }, { "from": "arb", "to": "log", "label": "Txn for queue head", "kind": "call" }, { "from": "log", "to": "arb", "label": "committed · token", "kind": "return" }, { "from": "arb", "to": "w", "label": "granted · token", "kind": "return" }, { "from": "arb", "to": "w", "label": "or: deadline → not granted", "kind": "error" } ], "note": "A notification only triggers an attempt. A lost event falls back to a 1 s jittered retry, never to a grant.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "17-posture-by-class", "title": "What Each Lock Class Guarantees When Things Break", "layout": "swimlane", "laneHeaderWidth": 150, "stages": [ "Normal", "Holder pauses", "Renewal fails", "Quorum lost", "After recovery" ], "lanes": [ { "title": "Correctness · fenced", "cells": [ [ { "label": "Grant + token", "kind": "app" } ], [ { "label": "Stale write rejected", "kind": "opportunity" } ], [ { "label": "Fence at local deadline", "kind": "security" } ], [ { "label": "No grants · self-fence", "kind": "security" } ], [ { "label": "Epoch advance first", "kind": "security" } ] ] }, { "title": "Correctness · advisory", "cells": [ [ { "label": "Grant + token", "kind": "app" } ], [ { "label": "Double write possible", "kind": "risk" } ], [ { "label": "Fence at local deadline", "kind": "security" } ], [ { "label": "No grants · self-fence", "kind": "security" } ], [ { "label": "Epoch advance first", "kind": "security" } ] ] }, { "title": "Efficiency", "cells": [ [ { "label": "Grant + token", "kind": "app" } ], [ { "label": "Duplicate work only", "kind": "external", "icon": "fa5_copy" } ], [ { "label": "Fence at local deadline", "kind": "security" } ], [ { "label": "Open if declared", "kind": "decision" } ], [ { "label": "Reconcile unprotected", "kind": "platform" } ] ] } ], "note": "Postures are declared on the class and reviewed by the resource owner. Nothing here is a runtime default.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "18-holder-when-unconfirmed", "title": "What a Holder Does When It Cannot Confirm Its Lease", "layout": "flow", "chain": false, "stages": [ { "title": "Hold", "nodes": [ { "id": "u-hold", "label": "Holding", "sub": "keepalive every 5 s", "kind": "app", "icon": "fa5_lock" } ] }, { "title": "Signal", "nodes": [ { "id": "u-miss", "label": "Keepalive fails", "sub": "timeout or lost", "kind": "risk" } ] }, { "title": "Decide", "nodes": [ { "id": "u-dec", "label": "Confirmed in time?", "kind": "decision" } ] }, { "title": "Drain", "nodes": [ { "id": "u-ok", "label": "Resume holding", "sub": "new deadline", "kind": "app" }, { "id": "u-stop", "label": "Stop starting work", "sub": "finish in-flight", "kind": "security" } ] }, { "title": "Fence", "nodes": [ { "id": "u-fence", "label": "onFence callback", "sub": "send time + TTL − 1 s", "kind": "security", "icon": "fa5_hand_paper" }, { "id": "u-re", "label": "Reacquire", "sub": "new, higher token", "kind": "app" } ] } ], "edges": [ { "from": "u-hold", "to": "u-miss" }, { "from": "u-miss", "to": "u-dec" }, { "from": "u-dec", "to": "u-ok", "label": "yes" }, { "from": "u-dec", "to": "u-stop", "label": "no" }, { "from": "u-stop", "to": "u-fence", "label": "deadline" }, { "from": "u-fence", "to": "u-re", "label": "later" } ], "note": "The deadline runs from when the last successful keepalive was sent, on the monotonic clock, never from its reply.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "19-deployment-architecture", "title": "Deployment — One Region, Three Failure Domains", "layout": "nested", "boxes": [ { "title": "Data centre A · on-premises", "kind": "onprem", "dir": "row", "children": [ { "title": "Hall 1 · power + network domain", "kind": "boundary", "nodes": [ { "id": "d1-coord", "label": "coord members", "sub": "etcd × 2", "kind": "store", "icon": "fa5_server" }, { "id": "d1-ent", "label": "entity-01 member", "sub": "etcd · NVMe · LUKS", "kind": "store", "icon": "fa5_server" }, { "id": "d1-k8s", "label": "Worker nodes", "sub": "Envoy · arbiter", "kind": "platform", "icon": "kubernetes" } ] }, { "title": "Hall 2 · power + network domain", "kind": "boundary", "nodes": [ { "id": "d2-coord", "label": "coord members", "sub": "etcd × 2", "kind": "store", "icon": "fa5_server" }, { "id": "d2-ent", "label": "entity-01 member", "sub": "etcd · NVMe · LUKS", "kind": "store", "icon": "fa5_server" }, { "id": "d2-k8s", "label": "Worker nodes", "sub": "Envoy · arbiter", "kind": "platform", "icon": "kubernetes" } ] }, { "title": "Hall 3 · power + network domain", "kind": "boundary", "nodes": [ { "id": "d3-coord", "label": "coord member", "sub": "etcd × 1", "kind": "store", "icon": "fa5_server" }, { "id": "d3-ent", "label": "entity-01 member", "sub": "etcd · NVMe · LUKS", "kind": "store", "icon": "fa5_server" }, { "id": "d3-k8s", "label": "Worker nodes", "sub": "Envoy · arbiter", "kind": "platform", "icon": "kubernetes" } ] } ] } ], "outside": [ { "id": "d-snap", "label": "Snapshot vault", "sub": "MinIO · data centre B", "kind": "store", "icon": "minio" }, { "id": "d-tang", "label": "Tang servers", "sub": "disk unlock keys", "kind": "security", "icon": "fa5_key" } ], "edges": [ { "from": "d2-ent", "to": "d-snap", "label": "etcdutl · 30 min", "kind": "batch" } ], "note": "These are not the Kubernetes control plane's etcd. Losing any one hall loses no quorum.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "20-cicd-and-release-gates", "title": "CI/CD — No Release Without a Checked History", "layout": "flow", "chain": true, "stages": [ { "title": "Source", "nodes": [ { "id": "c-pr", "label": "Pull request", "sub": "Gitea", "kind": "platform", "icon": "git" }, { "id": "c-buf", "label": "Proto check", "sub": "buf breaking", "kind": "platform", "icon": "fa5_file_contract" } ] }, { "title": "Build", "nodes": [ { "id": "c-build", "label": "Build + test", "sub": "Gitea Actions", "kind": "platform", "icon": "git" }, { "id": "c-sign", "label": "Sign image", "sub": "cosign → Harbor", "kind": "security", "icon": "fa5_file_signature" } ] }, { "title": "Prove", "nodes": [ { "id": "c-chaos", "label": "Fault injection", "sub": "Chaos Mesh", "kind": "platform", "icon": "fa5_bolt" }, { "id": "c-lin", "label": "History checker", "sub": "Porcupine", "kind": "app", "icon": "fa5_microscope" }, { "id": "c-gate", "label": "Zero double grants?", "kind": "decision" } ] }, { "title": "Soak", "nodes": [ { "id": "c-stage", "label": "Staging clusters", "sub": "72 h replayed load", "kind": "app", "icon": "fa5_flask" } ] }, { "title": "Release", "nodes": [ { "id": "c-argo", "label": "Argo CD sync", "kind": "platform", "icon": "argocd" }, { "id": "c-canary", "label": "Canary arbiter", "sub": "1 of 9", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "c-roll", "label": "etcd rolling", "sub": "one member at a time", "kind": "store", "icon": "fa5_sync_alt" } ] } ], "edges": [ { "from": "c-chaos", "to": "c-lin", "label": "histories" }, { "from": "c-lin", "to": "c-gate" } ], "note": "A failed history check blocks the release outright. It is not a flaky test to retry.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "21-observability", "title": "Observability — Signals and What Acts on Them", "layout": "grid", "laneHeaderWidth": 150, "columns": [ "Emit", "Collect", "Store", "Watch", "Act" ], "rows": [ { "title": "Lock SLIs", "cells": [ [ { "label": "Grant latency", "sub": "per ns · class", "kind": "app" } ], [ { "label": "Prometheus scrape", "kind": "integration", "icon": "prometheus" } ], [ { "label": "Prometheus", "sub": "30 d local", "kind": "store", "icon": "prometheus" } ], [ { "label": "SLO burn rate", "sub": "Grafana", "kind": "platform", "icon": "grafana" } ], [ { "label": "Page on-call", "sub": "Alertmanager", "kind": "security" } ] ] }, { "title": "Holder health", "cells": [ [ { "label": "Expiry without release", "kind": "app" } ], [ { "label": "Prometheus scrape", "kind": "integration", "icon": "prometheus" } ], [ { "label": "Prometheus", "kind": "store", "icon": "prometheus" } ], [ { "label": "Longest hold per ns", "kind": "platform" } ], [ { "label": "Client-health ticket", "kind": "security" } ] ] }, { "title": "Consensus", "cells": [ [ { "label": "WAL fsync p99", "sub": "etcd", "kind": "store" } ], [ { "label": "Prometheus scrape", "kind": "integration", "icon": "prometheus" } ], [ { "label": "Prometheus", "kind": "store", "icon": "prometheus" } ], [ { "label": "Leader changes · DB size", "kind": "platform" } ], [ { "label": "Page platform team", "kind": "security" } ] ] }, { "title": "Traces and logs", "cells": [ [ { "label": "OTel SDK", "sub": "no token fields", "kind": "app", "icon": "opentelemetry" } ], [ { "label": "OTel Collector", "sub": "redaction processor", "kind": "integration", "icon": "opentelemetry" } ], [ { "label": "Tempo · Loki", "kind": "store", "icon": "tempo" } ], [ { "label": "Slow grant analysis", "kind": "platform" } ], [] ] }, { "title": "Cost and design", "cells": [ [ { "label": "Audit events", "kind": "queue", "icon": "fa5_history" } ], [ { "label": "Audit tailer", "kind": "integration", "icon": "fa5_history" } ], [ { "label": "ClickHouse", "kind": "store", "icon": "clickhouse" } ], [ { "label": "Contention ratio", "kind": "platform" } ], [ { "label": "Monthly owner review", "kind": "security" } ] ] } ], "note": "No metric is labelled by lock key. Four million keys a day would end Prometheus; per-key lives in the inspection API.", "meta": { "v": "1.0", "owner": "SRE", "date": "2026-09" } }, { "id": "22-growth-path", "title": "Growth Path — Scaling Out Without Re-Platforming", "layout": "flow", "chain": true, "stages": [ { "title": "MVP", "nodes": [ { "id": "g1", "label": "One entity cluster", "sub": "3 members · 1 region", "kind": "store", "icon": "fa5_server" } ] }, { "title": "Phase 2", "nodes": [ { "id": "g2a", "label": "Coord cluster split", "sub": "5 members", "kind": "store", "icon": "fa5_server" }, { "id": "g2b", "label": "Lock classes", "sub": "TTL · mode · posture", "kind": "platform", "icon": "fa5_clipboard_list" } ] }, { "title": "Phase 3", "nodes": [ { "id": "g3a", "label": "Entity shards", "sub": "rendezvous hash", "kind": "store", "icon": "fa5_layer_group" }, { "id": "g3b", "label": "Rehome a key range", "sub": "epoch advance", "kind": "security" } ] }, { "title": "Regions", "nodes": [ { "id": "g4", "label": "Region-local clusters", "sub": "no stretched quorum", "kind": "store", "icon": "fa5_globe" } ] }, { "title": "If ever global", "nodes": [ { "id": "g5", "label": "Stretched coord", "sub": "p99 ≤ 150 ms, priced", "kind": "external", "icon": "fa5_globe" } ] } ], "note": "Each step adds clusters behind the same protocol and token format. No step changes what a caller sends.", "meta": { "v": "1.0", "owner": "Platform Architecture", "date": "2026-09" } }, { "id": "23-lock-hygiene-loop", "title": "Lock Hygiene Loop — Designing Locks Out", "layout": "cycle", "centre": { "label": "Lock review", "sub": "monthly", "icon": "fa5_calendar_check" }, "nodes": [ { "id": "o1", "label": "Signal", "sub": "longest hold · rate", "kind": "platform", "icon": "fa5_bell" }, { "id": "o2", "label": "Inspect", "sub": "holder · token · queue", "kind": "app", "icon": "fa5_search" }, { "id": "o3", "label": "Classify", "sub": "leak · pause · design", "kind": "decision" }, { "id": "o4", "label": "Act", "sub": "fix client or release", "kind": "security" }, { "id": "o5", "label": "Verify", "sub": "queue drained", "kind": "app", "icon": "fa5_check" }, { "id": "o6", "label": "Design out", "sub": "CAS instead of lock", "kind": "opportunity", "icon": "fa5_seedling" } ], "ringLabels": [ "alert", "evidence", "cause", "change", "cheaper path?", "new baseline" ], "rx": 430, "ry": 220, "meta": { "v": "1.0", "owner": "SRE", "date": "2026-09" } }, { "id": "24-security-trust-zones", "title": "Security — Trust Zones", "layout": "zones", "zones": [ { "title": "Workload and operator networks", "kind": "trust", "nodes": [ { "id": "z-lib", "label": "Client library", "sub": "SVID per pod", "kind": "app", "icon": "fa5_puzzle_piece" }, { "id": "z-hog", "label": "Runaway acquirer", "kind": "risk", "icon": "fa5_fire" }, { "id": "z-op", "label": "Operator", "kind": "actor", "icon": "fa5_user_cog" } ] }, { "title": "Access", "kind": "trust", "nodes": [ { "id": "z-envoy", "label": "Envoy fan-in", "sub": "per-identity limit", "kind": "integration", "icon": "envoy" }, { "id": "z-kc", "label": "Keycloak", "sub": "MFA", "kind": "security", "icon": "keycloak" } ] }, { "title": "Coordination", "kind": "trust", "nodes": [ { "id": "z-arb", "label": "Lock arbiter", "sub": "prefix grants", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "z-admin", "label": "Admin API", "sub": "two-person rule", "kind": "app", "icon": "fa5_user_shield" } ] }, { "title": "Consensus · restricted", "kind": "trust", "nodes": [ { "id": "z-etcd", "label": "Lock log", "sub": "client-cert RBAC", "kind": "store", "icon": "fa5_server" }, { "id": "z-tang", "label": "Disk encryption", "sub": "LUKS + Tang", "kind": "security", "icon": "fa5_key" } ] }, { "title": "Evidence · append-only", "kind": "trust", "nodes": [ { "id": "z-tail", "label": "Audit tailer", "sub": "read-only cert", "kind": "app", "icon": "fa5_history" }, { "id": "z-arch", "label": "Audit archive", "sub": "object lock", "kind": "store", "icon": "minio" } ] } ], "edges": [ { "from": "z-lib", "to": "z-envoy", "label": "mTLS" }, { "from": "z-hog", "to": "z-envoy", "label": "throttled", "kind": "error" }, { "from": "z-op", "to": "z-kc", "label": "OIDC" }, { "from": "z-envoy", "to": "z-arb", "label": "mTLS" }, { "from": "z-kc", "to": "z-admin", "label": "JWT" }, { "from": "z-arb", "to": "z-etcd", "label": "TLS" }, { "from": "z-etcd", "to": "z-tail", "label": "watch" }, { "from": "z-admin", "to": "z-etcd", "label": "delete-only" } ], "note": "No identity outside Coordination holds an etcd write credential. Tokens never appear in logs, traces or errors.", "meta": { "v": "1.0", "owner": "Security Architecture", "date": "2026-09" } }, { "id": "25-force-release-sequence", "title": "Force-Release — Two People, One Reason, One Trail", "layout": "sequence", "lifelines": [ { "id": "sre", "label": "On-call SRE", "kind": "actor" }, { "id": "kc", "label": "Keycloak", "kind": "security", "icon": "keycloak" }, { "id": "adm", "label": "Admin API", "kind": "app", "icon": "fa5_user_shield" }, { "id": "peer", "label": "Second approver", "kind": "actor" }, { "id": "log", "label": "Lock log", "kind": "store", "icon": "fa5_server" }, { "id": "tail", "label": "Audit tailer", "kind": "app", "icon": "fa5_history" } ], "messages": [ { "from": "sre", "to": "kc", "label": "sign in · MFA", "kind": "call" }, { "from": "kc", "to": "sre", "label": "JWT · role lock-operator", "kind": "return" }, { "from": "sre", "to": "adm", "label": "force-release key · reason", "kind": "call" }, { "from": "adm", "to": "adm", "label": "look up class · fenced?", "kind": "self" }, { "from": "adm", "to": "sre", "label": "warning if advisory", "kind": "return" }, { "from": "adm", "to": "peer", "label": "approval request", "kind": "async" }, { "from": "peer", "to": "adm", "label": "approve · different subject", "kind": "call" }, { "from": "adm", "to": "log", "label": "Txn: delete holder · put reason", "kind": "call" }, { "from": "log", "to": "adm", "label": "committed · revision R", "kind": "return" }, { "from": "log", "to": "tail", "label": "delete event R", "kind": "async" }, { "from": "tail", "to": "tail", "label": "join reason · fenced flag", "kind": "self" }, { "from": "adm", "to": "sre", "label": "released · next token > R", "kind": "return" } ], "note": "Force-release is safe only when the class is fenced. The console says so before the second approval, not after.", "meta": { "v": "1.0", "owner": "Security Architecture", "date": "2026-09" } }, { "id": "26-failure-modes", "title": "Failure Modes — Detection, Containment, What Keeps It Safe", "layout": "grid", "laneHeaderWidth": 170, "columns": [ "Detect", "Contain", "Safety held by", "Recover" ], "rows": [ { "title": "Leader failure", "cells": [ [ { "label": "Heartbeat timeout", "kind": "platform" } ], [ { "label": "Grants pause ≤ 3 s", "kind": "security" } ], [ { "label": "Raft term · leases extended", "kind": "store" } ], [ { "label": "Re-election", "kind": "app" } ] ] }, { "title": "Quorum loss", "cells": [ [ { "label": "No majority ack", "kind": "platform" } ], [ { "label": "Fail closed", "kind": "security" } ], [ { "label": "Holders self-fence", "kind": "app" } ], [ { "label": "Restore + epoch advance", "kind": "store" } ] ] }, { "title": "Holder pause > TTL", "cells": [ [ { "label": "Session lapses", "kind": "platform" } ], [ { "label": "Re-grant, higher token", "kind": "app" } ], [ { "label": "Resource high-water", "kind": "external", "icon": "fa5_database" } ], [ { "label": "Stale write rejected", "kind": "opportunity" } ] ] }, { "title": "Snapshot restore", "cells": [ [ { "label": "Cluster ID mismatch", "kind": "platform" } ], [ { "label": "Sealed · no grants", "kind": "security" } ], [ { "label": "Epoch in every Txn", "kind": "store" } ], [ { "label": "Advance to E+1", "kind": "app", "icon": "fa5_sort_numeric_up" } ] ] }, { "title": "fsync failure", "cells": [ [ { "label": "WAL error", "kind": "platform" } ], [ { "label": "Member stops, exits", "kind": "security" } ], [ { "label": "Majority still durable", "kind": "store" } ], [ { "label": "Replace member", "kind": "app" } ] ] }, { "title": "Hot-key herd", "cells": [ [ { "label": "Queue depth", "kind": "platform" } ], [ { "label": "Reject past depth", "kind": "security" } ], [ { "label": "One Txn per release", "kind": "store" } ], [ { "label": "Owner redesign", "kind": "opportunity" } ] ] }, { "title": "Restart lease storm", "cells": [ [ { "label": "Session open rate", "kind": "platform" } ], [ { "label": "Jittered reconnect", "kind": "security" } ], [ { "label": "Shed acquire before renew", "kind": "app" } ], [ { "label": "Rate returns to base", "kind": "app" } ] ] } ], "note": "Every row fails towards refusing a grant. None fails towards issuing a second one.", "meta": { "v": "1.0", "owner": "SRE", "date": "2026-09" } } ]