[ { "id": "01-system-context", "title": "Spot Capacity Orchestrator — System Context", "layout": "context", "colWidth": 300, "system": { "label": "Spot Capacity Orchestrator", "sub": "acquire · drain · replace · fall back", "icon": "fa5_random" }, "groups": [ { "side": "top", "title": "Schedulers that place the work, and who gets paged", "nodes": [ { "id": "c-k8s", "label": "Kubernetes clusters", "sub": "services · Spark · CI", "kind": "external", "rel": "pending pods", "dir": "in", "icon": "kubernetes" }, { "id": "c-slurm", "label": "Slurm clusters", "sub": "batch · training", "kind": "external", "rel": "pending jobs", "dir": "in", "icon": "fa5_tasks" }, { "id": "c-inc", "label": "Incident platform", "sub": "pages · escalation", "kind": "external", "rel": "alerts", "kind2": "async", "icon": "fa5_bullhorn" } ] }, { "side": "left", "title": "People", "nodes": [ { "id": "c-owner", "label": "Workload owner", "kind": "actor", "rel": "contract", "dir": "in", "icon": "fa5_users" }, { "id": "c-fin", "label": "Capacity and FinOps lead", "kind": "actor", "rel": "floor · spend", "dir": "in", "icon": "fa5_balance_scale" }, { "id": "c-sre", "label": "Platform SRE", "kind": "actor", "rel": "operates", "dir": "in", "icon": "fa5_user_cog" } ] }, { "side": "right", "title": "The capacity provider, and the record", "nodes": [ { "id": "c-os", "label": "Private cloud", "sub": "OpenStack · 3 regions", "kind": "external", "rel": "launch · delete", "icon": "fa5_server" }, { "id": "c-reclaim", "label": "Reclaimer", "sub": "private cloud team", "kind": "external", "rel": "notices", "dir": "in", "kind2": "async", "icon": "fa5_bell" }, { "id": "c-commit", "label": "Committed reservations", "sub": "Blazar · paid for", "kind": "external", "rel": "floor", "icon": "fa5_calendar_alt" }, { "id": "c-audit", "label": "Audit log service", "sub": "independent record", "kind": "store", "rel": "actions", "kind2": "async", "icon": "fa5_clipboard_list" } ] }, { "side": "bottom", "title": "Estate services it relies on without owning", "nodes": [ { "id": "c-ceph", "label": "Ceph object storage", "sub": "checkpoints", "kind": "store", "rel": "bucket policy", "icon": "fa5_archive" }, { "id": "c-id", "label": "SPIRE and OpenBao", "sub": "identity · credentials", "kind": "security", "rel": "SVIDs", "dir": "in", "icon": "fa5_fingerprint" } ] } ], "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "02-high-level-architecture", "title": "High-Level Architecture — Two Planes Split at the Notice", "layout": "flow", "chain": true, "stages": [ { "title": "Declare", "nodes": [ { "id": "h-repo", "label": "Contract repository", "sub": "Forgejo · reviewed", "kind": "store", "icon": "git" }, { "id": "h-comp", "label": "Contract compiler", "sub": "CUE · feasibility", "kind": "app", "icon": "fa5_file_contract" } ] }, { "title": "Decide", "nodes": [ { "id": "h-score", "label": "Pool scorer", "sub": "every 5 min", "kind": "app", "icon": "fa5_chart_line" }, { "id": "h-plan", "label": "Portfolio planner", "sub": "caps · correlation", "kind": "app", "icon": "fa5_sitemap" }, { "id": "h-fall", "label": "Fallback arbiter", "sub": "spend authority", "kind": "app", "icon": "fa5_balance_scale" } ] }, { "title": "Acquire", "nodes": [ { "id": "h-acq", "label": "Acquirer", "sub": "rate-limited", "kind": "app", "icon": "fa5_cubes" }, { "id": "h-os", "label": "Private cloud", "sub": "OpenStack Nova · Ironic", "kind": "external", "icon": "fa5_server" } ] }, { "title": "Run and drain", "nodes": [ { "id": "h-agent", "label": "Node agent", "sub": "acts alone", "kind": "app", "icon": "fa5_microchip" }, { "id": "h-work", "label": "Workloads", "sub": "Kubernetes · Slurm", "kind": "external", "icon": "kubernetes" }, { "id": "h-ckpt", "label": "Checkpoint store", "sub": "Ceph RGW", "kind": "store", "icon": "fa5_archive" } ] }, { "title": "Account", "nodes": [ { "id": "h-events", "label": "Lifecycle events", "sub": "Kafka", "kind": "queue", "icon": "kafka" }, { "id": "h-ledger", "label": "Net savings ledger", "sub": "ClickHouse", "kind": "store", "icon": "clickhouse" } ] } ], "edges": [ { "from": "h-os", "to": "h-agent", "label": "reclaim notice", "kind": "async" }, { "from": "h-agent", "to": "h-events", "label": "drain outcome", "kind": "async" } ], "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "03-actors-and-journeys", "title": "Actors — Who It Serves and What Each of Them Gets to Do", "layout": "actors", "cardWidth": 280, "groups": [ { "title": "People who run work on it", "kind": "boundary", "actors": [ { "id": "a-data", "label": "Data engineer", "sub": "nightly Spark pipelines", "icon": "spark", "goal": "Run my pipeline for a third of the price without learning what a reclaim notice is.", "journeys": [ { "id": "j-onboard", "label": "Move a pipeline onto reclaimable capacity" }, { "label": "Tell a reclaim from a bug" } ] }, { "id": "a-ml", "label": "ML engineer", "sub": "multi-day training", "icon": "fa5_brain", "goal": "Lose an hour of a three-day run to a reclaim, never the whole run.", "journeys": [ { "label": "Resume from the last checkpoint" }, { "label": "Hit a deadline through a shortage" } ] }, { "id": "a-ci", "label": "CI platform owner", "sub": "build runners", "icon": "fa5_code_branch", "goal": "Keep build queues short on capacity that costs a third.", "journeys": [ { "label": "Requeue a build that was reclaimed" } ] } ] }, { "title": "People who pay for it and set its limits", "kind": "boundary", "actors": [ { "id": "a-fin", "label": "Capacity and FinOps lead", "sub": "rate card · floor", "icon": "fa5_balance_scale", "goal": "Know what each point of reliability costs before I sign for it.", "journeys": [ { "id": "j-floor", "label": "Put a price on the tier-1 floor" }, { "label": "Move a workload off reclaimable capacity" } ] }, { "id": "a-budget", "label": "Budget owner", "sub": "spend authority", "icon": "fa5_user_tie", "goal": "Approve fallback spend only when a deadline actually needs it.", "journeys": [ { "label": "Approve fallback beyond the cap" } ] } ] }, { "title": "People who keep it running", "kind": "onprem", "actors": [ { "id": "a-sre", "label": "Platform SRE", "sub": "on call", "icon": "fa5_user_cog", "goal": "Get through a hall-wide reclaim without waking anyone but me.", "journeys": [ { "id": "j-reclaim", "label": "Ride out a correlated reclamation" }, { "label": "Recover the control plane slowly" } ] }, { "id": "a-cloud", "label": "Private cloud team", "kind": "external", "sub": "the provider", "icon": "fa5_server", "goal": "Give capacity back to guaranteed tenants inside three minutes, every time.", "journeys": [ { "label": "Reclaim a pool with notice" }, { "label": "Publish prices and forward capacity" } ] } ] }, { "title": "Machines that act without being asked", "kind": "plain", "actors": [ { "id": "a-sched", "label": "Schedulers", "kind": "platform", "sub": "Kubernetes · Slurm", "icon": "kubernetes", "goal": "Place work on nodes that tell me before they go away.", "journeys": [ { "label": "Stop placing on a draining node" } ] }, { "id": "a-drill", "label": "Reclamation exercise", "kind": "platform", "sub": "daily · 1% of nodes", "icon": "fa5_flask", "goal": "Prove the drain path works on a day nothing is wrong.", "journeys": [ { "label": "Reclaim a sample through the real path" } ] } ] } ], "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "04-journey-move-a-pipeline-onto-reclaimable-capacity", "title": "Journey — A Data Engineer Moves a Nightly Pipeline", "layout": "journey", "actor": { "label": "Data engineer", "sub": "orders-daily · Spark", "icon": "spark", "goal": "Halve the pipeline's compute bill, same 06:00 deadline", "trigger": "Quarterly budget review flags the pipeline", "success": "Four weeks on tier 1, deadline met, net saving shown" }, "phases": [ { "title": "Pick a tier" }, { "title": "Submit contract" }, { "title": "First night" }, { "title": "First reclaim", "moment": true }, { "title": "Read the report" }, { "title": "Stop thinking about it" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Reads the tier table" } ], [ { "label": "Opens a pull request" } ], [ { "label": "Deploys unchanged job" } ], [ { "label": "Sees a stage retry" } ], [ { "label": "Opens workload report" } ], [ { "label": "Forgets it exists" } ] ] }, { "title": "System behind it", "kind": "system", "cells": [ [ { "label": "Tier catalogue", "icon": "fa5_layer_group" } ], [ { "label": "Contract compiler", "icon": "fa5_file_contract" } ], [ { "label": "Portfolio planner", "icon": "fa5_sitemap" } ], [ { "label": "Node agent · Spark decommission", "icon": "fa5_microchip" } ], [ { "label": "Net savings ledger", "icon": "clickhouse" } ], [ { "label": "Breach alarm", "icon": "fa5_bell" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Unsure", "Alarmed" ], "points": [ "Unsure", "Alarmed", "Confident", "Alarmed", "Confident", "Confident" ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "Which tier is mine?" } ], [ { "label": "Rejected: 300 s drain" } ], [], [ { "label": "Is this my bug?" } ], [], [ { "label": "Floor cost surprises" } ] ] }, { "title": "What the platform gives", "kind": "gain", "cells": [ [ { "label": "Tier 1 fits deadlines" } ], [ { "label": "Rejection says why" } ], [ { "label": "30% floor, 70% reclaimable" } ], [ { "label": "Event says reclaimed" } ], [ { "label": "Waste charged to reclaim" } ], [ { "label": "Alarm only on breach" } ] ] } ], "chain": true, "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "05-journey-ride-out-a-correlated-reclamation", "title": "Journey — The On-Call SRE Through a Hall-Wide Reclamation", "layout": "journey", "actor": { "label": "Platform SRE", "sub": "on call, 02:10", "icon": "fa5_user_cog", "goal": "Keep tier-1 deadlines without an unbounded bill", "trigger": "Guaranteed tenant takes back 40% of hall B", "success": "No tier-1 shortfall over 15 min; spend inside authority" }, "phases": [ { "title": "Paged" }, { "title": "Size it" }, { "title": "Let it drain" }, { "title": "Spend or wait", "moment": true }, { "title": "Return" }, { "title": "Review" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Acknowledges page" } ], [ { "label": "Opens blast-radius board" } ], [ { "label": "Watches, does nothing" } ], [ { "label": "Calls budget owner" } ], [ { "label": "Leaves hysteresis alone" } ], [ { "label": "Reads the event trail" } ] ] }, { "title": "System behind it", "kind": "system", "cells": [ [ { "label": "Exclusion watcher", "icon": "fa5_ban" } ], [ { "label": "Correlation groups", "icon": "fa5_project_diagram" } ], [ { "label": "Node agents · replacer", "icon": "fa5_microchip" } ], [ { "label": "Fallback arbiter", "icon": "fa5_balance_scale" } ], [ { "label": "Return controller", "icon": "fa5_undo" } ], [ { "label": "Lifecycle events", "icon": "kafka" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "In control", "Tense", "Out of depth" ], "points": [ "Tense", "Tense", "In control", "Out of depth", "In control", "In control" ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "Nine alerts, one cause" } ], [ { "label": "Pools share an aggregate" } ], [], [ { "label": "Two jobs past spend cap" } ], [ { "label": "Urge to force return" } ], [] ] }, { "title": "What the platform gives", "kind": "gain", "cells": [ [ { "label": "Grouped by pool group" } ], [ { "label": "Units at risk per workload" } ], [ { "label": "96% drained in window" } ], [ { "label": "Floor restored, no approval" } ], [ { "label": "60 min healthy, 20%/h" } ], [ { "label": "Reclaim vs defect split" } ] ] } ], "chain": true, "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "06-journey-price-the-tier-1-floor", "title": "Journey — The FinOps Lead Puts a Price on the Floor", "layout": "journey", "actor": { "label": "Capacity and FinOps lead", "sub": "annual plan", "icon": "fa5_balance_scale", "goal": "Decide whether tier 1 needs 30% or 20% guaranteed", "trigger": "Engineering asks to lower the floor to save money", "success": "A priced decision, signed, applied without an outage" }, "phases": [ { "title": "The ask" }, { "title": "Gather evidence", "moment": true }, { "title": "Price it" }, { "title": "Decide" }, { "title": "Apply" }, { "title": "Check a month on" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Receives a proposal" } ], [ { "label": "Pulls 13 months of pools" } ], [ { "label": "Reads marginal floor cost" } ], [ { "label": "Signs as governance" } ], [ { "label": "Merges policy change" } ], [ { "label": "Compares shortfall" } ] ] }, { "title": "System behind it", "kind": "system", "cells": [ [ { "label": "Contract repository", "icon": "git" } ], [ { "label": "Pool history", "icon": "clickhouse" } ], [ { "label": "Floor cost model", "icon": "fa5_calculator" } ], [ { "label": "Dual-signature gate", "icon": "fa5_file_signature" } ], [ { "label": "Floor keeper", "icon": "fa5_layer_group" } ], [ { "label": "Net savings ledger", "icon": "fa5_chart_bar" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Sure", "Unsure", "Blind" ], "points": [ "Unsure", "Blind", "Sure", "Unsure", "Sure", "Sure" ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "Saving claimed, risk not" } ], [ { "label": "Failures mix bugs and reclaims" } ], [], [ { "label": "No worst-week evidence" } ], [ { "label": "Fear of a mass drain" } ], [] ] }, { "title": "What the platform gives", "kind": "gain", "cells": [ [ { "label": "Priced diff in the PR" } ], [ { "label": "Loss cause on every event" } ], [ { "label": "€4.3k/month per 10%" } ], [ { "label": "Worst correlated week replayed" } ], [ { "label": "Floor lowered through churn" } ], [ { "label": "Shortfall vs interruption" } ] ] } ], "chain": true, "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "07-layered-architecture", "title": "Layered Architecture — The Node Plane Skips the Control Plane", "layout": "bands", "layerHeaderWidth": 160, "canvas": { "width": 1500 }, "bands": [ { "name": "Workloads", "nodes": [ { "id": "l-spark", "label": "Spark pipelines", "sub": "decommission-aware", "kind": "external", "icon": "spark" }, { "id": "l-train", "label": "Training jobs", "sub": "PyTorch DCP", "kind": "external", "icon": "fa5_brain" }, { "id": "l-ci", "label": "CI and services", "sub": "requeue · stateless", "kind": "external", "icon": "fa5_code_branch" } ] }, { "name": "Node plane", "nodes": [ { "id": "l-client", "label": "Checkpoint client", "sub": "Go · Python · JVM", "kind": "app", "icon": "fa5_save" }, { "id": "l-agent", "label": "Node agent", "sub": "systemd · acts alone", "kind": "app", "icon": "fa5_microchip" }, { "id": "l-bundle", "label": "Cached contract", "sub": "signed · on disk", "kind": "security", "icon": "fa5_file_signature" } ] }, { "name": "Control plane", "nodes": [ { "id": "l-intake", "label": "Demand intake", "sub": "normalised units", "kind": "app", "icon": "fa5_inbox" }, { "id": "l-plan", "label": "Portfolio planner", "sub": "caps · groups", "kind": "app", "icon": "fa5_sitemap" }, { "id": "l-arb", "label": "Fallback arbiter", "sub": "spend authority", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "l-acq", "label": "Acquirer", "sub": "token budget", "kind": "app", "icon": "fa5_cubes" } ] }, { "name": "Schedulers", "nodes": [ { "id": "l-k8s", "label": "Kubernetes", "sub": "Kyverno admission", "kind": "platform", "icon": "kubernetes" }, { "id": "l-slurm", "label": "Slurm", "sub": "job_submit filter", "kind": "platform", "icon": "fa5_tasks" } ] }, { "name": "Contract and policy", "nodes": [ { "id": "l-repo", "label": "Contract repository", "sub": "Forgejo", "kind": "store", "icon": "git" }, { "id": "l-comp", "label": "Contract compiler", "sub": "CUE · feasibility", "kind": "app", "icon": "fa5_file_contract" }, { "id": "l-dist", "label": "Contract distributor", "sub": "signed bundles", "kind": "integration", "icon": "fa5_share_alt" } ] }, { "name": "Provider", "nodes": [ { "id": "l-nova", "label": "Nova and Ironic", "sub": "OpenStack", "kind": "external", "icon": "fa5_server" }, { "id": "l-relay", "label": "Notice relay", "sub": "NATS per hall", "kind": "external", "icon": "fa5_bell" }, { "id": "l-blazar", "label": "Reservations", "sub": "Blazar · committed", "kind": "external", "icon": "fa5_calendar_alt" } ] }, { "name": "State and record", "nodes": [ { "id": "l-pg", "label": "Fleet state", "sub": "PostgreSQL · Patroni", "kind": "store", "icon": "postgresql" }, { "id": "l-kafka", "label": "Lifecycle events", "sub": "Kafka", "kind": "queue", "icon": "kafka" }, { "id": "l-ch", "label": "Ledger and history", "sub": "ClickHouse", "kind": "store", "icon": "clickhouse" }, { "id": "l-rgw", "label": "Checkpoints", "sub": "Ceph RGW · per workload", "kind": "store", "icon": "fa5_archive" } ] } ], "edges": [ { "from": "l-train", "to": "l-client" }, { "from": "l-bundle", "to": "l-dist", "label": "refresh 60 s", "kind": "batch", "route": "gutter" }, { "from": "l-intake", "to": "l-k8s", "label": "pending work" }, { "from": "l-k8s", "to": "l-comp", "label": "contract exists?" }, { "from": "l-acq", "to": "l-nova", "label": "launch · delete", "route": "gutter" } ], "note": "Nothing in the node plane depends on the control plane: the agent reads a bundle the contract layer signed, so a control-plane outage costs convergence, not drain.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "08-two-planes", "title": "Two Planes — Where Capacity Comes From, What Happens When It Goes", "layout": "nested", "canvas": { "width": 1240 }, "boxes": [ { "title": "Split on the interruption boundary", "kind": "plain", "dir": "row", "children": [ { "title": "Control plane · per region · guaranteed capacity", "kind": "boundary", "dir": "col", "nodes": [ { "id": "p-plan", "label": "Portfolio planner", "sub": "seconds to minutes", "kind": "app", "icon": "fa5_sitemap" }, { "id": "p-arb", "label": "Fallback arbiter", "sub": "spend authority", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "p-acq", "label": "Acquirer and replacer", "sub": "token budget", "kind": "app", "icon": "fa5_cubes" }, { "id": "p-dist", "label": "Contract distributor", "sub": "signed bundles", "kind": "integration", "icon": "fa5_share_alt" } ] }, { "title": "Node plane · every reclaimable instance", "kind": "trust", "dir": "col", "nodes": [ { "id": "p-cache", "label": "Cached contract", "sub": "drain · checkpoint · restart", "kind": "security", "icon": "fa5_file_signature" }, { "id": "p-agent", "label": "Node agent", "sub": "≤ 2 s to drain signal", "kind": "app", "icon": "fa5_microchip" }, { "id": "p-work", "label": "Workload", "sub": "≥ 115 s to finish", "kind": "external", "icon": "fa5_cube" } ] } ] } ], "outside": [ { "id": "p-nova", "label": "Nova and Ironic", "sub": "private cloud", "kind": "external", "icon": "fa5_server" }, { "id": "p-relay", "label": "Notice relay", "sub": "NATS · per hall", "kind": "external", "icon": "fa5_bell" } ], "edges": [ { "from": "p-dist", "to": "p-cache", "label": "bundle ahead of time", "kind": "batch" }, { "from": "p-cache", "to": "p-agent" }, { "from": "p-agent", "to": "p-work", "label": "drain signal" }, { "from": "p-relay", "to": "p-agent", "label": "hard notice", "kind": "async" }, { "from": "p-acq", "to": "p-nova", "label": "replacement" } ], "note": "No arrow runs from a notice into the control plane. Replacement is requested from the lifecycle event, after the node has already acted (view 18).", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "09-control-plane-containers", "title": "Control Plane — One Region, Partitioned by Scheduler Cluster", "layout": "nested", "canvas": { "width": 1400 }, "boxes": [ { "title": "Region management cluster · Kubernetes on guaranteed capacity · 3 halls", "kind": "onprem", "dir": "row", "children": [ { "title": "Signals", "kind": "boundary", "dir": "col", "nodes": [ { "id": "k-score", "label": "Pool scorer", "sub": "CronJob · 5 min", "kind": "app", "icon": "fa5_chart_line" }, { "id": "k-excl", "label": "Exclusion watcher", "sub": "stream · ≤ 60 s", "kind": "app", "icon": "fa5_ban" } ] }, { "title": "sco-controller · leader per partition", "kind": "boundary", "dir": "col", "nodes": [ { "id": "k-intake", "label": "Demand intake", "sub": "Kubernetes · Slurm adapters", "kind": "app", "icon": "fa5_inbox" }, { "id": "k-plan", "label": "Portfolio planner", "sub": "incremental", "kind": "app", "icon": "fa5_sitemap" }, { "id": "k-arb", "label": "Fallback arbiter", "sub": "floor keeper", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "k-acq", "label": "Acquirer and replacer", "sub": "quota · breakers", "kind": "app", "icon": "fa5_cubes" } ] }, { "title": "State", "kind": "boundary", "dir": "col", "nodes": [ { "id": "k-dist", "label": "Contract distributor", "sub": "3 replicas", "kind": "integration", "icon": "fa5_share_alt" }, { "id": "k-pg", "label": "Fleet state", "sub": "PostgreSQL · Patroni", "kind": "store", "icon": "postgresql" } ] } ] } ], "outside": [ { "id": "k-ch", "label": "Pool history", "sub": "ClickHouse", "kind": "store", "icon": "clickhouse" }, { "id": "k-kafka", "label": "Lifecycle events", "sub": "Kafka · estate cluster", "kind": "queue", "icon": "kafka" }, { "id": "k-bao", "label": "Provider credential", "sub": "OpenBao", "kind": "security", "icon": "fa5_key" }, { "id": "k-nova", "label": "Nova · Ironic · Placement", "sub": "private cloud APIs", "kind": "external", "icon": "fa5_server" } ], "edges": [ { "from": "k-score", "to": "k-ch", "label": "history" }, { "from": "k-excl", "to": "k-kafka", "label": "consume", "kind": "async" }, { "from": "k-plan", "to": "k-pg", "label": "plan generation" }, { "from": "k-acq", "to": "k-bao", "label": "app credential" }, { "from": "k-acq", "to": "k-nova", "label": "batched create" } ], "note": "A partition is one scheduler cluster in one region. Adding a region adds partitions; nothing reconciles the whole estate in one loop.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "10-node-agent", "title": "Node Agent — Everything the Notice Path Needs, Already on the Node", "layout": "nested", "canvas": { "width": 1400 }, "boxes": [ { "title": "One reclaimable instance", "kind": "onprem", "dir": "row", "children": [ { "title": "Held before any work lands", "kind": "boundary", "dir": "col", "nodes": [ { "id": "n-userdata", "label": "Launch user-data", "sub": "cloud-init · contract subset", "kind": "security", "icon": "fa5_file_code" }, { "id": "n-bundle", "label": "Contract bundle", "sub": "signed · never expires", "kind": "security", "icon": "fa5_file_signature" } ] }, { "title": "sco-agent · static Go · systemd", "kind": "trust", "dir": "col", "nodes": [ { "id": "n-watch", "label": "Notice watcher", "sub": "stream · 5 s poll", "kind": "app", "icon": "fa5_bell" }, { "id": "n-exec", "label": "Drain executor", "sub": "cordon · signal · evict", "kind": "app", "icon": "fa5_sign_out_alt" }, { "id": "n-verify", "label": "Drain verifier", "sub": "done or overrun", "kind": "app", "icon": "fa5_check_double" }, { "id": "n-spool", "label": "Event spool", "sub": "64 MB on disk", "kind": "store", "icon": "fa5_hdd" } ] }, { "title": "Scheduler and workload", "kind": "boundary", "dir": "col", "nodes": [ { "id": "n-kubelet", "label": "kubelet or slurmd", "sub": "graceful shutdown", "kind": "platform", "icon": "kubernetes" }, { "id": "n-client", "label": "Checkpoint client", "sub": "agent socket", "kind": "app", "icon": "fa5_save" }, { "id": "n-proc", "label": "Workload process", "sub": "restart-safe entry", "kind": "external", "icon": "fa5_cube" } ] } ] } ], "outside": [ { "id": "n-relay", "label": "Notice relay", "sub": "NATS · provider", "kind": "external", "icon": "fa5_bell" }, { "id": "n-api", "label": "Scheduler API", "sub": "API server · slurmctld", "kind": "external", "icon": "fa5_tasks" }, { "id": "n-kafka", "label": "Vector to Kafka", "sub": "when reachable", "kind": "queue", "icon": "vector" } ], "edges": [ { "from": "n-relay", "to": "n-watch", "label": "notice", "kind": "async" }, { "from": "n-bundle", "to": "n-exec", "label": "drain budget" }, { "from": "n-exec", "to": "n-kubelet", "label": "local fallback", "kind": "error" }, { "from": "n-exec", "to": "n-api", "label": "taint · evict" }, { "from": "n-client", "to": "n-proc", "label": "on_notice()" }, { "from": "n-spool", "to": "n-kafka", "label": "drain events", "kind": "async" } ], "note": "The agent holds no provider credential. It can cordon and drain its own node; it cannot launch, terminate or reach another node.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "11-integration-surface", "title": "Integration Surface — Every Interface In and Out", "layout": "hub", "left": { "title": "Who calls it", "nodes": [ { "id": "i-agents", "label": "Node agents", "sub": "bundle fetch", "kind": "app", "rel": "HTTPS · SVID", "icon": "fa5_microchip" }, { "id": "i-ci", "label": "Contract pipeline", "sub": "Forgejo Actions", "kind": "app", "rel": "signed publish", "icon": "fa5_code_branch" }, { "id": "i-ops", "label": "Operators and reports", "sub": "Grafana · CLI", "kind": "actor", "rel": "OIDC", "icon": "fa5_user_cog" } ] }, "centre": { "title": "Platform", "nodes": [ { "id": "i-core", "label": "Spot Capacity Orchestrator", "sub": "control plane · per region", "kind": "app", "icon": "fa5_random" } ] }, "right": { "title": "What it reaches", "nodes": [ { "id": "i-os", "label": "Private cloud APIs", "sub": "Nova · Ironic · Blazar · prices", "kind": "external", "rel": "REST · budgeted", "dir": "out", "icon": "fa5_server" }, { "id": "i-sched", "label": "Scheduler APIs", "sub": "Kubernetes · slurmrestd", "kind": "external", "rel": "watch · poll", "dir": "out", "icon": "kubernetes" }, { "id": "i-rec", "label": "Record", "sub": "Kafka · audit service", "kind": "queue", "rel": "events · outbox", "dir": "out", "kind2": "async", "icon": "kafka" } ] }, "note": "Workloads never call the orchestrator. They talk to the agent on their own node and to their own checkpoint bucket.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "12-contract-to-behaviour", "title": "The Contract Is the Interface — Each Field, Where It Acts", "layout": "grid", "laneHeaderWidth": 190, "columns": [ "Checked at compile", "Held on the node", "Decides", "A breach shows as" ], "rows": [ { "title": "Drain budget", "cells": [ [ { "label": "≤ 115 s, or advisory drain", "kind": "decision", "icon": "fa5_stopwatch" } ], [ { "label": "Yes · grace per unit", "kind": "security", "icon": "fa5_file_signature" } ], [ { "label": "Eviction grace", "kind": "app", "icon": "fa5_sign_out_alt" } ], [ { "label": "Drain overrun", "kind": "risk", "icon": "fa5_hourglass_end" } ] ] }, { "title": "Checkpoint mode", "cells": [ [ { "label": "Final write fits budget", "kind": "decision", "icon": "fa5_save" } ], [ { "label": "Yes · signal to send", "kind": "security", "icon": "fa5_file_signature" } ], [ { "label": "Resume or requeue", "kind": "app", "icon": "fa5_redo" } ], [ { "label": "Wasted work", "kind": "risk", "icon": "fa5_trash_alt" } ] ] }, { "title": "Restart semantics", "cells": [ [ { "label": "Declared, never inferred", "kind": "decision", "icon": "fa5_gavel" } ], [ { "label": "Yes", "kind": "security", "icon": "fa5_file_signature" } ], [ { "label": "Output commit or retry", "kind": "app", "icon": "fa5_check_double" } ], [ { "label": "Duplicate is a defect", "kind": "risk", "icon": "fa5_clone" } ] ] }, { "title": "Tolerated interruption", "cells": [ [ { "label": "Above best pool rate", "kind": "decision", "icon": "fa5_percentage" } ], [], [ { "label": "Eligible pools", "kind": "app", "icon": "fa5_sitemap" } ], [ { "label": "Breach alarm", "kind": "risk", "icon": "fa5_bell" } ] ] }, { "title": "Guaranteed floor", "cells": [ [ { "label": "Tier minimum · 2 signatures", "kind": "decision", "icon": "fa5_file_signature" } ], [], [ { "label": "Floor keeper", "kind": "app", "icon": "fa5_layer_group" } ], [ { "label": "Minutes below floor", "kind": "risk", "icon": "fa5_clock" } ] ] }, { "title": "Deadline", "cells": [ [ { "label": "Reachable at floor speed", "kind": "decision", "icon": "fa5_calendar_check" } ], [], [ { "label": "Deadline-risk fallback", "kind": "app", "icon": "fa5_balance_scale" } ], [ { "label": "Missed deadline", "kind": "risk", "icon": "fa5_exclamation_triangle" } ] ] }, { "title": "Spend authority", "cells": [ [ { "label": "Named approver exists", "kind": "decision", "icon": "fa5_user_tie" } ], [], [ { "label": "Automatic spend cap", "kind": "app", "icon": "fa5_coins" } ], [ { "label": "Escalation, not spend", "kind": "risk", "icon": "fa5_bullhorn" } ] ] } ], "note": "Empty cells are deliberate: only fields the notice path needs are cached on the node, so a stale cache can never change placement or spend.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "13-four-data-classes", "title": "Four Data Classes — Separated by What Losing Each Costs", "layout": "nested", "canvas": { "width": 1560 }, "boxes": [ { "title": "Held by the orchestrator, or deliberately not", "kind": "plain", "dir": "row", "children": [ { "title": "Contracts and policy · RPO 0", "kind": "boundary", "dir": "col", "nodes": [ { "id": "d-repo", "label": "Contract repository", "sub": "Forgejo · mirrored", "kind": "store", "icon": "git" }, { "id": "d-bundle", "label": "Compiled bundles", "sub": "signed · versioned", "kind": "security", "icon": "fa5_file_signature" } ] }, { "title": "Fleet and pool state · rebuildable", "kind": "boundary", "dir": "col", "nodes": [ { "id": "d-tags", "label": "Instance tags", "sub": "provider · authoritative", "kind": "external", "icon": "fa5_tags" }, { "id": "d-fleet", "label": "Fleet state", "sub": "PostgreSQL · Patroni", "kind": "store", "icon": "postgresql" } ] }, { "title": "Lifecycle events · 400 days", "kind": "boundary", "dir": "col", "nodes": [ { "id": "d-kafka", "label": "Event stream", "sub": "Kafka · RF 3", "kind": "queue", "icon": "kafka" }, { "id": "d-ch", "label": "Event history", "sub": "ClickHouse · 400 d", "kind": "store", "icon": "clickhouse" }, { "id": "d-arch", "label": "Cost archive", "sub": "Ceph Object Lock · 7 y", "kind": "store", "icon": "fa5_archive" } ] }, { "title": "Checkpoints · owned by the workload", "kind": "trust", "dir": "col", "nodes": [ { "id": "d-key", "label": "Workload key", "sub": "OpenBao transit", "kind": "security", "icon": "fa5_key" }, { "id": "d-rgw", "label": "Checkpoint bucket", "sub": "3 gen or 7 d", "kind": "store", "icon": "fa5_archive" } ] } ] } ], "edges": [ { "from": "d-repo", "to": "d-bundle", "label": "on merge" }, { "from": "d-tags", "to": "d-fleet", "label": "rebuild ≤ 5 min" }, { "from": "d-kafka", "to": "d-ch", "label": "≤ 30 s" }, { "from": "d-ch", "to": "d-arch", "label": "monthly", "kind": "batch" }, { "from": "d-key", "to": "d-rgw", "label": "SSE-KMS" } ], "note": "The orchestrator's identity has no read permission on any checkpoint bucket. It sees object counts and bytes from RGW usage statistics, never contents.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "14-data-model", "title": "Data Model — Contracts, Pools, Plans, Instances and Their Evidence", "layout": "er", "canvas": { "width": 1640, "cols": 4 }, "rowGap": 250, "entities": [ { "id": "e-contract", "name": "contract", "kind": "store", "row": 0, "col": 0, "attrs": [ "contract_id, version PK", "tier T1..T5 | bespoke", "drain_budget_s, checkpoint_mode", "restart_semantics declared", "max_interrupt_per_h, floor_pct", "deadline, authority_id FK", "repo_commit" ] }, { "id": "e-workload", "name": "workload", "kind": "store", "row": 0, "col": 1, "attrs": [ "workload_id PK", "owner_team NOT NULL", "partition_id", "contract_id, version FK", "eligible_attrs arch, gpu, nvme" ] }, { "id": "e-plan", "name": "placement_plan", "kind": "store", "row": 0, "col": 2, "attrs": [ "plan_gen PK", "workload_id FK", "ranking_id FK", "units_by_pool vCPU, GiB, GPU", "max_pool_share, max_group_share", "depth_premium recorded" ] }, { "id": "e-rank", "name": "pool_ranking", "kind": "store", "row": 0, "col": 3, "attrs": [ "ranking_id PK", "partition_id, computed_at", "inputs_ref prices, rates, signal", "scores per pool", "scorer_version" ] }, { "id": "e-auth", "name": "spend_authority", "kind": "store", "row": 1, "col": 0, "attrs": [ "authority_id PK", "owner, approver named", "cap_per_24h", "signed_by governance" ] }, { "id": "e-fall", "name": "fallback", "kind": "store", "row": 1, "col": 1, "attrs": [ "fallback_id PK", "workload_id FK", "trigger rate|shortfall|deadline", "spend_reserved, approved_by", "started_at, ended_at, cost" ] }, { "id": "e-inst", "name": "instance", "kind": "store", "row": 1, "col": 2, "attrs": [ "instance_uuid PK", "pool_id FK", "workload_id, plan_gen FK", "purchase_model", "launched_by identity" ] }, { "id": "e-pool", "name": "pool", "kind": "store", "row": 1, "col": 3, "attrs": [ "pool_id PK", "sku_family, size", "region, hall", "purchase_model", "group_id FK", "attributes arch, gpu, nvme, nic" ] }, { "id": "e-event", "name": "lifecycle_event", "kind": "store", "row": 2, "col": 2, "attrs": [ "event_id PK", "instance_uuid FK", "type notice|drained|overrun|no_notice", "cause reclaim|defect|operator|drill", "occurred_at, notice_at", "wasted_vcpu_s" ] }, { "id": "e-group", "name": "correlation_group", "kind": "store", "row": 2, "col": 3, "attrs": [ "group_id PK", "basis aggregate|hall|learned", "co_reclaim_phi", "computed_at" ] } ], "relations": [ { "from": "e-contract", "to": "e-workload", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-workload", "to": "e-plan", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-rank", "to": "e-plan", "label": "1 : N", "from_side": "w", "to_side": "e" }, { "from": "e-auth", "to": "e-fall", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-workload", "to": "e-fall", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "e-plan", "to": "e-inst", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "e-pool", "to": "e-inst", "label": "1 : N", "from_side": "w", "to_side": "e" }, { "from": "e-inst", "to": "e-event", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "e-group", "to": "e-pool", "label": "1 : N", "from_side": "n", "to_side": "s" } ], "note": "There is no checkpoint entity. Checkpoints belong to the workload; the platform records only the wasted work that a missing one caused.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "15-lifecycle-event-pipeline", "title": "Lifecycle Events — From a Node's Spool to the Finance Archive", "layout": "flow", "chain": true, "stages": [ { "title": "Emit", "nodes": [ { "id": "v-agent", "label": "Node agents", "sub": "local spool", "kind": "app", "icon": "fa5_microchip" }, { "id": "v-ctl", "label": "Controllers", "sub": "transactional outbox", "kind": "app", "icon": "fa5_random" }, { "id": "v-prov", "label": "Provider notifications", "sub": "reclaim · delete", "kind": "external", "icon": "fa5_bell" } ] }, { "title": "Transport", "nodes": [ { "id": "v-vector", "label": "Vector", "sub": "on each node", "kind": "integration", "icon": "vector" }, { "id": "v-kafka", "label": "sco.lifecycle", "sub": "Kafka · keyed by instance", "kind": "queue", "icon": "kafka" } ] }, { "title": "Classify", "nodes": [ { "id": "v-excl", "label": "Exclusion watcher", "sub": "sliding window", "kind": "app", "icon": "fa5_ban" }, { "id": "v-cause", "label": "Reclaim or defect?", "sub": "notice · exit code", "kind": "decision", "icon": "fa5_gavel" } ] }, { "title": "Store", "nodes": [ { "id": "v-ch", "label": "Event history", "sub": "ClickHouse · 400 d", "kind": "store", "icon": "clickhouse" }, { "id": "v-roll", "label": "Cost rollups", "sub": "per workload · 7 y", "kind": "store", "icon": "fa5_coins" } ] }, { "title": "Act and report", "nodes": [ { "id": "v-alert", "label": "Breach alarms", "sub": "Alertmanager", "kind": "platform", "icon": "fa5_bell" }, { "id": "v-graf", "label": "Workload reports", "sub": "Grafana", "kind": "platform", "icon": "grafana" }, { "id": "v-fin", "label": "Finance export", "sub": "Ceph Object Lock", "kind": "store", "icon": "fa5_archive" } ] } ], "edges": [ { "from": "v-agent", "to": "v-vector", "label": "on reconnect", "kind": "async" }, { "from": "v-ctl", "to": "v-kafka", "kind": "async" }, { "from": "v-roll", "to": "v-fin", "label": "monthly", "kind": "batch" } ], "note": "An event without a cause is not written to the ledger. Defects are charged to the workload's own report, reclaims to the platform's.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "16-pool-scoring-pipeline", "title": "Pool Scoring — Inputs, Filters, a Versioned Ranking", "layout": "flow", "chain": true, "stages": [ { "title": "Inputs · 5 min", "nodes": [ { "id": "s-price", "label": "Rate card", "sub": "reclaimable vs guaranteed", "kind": "external", "icon": "fa5_coins" }, { "id": "s-hist", "label": "Interruption history", "sub": "ClickHouse · 30 d", "kind": "store", "icon": "clickhouse" }, { "id": "s-fwd", "label": "Forward capacity", "sub": "Blazar calendar", "kind": "external", "icon": "fa5_calendar_alt" }, { "id": "s-quota", "label": "Quota headroom", "sub": "Nova · Placement", "kind": "external", "icon": "fa5_tachometer_alt" } ] }, { "title": "Filter", "nodes": [ { "id": "s-elig", "label": "Can it run there?", "sub": "arch · GPU · NVMe · NIC", "kind": "decision", "icon": "fa5_check_circle" }, { "id": "s-inv", "label": "Still a discount?", "sub": "rate ≤ 85% guaranteed", "kind": "decision", "icon": "fa5_percentage" }, { "id": "s-excl", "label": "Excluded pools", "sub": "cool-off · probation", "kind": "risk", "icon": "fa5_ban" } ] }, { "title": "Score", "nodes": [ { "id": "s-score", "label": "Composite score", "sub": "depth before price", "kind": "app", "icon": "fa5_chart_line" }, { "id": "s-group", "label": "Correlation groups", "sub": "aggregate · hall", "kind": "app", "icon": "fa5_project_diagram" } ] }, { "title": "Record", "nodes": [ { "id": "s-rank", "label": "Ranking r-N", "sub": "inputs kept with it", "kind": "store", "icon": "postgresql" }, { "id": "s-hist90", "label": "Ranking history", "sub": "90 d full · 13 mo agg", "kind": "store", "icon": "clickhouse" } ] }, { "title": "Use", "nodes": [ { "id": "s-plan", "label": "Portfolio planner", "sub": "only changed claims", "kind": "app", "icon": "fa5_sitemap" }, { "id": "s-audit", "label": "Why this pool?", "sub": "decision replay", "kind": "platform", "icon": "fa5_search" } ] } ], "edges": [ { "from": "s-rank", "to": "s-hist90", "label": "every run", "kind": "batch" } ], "note": "Exclusion is not part of the 5-minute run. The exclusion watcher applies it within 60 s, and the next ranking inherits it.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "17-placement-decision", "title": "Placement — From Pending Work to Diversified Capacity", "layout": "sequence", "lifelines": [ { "id": "q-sched", "label": "Kubernetes", "kind": "external", "icon": "kubernetes" }, { "id": "q-intake", "label": "Demand intake", "kind": "app", "icon": "fa5_inbox" }, { "id": "q-plan", "label": "Portfolio planner", "kind": "app", "icon": "fa5_sitemap" }, { "id": "q-pg", "label": "Fleet state", "kind": "store", "icon": "postgresql" }, { "id": "q-acq", "label": "Acquirer", "kind": "app", "icon": "fa5_cubes" }, { "id": "q-nova", "label": "Nova", "kind": "external", "icon": "fa5_server" }, { "id": "q-agent", "label": "Node agent", "kind": "app", "icon": "fa5_microchip" } ], "messages": [ { "from": "q-intake", "to": "q-sched", "label": "watch: 42 pods pending, contract T1", "kind": "call" }, { "from": "q-intake", "to": "q-intake", "label": "normalise: 168 vCPU · 672 GiB", "kind": "self" }, { "from": "q-intake", "to": "q-plan", "label": "claim delta for orders-daily", "kind": "async" }, { "from": "q-plan", "to": "q-pg", "label": "ranking r-8812 · contract · current units", "kind": "call" }, { "from": "q-pg", "to": "q-plan", "label": "71 eligible pools in 9 groups", "kind": "return" }, { "from": "q-plan", "to": "q-plan", "label": "floor first: 30% on committed capacity", "kind": "self" }, { "from": "q-plan", "to": "q-plan", "label": "fill: ≤ 15% per pool, ≤ 30% per group", "kind": "self" }, { "from": "q-plan", "to": "q-pg", "label": "commit plan gen 5531 · depth premium", "kind": "call" }, { "from": "q-plan", "to": "q-acq", "label": "diff: 6 pools gain units", "kind": "async" }, { "from": "q-acq", "to": "q-acq", "label": "token budget · quota headroom", "kind": "self" }, { "from": "q-acq", "to": "q-nova", "label": "multi-create · tags sco:* · user-data", "kind": "call" }, { "from": "q-nova", "to": "q-acq", "label": "9 instances building", "kind": "return" }, { "from": "q-agent", "to": "q-agent", "label": "boot: contract subset already on disk", "kind": "self" }, { "from": "q-agent", "to": "q-sched", "label": "node Ready · taint reclaimable", "kind": "call" } ], "note": "Steps 1 to 8 are the placement decision, p99 ≤ 10 s. The planner touches only the claim that changed, never the whole partition.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "18-notice-to-drain", "title": "Hard Notice — 120 Seconds, Spent on the Node", "layout": "sequence", "lifelines": [ { "id": "r-recl", "label": "Reclaimer", "kind": "external", "icon": "fa5_bell" }, { "id": "r-agent", "label": "Node agent", "kind": "app", "icon": "fa5_microchip" }, { "id": "r-api", "label": "Scheduler API", "kind": "external", "icon": "kubernetes" }, { "id": "r-work", "label": "Workload", "kind": "external", "icon": "fa5_cube" }, { "id": "r-rgw", "label": "Checkpoint bucket", "kind": "store", "icon": "fa5_archive" }, { "id": "r-events", "label": "Lifecycle events", "kind": "queue", "icon": "kafka" }, { "id": "r-acq", "label": "Acquirer", "kind": "app", "icon": "fa5_cubes" } ], "messages": [ { "from": "r-recl", "to": "r-agent", "label": "T0: terminate at T+120 (via relay)", "kind": "async" }, { "from": "r-agent", "to": "r-agent", "label": "bundle on disk: budget 110 s, SIGTERM", "kind": "self" }, { "from": "r-agent", "to": "r-api", "label": "taint NoSchedule (800 ms, else local)", "kind": "call" }, { "from": "r-agent", "to": "r-work", "label": "T+1.4 s: evict · grace 110 s · deadline file", "kind": "call" }, { "from": "r-agent", "to": "r-events", "label": "notice_received", "kind": "async" }, { "from": "r-events", "to": "r-acq", "label": "replace from another group", "kind": "async" }, { "from": "r-work", "to": "r-rgw", "label": "final checkpoint 1.4 GB", "kind": "call" }, { "from": "r-rgw", "to": "r-work", "label": "generation 18 committed", "kind": "return" }, { "from": "r-work", "to": "r-agent", "label": "exited 0 at T+71 s", "kind": "return" }, { "from": "r-agent", "to": "r-agent", "label": "verify: no pods, no job steps", "kind": "self" }, { "from": "r-agent", "to": "r-events", "label": "drained · 49 s spare", "kind": "async" }, { "from": "r-recl", "to": "r-recl", "label": "T+120: delete instance", "kind": "self" }, { "from": "r-agent", "to": "r-events", "label": "if still running at T+118: overrun", "kind": "error" } ], "note": "The acquirer's replacement runs in parallel with the drain and waits for nothing on the node. The deadline is the provider's and is never extended.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "19-drain-by-scheduler", "title": "One Agent, Four Drain Paths — Through the Scheduler, or Around It", "layout": "swimlane", "laneHeaderWidth": 190, "stages": [ "Notice · ≤ 2 s", "Stop new work", "Signal the work", "Preserve", "Verify", "At T+120" ], "lanes": [ { "title": "Kubernetes pods", "cells": [ [ { "label": "Budget from bundle", "kind": "security", "icon": "fa5_file_signature" } ], [ { "label": "Taint NoSchedule", "kind": "app", "icon": "kubernetes" } ], [ { "label": "Evict · grace = budget", "kind": "app", "icon": "fa5_sign_out_alt" } ], [ { "label": "preStop final write", "kind": "app", "icon": "fa5_save" } ], [ { "label": "Pods gone from node", "kind": "opportunity", "icon": "fa5_check_double" } ], [ { "label": "Job controller retries", "kind": "external", "icon": "fa5_redo" } ] ] }, { "title": "Spark on Kubernetes", "cells": [ [ { "label": "Budget from bundle", "kind": "security", "icon": "fa5_file_signature" } ], [ { "label": "Taint NoSchedule", "kind": "app", "icon": "kubernetes" } ], [ { "label": "Executor decommission", "kind": "app", "icon": "spark" } ], [ { "label": "Shuffle to fallback store", "kind": "app", "icon": "fa5_archive" } ], [ { "label": "Blocks migrated", "kind": "opportunity", "icon": "fa5_check_double" } ], [ { "label": "Stage continues", "kind": "external", "icon": "spark" } ] ] }, { "title": "Slurm jobs", "cells": [ [ { "label": "Budget from bundle", "kind": "security", "icon": "fa5_file_signature" } ], [ { "label": "Node state DRAIN", "kind": "app", "icon": "fa5_tasks" } ], [ { "label": "USR1 to job cgroup", "kind": "app", "icon": "fa5_bolt" } ], [ { "label": "Application checkpoint", "kind": "app", "icon": "fa5_save" } ], [ { "label": "Job steps exited", "kind": "opportunity", "icon": "fa5_check_double" } ], [ { "label": "Requeued, resumes", "kind": "external", "icon": "fa5_redo" } ] ] }, { "title": "Scheduler API unreachable", "cells": [ [ { "label": "800 ms timeout", "kind": "risk", "icon": "fa5_stopwatch" } ], [ { "label": "Scheduler not told", "kind": "risk", "icon": "fa5_exclamation_triangle" } ], [ { "label": "kubelet graceful shutdown", "kind": "decision", "icon": "fa5_power_off" } ], [ { "label": "Same final write", "kind": "app", "icon": "fa5_save" } ], [ { "label": "Outcome spooled", "kind": "app", "icon": "fa5_hdd" } ], [ { "label": "Scheduler learns late", "kind": "external", "icon": "fa5_clock" } ] ] } ], "note": "The agent decides and keeps the deadline. The scheduler's own eviction does the work when it answers in time, so two systems never evict the same pod.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" }, "canvas": { "width": 1800 } }, { "id": "20-checkpoint-and-restore", "title": "Checkpoint and Restore — Periodic Insurance, Final Write, Clean Resume", "layout": "sequence", "lifelines": [ { "id": "c-work", "label": "Workload", "kind": "external", "icon": "fa5_cube" }, { "id": "c-client", "label": "Checkpoint client", "kind": "app", "icon": "fa5_save" }, { "id": "c-agent", "label": "Node agent", "kind": "app", "icon": "fa5_microchip" }, { "id": "c-rgw", "label": "Ceph RGW bucket", "kind": "store", "icon": "fa5_archive" }, { "id": "c-sched", "label": "Scheduler", "kind": "external", "icon": "fa5_tasks" }, { "id": "c-new", "label": "Restored workload", "kind": "external", "icon": "fa5_redo" } ], "messages": [ { "from": "c-work", "to": "c-client", "label": "save() every 60 min", "kind": "call" }, { "from": "c-client", "to": "c-rgw", "label": "gen-17 parts, SSE-KMS", "kind": "call" }, { "from": "c-client", "to": "c-rgw", "label": "manifest gen-17 written last", "kind": "call" }, { "from": "c-agent", "to": "c-client", "label": "notice · deadline T+120", "kind": "async" }, { "from": "c-client", "to": "c-work", "label": "on_notice()", "kind": "call" }, { "from": "c-work", "to": "c-client", "label": "save(1.4 GB)", "kind": "call" }, { "from": "c-client", "to": "c-rgw", "label": "gen-18 parts + manifest", "kind": "call" }, { "from": "c-rgw", "to": "c-client", "label": "committed in 31 s", "kind": "return" }, { "from": "c-client", "to": "c-agent", "label": "checkpoint committed", "kind": "return" }, { "from": "c-client", "to": "c-agent", "label": "if RGW unreachable: release lease, no resume", "kind": "error" }, { "from": "c-sched", "to": "c-new", "label": "reschedule unit", "kind": "call" }, { "from": "c-new", "to": "c-rgw", "label": "latest manifest", "kind": "call" }, { "from": "c-rgw", "to": "c-new", "label": "gen-18", "kind": "return" }, { "from": "c-new", "to": "c-new", "label": "resume · outputs staged until done", "kind": "self" } ], "note": "A generation without its manifest does not exist. Outputs are published only when the unit completes, so a resumed unit cannot emit a result twice.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "21-correlated-reclamation", "title": "Correlated Reclamation — 800 Nodes in Hall B, Minute by Minute", "layout": "swimlane", "laneHeaderWidth": 180, "stages": [ "T0", "T+2 s", "T+60 s", "T+120 s", "T+3 min", "T+15 min" ], "lanes": [ { "title": "Provider", "cells": [ [ { "label": "800 notices · aggregate 7", "kind": "external", "icon": "fa5_bell" } ], [], [ { "label": "Advisory on 2 more", "kind": "external", "icon": "fa5_exclamation_circle" } ], [ { "label": "800 instances deleted", "kind": "external", "icon": "fa5_trash_alt" } ], [], [ { "label": "Guaranteed demand met", "kind": "external", "icon": "fa5_check_circle" } ] ] }, { "title": "Node agents", "cells": [ [ { "label": "Bundles already local", "kind": "security", "icon": "fa5_file_signature" } ], [ { "label": "800 drains signalled", "kind": "app", "icon": "fa5_microchip" } ], [ { "label": "Most drains finished", "kind": "app", "icon": "fa5_check_double" } ], [ { "label": "≤ 4 overruns counted", "kind": "risk", "icon": "fa5_hourglass_end" } ], [], [] ] }, { "title": "Control plane", "cells": [ [ { "label": "Events arrive", "kind": "queue", "icon": "kafka" } ], [], [ { "label": "Group 7 excluded", "kind": "decision", "icon": "fa5_ban" } ], [ { "label": "Replace in halls A, C, D", "kind": "app", "icon": "fa5_cubes" } ], [ { "label": "Serving ≤ 180 s p99", "kind": "app", "icon": "fa5_stopwatch" } ], [ { "label": "Fallback where breached", "kind": "decision", "icon": "fa5_balance_scale" } ] ] }, { "title": "Checkpoint store", "cells": [ [], [ { "label": "Final writes begin", "kind": "store", "icon": "fa5_archive" } ], [ { "label": "Peak ≈ 10 GB/s", "kind": "risk", "icon": "fa5_tachometer_alt" } ], [ { "label": "Writes committed", "kind": "store", "icon": "fa5_check_circle" } ], [ { "label": "Restores start", "kind": "store", "icon": "fa5_redo" } ], [] ] }, { "title": "Tier-1 workloads", "cells": [ [ { "label": "Floor untouched", "kind": "opportunity", "icon": "fa5_layer_group" } ], [], [], [ { "label": "Running at ≥ 30%", "kind": "opportunity", "icon": "fa5_layer_group" } ], [ { "label": "Units resumed", "kind": "app", "icon": "fa5_redo" } ], [ { "label": "Deadline risk rechecked", "kind": "decision", "icon": "fa5_calendar_check" } ] ] } ], "note": "The replacement rate is capped at the region's API token budget. A request storm against Nova would slow every tenant, including the one reclaiming.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" }, "canvas": { "width": 1800 } }, { "id": "22-fallback-and-return", "title": "Fallback and Return — Floor First, Spend Within Authority, Return Slowly", "layout": "flow", "align": "top", "stages": [ { "title": "Trigger", "nodes": [ { "id": "f-rate", "label": "Interruption breach", "sub": "above declared rate", "kind": "risk", "icon": "fa5_percentage" }, { "id": "f-short", "label": "Shortfall", "sub": "≥ 5 min unmet", "kind": "risk", "icon": "fa5_exclamation_triangle" }, { "id": "f-dead", "label": "Deadline risk", "sub": "ETA past deadline", "kind": "risk", "icon": "fa5_calendar_alt" } ] }, { "title": "Floor", "nodes": [ { "id": "f-below", "label": "Below floor?", "kind": "decision", "icon": "fa5_layer_group" }, { "id": "f-restore", "label": "Restore floor", "sub": "committed first · no approval", "kind": "app", "icon": "fa5_calendar_check" } ] }, { "title": "Authority", "nodes": [ { "id": "f-cap", "label": "Within 24 h cap?", "kind": "decision", "icon": "fa5_coins" }, { "id": "f-stagger", "label": "Staggered fallback", "sub": "priority · 40 per min", "kind": "app", "icon": "fa5_sort_amount_down" } ] }, { "title": "Beyond authority", "nodes": [ { "id": "f-esc", "label": "Page budget owner", "sub": "15 min to answer", "kind": "integration", "icon": "fa5_bullhorn" }, { "id": "f-record", "label": "Record shortfall", "sub": "not spend", "kind": "risk", "icon": "fa5_clipboard_list" } ] }, { "title": "Return", "nodes": [ { "id": "f-healthy", "label": "Healthy for 60 min?", "kind": "decision", "icon": "fa5_heartbeat" }, { "id": "f-back", "label": "Return 20% per hour", "sub": "newest fallback first", "kind": "app", "icon": "fa5_undo" } ] } ], "edges": [ { "from": "f-short", "to": "f-below" }, { "from": "f-below", "to": "f-restore", "label": "yes" }, { "from": "f-restore", "to": "f-cap" }, { "from": "f-cap", "to": "f-stagger", "label": "yes" }, { "from": "f-cap", "to": "f-esc", "label": "no", "kind": "error" }, { "from": "f-esc", "to": "f-record", "label": "no answer", "kind": "error" }, { "from": "f-healthy", "to": "f-back", "label": "yes" } ], "note": "All three triggers enter at the floor check. Every active fallback is tested for return each cycle, and records its trigger, duration and incremental cost.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "23-control-plane-outage", "title": "Control-Plane Outage — The Fleet Keeps Running Exactly as It Was", "layout": "grid", "laneHeaderWidth": 190, "columns": [ "0 – 5 min", "5 min – 1 h", "1 – 12 h", "Beyond 12 h", "On recovery" ], "rows": [ { "title": "Running capacity", "cells": [ [ { "label": "Untouched", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Untouched", "kind": "opportunity", "icon": "fa5_check_circle" } ], [ { "label": "Shrinks as reclaimed", "kind": "risk", "icon": "fa5_compress_arrows_alt" } ], [ { "label": "Converges to floor", "kind": "risk", "icon": "fa5_layer_group" } ], [ { "label": "No mass terminate", "kind": "decision", "icon": "fa5_ban" } ] ] }, { "title": "Notice and drain", "cells": [ [ { "label": "Node-local as always", "kind": "opportunity", "icon": "fa5_microchip" } ], [ { "label": "Node-local as always", "kind": "opportunity", "icon": "fa5_microchip" } ], [ { "label": "Node-local as always", "kind": "opportunity", "icon": "fa5_microchip" } ], [ { "label": "Bundle age reported", "kind": "decision", "icon": "fa5_clock" } ], [ { "label": "Bundles refreshed", "kind": "app", "icon": "fa5_file_signature" } ] ] }, { "title": "New demand", "cells": [ [ { "label": "Waits in scheduler", "kind": "external", "icon": "fa5_hourglass_half" } ], [ { "label": "Pending work grows", "kind": "risk", "icon": "fa5_hourglass_half" } ], [ { "label": "Tier-1 shortfall incident", "kind": "risk", "icon": "fa5_bullhorn" } ], [ { "label": "Throughput at floor", "kind": "risk", "icon": "fa5_layer_group" } ], [ { "label": "Tier 1 served first", "kind": "app", "icon": "fa5_sort_amount_down" } ] ] }, { "title": "Replacement and fallback", "cells": [ [ { "label": "Paused", "kind": "decision", "icon": "fa5_pause" } ], [ { "label": "Paused", "kind": "decision", "icon": "fa5_pause" } ], [ { "label": "Floor already paid for", "kind": "opportunity", "icon": "fa5_calendar_check" } ], [ { "label": "Manual floor via runbook", "kind": "integration", "icon": "fa5_tools" } ], [ { "label": "≤ 5% of fleet per 5 min", "kind": "app", "icon": "fa5_tachometer_alt" } ] ] }, { "title": "Evidence", "cells": [ [ { "label": "Spooled on nodes", "kind": "store", "icon": "fa5_hdd" } ], [ { "label": "Spooled on nodes", "kind": "store", "icon": "fa5_hdd" } ], [ { "label": "Lost with reclaimed nodes", "kind": "risk", "icon": "fa5_trash_alt" } ], [ { "label": "Gap marked, not zero", "kind": "decision", "icon": "fa5_exclamation_circle" } ], [ { "label": "Rebuilt from provider", "kind": "app", "icon": "fa5_tags" } ] ] } ], "note": "Recovery starts read-only: two reconcile cycles rebuild state from provider tags before the first action is allowed.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "24-pool-exclusion-cycle", "title": "Pool Exclusion — Out Fast, Back Slowly, Never Permanently", "layout": "cycle", "centre": { "label": "One pool", "icon": "fa5_server" }, "nodes": [ { "id": "x1", "label": "Eligible", "sub": "ranked every 5 min", "kind": "app", "icon": "fa5_check_circle" }, { "id": "x2", "label": "Tripped", "sub": "3 reclaims in 30 min", "kind": "risk", "icon": "fa5_exclamation_triangle" }, { "id": "x3", "label": "Excluded", "sub": "≤ 60 s · no new units", "kind": "decision", "icon": "fa5_ban" }, { "id": "x4", "label": "Cool-off", "sub": "2 h, doubles on repeat", "kind": "app", "icon": "fa5_hourglass_half" }, { "id": "x5", "label": "Probation", "sub": "≤ 5% of any workload", "kind": "security", "icon": "fa5_eye" }, { "id": "x6", "label": "Restored", "sub": "back-off reset after 24 h", "kind": "opportunity", "icon": "fa5_undo" } ], "ringLabels": [ "threshold crossed", "planner skips it", "timer starts", "capped trial", "1 h clean", "next ranking" ], "rx": 420, "ry": 210, "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "25-deployment-architecture", "title": "Deployment — One Region of Three, Four Halls", "layout": "nested", "canvas": { "width": 1600 }, "boxes": [ { "title": "Region r1 · r2 and r3 are identical and independent", "kind": "onprem", "dir": "row", "children": [ { "title": "Hall A", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-ctl-a", "label": "sco-controller", "sub": "leader · partitions 1–4", "kind": "app", "icon": "fa5_random" }, { "id": "y-pg-a", "label": "PostgreSQL primary", "sub": "Patroni", "kind": "store", "icon": "postgresql" }, { "id": "y-pool-a", "label": "Reclaimable pools", "sub": "9 SKUs × 4 sizes", "kind": "external", "icon": "fa5_server" } ] }, { "title": "Hall B", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-ctl-b", "label": "sco-controller", "sub": "standby", "kind": "app", "icon": "fa5_random" }, { "id": "y-pg-b", "label": "PostgreSQL replica", "sub": "synchronous", "kind": "store", "icon": "postgresql" }, { "id": "y-pool-b", "label": "Reclaimable pools", "sub": "9 SKUs × 4 sizes", "kind": "external", "icon": "fa5_server" } ] }, { "title": "Hall C", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-ctl-c", "label": "sco-controller", "sub": "standby", "kind": "app", "icon": "fa5_random" }, { "id": "y-pg-c", "label": "PostgreSQL replica", "sub": "asynchronous", "kind": "store", "icon": "postgresql" }, { "id": "y-pool-c", "label": "Reclaimable pools", "sub": "9 SKUs × 4 sizes", "kind": "external", "icon": "fa5_server" } ] }, { "title": "Hall D", "kind": "boundary", "dir": "col", "nodes": [ { "id": "y-floor", "label": "Guaranteed floor", "sub": "spread over 3+ halls", "kind": "external", "icon": "fa5_layer_group" }, { "id": "y-pool-d", "label": "Reclaimable pools", "sub": "9 SKUs × 4 sizes", "kind": "external", "icon": "fa5_server" } ] } ] } ], "outside": [ { "id": "y-estate", "label": "Estate data services", "sub": "Kafka · ClickHouse · Ceph", "kind": "store", "icon": "kafka" }, { "id": "y-prom", "label": "Prometheus and Thanos", "sub": "per region · global query", "kind": "platform", "icon": "prometheus" }, { "id": "y-git", "label": "Forgejo and CI", "sub": "mirrored to r2", "kind": "store", "icon": "git" } ], "edges": [ { "from": "y-pg-a", "to": "y-pg-b", "label": "sync commit" }, { "from": "y-pg-b", "to": "y-pg-c", "label": "streaming", "kind": "async" } ], "note": "The management cluster spans halls A to C on guaranteed capacity. Losing any one hall leaves a leader, a synchronous replica and the whole node plane working.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "26-release-pipeline", "title": "Change — Contracts and Code Both Reach the Fleet Through a Canary", "layout": "flow", "chain": true, "stages": [ { "title": "Propose", "nodes": [ { "id": "w-contract", "label": "Contract change", "sub": "Forgejo pull request", "kind": "store", "icon": "git" }, { "id": "w-code", "label": "Agent or controller change", "sub": "Go · pull request", "kind": "app", "icon": "go" } ] }, { "title": "Check", "nodes": [ { "id": "w-cue", "label": "Schema and feasibility", "sub": "CUE · 30-day replay", "kind": "app", "icon": "fa5_file_contract" }, { "id": "w-gov", "label": "Floor or spend changed?", "kind": "decision", "icon": "fa5_coins" } ] }, { "title": "Authorise", "nodes": [ { "id": "w-sig", "label": "Governance signature", "sub": "second key", "kind": "security", "icon": "fa5_file_signature" }, { "id": "w-sign", "label": "Build and sign", "sub": "cosign · image bake", "kind": "security", "icon": "fa5_stamp" } ] }, { "title": "Canary", "nodes": [ { "id": "w-part", "label": "One partition", "sub": "bundle · 24 h", "kind": "app", "icon": "fa5_flask" }, { "id": "w-launch", "label": "New launches only", "sub": "5% of pools", "kind": "app", "icon": "fa5_microchip" } ] }, { "title": "Gate", "nodes": [ { "id": "w-gate", "label": "Canary clean?", "sub": "overruns · waste · alarms", "kind": "decision", "icon": "fa5_traffic_light" } ] }, { "title": "Fleet", "nodes": [ { "id": "w-all", "label": "All partitions", "sub": "bundle pushed", "kind": "app", "icon": "fa5_share_alt" }, { "id": "w-churn", "label": "Fleet replaced by churn", "sub": "~ 72 h to 95%", "kind": "opportunity", "icon": "fa5_sync_alt" } ] } ], "edges": [ { "from": "w-gate", "to": "w-launch", "kind": "error" } ], "note": "A dirty canary reverts the bundle or stops launching the new image. Agents are never upgraded in place on a draining node. The fleet turns over by reclamation and scale-down, which is the rollout.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "27-observability", "title": "Observability — What Is Measured, and Who Is Woken", "layout": "grid", "laneHeaderWidth": 180, "columns": [ "Emit", "Collect", "Store", "Consume", "Act" ], "rows": [ { "title": "Interruption", "cells": [ [ { "label": "Agent drain events", "kind": "app", "icon": "fa5_microchip" } ], [ { "label": "Vector to Kafka", "kind": "queue", "icon": "vector" } ], [ { "label": "ClickHouse · 400 d", "kind": "store", "icon": "clickhouse" } ], [ { "label": "Overrun rate per pool", "kind": "platform", "icon": "grafana" } ], [ { "label": "Contract flagged", "kind": "decision", "icon": "fa5_flag" } ] ] }, { "title": "Capacity", "cells": [ [ { "label": "Claims · shortfall", "kind": "app", "icon": "fa5_inbox" } ], [ { "label": "Prometheus", "kind": "platform", "icon": "prometheus" } ], [ { "label": "Thanos · 13 mo", "kind": "store", "icon": "prometheus" } ], [ { "label": "Shortfall apart from reclaim", "kind": "platform", "icon": "grafana" } ], [ { "label": "Page at 15 min, tier 1", "kind": "decision", "icon": "fa5_bullhorn" } ] ] }, { "title": "Placement quality", "cells": [ [ { "label": "Pool and group shares", "kind": "app", "icon": "fa5_sitemap" } ], [ { "label": "Prometheus", "kind": "platform", "icon": "prometheus" } ], [ { "label": "Rankings · 90 d", "kind": "store", "icon": "clickhouse" } ], [ { "label": "Blast-radius board", "kind": "platform", "icon": "grafana" } ], [ { "label": "Replan on cap breach", "kind": "decision", "icon": "fa5_random" } ] ] }, { "title": "Money", "cells": [ [ { "label": "Rate card · spend ledger", "kind": "app", "icon": "fa5_coins" } ], [ { "label": "Attribution · hourly", "kind": "app", "icon": "fa5_calculator" } ], [ { "label": "Cost rollups · 7 y", "kind": "store", "icon": "clickhouse" } ], [ { "label": "Net saving per workload", "kind": "platform", "icon": "grafana" } ], [ { "label": "Move-off recommendation", "kind": "decision", "icon": "fa5_exchange_alt" } ] ] }, { "title": "Platform health", "cells": [ [ { "label": "Leader · bundle age", "kind": "app", "icon": "fa5_heartbeat" } ], [ { "label": "Prometheus", "kind": "platform", "icon": "prometheus" } ], [ { "label": "Thanos · 13 mo", "kind": "store", "icon": "prometheus" } ], [ { "label": "Own cost vs saving", "kind": "platform", "icon": "grafana" } ], [ { "label": "Page SRE, never workloads", "kind": "decision", "icon": "fa5_user_cog" } ] ] } ], "note": "Workload owners are alarmed only on their own contract breach. Pool behaviour pages the platform, because it is the platform's to fix.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "28-net-savings-ledger", "title": "Net Savings — Every Cost of the Discount, Charged Against It", "layout": "grid", "laneHeaderWidth": 190, "columns": [ "Measured from", "Planning figure, per month", "Charged to" ], "rows": [ { "title": "Counterfactual", "cells": [ [ { "label": "Useful vCPU-h × guaranteed rate", "kind": "app", "icon": "fa5_calculator" } ], [ { "label": "€335k · 11.2 M vCPU-h", "kind": "opportunity", "icon": "fa5_coins" } ], [ { "label": "Baseline, not a charge", "kind": "external", "icon": "fa5_equals" } ] ] }, { "title": "Reclaimable spend", "cells": [ [ { "label": "Instance hours × pool rate", "kind": "app", "icon": "fa5_server" } ], [ { "label": "− €115k", "kind": "risk", "icon": "fa5_coins" } ], [ { "label": "The workload", "kind": "external", "icon": "fa5_users" } ] ] }, { "title": "Wasted work", "cells": [ [ { "label": "Redone vCPU-s, by cause", "kind": "app", "icon": "fa5_trash_alt" } ], [ { "label": "2.5% · left out of useful", "kind": "risk", "icon": "fa5_percentage" } ], [ { "label": "Reclaim here, defect to owner", "kind": "external", "icon": "fa5_gavel" } ] ] }, { "title": "Fallback", "cells": [ [ { "label": "Fallback hours × guaranteed", "kind": "app", "icon": "fa5_balance_scale" } ], [ { "label": "− €13.5k", "kind": "risk", "icon": "fa5_coins" } ], [ { "label": "Workload, by trigger", "kind": "external", "icon": "fa5_users" } ] ] }, { "title": "Warm headroom", "cells": [ [ { "label": "Idle warm vCPU-h", "kind": "app", "icon": "fa5_hourglass_half" } ], [ { "label": "− €2.3k", "kind": "risk", "icon": "fa5_coins" } ], [ { "label": "Contracts that asked", "kind": "external", "icon": "fa5_file_contract" } ] ] }, { "title": "Checkpoint storage", "cells": [ [ { "label": "RGW bytes × storage rate", "kind": "app", "icon": "fa5_archive" } ], [ { "label": "− €1.4k", "kind": "risk", "icon": "fa5_coins" } ], [ { "label": "Bucket owner", "kind": "external", "icon": "fa5_users" } ] ] }, { "title": "Orchestrator", "cells": [ [ { "label": "Own pods, DB, agents, share", "kind": "app", "icon": "fa5_random" } ], [ { "label": "− €3.2k · 1.6% of saving", "kind": "risk", "icon": "fa5_coins" } ], [ { "label": "Pro rata to saving", "kind": "external", "icon": "fa5_balance_scale" } ] ] }, { "title": "Net saving", "cells": [ [ { "label": "Counterfactual − all spend", "kind": "decision", "icon": "fa5_equals" } ], [ { "label": "€199k · 59%", "kind": "opportunity", "icon": "fa5_chart_line" } ], [ { "label": "Reported per workload", "kind": "platform", "icon": "grafana" } ] ] } ], "note": "Figures are planning estimates on an assumed rate card of €0.030 per guaranteed vCPU-h and €0.0105 average reclaimable. The floor premium is priced separately (view 06).", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "29-roadmap", "title": "Roadmap — Prove the Notice Path, Then One Region, Then Breadth", "layout": "flow", "chain": true, "stages": [ { "title": "Proof · 8 weeks", "nodes": [ { "label": "Notice relay p99 ≤ 1 s", "sub": "5,000 subscribers", "kind": "decision", "icon": "fa5_bell" }, { "label": "800-node drain", "sub": "evict inside budget", "kind": "decision", "icon": "fa5_microchip" }, { "label": "RGW at 10 GB/s", "sub": "800 final writes", "kind": "decision", "icon": "fa5_archive" } ] }, { "title": "MVP · one region", "nodes": [ { "label": "Contracts · 5 tiers", "sub": "admission refuses", "kind": "app", "icon": "fa5_file_contract" }, { "label": "Pools · caps · exclusion", "kind": "app", "icon": "fa5_sitemap" }, { "label": "Agent · Kubernetes · Slurm", "kind": "app", "icon": "fa5_microchip" }, { "label": "Checkpoint client", "sub": "Spark · PyTorch", "kind": "app", "icon": "fa5_save" }, { "label": "Floor · capped fallback", "kind": "app", "icon": "fa5_balance_scale" }, { "label": "Daily 1% drill", "sub": "moved up from phase 2", "kind": "opportunity", "icon": "fa5_flask" } ] }, { "title": "Phase 2", "nodes": [ { "label": "Learned correlation", "kind": "app", "icon": "fa5_project_diagram" }, { "label": "Advisory drain", "sub": "per contract", "kind": "app", "icon": "fa5_exclamation_circle" }, { "label": "Warm headroom", "kind": "app", "icon": "fa5_hourglass_half" }, { "label": "Three regions · quotas", "kind": "app", "icon": "fa5_building" }, { "label": "Commitment-aware fallback", "kind": "app", "icon": "fa5_calendar_check" }, { "label": "Poison-unit quarantine", "kind": "app", "icon": "fa5_biohazard" } ] }, { "title": "Phase 3", "nodes": [ { "label": "Generic checkpoint", "sub": "CRIU, where it fits", "kind": "app", "icon": "fa5_camera" }, { "label": "Predictive scoring", "kind": "app", "icon": "fa5_chart_line" }, { "label": "Contract recommendations", "kind": "app", "icon": "fa5_exchange_alt" }, { "label": "Contract-priced chargeback", "kind": "opportunity", "icon": "fa5_file_invoice_dollar" } ] } ], "note": "A failed proof item changes the design before the MVP starts: a slow relay moves notice delivery onto the hypervisor, a slow store moves final writes to requeue.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "30-security-trust-zones", "title": "Security — Who Can Acquire, Terminate and Drain", "layout": "zones", "zones": [ { "title": "Workloads", "kind": "trust", "nodes": [ { "id": "z-pod", "label": "Workload", "sub": "no capacity rights", "kind": "external", "icon": "fa5_cube" }, { "id": "z-bad", "label": "Compromised workload", "sub": "the assumed attacker", "kind": "risk", "icon": "fa5_user_secret" } ] }, { "title": "Node", "kind": "trust", "nodes": [ { "id": "z-agent", "label": "Node agent", "sub": "own node only", "kind": "app", "icon": "fa5_microchip" }, { "id": "z-spire", "label": "SPIRE agent", "sub": "node-bound SVID", "kind": "security", "icon": "fa5_fingerprint" } ] }, { "title": "Control plane", "kind": "trust", "nodes": [ { "id": "z-ctl", "label": "sco-controller", "sub": "only acquirer", "kind": "app", "icon": "fa5_random" }, { "id": "z-bao", "label": "OpenBao", "sub": "credential · 1 h", "kind": "security", "icon": "fa5_key" } ] }, { "title": "Provider", "kind": "trust", "nodes": [ { "id": "z-nova", "label": "Nova · sco projects", "sub": "nobody else has a role", "kind": "external", "icon": "fa5_server" }, { "id": "z-recl", "label": "Reclaimer", "sub": "notices, never commands", "kind": "external", "icon": "fa5_bell" } ] } ], "edges": [ { "from": "z-pod", "to": "z-agent", "label": "unix socket" }, { "from": "z-agent", "to": "z-ctl", "label": "bundle only" }, { "from": "z-ctl", "to": "z-nova", "label": "scoped" }, { "from": "z-bad", "to": "z-spire", "label": "own SVID only", "kind": "error" } ], "note": "Reclaimable instances live in projects where only the orchestrator holds a role. Every capacity action is written to the audit log service through the controller's outbox (view 31).", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "31-spend-authority", "title": "Spend Authority — Who May Turn a Shortage Into a Bill", "layout": "sequence", "lifelines": [ { "id": "g-arb", "label": "Fallback arbiter", "kind": "app", "icon": "fa5_balance_scale" }, { "id": "g-ledger", "label": "Spend ledger", "kind": "store", "icon": "postgresql" }, { "id": "g-inc", "label": "Incident platform", "kind": "external", "icon": "fa5_bullhorn" }, { "id": "g-owner", "label": "Budget owner", "kind": "actor", "icon": "fa5_user_tie" }, { "id": "g-bao", "label": "OpenBao", "kind": "security", "icon": "fa5_key" }, { "id": "g-nova", "label": "Nova", "kind": "external", "icon": "fa5_server" }, { "id": "g-audit", "label": "Audit log", "kind": "store", "icon": "fa5_clipboard_list" } ], "messages": [ { "from": "g-arb", "to": "g-arb", "label": "deadline risk: needs €2,400 more", "kind": "self" }, { "from": "g-arb", "to": "g-ledger", "label": "reserve against 24 h cap", "kind": "call" }, { "from": "g-ledger", "to": "g-arb", "label": "cap exhausted", "kind": "return" }, { "from": "g-arb", "to": "g-inc", "label": "approval request · expires 15 min", "kind": "async" }, { "from": "g-inc", "to": "g-owner", "label": "page with priced options", "kind": "call" }, { "from": "g-owner", "to": "g-arb", "label": "approve €2,400 · OIDC + WebAuthn", "kind": "call" }, { "from": "g-arb", "to": "g-arb", "label": "approver named in contract?", "kind": "self" }, { "from": "g-arb", "to": "g-ledger", "label": "reserve with approver id", "kind": "call" }, { "from": "g-arb", "to": "g-audit", "label": "fallback.approved by owner", "kind": "async" }, { "from": "g-arb", "to": "g-bao", "label": "provider credential", "kind": "call" }, { "from": "g-bao", "to": "g-arb", "label": "1 h · sco projects only", "kind": "return" }, { "from": "g-arb", "to": "g-nova", "label": "launch guaranteed · tag sco:fallback", "kind": "call" }, { "from": "g-arb", "to": "g-audit", "label": "capacity.launch · policy + approver", "kind": "async" }, { "from": "g-arb", "to": "g-ledger", "label": "no answer at 15 min: shortfall, no spend", "kind": "error" } ], "note": "Restoring the floor never reaches step 4: it is pre-authorised by the tier and drawn from committed capacity already paid for.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } }, { "id": "32-failure-modes", "title": "Failure Modes — Thirteen Classes, Each Detected, Contained, Recovered", "layout": "grid", "laneHeaderWidth": 220, "columns": [ "Detected by", "Contained by", "Recovered by" ], "rows": [ { "title": "Single-pool reclaim", "cells": [ [ { "label": "Exclusion watcher", "kind": "app", "icon": "fa5_ban" } ], [ { "label": "≤ 15% per pool", "kind": "decision", "icon": "fa5_percentage" } ], [ { "label": "Replace in other groups", "kind": "opportunity", "icon": "fa5_cubes" } ] ] }, { "title": "Correlated reclaim", "cells": [ [ { "label": "Group reclaim rate", "kind": "app", "icon": "fa5_project_diagram" } ], [ { "label": "≤ 30% per group", "kind": "decision", "icon": "fa5_percentage" } ], [ { "label": "Floor holds · fallback", "kind": "opportunity", "icon": "fa5_layer_group" } ] ] }, { "title": "No-notice termination", "cells": [ [ { "label": "NotReady · lease expiry", "kind": "app", "icon": "fa5_heartbeat" } ], [ { "label": "Periodic checkpoint", "kind": "decision", "icon": "fa5_save" } ], [ { "label": "Unit requeued", "kind": "opportunity", "icon": "fa5_redo" } ] ] }, { "title": "Drain overrun", "cells": [ [ { "label": "Agent at T+118 s", "kind": "app", "icon": "fa5_stopwatch" } ], [ { "label": "Counted · contract flagged", "kind": "decision", "icon": "fa5_flag" } ], [ { "label": "Shorter interval proposed", "kind": "opportunity", "icon": "fa5_file_contract" } ] ] }, { "title": "Checkpoint store down", "cells": [ [ { "label": "Client write error", "kind": "app", "icon": "fa5_exclamation_circle" } ], [ { "label": "Lease released, recorded", "kind": "decision", "icon": "fa5_clipboard_list" } ], [ { "label": "Resume from last generation", "kind": "opportunity", "icon": "fa5_redo" } ] ] }, { "title": "No capacity at any price", "cells": [ [ { "label": "Shortfall ≥ 5 min", "kind": "app", "icon": "fa5_exclamation_triangle" } ], [ { "label": "Floor already running", "kind": "decision", "icon": "fa5_layer_group" } ], [ { "label": "Fallback within authority", "kind": "opportunity", "icon": "fa5_balance_scale" } ] ] }, { "title": "Fallback storm", "cells": [ [ { "label": "Arbiter queue depth", "kind": "app", "icon": "fa5_sort_amount_down" } ], [ { "label": "40 per min, by priority", "kind": "decision", "icon": "fa5_tachometer_alt" } ], [ { "label": "Floor was pre-provisioned", "kind": "opportunity", "icon": "fa5_calendar_check" } ] ] }, { "title": "Acquisition thrash", "cells": [ [ { "label": "Lost within 15 min", "kind": "app", "icon": "fa5_sync_alt" } ], [ { "label": "Back-off doubles", "kind": "decision", "icon": "fa5_hourglass_half" } ], [ { "label": "Probation return", "kind": "opportunity", "icon": "fa5_undo" } ] ] }, { "title": "Quota exhaustion", "cells": [ [ { "label": "Headroom ≤ 20%", "kind": "app", "icon": "fa5_tachometer_alt" } ], [ { "label": "Alarm before it binds", "kind": "decision", "icon": "fa5_bell" } ], [ { "label": "Quota raised by cloud team", "kind": "opportunity", "icon": "fa5_server" } ] ] }, { "title": "Price inversion", "cells": [ [ { "label": "Rate ≥ 85% guaranteed", "kind": "app", "icon": "fa5_coins" } ], [ { "label": "Not a discount pool", "kind": "decision", "icon": "fa5_ban" } ], [ { "label": "Move-off recommended", "kind": "opportunity", "icon": "fa5_exchange_alt" } ] ] }, { "title": "Control-plane outage", "cells": [ [ { "label": "Leader lease lost", "kind": "app", "icon": "fa5_heartbeat" } ], [ { "label": "Fail static", "kind": "decision", "icon": "fa5_pause" } ], [ { "label": "Read-only rebuild first", "kind": "opportunity", "icon": "fa5_tags" } ] ] }, { "title": "Poison work item", "cells": [ [ { "label": "Attempts ≥ 4", "kind": "app", "icon": "fa5_biohazard" } ], [ { "label": "Quarantine queue", "kind": "decision", "icon": "fa5_ban" } ], [ { "label": "Owner notified", "kind": "opportunity", "icon": "fa5_envelope" } ] ] }, { "title": "Hall loss", "cells": [ [ { "label": "Provider hall status", "kind": "app", "icon": "fa5_building" } ], [ { "label": "Hall excluded · caps rescaled", "kind": "decision", "icon": "fa5_percentage" } ], [ { "label": "Floor rebuilt ≤ 15 min", "kind": "opportunity", "icon": "fa5_layer_group" } ] ] } ], "note": "Every row is exercised: the daily drill covers the first four, the quarterly game day the hall loss and the control-plane outage together.", "meta": { "v": "1.0", "owner": "Compute Platform Architecture", "date": "2026-09" } } ]