[ { "id": "01-system-context", "title": "Data Quality Service — System Context", "layout": "context", "canvas": { "width": 1700 }, "colWidth": 270, "system": { "label": "Data Quality Service", "sub": "judge · record · gate", "icon": "databricks" }, "groups": [ { "side": "left", "title": "People it serves", "nodes": [ { "id": "owner", "label": "Data product owner", "kind": "actor", "sub": "42 domains", "rel": "owns tiers, SLAs", "dir": "in" }, { "id": "deng", "label": "Data engineer", "kind": "actor", "sub": "~180", "rel": "authors assertions", "dir": "in" }, { "id": "steward", "label": "Data steward", "kind": "actor", "sub": "per domain", "rel": "overrides", "dir": "in" } ] }, { "side": "right", "title": "Lakehouse it judges", "nodes": [ { "id": "lake", "label": "Delta Lake tables", "kind": "store", "sub": "2.4 PB · 12,000 datasets", "rel": "versions", "kind2": "sync" }, { "id": "uc", "label": "Unity Catalog", "kind": "platform", "sub": "governance + lineage", "rel": "lineage", "kind2": "sync" }, { "id": "jobs", "label": "Databricks Jobs / SQL", "kind": "platform", "sub": "push-down compute", "rel": "SQL plans", "kind2": "sync" } ] }, { "side": "top", "title": "Producers and orchestration", "nodes": [ { "id": "pipes", "label": "Ingestion pipelines", "kind": "external", "sub": "orders · rides · payments", "rel": "commit events", "dir": "in", "kind2": "async" }, { "id": "orch", "label": "Orchestrator", "kind": "external", "sub": "reads state before run", "rel": "gate check", "kind2": "sync" } ] }, { "side": "bottom", "title": "Consumers and signalling", "nodes": [ { "id": "analyst", "label": "Analyst / scientist", "kind": "actor", "sub": "~2,400", "rel": "state", "kind2": "sync" }, { "id": "bi", "label": "BI and reporting", "kind": "external", "sub": "regulatory + exec", "rel": "scorecards", "kind2": "sync" }, { "id": "fs", "label": "Feature store", "kind": "external", "sub": "pricing · ranking", "rel": "known-good", "kind2": "sync" }, { "id": "page", "label": "Paging and ticketing", "kind": "external", "sub": "Tier 1 breaches", "rel": "incidents", "dir": "in", "kind2": "async" } ] } ], "note": "Quarantine tables are omitted here and drawn in views 09 and 13. Out of scope: scheduling pipelines, owning producer contracts, and repairing data — the service judges, records and gates, and never writes a corrected value.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "02-high-level-architecture", "title": "Data Quality Service — High-Level Architecture", "layout": "flow", "canvas": { "width": 1960 }, "chain": true, "align": "middle", "nodeWidth": 185, "stageGap": 46, "stages": [ { "title": "1 · Declare", "nodes": [ { "id": "dsl", "label": "Rule DSL in Git", "kind": "app", "sub": "versioned assertions" }, { "id": "tier", "label": "Tier + policy registry", "kind": "platform", "sub": "who may block" } ] }, { "title": "2 · Trigger", "nodes": [ { "id": "commit", "label": "Table-version commit", "kind": "queue", "sub": "event stream" }, { "id": "sched", "label": "Schedule", "kind": "platform", "sub": "5 min – hourly" }, { "id": "sweep", "label": "Reconciliation sweep", "kind": "platform", "sub": "catches missed events" } ] }, { "title": "3 · Plan", "nodes": [ { "id": "plan", "label": "Plan compiler", "kind": "app", "sub": "fuse per scan" }, { "id": "cost", "label": "Cost estimator", "kind": "app", "sub": "budget per dataset" } ] }, { "title": "4 · Evaluate", "nodes": [ { "id": "meta", "label": "Metadata evaluator", "kind": "app", "sub": "no scan", "icon": "databricks" }, { "id": "scan", "label": "Scan evaluator", "kind": "app", "sub": "pushed down", "icon": "databricks" }, { "id": "dist", "label": "Distribution evaluator", "kind": "app", "sub": "vs baseline" } ] }, { "title": "5 · Record", "nodes": [ { "id": "vs", "label": "Verdict store", "kind": "store", "sub": "append-only · 25 mo" }, { "id": "csi", "label": "Current-state index", "kind": "store", "sub": "4k reads/s", "icon": "dynamodb" } ] }, { "title": "6 · Gate", "nodes": [ { "id": "gate", "label": "Circulation gate", "kind": "decision", "sub": "promote or not" }, { "id": "qout", "label": "Quarantine divert", "kind": "store", "sub": "≤ 2% of batch", "icon": "aws_s3" } ] }, { "title": "7 · Answer", "nodes": [ { "id": "cons", "label": "Consumers", "kind": "external", "sub": "BI · features · reports" }, { "id": "sig", "label": "Signalling", "kind": "external", "sub": "pages · debt · scorecards" } ] } ], "edges": [ { "from": "vs", "to": "csi", "label": "derives", "kind": "async" }, { "from": "csi", "to": "gate" }, { "from": "gate", "to": "cons", "label": "known-good version" } ], "note": "Stage 5 is the pivot: the verdict is written against a data version, not a run — which is what makes stage 4 re-runnable over history.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "03-actors-and-journeys", "title": "Actors and What Each of Them Gets to Do", "layout": "actors", "canvas": { "width": 1760 }, "cardWidth": 285, "groups": [ { "title": "Inside the data platform", "kind": "boundary", "actors": [ { "id": "deng", "label": "Data engineer", "sub": "~180, across 42 domains", "goal": "Write a rule that catches the thing that broke last quarter, and know before I turn it on whether it would have stopped the pipeline every night since.", "journeys": [ { "id": "j-author", "label": "Author and promote a rule", "sub": "probation → block" }, { "label": "See what a rule costs to run" }, { "label": "Replay corrected rows from quarantine" } ] }, { "id": "owner", "label": "Data product owner", "sub": "one per domain", "goal": "Say out loud which of my tables are Tier 1, and be told when my domain's quality debt is aging rather than shrinking.", "journeys": [ { "label": "Set a dataset's tier and SLA" }, { "label": "Read the domain scorecard" }, { "label": "Pay down aged debt" } ] }, { "id": "steward", "label": "Data steward", "sub": "override authority", "goal": "Let a blocked release through when I have weighed the risk, on the record, with an expiry — and never be able to do it silently.", "journeys": [ { "label": "Grant a time-boxed override" }, { "label": "Review expiring suppressions" } ] } ] }, { "title": "Downstream of it", "kind": "cloud", "actors": [ { "id": "analyst", "label": "Analyst / data scientist", "sub": "~2,400 monthly", "goal": "Know, before I put a number in front of the board, whether the table I just queried is trusted — and be told afterwards if it turns out it was not.", "journeys": [ { "id": "j-silent", "label": "Find out a report was wrong", "sub": "retroactive verdict" }, { "label": "Check a table's state and coverage" } ] }, { "id": "mle", "label": "ML engineer", "sub": "pricing and ranking", "goal": "Train on a snapshot I can later prove was clean, and be told which model versions read a window that has since been condemned.", "journeys": [ { "label": "Pin a known-good version" }, { "label": "Trace a condemned window to a model" } ] }, { "id": "audit", "label": "Auditor", "kind": "external", "sub": "2 cycles / year", "goal": "Ask who overrode what, when, on whose authority — and get the answer from a log nobody could have edited.", "journeys": [ { "label": "Pull the override audit trail" } ] } ] }, { "title": "Machines in the cast", "kind": "plain", "actors": [ { "id": "orch", "label": "Orchestrator", "kind": "platform", "sub": "~9,000 runs / day", "goal": "Ask one cheap question before I start a job — is my input safe to read — and skip rather than cascade when the answer is no.", "journeys": [ { "label": "Read state before a dependent run" } ] }, { "id": "sweeper", "label": "Reconciliation sweeper", "kind": "platform", "sub": "every 10 min", "goal": "Make sure no table version ever went unjudged because an event went missing.", "journeys": [ { "label": "Enqueue unjudged versions" } ] } ] } ], "note": "Eight actors. The two machines are in the cast deliberately: the orchestrator is the biggest single reader of state, and the sweeper is why a lost event is not a silent pass.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "04-journey-silent-corruption", "title": "Journey — The Report That Was Wrong for Three Weeks", "layout": "journey", "canvas": { "width": 1760 }, "actor": { "label": "Analyst", "sub": "merchant finance", "goal": "Trust the margin number I publish weekly", "trigger": "A merchant queries an invoice that does not match their own books", "success": "Knows exactly which weeks to disbelieve, and who else read them" }, "phases": [ { "title": "Publish", "sub": "week 1" }, { "title": "Doubt", "sub": "week 4" }, { "title": "Prove", "moment": true }, { "title": "Scope", "moment": true }, { "title": "Recover" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Publishes margin report" } ], [ { "label": "Merchant disputes it" }, { "label": "Re-queries the table" } ], [ { "label": "Writes a sum-reconciles rule" } ], [ { "label": "Reads the blast radius" } ], [ { "label": "Reissues weeks 1–3" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "All assertions pass" }, { "label": "No coverage on currency" } ], [ { "label": "State still reads good" } ], [ { "label": "Backfills 30 days", "kind": "opportunity" }, { "label": "18 versions fail" } ], [ { "label": "Emits corruption window", "kind": "opportunity" }, { "label": "Marks 11 tables suspect" } ], [ { "label": "Re-judges after fix" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Uneasy", "Alarmed" ], "points": [ 0, 1, 2, 1, 0 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "Green with no coverage" } ], [ { "label": "Nothing to appeal to" } ], [ { "label": "Was it a discovery or a bad rule?" } ], [ { "label": "Lineage is only as good as its gaps" } ], [] ] } ], "chain": true, "note": "The trough is at 'Prove', not at 'Doubt' — the platform's answer is retroactive evaluation, and its risk is that a new rule condemning 18 past versions is as likely to be a bad rule as a discovery.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "05-journey-author-a-rule", "title": "Journey — Turning a Rule On Without Taking a Pipeline Down", "layout": "journey", "canvas": { "width": 1760 }, "actor": { "label": "Data engineer", "sub": "payments domain", "goal": "Stop the paise/rupee class of defect reaching the feature store", "trigger": "A postmortem action item with their name on it", "success": "The rule blocks in production and has not paged anyone falsely in 30 days" }, "phases": [ { "title": "Draft", "sub": "in Git" }, { "title": "Price" }, { "title": "Backfill", "moment": true }, { "title": "Probation", "sub": "30 days" }, { "title": "Enforce", "moment": true } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Writes the assertion" }, { "label": "Picks severity and tier" } ], [ { "label": "Reads the dry-run cost" } ], [ { "label": "Reviews 30 days of results" } ], [ { "label": "Watches false-positive rate" } ], [ { "label": "Promotes to block" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "Compiles to a bounded plan" }, { "label": "Refuses block on Tier 3", "kind": "pain" } ], [ { "label": "Estimates $/run" } ], [ { "label": "Auto-backfills lookback", "kind": "opportunity" } ], [ { "label": "Runs as advise only" }, { "label": "Tracks FP rate" } ], [ { "label": "Requires owner approval" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "In control", "Unsure", "Blocked" ], "points": [ 0, 0, 2, 1, 0 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Cost is a guess until it runs" } ], [ { "label": "12 of 30 days failed" } ], [ { "label": "Nobody watches advise verdicts" } ], [] ] } ], "chain": true, "note": "Probation is the answer to 'a bad rule must not take a pipeline down'; its own weakness — that advise verdicts go unread — is why the false-positive rate is a tracked number rather than a feeling.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "06-layered-architecture", "title": "Layered Architecture — and What Each Layer May Not Know", "layout": "bands", "canvas": { "width": 1740 }, "layerHeaderWidth": 160, "bands": [ { "name": "Experience", "nodes": [ { "id": "ui", "label": "Quality console", "kind": "app", "sub": "state · scorecards" }, { "id": "debtui", "label": "Debt register UI", "kind": "app", "sub": "owners · expiries" }, { "id": "api", "label": "State API", "kind": "integration", "sub": "read-only" }, { "id": "cli", "label": "Rule CLI", "kind": "app", "sub": "dry-run · lint" } ] }, { "name": "Authoring", "nodes": [ { "id": "dsl", "label": "Rule DSL", "kind": "app", "sub": "declarative" }, { "id": "prop", "label": "Profiling proposer", "kind": "app", "sub": "suggests bounds" }, { "id": "cimp", "label": "Contract importer", "kind": "integration", "sub": "from catalogue" }, { "id": "rev", "label": "Review + promotion", "kind": "security", "sub": "owner approval" } ] }, { "name": "Control", "nodes": [ { "id": "cat", "label": "Rule catalogue", "kind": "store", "sub": "versioned" }, { "id": "pol", "label": "Tier + policy registry", "kind": "platform", "sub": "who may block" }, { "id": "sched", "label": "Scheduler", "kind": "platform", "sub": "priority lanes" }, { "id": "comp", "label": "Plan compiler", "kind": "app", "sub": "bounded cost" }, { "id": "swp", "label": "Reconciliation sweeper", "kind": "platform", "sub": "every 10 min" } ] }, { "name": "Evaluation", "nodes": [ { "id": "metav", "label": "Metadata evaluator", "kind": "app", "sub": "no scan" }, { "id": "scanv", "label": "Scan evaluator", "kind": "app", "sub": "pushed down", "icon": "databricks" }, { "id": "distv", "label": "Distribution evaluator", "kind": "app", "sub": "vs baseline" }, { "id": "recv", "label": "Reconciliation runner", "kind": "app", "sub": "control totals" }, { "id": "bfl", "label": "Backfill lane", "kind": "app", "sub": "low priority" } ] }, { "name": "Verdict + state", "nodes": [ { "id": "vst", "label": "Verdict store", "kind": "store", "sub": "append-only" }, { "id": "csi", "label": "Current-state index", "kind": "store", "sub": "hot read path", "icon": "dynamodb" }, { "id": "base", "label": "Baseline store", "kind": "store", "sub": "28-day window" }, { "id": "blast", "label": "Blast-radius resolver", "kind": "app", "sub": "lineage walk" } ] }, { "name": "Enforcement", "nodes": [ { "id": "gate", "label": "Circulation gate", "kind": "decision", "sub": "promote / hold" }, { "id": "qdiv", "label": "Quarantine diverter", "kind": "integration", "sub": "row-level only" }, { "id": "ovr", "label": "Override service", "kind": "security", "sub": "attributed · expiring" } ] }, { "name": "Platform", "nodes": [ { "id": "uc", "label": "Unity Catalog", "kind": "platform", "sub": "grants + lineage" }, { "id": "lake", "label": "Delta Lake on S3", "kind": "store", "sub": "the judged estate", "icon": "aws_s3" }, { "id": "obs", "label": "Observability", "kind": "platform", "sub": "MTTD · FP rate" }, { "id": "aud", "label": "Audit log", "kind": "security", "sub": "immutable · 7 yr" } ] } ], "edges": [ { "from": "dsl", "to": "cat", "label": "publish", "kind": "sync" }, { "from": "comp", "to": "scanv", "label": "plan", "kind": "sync" }, { "from": "scanv", "to": "vst", "label": "verdict", "kind": "async" }, { "from": "vst", "to": "csi", "label": "derive", "kind": "async" }, { "from": "csi", "to": "gate", "label": "state", "kind": "sync" }, { "from": "gate", "to": "lake", "label": "promote", "kind": "sync" } ], "note": "The evaluation layer may not know who owns a dataset or what a verdict will cause; the enforcement layer may not recompute one. That separation is what lets history be re-judged without re-running enforcement.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "07-container-view", "title": "Container View — Control, Evaluation and Verdict Planes", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "AWS eu-west-1 · platform account", "kind": "cloud", "dir": "col", "children": [ { "title": "Control plane — EKS, private subnets", "kind": "boundary", "nodes": [ { "id": "cat", "label": "Rule catalogue API", "kind": "app", "sub": "Go · Aurora-backed" }, { "id": "sched", "label": "Scheduler", "kind": "app", "sub": "priority + quota" }, { "id": "comp", "label": "Plan compiler", "kind": "app", "sub": "DSL → SQL" }, { "id": "swp", "label": "Reconciliation sweeper", "kind": "platform", "sub": "10-min cadence" }, { "id": "ovr", "label": "Override service", "kind": "security", "sub": "expiry enforced" } ] }, { "title": "Evaluation plane — Databricks workspace", "kind": "boundary", "nodes": [ { "id": "meta", "label": "Metadata evaluator", "kind": "app", "sub": "catalogue reads" }, { "id": "sql", "label": "SQL warehouse pool", "kind": "app", "sub": "fused scans", "icon": "databricks" }, { "id": "distj", "label": "Distribution job", "kind": "app", "sub": "baseline compare", "icon": "databricks" }, { "id": "bfl", "label": "Backfill lane", "kind": "app", "sub": "separate pool", "icon": "databricks" } ] }, { "title": "Verdict + state plane", "kind": "boundary", "nodes": [ { "id": "vst", "label": "Verdict store", "kind": "store", "sub": "Delta · append-only" }, { "id": "csi", "label": "Current-state index", "kind": "store", "sub": "DynamoDB global", "icon": "dynamodb" }, { "id": "base", "label": "Baseline store", "kind": "store", "sub": "Delta · versioned" }, { "id": "blast", "label": "Blast-radius resolver", "kind": "app", "sub": "lineage walk" }, { "id": "aur", "label": "Catalogue + debt DB", "kind": "store", "sub": "Aurora PostgreSQL", "icon": "aurora" } ] } ] } ], "outside": [ { "id": "lake", "label": "Delta Lake on S3", "kind": "store", "sub": "2.4 PB judged estate", "icon": "aws_s3" }, { "id": "uc", "label": "Unity Catalog", "kind": "platform", "sub": "grants · lineage" }, { "id": "bus", "label": "Commit event stream", "kind": "queue", "sub": "MSK", "icon": "amazon msk" }, { "id": "orch", "label": "Orchestrator", "kind": "external", "sub": "reads state" } ], "edges": [ { "from": "bus", "to": "sched", "label": "commit events", "kind": "async" }, { "from": "sched", "to": "comp", "label": "work item" }, { "from": "comp", "to": "sql", "label": "compiled plan" }, { "from": "sql", "to": "vst", "label": "verdicts", "kind": "async" }, { "from": "vst", "to": "csi", "kind": "async" }, { "from": "csi", "to": "orch" }, { "from": "ovr", "to": "aur", "label": "expiry record" } ], "note": "Seven of about thirty edges are drawn; the in-place scan and the sweeper's catalogue poll are omitted for legibility. The edge that matters is absent by design: nothing in the control or verdict plane writes to the judged estate.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "08-integration-surface", "title": "Integration Surface — Every Interface the Service Commits To", "layout": "hub", "canvas": { "width": 1740 }, "left": { "title": "Callers and producers", "nodes": [ { "id": "orch", "label": "Orchestrator", "kind": "external", "sub": "pre-run gate check", "rel": "state", "kind2": "sync" }, { "id": "cons", "label": "BI + feature store", "kind": "external", "sub": "known-good pointer", "rel": "state", "kind2": "sync" }, { "id": "cicd", "label": "Rules CI", "kind": "integration", "sub": "Git merge", "rel": "rules", "kind2": "sync" }, { "id": "bus", "label": "Commit event stream", "kind": "queue", "sub": "MSK topic", "rel": "versions", "kind2": "async" }, { "id": "src", "label": "Source control totals", "kind": "external", "sub": "ledger · OLTP counts", "rel": "totals", "kind2": "batch" } ] }, "centre": { "title": "Data Quality Service", "nodes": [ { "id": "stateapi", "label": "State API", "kind": "integration", "sub": "≤ 20 ms p99" }, { "id": "ruleapi", "label": "Rule + override API", "kind": "integration", "sub": "owner-scoped" }, { "id": "evt", "label": "Verdict event feed", "kind": "queue", "sub": "at-least-once" }, { "id": "query", "label": "Verdict query API", "kind": "integration", "sub": "history · scorecards" } ] }, "right": { "title": "Dependencies", "nodes": [ { "id": "uc", "label": "Unity Catalog", "kind": "platform", "sub": "read: grants · lineage", "rel": " ", "dir": "out" }, { "id": "dbx", "label": "Databricks Jobs / SQL", "kind": "platform", "sub": "run: fused scan plans", "rel": " ", "dir": "out", "icon": "databricks" }, { "id": "page", "label": "Paging + ticketing", "kind": "external", "sub": "Tier 1 only" } ] }, "note": "Direction is on the arrows and the interaction is on each dependency's own line; identity is drawn in view 20 and the outbound paging integration in view 18. The State API is the contract the rest of the platform depends on — everything else can degrade, but if that read stops answering, every consumer needs an answer that is not 'assume good'.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "09-storage-zones", "title": "Storage Zones — Divided by What Can Be Rebuilt", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Irreplaceable — losing it loses the ability to answer an auditor", "kind": "trust", "dir": "row", "children": [ { "title": "Verdict record", "kind": "boundary", "nodes": [ { "id": "vst", "label": "Verdict store", "kind": "store", "sub": "append-only · 25 mo" }, { "id": "aud", "label": "Override audit log", "kind": "security", "sub": "immutable · 7 yr" } ] }, { "title": "Declared intent", "kind": "boundary", "nodes": [ { "id": "cat", "label": "Rule catalogue", "kind": "store", "sub": "Aurora · Git-sourced", "icon": "aurora" }, { "id": "pol", "label": "Tier + policy registry", "kind": "store", "sub": "Aurora", "icon": "aurora" } ] } ] }, { "title": "Rebuildable — derived, and cheaper to recompute than to restore", "kind": "boundary", "dir": "row", "children": [ { "title": "Serving projections", "kind": "plain", "nodes": [ { "id": "csi", "label": "Current-state index", "kind": "store", "sub": "DynamoDB · RTO 10 min", "icon": "dynamodb" }, { "id": "score", "label": "Scorecard rollups", "kind": "store", "sub": "nightly materialised" } ] }, { "title": "Learned state", "kind": "plain", "nodes": [ { "id": "base", "label": "Baseline store", "kind": "store", "sub": "28-day window · 13 mo" }, { "id": "plans", "label": "Compiled plan cache", "kind": "store", "sub": "disposable" } ] } ] }, { "title": "Custodial — other people's data, held under their classification", "kind": "trust", "dir": "row", "children": [ { "title": "Quarantine", "kind": "boundary", "nodes": [ { "id": "quar", "label": "Quarantine tables", "kind": "store", "sub": "Delta · 90 days", "icon": "aws_s3" }, { "id": "samp", "label": "Violation samples", "kind": "store", "sub": "redacted by default" } ] }, { "title": "Working state", "kind": "plain", "nodes": [ { "id": "debt", "label": "Debt register", "kind": "store", "sub": "Aurora · mutable", "icon": "aurora" }, { "id": "buf", "label": "Verdict write buffer", "kind": "queue", "sub": "replayed on outage" } ] } ] } ], "outside": [ { "id": "lake", "label": "Delta Lake on S3 — the judged estate", "kind": "store", "sub": "read-only to this service", "icon": "aws_s3" } ], "edges": [ { "from": "vst", "to": "csi", "label": "derives", "kind": "async" }, { "from": "lake", "to": "quar", "label": "rows diverted", "kind": "async" } ], "note": "Quarantine carries the source dataset's classification and masking policy. It is a custodial copy of regulated data, not a lower-governance staging area.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "10-data-model", "title": "Data Model — The Verdict and What It Points At", "layout": "er", "canvas": { "width": 1700, "cols": 4 }, "rowGap": 235, "entities": [ { "id": "ds", "name": "dataset", "kind": "store", "row": 1, "col": 0, "attrs": [ "dataset_id PK", "catalogue_fqn UQ", "tier 1|2|3", "owner_team", "staleness_sla_min" ] }, { "id": "dv", "name": "dataset_version", "kind": "store", "row": 0, "col": 0, "attrs": [ "version_id PK", "dataset_id FK", "commit_ts", "watermark_ts", "row_count", "promoted bool" ] }, { "id": "asr", "name": "assertion", "kind": "store", "row": 0, "col": 2, "attrs": [ "assertion_id PK", "dataset_id FK", "type", "action advise|quar|block", "owner_team", "status" ] }, { "id": "av", "name": "assertion_version", "kind": "store", "row": 0, "col": 1, "attrs": [ "assertion_ver_id PK", "assertion_id FK", "expr_hash", "published_ts", "probation_until" ] }, { "id": "vd", "name": "verdict", "kind": "store", "row": 1, "col": 1, "attrs": [ "verdict_id PK", "version_id FK", "assertion_ver_id FK", "outcome", "retroactive bool", "sample_fraction", "evaluated_ts" ] }, { "id": "st", "name": "dataset_state", "kind": "store", "row": 2, "col": 0, "attrs": [ "dataset_id PK", "partition_key PK", "state", "last_known_good FK", "as_of_ts" ] }, { "id": "bl", "name": "baseline", "kind": "store", "row": 0, "col": 3, "attrs": [ "baseline_id PK", "assertion_id FK", "trained_from", "trained_to", "stats_json" ] }, { "id": "qb", "name": "quarantine_batch", "kind": "store", "row": 1, "col": 2, "attrs": [ "batch_id PK", "verdict_id FK", "row_count", "pct_of_batch", "expires_ts" ] }, { "id": "cw", "name": "corruption_window", "kind": "risk", "row": 2, "col": 1, "attrs": [ "window_id PK", "dataset_id FK", "first_failing_ver", "first_passing_ver", "discovered_ts" ] }, { "id": "ov", "name": "override", "kind": "security", "row": 2, "col": 2, "attrs": [ "override_id PK", "verdict_id FK", "granted_by", "justification", "expires_ts" ] }, { "id": "dbt", "name": "debt_item", "kind": "risk", "row": 2, "col": 3, "attrs": [ "debt_id PK", "kind", "subject_id", "owner_team", "opened_ts", "age_days" ] }, { "id": "sus", "name": "suspect_mark", "kind": "risk", "row": 3, "col": 0, "attrs": [ "mark_id PK", "window_id FK", "downstream_dataset_id FK", "hops", "cleared_ts null" ] } ], "relations": [ { "from": "ds", "to": "dv", "label": "1 : N", "from_side": "n", "to_side": "s" }, { "from": "ds", "to": "st", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "dv", "to": "vd", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "av", "to": "vd", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "asr", "to": "av", "label": "1 : N", "from_side": "w", "to_side": "e" }, { "from": "asr", "to": "bl", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "vd", "to": "qb", "label": "0 : 1", "from_side": "e", "to_side": "w", "kind": "optional" }, { "from": "vd", "to": "cw", "label": "0 : N", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "vd", "to": "ov", "label": "0 : 1", "from_side": "s3", "to_side": "n", "kind": "optional" }, { "from": "cw", "to": "sus", "label": "1 : N", "from_side": "s", "to_side": "e" }, { "from": "ov", "to": "dbt", "label": "1 : 1", "from_side": "e", "to_side": "w" } ], "note": "verdict.version_id is the load-bearing key: it binds a judgement to an immutable data version rather than to a pipeline run, which is what makes retroactive evaluation and corruption_window expressible at all.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "11-verdict-data-flow", "title": "Data Flow — From a Committed Version to an Answerable State", "layout": "flow", "canvas": { "width": 1900 }, "chain": true, "align": "top", "nodeWidth": 200, "stageGap": 48, "stages": [ { "title": "Sources", "nodes": [ { "id": "ing", "label": "Ingestion pipelines", "kind": "external", "sub": "~40 TB/day" }, { "id": "ledger", "label": "Source control totals", "kind": "external", "sub": "ledger counts" } ] }, { "title": "Landing", "nodes": [ { "id": "tbl", "label": "Delta table version", "kind": "store", "sub": "written, unpromoted", "icon": "aws_s3" }, { "id": "evt", "label": "Commit event", "kind": "queue", "sub": "MSK", "icon": "amazon msk" } ] }, { "title": "Judging", "nodes": [ { "id": "fuse", "label": "Fused scan", "kind": "app", "sub": "N assertions, 1 pass", "icon": "databricks" }, { "id": "metaonly", "label": "Metadata checks", "kind": "app", "sub": "schema · freshness" } ] }, { "title": "Recording", "nodes": [ { "id": "vd", "label": "Verdicts", "kind": "store", "sub": "immutable rows" }, { "id": "qrows", "label": "Quarantined rows", "kind": "store", "sub": "+ verdict context", "icon": "aws_s3" } ] }, { "title": "Deriving", "nodes": [ { "id": "state", "label": "Current state", "kind": "store", "sub": "per dataset+partition", "icon": "dynamodb" }, { "id": "score", "label": "Scorecards", "kind": "store", "sub": "domain rollup" }, { "id": "debt", "label": "Debt register", "kind": "store", "sub": "aged", "icon": "aurora" } ] }, { "title": "Serving", "nodes": [ { "id": "gate", "label": "Gate decision", "kind": "decision", "sub": "promote / hold" }, { "id": "feed", "label": "Verdict event feed", "kind": "queue", "sub": "subscribers" } ] }, { "title": "Consumers", "nodes": [ { "id": "bi", "label": "BI + reporting", "kind": "external" }, { "id": "fs", "label": "Feature store", "kind": "external" }, { "id": "orch", "label": "Orchestrator", "kind": "external" } ] } ], "edges": [ { "from": "ledger", "to": "fuse", "label": "reconcile", "kind": "batch", "route": "gutter" }, { "from": "vd", "to": "state", "label": "derive", "kind": "async" }, { "from": "gate", "to": "tbl", "label": "promote", "route": "gutter" } ], "note": "Two arrows leave this diagram going backwards: the reconcile feed and the promote decision. Everything else moves one way, which is the property that makes a verdict re-derivable.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "12-critical-flow-commit-to-gate", "title": "Critical Flow — Table Version Committed to Gate Answered", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "pipe", "label": "Ingestion pipeline", "kind": "external" }, { "id": "lake", "label": "Delta table", "kind": "store" }, { "id": "sched", "label": "Scheduler", "kind": "app" }, { "id": "eval", "label": "Evaluator", "kind": "app" }, { "id": "vst", "label": "Verdict store", "kind": "store" }, { "id": "csi", "label": "State index", "kind": "store" }, { "id": "orch", "label": "Orchestrator", "kind": "external" } ], "messages": [ { "from": "pipe", "to": "lake", "label": "commit version 41207 (unpromoted)", "kind": "call" }, { "from": "lake", "to": "sched", "label": "commit event", "kind": "async" }, { "from": "sched", "to": "csi", "label": "mark unknown", "kind": "call" }, { "from": "sched", "to": "sched", "label": "fuse assertions for this partition", "kind": "self" }, { "from": "sched", "to": "eval", "label": "metadata set first", "kind": "call" }, { "from": "eval", "to": "vst", "label": "schema + freshness verdicts", "kind": "call" }, { "from": "eval", "to": "eval", "label": "schema passed — proceed to scan", "kind": "self" }, { "from": "sched", "to": "eval", "label": "fused scan plan, budget attached", "kind": "call" }, { "from": "eval", "to": "lake", "label": "single pass over the partition", "kind": "call" }, { "from": "lake", "to": "eval", "label": "aggregate + row results", "kind": "return" }, { "from": "eval", "to": "vst", "label": "verdicts bound to v41207", "kind": "call" }, { "from": "vst", "to": "csi", "label": "derive state: blocked", "kind": "async" }, { "from": "csi", "to": "lake", "label": "withhold promotion", "kind": "call" }, { "from": "orch", "to": "csi", "label": "is my input safe to read?", "kind": "call" }, { "from": "csi", "to": "orch", "label": "blocked · last known good v41198", "kind": "return" }, { "from": "orch", "to": "orch", "label": "skip dependent run, do not cascade", "kind": "self" }, { "from": "eval", "to": "vst", "label": "budget exhausted on a Tier 2 dataset", "kind": "error" }, { "from": "vst", "to": "csi", "label": "degraded, not pass", "kind": "error" } ], "note": "The last two messages are the alternate path: a verdict that could not be fully reached is recorded as degraded. Nothing in this flow ever writes 'pass' because evaluation did not finish.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "13-advise-quarantine-block", "title": "Three Verdict Actions, and What Each One Does to the Data", "layout": "swimlane", "canvas": { "width": 1820 }, "laneHeaderWidth": 150, "stages": [ "Violation found", "Attribution", "What happens to the data", "Who is told", "Way back" ], "lanes": [ { "title": "Advise", "cells": [ [ { "label": "Low severity", "kind": "journey" }, { "label": "Rule in probation", "kind": "journey" } ], [ { "label": "Dataset or rows", "kind": "app" } ], [ { "label": "Circulates unchanged", "kind": "opportunity" } ], [ { "label": "Scorecard only", "kind": "app" } ], [ { "label": "Promote to block", "kind": "decision" } ] ] }, { "title": "Quarantine", "cells": [ [ { "label": "Row predicate fails", "kind": "risk" }, { "label": "Orphan FK", "kind": "risk" } ], [ { "label": "Row-attributable", "kind": "app" } ], [ { "label": "Rows diverted", "kind": "store" }, { "label": "Remainder promoted", "kind": "opportunity" } ], [ { "label": "Owner ticket", "kind": "app" } ], [ { "label": "Re-admit via ingestion", "kind": "decision" } ] ] }, { "title": "Quarantine over ceiling", "cells": [ [ { "label": "> 2% of the batch", "kind": "risk" } ], [ { "label": "Row-attributable", "kind": "app" } ], [ { "label": "Escalates to block", "kind": "risk" } ], [ { "label": "Page the owner", "kind": "risk" } ], [ { "label": "Fix upstream", "kind": "decision" } ] ] }, { "title": "Block", "cells": [ [ { "label": "Aggregate fails", "kind": "risk" }, { "label": "Reconciliation gap", "kind": "risk" } ], [ { "label": "Dataset-level only", "kind": "app" } ], [ { "label": "Version not promoted", "kind": "risk" }, { "label": "Last known good served", "kind": "opportunity" } ], [ { "label": "Page + orchestrator skip", "kind": "risk" } ], [ { "label": "Override, expiring", "kind": "security" } ] ] }, { "title": "Unknown", "cells": [ [ { "label": "Evaluation failed", "kind": "risk" }, { "label": "No assertion exists", "kind": "risk" } ], [ { "label": "Nothing to attribute", "kind": "app" } ], [ { "label": "Tier 1: not promoted", "kind": "risk" }, { "label": "Tier 2-3: circulates", "kind": "journey" } ], [ { "label": "Coverage gap on scorecard", "kind": "app" } ], [ { "label": "Author an assertion", "kind": "decision" } ] ] } ], "note": "The bottom lane is the failure class the whole design is aimed at: unknown is rendered as its own state, never folded into pass, and on Tier 1 it withholds promotion exactly as a block does.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "14-retroactive-evaluation", "title": "Retroactive Evaluation — Re-Judging History and Scoping the Damage", "layout": "flow", "canvas": { "width": 1880 }, "chain": true, "align": "top", "nodeWidth": 205, "stageGap": 48, "stages": [ { "title": "Trigger", "nodes": [ { "id": "newrule", "label": "New assertion published", "kind": "app", "sub": "or revised" }, { "id": "suspicion", "label": "Reported discrepancy", "kind": "external", "sub": "a human noticed" } ] }, { "title": "Enumerate", "nodes": [ { "id": "vers", "label": "Historical versions", "kind": "store", "sub": "lookback 30 days", "icon": "aws_s3" }, { "id": "prio", "label": "Backfill lane", "kind": "queue", "sub": "below live work" } ] }, { "title": "Re-judge", "nodes": [ { "id": "replay", "label": "Evaluate per version", "kind": "app", "sub": "time-travel read", "icon": "databricks" }, { "id": "mark", "label": "Verdicts marked retroactive", "kind": "store", "sub": "never overwrite" } ] }, { "title": "Decide", "nodes": [ { "id": "judge", "label": "Discovery or bad rule?", "kind": "decision", "sub": "owner must answer" }, { "id": "fprate", "label": "FP rate check", "kind": "app", "sub": "≤ 5% / 30 days" } ] }, { "title": "Scope", "nodes": [ { "id": "window", "label": "Corruption window", "kind": "risk", "sub": "first fail → first pass" }, { "id": "lin", "label": "Lineage walk", "kind": "app", "sub": "one hop" } ] }, { "title": "Mark", "nodes": [ { "id": "sus", "label": "Downstream suspect", "kind": "risk", "sub": "persists until cleared" }, { "id": "reads", "label": "Reports + snapshots", "kind": "risk", "sub": "who read the window" } ] }, { "title": "Recover", "nodes": [ { "id": "fix", "label": "Upstream fix", "kind": "external", "sub": "producer's job" }, { "id": "rederive", "label": "Re-derive downstream", "kind": "app", "sub": "then re-judge" }, { "id": "close", "label": "Window closed", "kind": "opportunity", "sub": "explicitly, by a human" } ] } ], "edges": [ { "from": "judge", "to": "newrule", "label": "rule withdrawn", "kind": "error", "route": "gutter" }, { "from": "close", "to": "sus", "label": "clears marks", "kind": "async", "route": "gutter" } ], "note": "A suspect mark is never cleared by the next successful run — only by re-derivation or an explicit human dismissal, because the next run passing says nothing about the window that failed.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "15-assertion-lifecycle", "title": "Assertion Lifecycle — The Loop That Keeps Rules Credible", "layout": "cycle", "canvas": { "width": 1420 }, "centre": { "label": "Assertion", "sub": "versioned" }, "nodes": [ { "id": "c1", "label": "Authored", "kind": "app", "sub": "DSL in Git" }, { "id": "c2", "label": "Priced", "kind": "app", "sub": "dry-run" }, { "id": "c3", "label": "Backfilled", "kind": "app", "sub": "30-day lookback" }, { "id": "c4", "label": "Probation", "kind": "journey", "sub": "advise only" }, { "id": "c5", "label": "Enforcing", "kind": "opportunity", "sub": "quarantine or block" }, { "id": "c6", "label": "Demoted", "kind": "risk", "sub": "FP > 5%" } ], "ringLabels": [ "bounded plan", "re-judged", "owner review", "FP under 5%", "cries wolf", "reworked" ], "note": "Demotion is automatic and retirement is not: a rule that cries wolf is silenced by the platform, but only a person may decide a rule is no longer worth having — and its verdicts survive either way.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "16-deployment", "title": "Deployment — Two Regions, One Judged Estate", "layout": "nested", "canvas": { "width": 1780 }, "boxes": [ { "title": "AWS eu-west-1 · primary", "kind": "cloud", "dir": "col", "children": [ { "title": "Control plane — EKS across three AZs", "kind": "boundary", "nodes": [ { "id": "eks", "label": "EKS node groups", "kind": "platform", "sub": "3 AZ · 12 nodes" }, { "id": "apis", "label": "Catalogue + state APIs", "kind": "app", "sub": "HPA · 6–40 pods" }, { "id": "schedp", "label": "Scheduler", "kind": "app", "sub": "leader-elected" }, { "id": "aur", "label": "Aurora PostgreSQL", "kind": "store", "sub": "multi-AZ writer", "icon": "aurora" } ] }, { "title": "Evaluation plane — Databricks workspace", "kind": "boundary", "nodes": [ { "id": "wh", "label": "SQL warehouses", "kind": "app", "sub": "live lane · autoscaled", "icon": "databricks" }, { "id": "bfw", "label": "Backfill warehouse", "kind": "app", "sub": "spot · low priority", "icon": "databricks" }, { "id": "jobs", "label": "Job clusters", "kind": "app", "sub": "distribution + reconcile", "icon": "databricks" } ] }, { "title": "Regional data", "kind": "boundary", "nodes": [ { "id": "s3", "label": "S3 — verdicts, baselines, quarantine", "kind": "store", "sub": "versioned · Object Lock on audit", "icon": "aws_s3" }, { "id": "msk", "label": "MSK", "kind": "queue", "sub": "commit + verdict topics", "icon": "amazon msk" } ] } ] }, { "title": "AWS eu-central-1 · read replica of state", "kind": "cloud", "dir": "row", "children": [ { "title": "State read path only", "kind": "boundary", "nodes": [ { "id": "ddb2", "label": "DynamoDB global table", "kind": "store", "sub": "replica · ≤ 2 s lag", "icon": "dynamodb" }, { "id": "api2", "label": "State API replica", "kind": "app", "sub": "reads only" } ] }, { "title": "Cold recovery", "kind": "plain", "nodes": [ { "id": "s3r", "label": "S3 cross-region replication", "kind": "store", "sub": "verdicts + audit log", "icon": "aws_s3" } ] } ] } ], "outside": [ { "id": "ddb", "label": "DynamoDB global table · primary", "kind": "store", "sub": "current-state index", "icon": "dynamodb" }, { "id": "lake", "label": "Delta Lake on S3 — the judged estate", "kind": "store", "sub": "not owned by this service", "icon": "aws_s3" }, { "id": "uc", "label": "Unity Catalog", "kind": "platform", "sub": "account-level" } ], "edges": [ { "from": "apis", "to": "ddb", "label": "state writes" }, { "from": "ddb", "to": "ddb2", "label": "replication", "kind": "async" }, { "from": "s3", "to": "s3r", "label": "CRR", "kind": "batch" } ], "note": "Four of the region's edges are drawn; the in-place scan of the judged estate is omitted here and drawn in views 07 and 11. The second region carries the state read path and the durable record, not the ability to evaluate: a regional loss still answers 'is this safe to read' within 2 s of lag, but produces no new verdicts.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "17-rules-as-code-pipeline", "title": "Rules as Code — From a Pull Request to an Enforcing Assertion", "layout": "flow", "canvas": { "width": 1880 }, "chain": true, "align": "top", "nodeWidth": 200, "stageGap": 48, "stages": [ { "title": "Author", "nodes": [ { "id": "pr", "label": "Pull request", "kind": "app", "sub": "rule DSL" }, { "id": "lint", "label": "Lint + schema check", "kind": "app", "sub": "pre-commit" } ] }, { "title": "Compile", "nodes": [ { "id": "comp", "label": "Plan compiler", "kind": "app", "sub": "must be bounded" }, { "id": "est", "label": "Cost estimate", "kind": "app", "sub": "$ / run, posted to PR" } ] }, { "title": "Gate", "nodes": [ { "id": "tierg", "label": "Tier policy check", "kind": "decision", "sub": "may this block?" }, { "id": "own", "label": "Owner approval", "kind": "security", "sub": "CODEOWNERS" } ] }, { "title": "Publish", "nodes": [ { "id": "cat", "label": "Rule catalogue", "kind": "store", "sub": "new assertion version", "icon": "aurora" }, { "id": "prob", "label": "Probation set", "kind": "platform", "sub": "advise for 30 days" } ] }, { "title": "Prove", "nodes": [ { "id": "bf", "label": "Automatic backfill", "kind": "app", "sub": "30-day lookback" }, { "id": "rep", "label": "Would-have-failed report", "kind": "app", "sub": "posted to the PR thread" } ] }, { "title": "Enforce", "nodes": [ { "id": "prom", "label": "Promote action", "kind": "decision", "sub": "advise → quarantine/block" }, { "id": "live", "label": "Live in the gate", "kind": "opportunity", "sub": "with FP tracking" } ] }, { "title": "Watch", "nodes": [ { "id": "fp", "label": "FP rate monitor", "kind": "platform", "sub": "trailing 30 days" }, { "id": "demote", "label": "Auto-demote", "kind": "risk", "sub": "> 5% → advise" } ] } ], "edges": [ { "from": "demote", "to": "pr", "label": "rework", "kind": "error", "route": "gutter" }, { "from": "tierg", "to": "pr", "label": "refused", "kind": "error", "route": "gutter" } ], "note": "Nothing reaches the gate without having been run against history first. The would-have-failed report is the artefact that forces the discovery-or-bad-rule question to be answered by a person before enforcement.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "18-observability", "title": "Observability — Signals by Plane, and the Ones Nobody Usually Has", "layout": "grid", "canvas": { "width": 1840 }, "laneHeaderWidth": 170, "columns": [ "Control plane", "Evaluation plane", "Verdict + state", "Enforcement", "The practice itself" ], "rows": [ { "title": "Metrics", "cells": [ [ { "label": "Queue depth by tier", "kind": "platform" }, { "label": "Sweeper catch rate", "kind": "platform" } ], [ { "label": "Fusion ratio", "kind": "platform" }, { "label": "Full-scan fraction", "kind": "platform" } ], [ { "label": "Verdict lag p95", "kind": "platform" }, { "label": "State read p99", "kind": "platform" } ], [ { "label": "Blocks per day", "kind": "platform" }, { "label": "Quarantine % of batch", "kind": "platform" } ], [ { "label": "MTTD by assertion type", "kind": "opportunity" }, { "label": "FP rate per assertion", "kind": "opportunity" } ] ] }, { "title": "Logs and traces", "cells": [ [ { "label": "Plan compilations", "kind": "app" } ], [ { "label": "Per-assertion runtime", "kind": "app" }, { "label": "Circuit-break events", "kind": "app" } ], [ { "label": "Write buffer replays", "kind": "app" } ], [ { "label": "Gate decisions", "kind": "app" }, { "label": "Override grants", "kind": "security" } ], [ { "label": "Discovery vs bad-rule calls", "kind": "app" } ] ] }, { "title": "Alerts", "cells": [ [ { "label": "Scheduler leader lost", "kind": "risk" } ], [ { "label": "Tier 1 budget exhausted", "kind": "risk" }, { "label": "Poison assertion", "kind": "risk" } ], [ { "label": "State index stale > 60 s", "kind": "risk" } ], [ { "label": "Tier 1 freshness breach", "kind": "risk" }, { "label": "Ceiling escalation", "kind": "risk" } ], [ { "label": "Coverage lost on Tier 1", "kind": "risk" } ] ] }, { "title": "Reports", "cells": [ [ { "label": "Cost by domain", "kind": "store" } ], [ { "label": "Assertions by cost", "kind": "store" } ], [ { "label": "Verdict history queries", "kind": "store" } ], [ { "label": "Overrides expiring", "kind": "store" } ], [ { "label": "Debt age by team", "kind": "store" }, { "label": "Coverage vs contract", "kind": "store" } ] ] }, { "title": "Absence checks", "cells": [ [ { "label": "No work enqueued", "kind": "risk" } ], [ { "label": "No verdict in 2 cadences", "kind": "risk" } ], [ { "label": "Unknown state count", "kind": "risk" } ], [ { "label": "No gate reads from a consumer", "kind": "risk" } ], [ { "label": "Datasets with zero assertions", "kind": "risk" } ] ] } ], "note": "The bottom row is the one most quality platforms lack: every cell answers 'is something that should be happening not happening', which is the only class of signal that catches a silent pass.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "19-security-zones", "title": "Security Zones — A Verdict Must Not Become a Side Channel", "layout": "zones", "canvas": { "width": 1740 }, "zones": [ { "title": "Zone 0 · Corporate — people and their groups", "kind": "trust", "nodes": [ { "id": "eng", "label": "Data engineer", "kind": "actor", "sub": "authors rules" }, { "id": "stew", "label": "Data steward", "kind": "actor", "sub": "grants overrides" }, { "id": "an", "label": "Analyst", "kind": "actor", "sub": "reads state" }, { "id": "aud", "label": "Auditor", "kind": "actor", "sub": "reads the log" } ] }, { "title": "Zone 1 · Presentation — authenticated, authorised, no data", "kind": "trust", "nodes": [ { "id": "idp", "label": "Identity provider", "kind": "security", "sub": "OIDC · groups" }, { "id": "console", "label": "Quality console", "kind": "app", "sub": "state · scorecards" }, { "id": "sapi", "label": "State API", "kind": "integration", "sub": "no row values" } ] }, { "title": "Zone 2 · Control — intent, never data", "kind": "trust", "nodes": [ { "id": "cat", "label": "Rule catalogue", "kind": "store", "sub": "definitions only" }, { "id": "ovr", "label": "Override service", "kind": "security", "sub": "owner-scoped" }, { "id": "audl", "label": "Audit log", "kind": "security", "sub": "Object Lock · 7 yr" } ] }, { "title": "Zone 3 · Evaluation — the only zone that sees rows", "kind": "trust", "nodes": [ { "id": "wl", "label": "Workload identity", "kind": "security", "sub": "scoped to targets" }, { "id": "ev", "label": "Evaluators", "kind": "app", "sub": "UC grants enforced", "icon": "databricks" }, { "id": "red", "label": "Sample redactor", "kind": "security", "sub": "ids, not values" } ] }, { "title": "Zone 4 · Data — classified, masked, custodial", "kind": "trust", "nodes": [ { "id": "lake", "label": "Delta Lake on S3", "kind": "store", "sub": "read-only to this service", "icon": "aws_s3" }, { "id": "quar", "label": "Quarantine tables", "kind": "store", "sub": "source classification kept", "icon": "aws_s3" }, { "id": "vst", "label": "Verdict store", "kind": "store", "sub": "outcomes, not payloads" } ] } ], "edges": [ { "from": "eng", "to": "idp", "label": "authenticate" }, { "from": "an", "to": "sapi", "label": "state read" }, { "from": "stew", "to": "ovr", "label": "override" }, { "from": "ovr", "to": "audl", "label": "immutable record", "kind": "async" }, { "from": "ev", "to": "lake" }, { "from": "red", "to": "vst", "label": "redacted samples" }, { "from": "ev", "to": "quar", "kind": "async" } ], "note": "Arrows out of Zone 3 are unlabelled because their direction is the whole statement: rows only ever move downward, into the data zone. The boundary that is easy to miss is inside Zone 3 — an author who cannot read a column must not be able to write an assertion whose failure message prints it, so redaction happens before a verdict crosses into anything a broader audience can query.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } }, { "id": "20-identity-and-override-flow", "title": "Identity Flow — Who May Block, and Who May Let It Through", "layout": "sequence", "canvas": { "width": 1720 }, "lifelines": [ { "id": "stew", "label": "Data steward", "kind": "actor" }, { "id": "idp", "label": "Identity provider", "kind": "security" }, { "id": "api", "label": "Rule + override API", "kind": "integration" }, { "id": "pol", "label": "Tier + policy registry", "kind": "platform" }, { "id": "ovr", "label": "Override service", "kind": "security" }, { "id": "audl", "label": "Audit log", "kind": "security" }, { "id": "csi", "label": "State index", "kind": "store" } ], "messages": [ { "from": "stew", "to": "idp", "label": "OIDC sign-in", "kind": "call" }, { "from": "idp", "to": "api", "label": "token + group claims", "kind": "return" }, { "from": "api", "to": "pol", "label": "is this actor the dataset's owner or delegate?", "kind": "call" }, { "from": "pol", "to": "api", "label": "yes — payments domain steward", "kind": "return" }, { "from": "stew", "to": "api", "label": "override block on v41207, 48 h", "kind": "call" }, { "from": "api", "to": "api", "label": "require justification, reject if absent", "kind": "self" }, { "from": "api", "to": "ovr", "label": "create override with expiry", "kind": "call" }, { "from": "ovr", "to": "audl", "label": "actor · reason · expiry · approver", "kind": "call" }, { "from": "ovr", "to": "csi", "label": "state: blocked, override active", "kind": "call" }, { "from": "csi", "to": "stew", "label": "version promotable until expiry", "kind": "return" }, { "from": "ovr", "to": "ovr", "label": "at expiry: state reverts, debt raised", "kind": "self" }, { "from": "stew", "to": "api", "label": "override on another team's dataset", "kind": "error" }, { "from": "pol", "to": "stew", "label": "refused — not the owner", "kind": "error" } ], "note": "The override is an object with an owner and an expiry, not a flag. At expiry the state reverts by itself and the debt register carries the item — which is why nothing here has a path to a permanent exemption.", "meta": { "v": "1.0", "owner": "Data Platform Architecture", "date": "2026-09" } } ]