[ { "id": "01-system-context", "title": "Audit Log Service — System Context", "layout": "context", "system": { "label": "Audit Log Service", "sub": "capture · custody · disposal" }, "colWidth": 300, "canvas": { "width": 1820 }, "groups": [ { "side": "left", "title": "People it serves", "nodes": [ { "id": "ops", "label": "Platform Operator", "kind": "actor", "sub": "runs it, reads nothing", "rel": "operates", "dir": "in", "icon": "role_devops" }, { "id": "inv", "label": "Investigator", "kind": "actor", "sub": "read within scope", "rel": "queries", "dir": "in", "icon": "role_secops" }, { "id": "comp", "label": "Compliance Officer", "kind": "actor", "sub": "read, export, hold", "rel": "exports", "dir": "in", "icon": "person_approver" }, { "id": "legal", "label": "Legal Hold Officer", "kind": "actor", "sub": "segregated role", "rel": "places holds", "dir": "in", "icon": "fa5_gavel" } ] }, { "side": "top", "title": "Producers — write only", "nodes": [ { "id": "apps", "label": "Business Applications", "kind": "external", "sub": "410 services", "rel": "POST events", "dir": "in", "icon": "fa5_cube" }, { "id": "cons", "label": "Admin Consoles", "kind": "external", "sub": "privileged actions", "rel": "POST events", "dir": "in", "icon": "fa5_desktop" }, { "id": "plat", "label": "Platform Services", "kind": "external", "sub": "IAM, payments, HR", "rel": "stream", "dir": "in", "kind2": "async", "icon": "fa5_cogs" }, { "id": "azact", "label": "Azure Activity Log", "kind": "external", "sub": "cloud control plane", "rel": "diagnostics", "dir": "in", "kind2": "async", "icon": "azure_activity-log" } ] }, { "side": "right", "title": "External assurance", "nodes": [ { "id": "aud", "label": "External Auditor", "kind": "actor", "sub": "trusts nothing", "rel": "verifies offline", "dir": "out", "icon": "person_approver" }, { "id": "reg", "label": "Regulator", "kind": "external", "sub": "evidence on demand", "rel": "signed bundles", "dir": "out", "icon": "person_approver" }, { "id": "wit", "label": "Independent Witnesses", "kind": "external", "sub": "RFC 3161 TSA · Rekor", "rel": "anchors heads", "dir": "out" } ] }, { "side": "bottom", "title": "Custody held elsewhere — deliberately", "nodes": [ { "id": "acl", "label": "Confidential Ledger", "kind": "store", "sub": "separate subscription", "rel": "chain heads", "dir": "out", "icon": "azure_confidential-ledgers" }, { "id": "mhsm", "label": "Managed HSM", "kind": "security", "sub": "non-exportable keys", "rel": "signs heads", "dir": "in", "icon": "azure_AzureKeyVault__Security" }, { "id": "entra", "label": "Microsoft Entra ID", "kind": "security", "sub": "workload identity", "rel": "authenticates", "dir": "in", "icon": "azure_AzureActiveDirectory__Identity" } ] } ], "note": "Out of scope: application and operational logging, SIEM detection and correlation, and the business records themselves. Regulators receive evidence through the auditor's bundle, not through an interface of their own.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "02-high-level-architecture", "title": "Audit Log Service — High-Level Architecture", "layout": "flow", "chain": true, "align": "middle", "canvas": { "width": 1790 }, "nodeWidth": 205, "stages": [ { "title": "Produce", "nodes": [ { "id": "sdk", "label": "Producer SDK", "kind": "external", "sub": "spool ≥ 1 h", "icon": "fa5_code" }, { "id": "wid", "label": "Workload Identity", "kind": "security", "sub": "Entra · SPIFFE", "icon": "azure_AzureActiveDirectory__Identity" } ] }, { "title": "Accept", "nodes": [ { "id": "ing", "label": "Ingest API", "kind": "app", "sub": "Rust · p99 150 ms", "icon": "rust" }, { "id": "val", "label": "Validate + Detect", "kind": "app", "sub": "schema · secrets", "icon": "fa5_clipboard_check" }, { "id": "dedup", "label": "Dedup Window", "kind": "store", "sub": "Valkey · 24 h", "icon": "redis" } ] }, { "title": "Buffer", "nodes": [ { "id": "kafka", "label": "Apache Kafka", "kind": "queue", "sub": "72 h · RF 3", "icon": "kafka" } ] }, { "title": "Seal", "nodes": [ { "id": "seal", "label": "Sealer", "kind": "app", "sub": "Parquet · ≤ 60 s", "icon": "fa5_stamp" }, { "id": "chain", "label": "Chain Service", "kind": "app", "sub": "Merkle per tenant", "icon": "fa5_link" }, { "id": "sign", "label": "Checkpoint Signer", "kind": "security", "sub": "HSM · 15 min", "icon": "fa5_key" } ] }, { "title": "Custody", "nodes": [ { "id": "seg", "label": "Immutable Segments", "kind": "store", "sub": "Blob WORM · locked", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "ledger", "label": "Checkpoint Ledger", "kind": "store", "sub": "separate custody", "icon": "azure_confidential-ledgers" } ] }, { "title": "Find", "nodes": [ { "id": "idx", "label": "Indexer", "kind": "app", "sub": "commit → 60 s", "icon": "fa5_sitemap" }, { "id": "ch", "label": "ClickHouse Index", "kind": "store", "sub": "derived · rebuildable", "icon": "clickhouse" }, { "id": "qapi", "label": "Query API", "kind": "app", "sub": "every read audited", "icon": "fa5_search" } ] }, { "title": "Prove", "nodes": [ { "id": "ver", "label": "Verifier", "kind": "app", "sub": "intact · broken here", "icon": "fa5_check_double" }, { "id": "exp", "label": "Export Service", "kind": "app", "sub": "signed bundles", "icon": "fa5_file_export" }, { "id": "offl", "label": "Offline Verifier", "kind": "external", "sub": "published binary", "icon": "fa5_check_double" } ] } ], "edges": [ { "from": "qapi", "to": "ing", "label": "read is a record", "kind": "async", "route": "gutter" }, { "from": "seg", "to": "idx", "label": "rebuild source", "kind": "batch", "route": "gutter" } ], "note": "The acknowledgement is given at the buffer, not at the segment. What that promise means, and what it does not, is the subject of view 20.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "03-three-planes", "title": "Four Planes — No Principal Can Both Write and Remove", "layout": "nested", "canvas": { "width": 1780 }, "boxes": [ { "title": "Capture plane — may create a record, may never remove one", "kind": "boundary", "dir": "row", "children": [ { "title": "Boundary", "kind": "plain", "nodes": [ { "id": "cp-api", "label": "Ingest API", "kind": "app", "sub": "write only", "icon": "rust" }, { "id": "cp-val", "label": "Validator", "kind": "app", "sub": "schema + secrets", "icon": "fa5_clipboard_check" } ] }, { "title": "Hand-off", "kind": "plain", "nodes": [ { "id": "cp-buf", "label": "Durable Buffer", "kind": "queue", "sub": "no delete verb", "icon": "kafka" }, { "id": "cp-q", "label": "Quarantine", "kind": "store", "sub": "rejections, audited", "icon": "fa5_biohazard" } ] } ] }, { "title": "Custody plane — may retain and expire, may never create a record", "kind": "trust", "dir": "row", "children": [ { "title": "Seal and chain", "kind": "plain", "nodes": [ { "id": "cu-seal", "label": "Sealer", "kind": "app", "sub": "append only", "icon": "fa5_stamp" }, { "id": "cu-chain", "label": "Chain Service", "kind": "app", "sub": "hash forward", "icon": "fa5_link" }, { "id": "cu-seg", "label": "Segment Store", "kind": "store", "sub": "locked retention", "icon": "azure_AzureBlobStorage__Storage" } ] }, { "title": "Expire", "kind": "plain", "nodes": [ { "id": "cu-ret", "label": "Retention Engine", "kind": "app", "sub": "extend, never shorten", "icon": "fa5_hourglass_half" }, { "id": "cu-hold", "label": "Legal Hold Registry", "kind": "store", "sub": "legal role only", "icon": "postgresql" }, { "id": "cu-disp", "label": "Disposal Service", "kind": "app", "sub": "certificate or nothing", "icon": "fa5_trash_alt" } ] } ] }, { "title": "Query plane — derived, authoritative for nothing", "kind": "boundary", "dir": "row", "children": [ { "title": "Projection", "kind": "plain", "nodes": [ { "id": "qp-idx", "label": "Indexer", "kind": "app", "sub": "rebuildable", "icon": "fa5_sitemap" }, { "id": "qp-ch", "label": "Search Index", "kind": "store", "sub": "disposable", "icon": "clickhouse" } ] }, { "title": "Access", "kind": "plain", "nodes": [ { "id": "qp-api", "label": "Query API", "kind": "app", "sub": "tenant scoped", "icon": "fa5_search" }, { "id": "qp-arc", "label": "Archive Query", "kind": "app", "sub": "async jobs", "icon": "trino" } ] } ] }, { "title": "Evidence plane — may read and prove, may never write a record", "kind": "boundary", "dir": "row", "children": [ { "title": "Proof", "kind": "plain", "nodes": [ { "id": "ev-ver", "label": "Verifier", "kind": "app", "sub": "recompute chain", "icon": "fa5_check_double" }, { "id": "ev-pf", "label": "Proof Service", "kind": "app", "sub": "inclusion, absence", "icon": "fa5_stamp" } ] }, { "title": "Delivery", "kind": "plain", "nodes": [ { "id": "ev-exp", "label": "Export Service", "kind": "app", "sub": "watermark, cap", "icon": "fa5_file_export" }, { "id": "ev-cli", "label": "Offline Verifier", "kind": "external", "sub": "no service needed", "icon": "fa5_check_double" } ] } ] } ], "outside": [ { "id": "ox-hsm", "label": "Managed HSM", "kind": "security", "sub": "key custody team", "icon": "azure_AzureKeyVault__Security" }, { "id": "ox-led", "label": "Confidential Ledger", "kind": "store", "sub": "checkpoint custody", "icon": "azure_confidential-ledgers" }, { "id": "ox-ent", "label": "Microsoft Entra ID", "kind": "security", "sub": "identity team", "icon": "azure_AzureActiveDirectory__Identity" } ], "edges": [ { "from": "cp-buf", "to": "cu-seal", "label": "sealed hand-off", "kind": "async" }, { "from": "cu-seg", "to": "qp-idx", "label": "projection", "kind": "async" }, { "from": "cu-chain", "to": "ox-led", "label": "signed head", "kind": "sync" }, { "from": "ox-hsm", "to": "cu-chain", "label": "sign, never export", "kind": "sync" } ], "note": "Four planes, four managed identities, four Entra groups, no membership overlap. The capture plane holds no delete verb in its data-access layer; the custody plane holds no insert verb for records.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "04-actors-and-journeys", "title": "Who the Record Is For, and What Each of Them Gets to Do", "layout": "actors", "cardWidth": 268, "canvas": { "width": 1800 }, "groups": [ { "title": "The people the record is for", "kind": "boundary", "actors": [ { "id": "inv", "label": "Investigator", "sub": "38 named, scoped", "goal": "Show me every privileged action this account took last March, and tell me honestly whether I am seeing all of them.", "journeys": [ { "id": "j-investigate", "label": "Trace an account", "sub": "view 05" }, { "label": "Scope a breach window" } ], "icon": "role_secops" }, { "id": "comp", "label": "Compliance Officer", "sub": "12 named", "goal": "Give me something I can hand to a regulator that they can check without ever asking us a question.", "journeys": [ { "label": "Produce an evidence bundle" }, { "label": "Certify a disposal" } ], "icon": "person_approver" }, { "id": "aud", "label": "External Auditor", "kind": "external", "sub": "annual and ad hoc", "goal": "Prove to me that the record could not have been altered, without me having to trust you or your operators.", "journeys": [ { "id": "j-evidence", "label": "Verify a bundle offline", "sub": "view 06" }, { "label": "Challenge a gap in a range" } ], "icon": "person_approver" } ] }, { "title": "The people who run it", "kind": "trust", "actors": [ { "id": "ops", "label": "Platform Operator", "sub": "SRE, on call 24×7", "goal": "Hold ingest above 99.99% without ever being able to read a single customer's audit record.", "journeys": [ { "label": "Absorb a 4× burst" }, { "label": "Rebuild the index" } ], "icon": "role_devops" }, { "id": "sec", "label": "Security Engineer", "sub": "detection and response", "goal": "Be told within fifteen minutes if anyone has touched a segment, and that includes me.", "journeys": [ { "label": "Answer a divergence alert" }, { "label": "Review export anomalies" } ], "icon": "role_secops" }, { "id": "legal", "label": "Legal Hold Officer", "kind": "security", "sub": "segregated, 4 named", "goal": "Freeze everything touching this matter today, and be certain nothing expired overnight while I slept.", "journeys": [ { "label": "Place and lift a hold" }, { "label": "Prove nothing was lost" } ], "icon": "fa5_gavel" } ] }, { "title": "The systems that write to it", "kind": "cloud", "actors": [ { "id": "app", "label": "Business Application", "kind": "external", "sub": "410 services, one SDK", "goal": "Emit my audit events without your outage becoming my outage.", "journeys": [ { "id": "j-produce", "label": "Onboard an event class", "sub": "view 07" }, { "label": "Survive an audit outage" } ], "icon": "fa5_cube" }, { "id": "cons", "label": "Admin Console", "kind": "external", "sub": "privileged surfaces", "goal": "Record the privileged action I just performed, including the one that was refused.", "journeys": [ { "label": "Record a denied action" } ], "icon": "fa5_desktop" }, { "id": "azure", "label": "Azure Control Plane", "kind": "external", "sub": "Activity Log export", "goal": "Land every subscription-level change in the same record as the applications, not in a portal of its own.", "journeys": [ { "label": "Export control-plane events", "icon": "azure_activity-log" } ], "icon": "azure_activity-log" }, { "id": "self", "label": "The Service Itself", "sub": "reads, exports, holds", "goal": "Audit my own use into myself, so nobody reads this log without leaving a record of having read it.", "journeys": [ { "label": "Write a read record" }, { "label": "Write a key-use record" } ] } ] }, { "title": "The machines that act without being asked", "kind": "lane", "actors": [ { "id": "ver", "label": "Rolling Verifier", "kind": "platform", "sub": "full corpus / 7 days", "goal": "Walk the whole chain every week and name the first divergence before anyone else finds it.", "journeys": [ { "label": "Complete a sweep" } ], "icon": "fa5_check_double" }, { "id": "ret", "label": "Retention Scheduler", "kind": "platform", "sub": "nightly window", "goal": "Dispose of exactly what expired, nothing that is held, and leave a signed certificate behind.", "journeys": [ { "label": "Run a disposal window" } ], "icon": "fa5_hourglass_half" }, { "id": "rec", "label": "Index Reconciler", "kind": "platform", "sub": "continuous", "goal": "Find the record that exists but cannot be found, before an auditor does.", "journeys": [ { "label": "Repair a divergence" } ], "icon": "fa5_balance_scale" }, { "id": "rep", "label": "Cross-Region Replicator", "kind": "platform", "sub": "continuous, one way", "goal": "Keep the second region inside RPO 0 while holding no right to delete anything in the first.", "journeys": [ { "label": "Replicate segments" } ], "icon": "fa5_sync_alt" } ] } ], "note": "Four groups, and the fourth is the one usually left off. A weekly verifier and a nightly disposer have SLAs, failure paths and blast radius of their own; leaving them off the cast list is how they end up with no owner.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "05-journey-investigation", "title": "Investigator — Trace an Account Through Thirteen Months", "layout": "journey", "canvas": { "width": 1720 }, "actor": { "label": "Investigator", "sub": "Security incident response", "goal": "Establish who granted this role, under whose authority, and whether the record is complete", "trigger": "An alert: a service principal acquired Owner on a production subscription overnight", "success": "A defensible timeline handed to the incident lead, with a stated completeness bound", "icon": "role_secops" }, "phases": [ { "title": "Suspect", "sub": "the alert lands" }, { "title": "Scope", "sub": "actor and window" }, { "title": "Search", "moment": true }, { "title": "Corroborate", "moment": true }, { "title": "Hand over", "sub": "to incident lead" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Reads the alert" } ], [ { "label": "Fixes actor + window" }, { "label": "Cites justification" } ], [ { "label": "Queries by actor" }, { "label": "Widens to resource" } ], [ { "label": "Checks denied too" }, { "label": "Pulls before/after" } ], [ { "label": "Exports the range" } ] ] }, { "title": "Where they do it", "kind": "touch", "cells": [ [ { "label": "Alert console", "icon": "fa5_desktop" } ], [ { "label": "Query console" } ], [ { "label": "Query API", "icon": "fa5_search" } ], [ { "label": "Query API", "icon": "fa5_search" } ], [ { "label": "Export console", "icon": "fa5_file_export" } ] ] }, { "title": "Behind the glass", "kind": "system", "cells": [ [ { "label": "Alertmanager", "icon": "fa5_bell" } ], [ { "label": "RBAC scope check" } ], [ { "label": "ClickHouse index" } ], [ { "label": "Segment fetch", "icon": "azure_AzureBlobStorage__Storage" } ], [ { "label": "Export service", "icon": "fa5_file_export" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Working", "Doubting" ], "points": [ 1, 1, 0, 1, 2 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Scope check is slow" } ], [ { "label": "Is the index complete?" } ], [ { "label": "Payload not in index" } ], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [], [ { "label": "Precomputed scope" } ], [ { "label": "Completeness bound" } ], [ { "label": "Fetched on demand" } ], [ { "label": "Signed bundle", "icon": "fa5_file_contract" } ] ] } ], "note": "The trough is at Search, and it is not about speed. The investigator cannot tell whether an empty result means nothing happened or the index has not caught up — which is why every result carries the sealed segment it is complete to. View 22.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "06-journey-external-auditor", "title": "External Auditor — Verify Without Trusting the Operator", "layout": "journey", "canvas": { "width": 1720 }, "actor": { "label": "External Auditor", "sub": "Appointed by the regulator", "goal": "Satisfy myself that the record could not have been altered, on evidence I can check myself", "trigger": "The annual controls audit, plus one range subpoenaed under litigation", "success": "A signed opinion that the range is intact — or a named, dated divergence", "icon": "person_approver" }, "phases": [ { "title": "Request", "sub": "names a range" }, { "title": "Receive", "sub": "the bundle" }, { "title": "Verify offline", "moment": true }, { "title": "Challenge", "moment": true }, { "title": "Conclude", "sub": "opinion issued" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Names tenant + range" } ], [ { "label": "Takes the bundle" } ], [ { "label": "Runs the verifier" }, { "label": "Checks the anchor" } ], [ { "label": "Asks about a gap" } ], [ { "label": "Signs the opinion" } ] ] }, { "title": "Where they do it", "kind": "touch", "cells": [ [ { "label": "Evidence request", "icon": "fa5_file_contract" } ], [ { "label": "Signed bundle", "icon": "fa5_file_contract" } ], [ { "label": "audit-verify CLI", "icon": "fa5_terminal" } ], [ { "label": "Disposal certificate", "icon": "fa5_file_contract" } ], [ { "label": "Opinion letter", "icon": "fa5_file_contract" } ] ] }, { "title": "Behind the glass", "kind": "system", "cells": [ [ { "label": "Export service", "icon": "fa5_file_export" } ], [ { "label": "Proof service" } ], [ { "label": "Published key", "icon": "fa5_key" } ], [ { "label": "Tombstone record", "icon": "fa5_archive" } ], [] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Satisfied", "Working", "Sceptical" ], "points": [ 1, 1, 2, 0, 2 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "Trusts nothing said" } ], [], [ { "label": "Whose key is this?" } ], [ { "label": "A gap reads as tamper" } ], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [], [ { "label": "Bundle self-contained" } ], [ { "label": "Third-party anchor" } ], [ { "label": "Lawfully disposed" } ], [ { "label": "No service call made" } ] ] } ], "note": "The only phase that rises is the one performed offline. Everything the service says about itself is discounted; what carries the audit is a bundle that verifies with a published algorithm and a public key. Views 24 and 30.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "07-journey-producer-engineer", "title": "Producer Engineer — Emit Without Inheriting Our Outage", "layout": "journey", "canvas": { "width": 1720 }, "actor": { "label": "Producer Engineer", "sub": "One of 410 owning teams", "goal": "Emit audit events for a new privileged action without taking on a new availability dependency", "trigger": "A role-grant screen ships next sprint and legal has classed it as auditable", "success": "Events land, and I can prove afterwards which ones were sealed", "icon": "role_developer" }, "phases": [ { "title": "Model", "sub": "the event class" }, { "title": "Onboard", "sub": "identity" }, { "title": "Emit", "moment": true }, { "title": "Outage", "moment": true }, { "title": "Reconcile", "sub": "prove it landed" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Picks an action verb" }, { "label": "Declares the class" } ], [ { "label": "Federates identity" } ], [ { "label": "POST /v1/events" } ], [ { "label": "Spools locally" } ], [ { "label": "Checks receipts" } ] ] }, { "title": "Where they do it", "kind": "touch", "cells": [ [ { "label": "Schema registry", "icon": "fa5_file_code" } ], [ { "label": "Entra federation", "icon": "azure_AzureActiveDirectory__Identity" } ], [ { "label": "Ingest SDK", "icon": "fa5_code" } ], [ { "label": "Local spool", "icon": "fa5_hdd" } ], [ { "label": "Receipt API", "icon": "fa5_receipt" } ] ] }, { "title": "Behind the glass", "kind": "system", "cells": [ [ { "label": "Apicurio + review" } ], [ { "label": "No shared secret" } ], [ { "label": "Dedup + quota" } ], [ { "label": "72 h buffer" } ], [ { "label": "Seal receipt", "icon": "fa5_receipt" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Working", "Blocked" ], "points": [ 1, 1, 2, 0, 1 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [ { "label": "Verb must be agreed" } ], [], [], [ { "label": "Open or closed?" } ], [ { "label": "Was it really sealed?" } ] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Vocabulary PR, 1 day" } ], [ { "label": "Short-lived tokens" } ], [ { "label": "p99 150 ms" } ], [ { "label": "Declared per class" } ], [ { "label": "Seal receipt API" } ] ] } ], "note": "The trough is the outage, and it is a design question rather than a UX one: the producer must know before it ships whether this class fails open with a gap marker or fails closed and stops the business action. View 27 makes that a declaration, not a default.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "08-layered-architecture", "title": "Layered Architecture — Dependencies Point One Way", "layout": "bands", "canvas": { "width": 1740 }, "layerHeaderWidth": 160, "bands": [ { "name": "Access and evidence", "nodes": [ { "id": "l-con", "label": "Query Console", "kind": "app", "sub": "scoped, justified", "icon": "fa5_desktop" }, { "id": "l-exp", "label": "Export Service", "kind": "app", "sub": "watermarked", "icon": "fa5_file_export" }, { "id": "l-pf", "label": "Proof Service", "kind": "app", "sub": "inclusion, absence", "icon": "fa5_stamp" }, { "id": "l-cli", "label": "Offline Verifier", "kind": "external", "sub": "published binary", "icon": "fa5_check_double" }, { "id": "l-rcpt", "label": "Receipt API", "kind": "app", "sub": "commit, seal", "icon": "fa5_receipt" } ] }, { "name": "Service", "nodes": [ { "id": "l-ing", "label": "Ingest API", "kind": "app", "sub": "write only", "icon": "rust" }, { "id": "l-q", "label": "Query API", "kind": "app", "sub": "read only", "icon": "fa5_search" }, { "id": "l-ver", "label": "Verification Service", "kind": "app", "sub": "range and rolling", "icon": "fa5_check_double" }, { "id": "l-ret", "label": "Retention Engine", "kind": "app", "sub": "extend only", "icon": "fa5_hourglass_half" }, { "id": "l-disp", "label": "Disposal Service", "kind": "app", "sub": "certificate signed", "icon": "fa5_trash_alt" } ] }, { "name": "Processing", "nodes": [ { "id": "l-val", "label": "Validator", "kind": "app", "sub": "JSON Schema", "icon": "fa5_clipboard_check" }, { "id": "l-sec", "label": "Secret Detector", "kind": "app", "sub": "pre-commit", "icon": "fa5_user_secret" }, { "id": "l-seal", "label": "Sealer", "kind": "app", "sub": "Parquet · zstd", "icon": "fa5_stamp" }, { "id": "l-chain", "label": "Chain Service", "kind": "app", "sub": "Merkle", "icon": "fa5_link" }, { "id": "l-idx", "label": "Indexer", "kind": "app", "sub": "at-least-once", "icon": "fa5_sitemap" }, { "id": "l-rec", "label": "Reconciler", "kind": "app", "sub": "segment vs index", "icon": "fa5_balance_scale" } ] }, { "name": "Data", "nodes": [ { "id": "l-kaf", "label": "Kafka Buffer", "kind": "queue", "sub": "72 h", "icon": "kafka" }, { "id": "l-seg", "label": "Segment Store", "kind": "store", "sub": "Blob WORM", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "l-led", "label": "Checkpoint Ledger", "kind": "store", "sub": "append only", "icon": "azure_confidential-ledgers" }, { "id": "l-ch", "label": "ClickHouse Index", "kind": "store", "sub": "derived", "icon": "clickhouse" }, { "id": "l-pg", "label": "Segment Registry", "kind": "store", "sub": "PostgreSQL", "icon": "postgresql" }, { "id": "l-val2", "label": "Dedup Cache", "kind": "store", "sub": "Valkey · 24 h", "icon": "redis" } ] }, { "name": "Platform", "nodes": [ { "id": "l-aks", "label": "AKS", "kind": "platform", "sub": "3 zones", "icon": "kubernetes" }, { "id": "l-ent", "label": "Microsoft Entra ID", "kind": "security", "sub": "workload identity", "icon": "azure_AzureActiveDirectory__Identity" }, { "id": "l-hsm", "label": "Managed HSM", "kind": "security", "sub": "FIPS 140-3 L3", "icon": "azure_AzureKeyVault__Security" }, { "id": "l-opa", "label": "OPA", "kind": "security", "sub": "authorization", "icon": "opa" }, { "id": "l-obs", "label": "Prometheus + Loki", "kind": "platform", "sub": "no record content", "icon": "prometheus" }, { "id": "l-argo", "label": "Argo CD", "kind": "platform", "sub": "GitOps", "icon": "argocd" } ] } ], "edges": [ { "from": "l-q", "to": "l-ing", "label": "read is a record", "kind": "async" }, { "from": "l-disp", "to": "l-ing", "label": "certificate is a record", "kind": "async" } ], "note": "The two lateral arrows are the log auditing its own use; key use in the HSM re-enters the same way. Nothing points up: a layer calls only downwards, and the data layer calls nothing at all.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "09-container-architecture", "title": "Container Architecture — Deployable Units (C4 L2)", "layout": "nested", "canvas": { "width": 1790 }, "boxes": [ { "title": "Capture plane — namespace audit-ingest · managed identity ingest-mi", "kind": "boundary", "nodes": [ { "id": "c-gw", "label": "envoy-gateway", "kind": "integration", "sub": "Envoy · mTLS · 12 pods", "icon": "envoy" }, { "id": "c-api", "label": "ingest-api", "kind": "app", "sub": "Rust axum · 24 pods", "icon": "rust" }, { "id": "c-adm", "label": "admission", "kind": "app", "sub": "schema · secrets · size" }, { "id": "c-quo", "label": "quota-governor", "kind": "app", "sub": "OPA · per tenant", "icon": "opa" }, { "id": "c-qua", "label": "quarantine-writer", "kind": "app", "sub": "rejections, audited", "icon": "fa5_biohazard" } ] }, { "title": "Durable buffer — Strimzi on dedicated node pool", "kind": "boundary", "nodes": [ { "id": "c-kaf", "label": "kafka-broker", "kind": "queue", "sub": "12 × D16s v5 · RF 3", "icon": "kafka" }, { "id": "c-kraft", "label": "kraft-controller", "kind": "platform", "sub": "3 pods, 3 zones", "icon": "fa5_project_diagram" }, { "id": "c-tier", "label": "tiered-storage", "kind": "app", "sub": "6 h local, 72 h Blob", "icon": "fa5_layer_group" } ] }, { "title": "Custody plane — namespace audit-custody · managed identity custody-mi", "kind": "trust", "nodes": [ { "id": "c-seal", "label": "sealer", "kind": "app", "sub": "Go · 32 pods", "icon": "fa5_stamp" }, { "id": "c-chain", "label": "chain-service", "kind": "app", "sub": "Go · 6, leader per shard", "icon": "fa5_link" }, { "id": "c-sign", "label": "checkpoint-signer", "kind": "security", "sub": "Go · 3 pods", "icon": "fa5_key" }, { "id": "c-comp", "label": "compactor", "kind": "app", "sub": "chain-preserving", "icon": "fa5_compress_arrows_alt" }, { "id": "c-ret", "label": "retention-engine", "kind": "app", "sub": "Go · 3 pods", "icon": "fa5_hourglass_half" }, { "id": "c-disp", "label": "disposal-service", "kind": "app", "sub": "Go · 2 pods", "icon": "fa5_trash_alt" } ] }, { "title": "Query plane — namespace audit-query · managed identity query-mi", "kind": "boundary", "nodes": [ { "id": "c-idx", "label": "indexer", "kind": "app", "sub": "Go · 24 pods", "icon": "fa5_sitemap" }, { "id": "c-ch", "label": "clickhouse", "kind": "store", "sub": "16 × E16ds v5" }, { "id": "c-qapi", "label": "query-api", "kind": "app", "sub": "Go · 12 pods", "icon": "fa5_search" }, { "id": "c-rec", "label": "reconciler", "kind": "app", "sub": "Go · 4 pods", "icon": "fa5_balance_scale" }, { "id": "c-trino", "label": "trino", "kind": "app", "sub": "archive, autoscaled", "icon": "trino" } ] }, { "title": "Evidence plane — namespace audit-evidence · managed identity evidence-mi", "kind": "boundary", "nodes": [ { "id": "c-ver", "label": "verifier", "kind": "app", "sub": "Go · 16 pods", "icon": "fa5_check_double" }, { "id": "c-pf", "label": "proof-service", "kind": "app", "sub": "Go · 6 pods", "icon": "fa5_stamp" }, { "id": "c-exp", "label": "export-service", "kind": "app", "sub": "Go · 6 pods", "icon": "fa5_file_export" } ] } ], "outside": [ { "id": "c-blob", "label": "Azure Blob Storage", "kind": "store", "sub": "immutable containers", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "c-led", "label": "Confidential Ledger", "kind": "store", "sub": "separate subscription", "icon": "azure_confidential-ledgers" }, { "id": "c-hsm", "label": "Managed HSM", "kind": "security", "sub": "per-tenant keys", "icon": "azure_AzureKeyVault__Security" }, { "id": "c-pg", "label": "PostgreSQL Flexible", "kind": "store", "sub": "registry, holds", "icon": "postgresql" }, { "id": "c-valk", "label": "Valkey Cluster", "kind": "store", "sub": "dedup, quota", "icon": "redis" } ], "edges": [ { "from": "c-api", "to": "c-kaf", "label": "acks=all, isr 2", "kind": "sync" }, { "from": "c-kaf", "to": "c-seal", "label": "consume", "kind": "async" }, { "from": "c-seal", "to": "c-blob", "label": "PUT once", "kind": "sync" }, { "from": "c-sign", "to": "c-led", "label": "append head", "kind": "sync" }, { "from": "c-kaf", "to": "c-idx", "label": "consume", "kind": "async" } ], "note": "Five namespaces, five managed identities, five Entra groups with no shared membership. Blob data-plane roles are granted per plane: the sealer may create a blob and no more; nothing anywhere holds Storage Blob Data Owner.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "10-ingest-plane", "title": "Ingest Plane — Everything That Happens Before the Acknowledgement", "layout": "nested", "canvas": { "width": 1780 }, "boxes": [ { "title": "Arrival", "kind": "boundary", "dir": "row", "children": [ { "title": "Network", "kind": "plain", "nodes": [ { "id": "i-pl", "label": "Private Endpoint", "kind": "integration", "sub": "in-estate producers", "icon": "azure_private-endpoints" }, { "id": "i-fd", "label": "Front Door + WAF", "kind": "integration", "sub": "off-estate only", "icon": "azure_AzureFrontDoor__Networking" } ] }, { "title": "Identity", "kind": "plain", "nodes": [ { "id": "i-mtls", "label": "mTLS Termination", "kind": "security", "sub": "Envoy", "icon": "envoy" }, { "id": "i-tok", "label": "Token Validation", "kind": "security", "sub": "Entra · SPIFFE", "icon": "azure_AzureActiveDirectory__Identity" }, { "id": "i-ten", "label": "Tenant Assignment", "kind": "security", "sub": "from identity only", "icon": "fa5_user_shield" } ] } ] }, { "title": "Admission — every check before the record can be accepted", "kind": "boundary", "dir": "row", "children": [ { "title": "Shape", "kind": "plain", "nodes": [ { "id": "i-sch", "label": "Schema Validation", "kind": "app", "sub": "versioned, Apicurio", "icon": "fa5_file_code" }, { "id": "i-voc", "label": "Vocabulary Check", "kind": "app", "sub": "controlled verbs", "icon": "fa5_spell_check" }, { "id": "i-siz", "label": "Size Cap", "kind": "decision", "sub": "256 KB, reject" } ] }, { "title": "Content", "kind": "plain", "nodes": [ { "id": "i-sec", "label": "Secret Detector", "kind": "app", "sub": "Presidio + rules", "icon": "fa5_user_secret" }, { "id": "i-pan", "label": "PAN / Credential", "kind": "decision", "sub": "reject, never redact", "icon": "fa5_user_secret" } ] }, { "title": "Governance", "kind": "plain", "nodes": [ { "id": "i-dedup", "label": "Idempotency Check", "kind": "store", "sub": "24 h window", "icon": "redis" }, { "id": "i-quota", "label": "Tenant Quota", "kind": "decision", "sub": "token bucket", "icon": "opa" }, { "id": "i-cls", "label": "Class Declaration", "kind": "app", "sub": "open or closed" } ] } ] }, { "title": "Commit — the acknowledgement is issued here and nowhere earlier", "kind": "trust", "dir": "row", "children": [ { "title": "Durable write", "kind": "plain", "nodes": [ { "id": "i-prod", "label": "Kafka Producer", "kind": "app", "sub": "acks=all, isr ≥ 2", "icon": "kafka" }, { "id": "i-part", "label": "Partition Router", "kind": "app", "sub": "tenant · resource" } ] }, { "title": "Promise", "kind": "plain", "nodes": [ { "id": "i-rcpt", "label": "Commit Receipt", "kind": "app", "sub": "event_id + offset", "icon": "fa5_receipt" }, { "id": "i-obs", "label": "received_at Stamp", "kind": "app", "sub": "service clock", "icon": "fa5_clock" } ] } ] }, { "title": "Rejection — a refusal is itself an auditable event", "kind": "boundary", "nodes": [ { "id": "i-err", "label": "Structured Error", "kind": "app", "sub": "field and reason", "icon": "fa5_exclamation_circle" }, { "id": "i-qtop", "label": "Quarantine Topic", "kind": "queue", "sub": "never blocks a partition", "icon": "kafka" }, { "id": "i-qseg", "label": "Quarantine Segments", "kind": "store", "sub": "same retention lock", "icon": "azure_AzureBlobStorage__Storage" } ] } ], "edges": [ { "from": "i-siz", "to": "i-err", "label": "oversized", "kind": "error" }, { "from": "i-pan", "to": "i-err", "label": "secret found", "kind": "error" }, { "from": "i-err", "to": "i-qtop", "label": "durable", "kind": "async" } ], "note": "Nothing here transforms the record. Validation and detection can only accept or reject; there is no redaction path, because a record that had to be edited to be storable was never a faithful account of the action.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "11-custody-plane", "title": "Custody Plane — Seal, Chain, Lock, Expire", "layout": "nested", "canvas": { "width": 1780 }, "boxes": [ { "title": "Assemble — a segment is closed at most 60 seconds after it opens", "kind": "boundary", "dir": "row", "children": [ { "title": "Read", "kind": "plain", "nodes": [ { "id": "u-part", "label": "Partition Reader", "kind": "app", "sub": "ordered, committed" }, { "id": "u-grp", "label": "Segment Grouping", "kind": "app", "sub": "tenant · date · class" } ] }, { "title": "Write", "kind": "plain", "nodes": [ { "id": "u-parq", "label": "Parquet Writer", "kind": "app", "sub": "zstd · per-column", "icon": "fa5_table" }, { "id": "u-seal", "label": "Seal Trigger", "kind": "decision", "sub": "60 s or 256 MB", "icon": "fa5_stamp" } ] } ] }, { "title": "Chain — hashing forward so that removing anything invalidates everything after", "kind": "trust", "dir": "row", "children": [ { "title": "Hash", "kind": "plain", "nodes": [ { "id": "u-leaf", "label": "Leaf Hashing", "kind": "app", "sub": "SHA-256, canonical", "icon": "fa5_hashtag" }, { "id": "u-tree", "label": "Segment Merkle Root", "kind": "app", "sub": "RFC 6962 style" }, { "id": "u-prev", "label": "Predecessor Link", "kind": "app", "sub": "prev_root in header", "icon": "fa5_link" } ] }, { "title": "Sign", "kind": "plain", "nodes": [ { "id": "u-ckpt", "label": "Checkpoint Builder", "kind": "app", "sub": "15 min, all tenants", "icon": "fa5_project_diagram" }, { "id": "u-hsm", "label": "HSM Signature", "kind": "security", "sub": "non-exportable", "icon": "azure_AzureKeyVault__Security" }, { "id": "u-tsa", "label": "TSA Countersign", "kind": "external", "sub": "RFC 3161", "icon": "fa5_clock" } ] } ] }, { "title": "Lock — immutability is a storage property, not an application rule", "kind": "trust", "dir": "row", "children": [ { "title": "Object", "kind": "plain", "nodes": [ { "id": "u-blob", "label": "Immutable Container", "kind": "store", "sub": "locked time-based", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "u-enc", "label": "Encryption Scope", "kind": "security", "sub": "per-tenant CMK", "icon": "fa5_lock" }, { "id": "u-ver", "label": "Version-Level Lock", "kind": "store", "sub": "no overwrite" } ] }, { "title": "Record of it", "kind": "plain", "nodes": [ { "id": "u-reg", "label": "Segment Registry", "kind": "store", "sub": "PostgreSQL, derived", "icon": "postgresql" }, { "id": "u-tier", "label": "Lifecycle Tiering", "kind": "app", "sub": "hot → cool → archive", "icon": "fa5_layer_group" } ] } ] }, { "title": "Expire — acting only on records this plane did not create", "kind": "boundary", "nodes": [ { "id": "u-cls", "label": "Retention Class", "kind": "app", "sub": "13 mo · 7 yr · 10 yr", "icon": "fa5_hourglass_half" }, { "id": "u-hold", "label": "Hold Check", "kind": "decision", "sub": "tenant, actor, resource", "icon": "fa5_gavel" }, { "id": "u-shred", "label": "Crypto-Shred", "kind": "app", "sub": "destroy segment DEK", "icon": "fa5_burn" }, { "id": "u-cert", "label": "Disposal Certificate", "kind": "app", "sub": "signed, re-ingested", "icon": "fa5_file_contract" }, { "id": "u-tomb", "label": "Tombstone Node", "kind": "app", "sub": "chain stays verifiable", "icon": "fa5_archive" } ] } ], "outside": [ { "id": "u-ledger", "label": "Confidential Ledger", "kind": "store", "sub": "chain heads only", "icon": "azure_confidential-ledgers" } ], "edges": [ { "from": "u-ckpt", "to": "u-ledger", "label": "signed head", "kind": "sync" } ], "note": "The lock is set by a locked time-based retention policy on the container. Once locked, the policy can be extended and cannot be shortened, deleted or bypassed — not by the subscription owner, not by a support ticket, and not by the plane that wrote the object.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "12-integration-surface", "title": "Integration Surface — Every Way In and Every Way Out", "layout": "hub", "canvas": { "width": 1780 }, "left": { "title": "Inbound", "nodes": [ { "id": "n-app", "label": "Business Applications", "kind": "external", "sub": "REST, sync", "rel": "mTLS · 25 k/s", "icon": "fa5_cube" }, { "id": "n-str", "label": "Platform Services", "kind": "queue", "sub": "Kafka protocol", "rel": "stream", "kind2": "async", "icon": "fa5_cogs" }, { "id": "n-az", "label": "Azure Activity Log", "kind": "external", "sub": "diagnostic settings", "rel": "Event Hub", "kind2": "async", "icon": "azure_activity-log" }, { "id": "n-legal", "label": "Legal Hold Console", "kind": "app", "sub": "segregated role", "rel": "hold API", "icon": "fa5_desktop" } ] }, "centre": { "title": "Audit Log Service", "nodes": [ { "id": "n-core", "label": "Audit Log Service", "kind": "app", "sub": "one versioned schema" } ] }, "right": { "title": "Outbound and dependencies", "nodes": [ { "id": "n-blob", "label": "Azure Blob Storage", "kind": "store", "sub": "immutable, ZRS", "rel": "write once", "dir": "out", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "n-led", "label": "Confidential Ledger", "kind": "store", "sub": "separate custody", "rel": "append", "dir": "out", "icon": "azure_confidential-ledgers" }, { "id": "n-hsm", "label": "Managed HSM", "kind": "security", "sub": "FIPS 140-3 L3", "rel": "sign", "icon": "azure_AzureKeyVault__Security" }, { "id": "n-ent", "label": "Microsoft Entra ID", "kind": "security", "sub": "workload identity", "rel": "validate", "icon": "azure_AzureActiveDirectory__Identity" } ] }, "note": "There is no interface that both writes and reads. A producer credential cannot query; a query credential cannot write anything except the read record the service writes on its behalf. Spool replay after an outage re-enters through the same inbound REST contract, carrying the original occurred_at.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "13-hash-chain-structure", "title": "The Shape of the Chain — Per-Tenant, Anchored Globally", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Level 1 — the record", "kind": "boundary", "nodes": [ { "id": "h-can", "label": "Canonical Bytes", "kind": "app", "sub": "stored form, never re-encoded" }, { "id": "h-leaf", "label": "Leaf Hash", "kind": "app", "sub": "SHA-256 with domain tag", "icon": "fa5_hashtag" }, { "id": "h-pos", "label": "Leaf Index", "kind": "app", "sub": "position within segment" } ] }, { "title": "Level 2 — the segment, sealed at most every 60 seconds", "kind": "boundary", "nodes": [ { "id": "h-tree", "label": "Merkle Tree", "kind": "app", "sub": "RFC 6962 hashing", "icon": "fa5_project_diagram" }, { "id": "h-root", "label": "Segment Root", "kind": "app", "sub": "one per sealed object", "icon": "fa5_project_diagram" }, { "id": "h-hdr", "label": "Segment Header", "kind": "app", "sub": "prev_root, count, range" }, { "id": "h-obj", "label": "Object Digest", "kind": "store", "sub": "binds bytes to root", "icon": "fa5_hashtag" } ] }, { "title": "Level 3 — one chain per tenant, so a proof reveals no other tenant", "kind": "trust", "nodes": [ { "id": "h-head", "label": "Tenant Head", "kind": "app", "sub": "hash of prev head + root", "icon": "fa5_link" }, { "id": "h-par", "label": "Parallel Sealing", "kind": "app", "sub": "500 chains, no contention" }, { "id": "h-blind", "label": "Tenant-Blind Proof", "kind": "app", "sub": "path stays inside tenant" }, { "id": "h-tomb", "label": "Tombstone Link", "kind": "app", "sub": "survives disposal" } ] }, { "title": "Level 4 — one global checkpoint every 15 minutes binds every tenant", "kind": "trust", "nodes": [ { "id": "h-gt", "label": "Checkpoint Tree", "kind": "app", "sub": "over all tenant heads", "icon": "fa5_project_diagram" }, { "id": "h-sig", "label": "HSM Signature", "kind": "security", "sub": "non-exportable key", "icon": "azure_AzureKeyVault__Security" }, { "id": "h-tsa", "label": "TSA Countersign", "kind": "external", "sub": "external clock", "icon": "fa5_clock" }, { "id": "h-pub", "label": "Ledger Publication", "kind": "store", "sub": "separate custody", "icon": "azure_confidential-ledgers" } ] } ], "outside": [ { "id": "h-wit", "label": "Public Witness Log", "kind": "external", "sub": "Rekor · phase 3" } ], "edges": [ { "from": "h-leaf", "to": "h-tree", "label": "leaves", "kind": "sync" }, { "from": "h-root", "to": "h-head", "label": "extends", "kind": "sync" }, { "from": "h-head", "to": "h-gt", "label": "anchors", "kind": "sync" }, { "from": "h-pub", "to": "h-wit", "label": "phase 3 anchor", "kind": "async" } ], "note": "Question 1 answered: per-tenant chains anchored into a global checkpoint. One global chain would serialise sealing at 100 k events/s and leak cross-tenant volume through every Merkle path; independent per-tenant chains would leave a tenant-scoped attacker a small structure to forge.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "14-data-architecture", "title": "Data Architecture — Four Tiers, Four Durability Contracts", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "System of record — losing any of this is unrecoverable", "kind": "trust", "nodes": [ { "id": "d-seg", "label": "Immutable Segments", "kind": "store", "sub": "Blob · 11 nines · RPO 0", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "d-led", "label": "Checkpoint Ledger", "kind": "store", "sub": "separate subscription", "icon": "azure_confidential-ledgers" }, { "id": "d-cert", "label": "Disposal Certificates", "kind": "store", "sub": "records inside the log" } ] }, { "title": "Durable hand-off — loss of an acknowledged event here is a Sev-1 defect", "kind": "boundary", "nodes": [ { "id": "d-kaf", "label": "Kafka Buffer", "kind": "queue", "sub": "72 h · RF 3 · isr 2", "icon": "kafka" }, { "id": "d-qua", "label": "Quarantine Topic", "kind": "queue", "sub": "sealed like any other", "icon": "kafka" } ] }, { "title": "Derived and rebuildable — deleting all of it loses no record", "kind": "boundary", "nodes": [ { "id": "d-ch", "label": "ClickHouse Index", "kind": "store", "sub": "90 d default, 13 mo max", "icon": "clickhouse" }, { "id": "d-reg", "label": "Segment Registry", "kind": "store", "sub": "PostgreSQL · rebuildable", "icon": "postgresql" }, { "id": "d-abs", "label": "Absence Filters", "kind": "store", "sub": "for non-existence proofs", "icon": "fa5_filter" } ] }, { "title": "Ephemeral — bounded loss accepted, stated on the contract", "kind": "plain", "nodes": [ { "id": "d-ded", "label": "Dedup Window", "kind": "store", "sub": "Valkey · 24 h TTL", "icon": "redis" }, { "id": "d-quo", "label": "Quota Counters", "kind": "store", "sub": "reset on loss", "icon": "opa" }, { "id": "d-res", "label": "Result Cursors", "kind": "store", "sub": "15 min resumable", "icon": "redis" } ] } ], "edges": [ { "from": "d-kaf", "to": "d-seg", "label": "seal ≤ 60 s", "kind": "async" }, { "from": "d-seg", "to": "d-ch", "label": "rebuild", "kind": "batch" } ], "note": "Losing the dedup window costs duplicate records, which are visible and reconcilable. Losing the index costs query latency for the length of a rebuild — 2.6 TB/day re-read from segments. Losing a segment costs the audit its meaning, which is why only that tier gets a locked retention policy.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "15-audit-record-model", "title": "The Audit Record — One Canonical, Versioned Schema", "layout": "er", "canvas": { "width": 1660, "cols": 3 }, "rowGap": 240, "entities": [ { "id": "tenant", "name": "tenant", "kind": "store", "row": 0, "col": 0, "attrs": [ "tenant_id PK", "residency_region", "retention_policy_id FK", "key_uri (HSM)", "index_window_days" ] }, { "id": "rclass", "name": "retention_class", "kind": "store", "row": 0, "col": 1, "attrs": [ "class_id PK", "online_months", "archive_years", "disposal_grace_days = 30", "extendable (never reduced)" ] }, { "id": "hold", "name": "legal_hold", "kind": "security", "row": 0, "col": 2, "attrs": [ "hold_id PK", "scope_type tenant|actor|resource|range", "scope_value", "matter_ref", "opened_by (legal role only)" ] }, { "id": "actor", "name": "actor", "kind": "store", "row": 1, "col": 0, "attrs": [ "actor_id PK", "principal_type human|service", "principal", "auth_method", "on_behalf_of (nullable)" ] }, { "id": "rec", "name": "audit_record", "kind": "app", "row": 1, "col": 1, "attrs": [ "event_id PK (service-assigned)", "tenant_id FK (from identity)", "occurred_at (producer asserts)", "received_at (service clock)", "clock_divergence_s", "action (controlled verb)", "outcome success|denied|error", "idempotency_key", "schema_version", "retention_class_id FK", "fail_mode open|closed" ] }, { "id": "res", "name": "resource", "kind": "store", "row": 1, "col": 2, "attrs": [ "resource_id PK", "resource_type", "stable_identifier", "tenant_id FK" ] }, { "id": "src", "name": "source", "kind": "store", "row": 2, "col": 0, "attrs": [ "source_id PK", "network_origin", "user_agent", "session_id", "request_correlation_id" ] }, { "id": "pay", "name": "payload", "kind": "store", "row": 2, "col": 1, "attrs": [ "payload_ref PK", "before_value", "after_value", "bytes (record cap 256 KB)", "secret_scan_verdict" ] }, { "id": "auth", "name": "authority", "kind": "security", "row": 2, "col": 2, "attrs": [ "authority_id PK", "authority_kind policy|role|grant", "policy_ref", "grant_ref", "approval_ref" ] } ], "relations": [ { "from": "tenant", "to": "rec", "label": "1 : N", "from_side": "s", "to_side": "n1" }, { "from": "rclass", "to": "rec", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "hold", "to": "rec", "label": "N : M", "from_side": "s", "to_side": "n3", "kind": "optional" }, { "from": "actor", "to": "rec", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "res", "to": "rec", "label": "1 : N", "from_side": "w", "to_side": "e" }, { "from": "src", "to": "rec", "label": "1 : N", "from_side": "n", "to_side": "s1" }, { "from": "pay", "to": "rec", "label": "1 : 1", "from_side": "n", "to_side": "s" }, { "from": "auth", "to": "rec", "label": "1 : N", "from_side": "n", "to_side": "s3" } ], "note": "tenant_id and received_at are assigned by the service and are the only fields a producer cannot influence. occurred_at is kept exactly as asserted, never corrected — the divergence is recorded beside it instead.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "16-segment-and-chain-model", "title": "Segment, Chain and Certificate — What Makes a Range Provable", "layout": "er", "canvas": { "width": 1700, "cols": 3 }, "rowGap": 250, "entities": [ { "id": "seg", "name": "segment", "kind": "store", "row": 0, "col": 1, "attrs": [ "segment_id PK", "tenant_id FK", "event_class · date_partition", "blob_uri (immutable container)", "object_digest SHA-256", "merkle_root", "prev_root (predecessor)", "record_count", "dek_id (HSM-wrapped)", "retention_until", "sealed_at" ] }, { "id": "head", "name": "chain_head", "kind": "app", "row": 1, "col": 0, "attrs": [ "head_id PK", "tenant_id FK", "sequence", "segment_id FK", "prev_head_hash", "head_hash" ] }, { "id": "ckpt", "name": "checkpoint", "kind": "security", "row": 1, "col": 1, "attrs": [ "checkpoint_id PK", "checkpoint_root (all tenant heads)", "tenant_head_count", "hsm_key_version", "signature", "tsa_token RFC 3161", "ledger_txn_id", "signed_at" ] }, { "id": "tomb", "name": "tombstone", "kind": "app", "row": 1, "col": 2, "attrs": [ "tombstone_id PK", "segment_id FK", "merkle_root (retained)", "certificate_id FK", "method crypto-shred", "disposed_at" ] }, { "id": "proof", "name": "inclusion_proof", "kind": "app", "row": 2, "col": 0, "attrs": [ "proof_id PK", "event_id FK", "segment_id FK", "leaf_index", "audit_path", "checkpoint_id FK" ] }, { "id": "bundle", "name": "evidence_bundle", "kind": "app", "row": 2, "col": 1, "attrs": [ "bundle_id PK", "query_hash", "record_count", "manifest_digest", "signature", "watermark (role, caller)" ] }, { "id": "cert", "name": "disposal_certificate", "kind": "security", "row": 2, "col": 2, "attrs": [ "certificate_id PK", "retention_class_id FK", "range_from · range_to", "record_count", "policy_ref", "signature", "event_id FK (written back in)" ] } ], "relations": [ { "from": "seg", "to": "head", "label": "1 : 1", "from_side": "s1", "to_side": "n" }, { "from": "seg", "to": "ckpt", "label": "N : 1", "from_side": "s", "to_side": "n" }, { "from": "seg", "to": "tomb", "label": "1 : 0..1", "from_side": "s3", "to_side": "n", "kind": "optional" }, { "from": "head", "to": "proof", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "ckpt", "to": "bundle", "label": "N : M", "from_side": "s", "to_side": "n" }, { "from": "tomb", "to": "cert", "label": "N : 1", "from_side": "s", "to_side": "n" }, { "from": "proof", "to": "bundle", "label": "N : 1", "from_side": "e", "to_side": "w" }, { "from": "cert", "to": "bundle", "label": "N : M", "from_side": "w", "to_side": "e", "kind": "optional" } ], "note": "The tombstone is the reason a disposed range verifies as lawfully disposed rather than as tampering: the segment bytes go, the Merkle root and the certificate that authorised their going both stay in the chain.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "17-commit-to-queryable", "title": "Data Flow — Accepted Record to Answerable Query", "layout": "flow", "chain": true, "align": "middle", "canvas": { "width": 1780 }, "nodeWidth": 215, "stages": [ { "title": "Accepted", "nodes": [ { "id": "f-json", "label": "JSON Record", "kind": "app", "sub": "p50 1.2 KB · p99 8 KB" }, { "id": "f-rcpt", "label": "Commit Receipt", "kind": "app", "sub": "event_id + offset", "icon": "fa5_receipt" } ] }, { "title": "Buffered", "nodes": [ { "id": "f-part", "label": "Ordered Partition", "kind": "queue", "sub": "240 partitions", "icon": "kafka" }, { "id": "f-rep", "label": "Zone Replication", "kind": "store", "sub": "3 AZ · isr 2" } ] }, { "title": "Sealed and chained", "nodes": [ { "id": "f-parq", "label": "Parquet Segment", "kind": "store", "sub": "zstd · ~4 : 1", "icon": "fa5_table" }, { "id": "f-root", "label": "Merkle Root", "kind": "app", "sub": "≤ 60 s window", "icon": "fa5_project_diagram" }, { "id": "f-ck", "label": "Signed Checkpoint", "kind": "security", "sub": "every 15 min", "icon": "fa5_signature" } ] }, { "title": "Locked", "nodes": [ { "id": "f-lock", "label": "Immutable Object", "kind": "store", "sub": "locked policy · ZRS" }, { "id": "f-tier", "label": "Tier Schedule", "kind": "app", "sub": "hot 30 d → cool", "icon": "fa5_layer_group" } ] }, { "title": "Indexed", "nodes": [ { "id": "f-proj", "label": "Query Projection", "kind": "app", "sub": "no payload bytes", "icon": "fa5_sitemap" }, { "id": "f-ins", "label": "ClickHouse Insert", "kind": "store", "sub": "p95 60 s from commit", "icon": "clickhouse" }, { "id": "f-recon", "label": "Reconciliation", "kind": "app", "sub": "count + root match", "icon": "fa5_balance_scale" } ] }, { "title": "Answerable", "nodes": [ { "id": "f-pt", "label": "Point Query", "kind": "app", "sub": "30 d · p95 2 s", "icon": "fa5_search" }, { "id": "f-agg", "label": "Aggregate Query", "kind": "app", "sub": "13 mo · p95 30 s", "icon": "fa5_search" }, { "id": "f-arc", "label": "Archive Job", "kind": "app", "sub": "Trino · ≤ 12 h", "icon": "trino" } ] } ], "edges": [ { "from": "f-lock", "to": "f-proj", "label": "rebuild source", "kind": "batch", "route": "gutter" }, { "from": "f-lock", "to": "f-arc", "label": "beyond the index", "kind": "batch", "route": "gutter" } ], "note": "Volume: 2.2 billion events/day, 2.6 TB raw, 650 GB compressed. The index carries no payload bytes — an investigator who needs before/after values pays a segment fetch for them, which keeps the dominant marginal cost proportional to the fields people actually filter on.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "18-retention-and-tiering", "title": "Retention and Tiering — Tier Is a Cost Decision, Never an Immutability One", "layout": "grid", "canvas": { "width": 1790 }, "laneHeaderWidth": 165, "columns": [ "0 – 30 days", "30 days – 13 months", "13 months – 7 years", "7 – 10 years (elevated)", "After expiry" ], "lanes": [ { "title": "Segment store", "cells": [ [ { "label": "Blob Hot ZRS", "kind": "store", "sub": "locked policy", "icon": "azure_AzureBlobStorage__Storage" } ], [ { "label": "Blob Cool ZRS", "kind": "store", "sub": "locked policy", "icon": "azure_AzureBlobStorage__Storage" } ], [ { "label": "Blob Archive", "kind": "store", "sub": "locked policy", "icon": "azure_AzureBlobStorage__Storage" } ], [ { "label": "Blob Archive", "kind": "store", "sub": "elevated class only", "icon": "azure_AzureBlobStorage__Storage" } ], [ { "label": "Crypto-Shred", "kind": "decision", "sub": "certificate signed", "icon": "fa5_burn" } ] ] }, { "title": "Query path", "cells": [ [ { "label": "ClickHouse NVMe", "kind": "store", "sub": "p95 2 s", "icon": "clickhouse" } ], [ { "label": "ClickHouse on Blob", "kind": "store", "sub": "per-tenant opt-in", "icon": "clickhouse" } ], [ { "label": "Trino over segments", "kind": "app", "sub": "async, ≤ 12 h", "icon": "trino" } ], [ { "label": "Trino over segments", "kind": "app", "sub": "async, ≤ 12 h", "icon": "trino" } ], [ { "label": "Rows Dropped", "kind": "app", "sub": "index only" } ] ] }, { "title": "Checkpoint ledger", "cells": [ [ { "label": "Confidential Ledger", "kind": "store", "sub": "append only", "icon": "azure_confidential-ledgers" } ], [ { "label": "Confidential Ledger", "kind": "store", "sub": "append only", "icon": "azure_confidential-ledgers" } ], [ { "label": "Confidential Ledger", "kind": "store", "sub": "append only", "icon": "azure_confidential-ledgers" } ], [ { "label": "Confidential Ledger", "kind": "store", "sub": "append only", "icon": "azure_confidential-ledgers" } ], [ { "label": "Outlives Records", "kind": "store", "sub": "proofs still verify" } ] ] }, { "title": "Ingest buffer", "cells": [ [ { "label": "Kafka · 72 h", "kind": "queue", "sub": "replay window", "icon": "kafka" } ], [], [], [], [] ] }, { "title": "Storage cost", "cells": [ [ { "label": "≈ $23 / TB-month", "kind": "app" } ], [ { "label": "≈ $10 / TB-month", "kind": "app" } ], [ { "label": "≈ $2 / TB-month", "kind": "app" } ], [ { "label": "≈ $2 / TB-month", "kind": "app" } ], [ { "label": "$0", "kind": "app", "sub": "and provably so", "icon": false } ] ] } ], "note": "Every cell in the top row carries the same locked retention policy. The tier changes what the storage costs and how long a read takes; it never changes what a principal is permitted to do to the object.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "19-tenancy-in-storage", "title": "Where the Tenant Boundary Sits in Storage", "layout": "nested", "canvas": { "width": 1780 }, "boxes": [ { "title": "486 tenants — shared containers, tenant as the first partition key", "kind": "boundary", "nodes": [ { "id": "t-cont", "label": "Shared Container", "kind": "store", "sub": "one locked policy", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "t-path", "label": "tenant / date / class", "kind": "app", "sub": "prefix is the boundary" }, { "id": "t-scope", "label": "Encryption Scope", "kind": "security", "sub": "per tenant, per blob", "icon": "fa5_lock" }, { "id": "t-kek", "label": "Per-Tenant KEK", "kind": "security", "sub": "Managed HSM", "icon": "azure_AzureKeyVault__Security" }, { "id": "t-filt", "label": "Query Tenant Filter", "kind": "app", "sub": "from identity, not input", "icon": "fa5_user_shield" } ] }, { "title": "14 tenants — dedicated accounts where residency or contract requires it", "kind": "trust", "nodes": [ { "id": "t-acct", "label": "Storage Account", "kind": "store", "sub": "one per tenant", "icon": "azure_storage-accounts" }, { "id": "t-reg", "label": "Own Region", "kind": "platform", "sub": "residency assignment" }, { "id": "t-pol", "label": "Own Locked Policy", "kind": "store", "sub": "own retention terms" }, { "id": "t-cust", "label": "Customer-Held Key", "kind": "security", "sub": "phase 3" } ] }, { "title": "Never shared, on either side of that line", "kind": "boundary", "nodes": [ { "id": "t-chain", "label": "Hash Chain", "kind": "app", "sub": "one per tenant" }, { "id": "t-proof", "label": "Proof Path", "kind": "app", "sub": "reveals no other tenant" }, { "id": "t-quo", "label": "Ingest Quota", "kind": "app", "sub": "largest is 35%", "icon": "opa" }, { "id": "t-part", "label": "Index Partition", "kind": "store", "sub": "pruned before scan", "icon": "fa5_sitemap" } ] }, { "title": "Residual risk, accepted and monitored", "kind": "plain", "nodes": [ { "id": "t-r1", "label": "Shared Policy Error", "kind": "risk", "sub": "one config, 486 tenants" }, { "id": "t-r2", "label": "Small-Object Tail", "kind": "risk", "sub": "compaction required" }, { "id": "t-r3", "label": "Ciphertext Only", "kind": "opportunity", "sub": "misroute yields nothing" } ] } ], "edges": [ { "from": "t-kek", "to": "t-scope", "label": "wraps segment DEK", "kind": "sync" } ], "note": "Question 7 answered as a split, not a principle. Physical separation for 500 tenants would multiply locked-policy and lifecycle configuration by 500 and shatter the long tail into objects too small to be economic; a shared container with a per-tenant encryption scope makes a misrouted read return bytes the reader holds no key for.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "20-ingest-sequence", "title": "Synchronous Ingest — What the Acknowledgement Actually Promises", "layout": "sequence", "canvas": { "width": 1720 }, "lifelines": [ { "id": "prod", "label": "Producer SDK", "kind": "external", "icon": "fa5_code" }, { "id": "gw", "label": "Envoy Gateway", "kind": "integration", "icon": "envoy" }, { "id": "api", "label": "Ingest API", "kind": "app", "icon": "rust" }, { "id": "idp", "label": "Entra / SPIRE", "kind": "security", "icon": "azure_AzureActiveDirectory__Identity" }, { "id": "ded", "label": "Valkey Dedup", "kind": "store", "icon": "redis" }, { "id": "kaf", "label": "Kafka Buffer", "kind": "queue", "icon": "kafka" }, { "id": "seal", "label": "Sealer", "kind": "app", "icon": "fa5_stamp" } ], "messages": [ { "from": "prod", "to": "gw", "label": "POST /v1/events — batch of 500", "kind": "call" }, { "from": "gw", "to": "idp", "label": "validate workload token (mTLS bound)", "kind": "call" }, { "from": "idp", "to": "gw", "label": "principal + tenant_id", "kind": "return" }, { "from": "gw", "to": "api", "label": "forward with observed identity", "kind": "call" }, { "from": "api", "to": "api", "label": "schema, vocabulary, 256 KB cap", "kind": "self" }, { "from": "api", "to": "api", "label": "secret scan — reject, never redact", "kind": "self" }, { "from": "api", "to": "ded", "label": "SETNX idempotency_key, TTL 24 h", "kind": "call" }, { "from": "ded", "to": "api", "label": "new (or duplicate: return first event_id)", "kind": "return" }, { "from": "api", "to": "kaf", "label": "produce · acks=all · isr ≥ 2", "kind": "call" }, { "from": "kaf", "to": "api", "label": "offset durable in 3 zones", "kind": "return" }, { "from": "api", "to": "prod", "label": "202 + commit receipt (event_id, offset)", "kind": "return" }, { "from": "kaf", "to": "seal", "label": "consume within the seal window", "kind": "async" }, { "from": "seal", "to": "seal", "label": "seal receipt queryable ≤ 60 s later", "kind": "self" } ], "note": "Question 3 answered. The acknowledgement at message 11 promises the record is durable across a zone loss and will be sealed; it does not yet promise the object is under a retention lock. That gap is bounded at 60 seconds, measured, and alerted — an acknowledged record that never reaches a segment is a Sev-1 defect, not a gap.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "21-seal-and-checkpoint-sequence", "title": "Seal, Chain, Sign, Publish — One 15-Minute Checkpoint", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "kaf", "label": "Kafka Buffer", "kind": "queue", "icon": "kafka" }, { "id": "seal", "label": "Sealer", "kind": "app", "icon": "fa5_stamp" }, { "id": "blob", "label": "Immutable Blob", "kind": "store", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "chain", "label": "Chain Service", "kind": "app", "icon": "fa5_link" }, { "id": "hsm", "label": "Managed HSM", "kind": "security", "icon": "azure_AzureKeyVault__Security" }, { "id": "tsa", "label": "Timestamp Authority", "kind": "external", "icon": "fa5_clock" }, { "id": "led", "label": "Confidential Ledger", "kind": "store", "icon": "azure_confidential-ledgers" } ], "messages": [ { "from": "kaf", "to": "seal", "label": "poll committed offsets, ordered", "kind": "call" }, { "from": "seal", "to": "seal", "label": "group by tenant · date · event class", "kind": "self" }, { "from": "seal", "to": "seal", "label": "close at 60 s or 256 MB, whichever first", "kind": "self" }, { "from": "seal", "to": "blob", "label": "PUT segment — create-only permission", "kind": "call" }, { "from": "blob", "to": "seal", "label": "object digest + immutable-until", "kind": "return" }, { "from": "seal", "to": "chain", "label": "merkle_root, prev_root, object digest", "kind": "call" }, { "from": "chain", "to": "chain", "label": "extend this tenant's head", "kind": "self" }, { "from": "chain", "to": "hsm", "label": "sign checkpoint over all tenant heads", "kind": "call" }, { "from": "hsm", "to": "chain", "label": "signature — key never leaves the HSM", "kind": "return" }, { "from": "chain", "to": "tsa", "label": "timestamp request over checkpoint root", "kind": "call" }, { "from": "tsa", "to": "chain", "label": "RFC 3161 token", "kind": "return" }, { "from": "chain", "to": "led", "label": "append signed head", "kind": "call" }, { "from": "led", "to": "chain", "label": "ledger transaction receipt", "kind": "return" }, { "from": "chain", "to": "kaf", "label": "key-use event, back into the log", "kind": "async" } ], "note": "If the HSM is unreachable, messages 1 to 7 continue and 8 onwards pause: ingest never fails because sealing cannot proceed. The unsigned interval is visible in verification as pending, never as a break.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "22-query-sequence", "title": "An Investigative Query — and the Record It Leaves Behind", "layout": "sequence", "canvas": { "width": 1700 }, "lifelines": [ { "id": "inv", "label": "Investigator", "kind": "actor", "icon": "role_secops" }, { "id": "q", "label": "Query API", "kind": "app", "icon": "fa5_search" }, { "id": "opa", "label": "OPA + Entra", "kind": "security", "icon": "opa" }, { "id": "ing", "label": "Ingest API", "kind": "app", "icon": "rust" }, { "id": "ch", "label": "ClickHouse", "kind": "store", "icon": "clickhouse" }, { "id": "seg", "label": "Segment Store", "kind": "store", "icon": "azure_AzureBlobStorage__Storage" } ], "messages": [ { "from": "inv", "to": "q", "label": "actor + range + justification reference", "kind": "call" }, { "from": "q", "to": "opa", "label": "role, tenant scope, query budget", "kind": "call" }, { "from": "opa", "to": "q", "label": "permitted · 30 s budget · 50 k row cap", "kind": "return" }, { "from": "q", "to": "ing", "label": "record.read written BEFORE execution", "kind": "call" }, { "from": "ing", "to": "q", "label": "committed", "kind": "return" }, { "from": "q", "to": "ch", "label": "scan — tenant partition pruned first", "kind": "call" }, { "from": "ch", "to": "q", "label": "rows + last reconciled segment id", "kind": "return" }, { "from": "q", "to": "seg", "label": "fetch payload bytes for this page only", "kind": "call" }, { "from": "seg", "to": "q", "label": "decrypt under this tenant's key", "kind": "return" }, { "from": "q", "to": "inv", "label": "page + cursor + completeness bound", "kind": "return" }, { "from": "q", "to": "ing", "label": "result count and truncation flag", "kind": "async" } ], "note": "The read record is written before the query runs, so a query that times out, is cancelled or returns nothing is still on the record. Writing it afterwards would make the interesting case the one that leaves no trace.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "23-verification-flow", "title": "Verification — Intact, Broken Here, or Lawfully Disposed", "layout": "flow", "chain": true, "align": "middle", "canvas": { "width": 1780 }, "nodeWidth": 215, "stages": [ { "title": "Trigger", "nodes": [ { "id": "v-req", "label": "On-Demand Range", "kind": "app", "sub": "24 h in ≤ 90 s" }, { "id": "v-roll", "label": "Rolling Sweep", "kind": "platform", "sub": "full corpus / 7 days" }, { "id": "v-exp", "label": "Export Trigger", "kind": "app", "sub": "before every bundle", "icon": "fa5_file_export" } ] }, { "title": "Fetch", "nodes": [ { "id": "v-reg", "label": "Segment List", "kind": "store", "sub": "from registry", "icon": "postgresql" }, { "id": "v-byt", "label": "Object Bytes", "kind": "store", "sub": "read-only role", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "v-ck", "label": "Checkpoints", "kind": "store", "sub": "from the ledger" } ] }, { "title": "Recompute", "nodes": [ { "id": "v-leaf", "label": "Leaf Hashes", "kind": "app", "sub": "canonical bytes", "icon": "fa5_hashtag" }, { "id": "v-root", "label": "Merkle Roots", "kind": "app", "sub": "per segment", "icon": "fa5_project_diagram" }, { "id": "v-head", "label": "Head Chain", "kind": "app", "sub": "per tenant", "icon": "fa5_link" } ] }, { "title": "Compare", "nodes": [ { "id": "v-c1", "label": "Root vs Header", "kind": "decision", "sub": "segment integrity" }, { "id": "v-c2", "label": "Head vs Checkpoint", "kind": "decision", "sub": "ordering integrity" }, { "id": "v-c3", "label": "Signature + Token", "kind": "decision", "sub": "authenticity, time" } ] }, { "title": "Verdict", "nodes": [ { "id": "v-ok", "label": "Intact Through", "kind": "opportunity", "sub": "named segment, time" }, { "id": "v-bad", "label": "Broken Here", "kind": "risk", "sub": "first divergence named" }, { "id": "v-disp", "label": "Lawfully Disposed", "kind": "app", "sub": "under certificate X" } ] }, { "title": "Respond", "nodes": [ { "id": "v-alert", "label": "Divergence Alert", "kind": "platform", "sub": "≤ 15 min", "icon": "fa5_bell" }, { "id": "v-idx", "label": "Index Repair", "kind": "app", "sub": "automatic, from segments" }, { "id": "v-inc", "label": "Segment Incident", "kind": "risk", "sub": "never auto-repaired" } ] } ], "edges": [ { "from": "v-bad", "to": "v-alert", "label": "page security", "kind": "error" }, { "from": "v-ok", "to": "v-req", "label": "result is a record", "kind": "async", "route": "gutter" } ], "note": "The verdict is never a bare boolean. Intact carries the segment and timestamp it is intact through; broken carries the first divergence and its position; disposed carries the certificate that authorised it. An auditor can act on all three and on none of a true/false.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "24-export-and-offline-verification", "title": "Evidence Bundle — Verified by Someone Who Trusts Nobody", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "comp", "label": "Compliance Officer", "kind": "actor", "icon": "person_approver" }, { "id": "exp", "label": "Export Service", "kind": "app", "icon": "fa5_file_export" }, { "id": "pf", "label": "Proof Service", "kind": "app", "icon": "fa5_stamp" }, { "id": "led", "label": "Confidential Ledger", "kind": "store", "icon": "azure_confidential-ledgers" }, { "id": "ing", "label": "Ingest API", "kind": "app", "icon": "rust" }, { "id": "aud", "label": "External Auditor", "kind": "actor", "icon": "person_approver" }, { "id": "cli", "label": "audit-verify CLI", "kind": "external", "icon": "fa5_terminal" } ], "messages": [ { "from": "comp", "to": "exp", "label": "export this result — matter reference", "kind": "call" }, { "from": "exp", "to": "exp", "label": "per-role export budget check", "kind": "self" }, { "from": "exp", "to": "pf", "label": "inclusion proofs for N records", "kind": "call" }, { "from": "pf", "to": "pf", "label": "leaf index + audit path per record", "kind": "self" }, { "from": "pf", "to": "led", "label": "covering checkpoints for the range", "kind": "call" }, { "from": "led", "to": "pf", "label": "signed heads + TSA tokens", "kind": "return" }, { "from": "exp", "to": "exp", "label": "manifest, watermark, HSM signature", "kind": "self" }, { "from": "exp", "to": "ing", "label": "record.export — who, what, how many", "kind": "async" }, { "from": "exp", "to": "comp", "label": "bundle + manifest digest", "kind": "return" }, { "from": "comp", "to": "aud", "label": "delivered out of band", "kind": "call" }, { "from": "aud", "to": "cli", "label": "audit-verify bundle --key published.pem", "kind": "call" }, { "from": "cli", "to": "aud", "label": "intact — no service was contacted", "kind": "return" } ], "note": "Messages 11 and 12 involve nothing this organisation runs. The bundle is self-contained, the algorithm is published, the key is published and anchored, and the auditor's laptop is offline — which is the only configuration in which the verdict means anything.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "25-disposal-lifecycle", "title": "Disposal — A Governed, Evidenced Act", "layout": "cycle", "canvas": { "width": 1500 }, "centre": { "label": "Disposal", "sub": "within 30 days of expiry" }, "nodes": [ { "id": "y1", "label": "Class Assigned", "kind": "app", "sub": "at commit, in storage" }, { "id": "y2", "label": "Expiry Computed", "kind": "app", "sub": "retention_until", "icon": "fa5_hourglass_half" }, { "id": "y3", "label": "Hold Check", "kind": "decision", "sub": "4 granularities", "icon": "fa5_gavel" }, { "id": "y4", "label": "Key Destroyed", "kind": "security", "sub": "segment DEK in HSM", "icon": "fa5_burn" }, { "id": "y5", "label": "Certificate Signed", "kind": "security", "sub": "class, range, count", "icon": "fa5_file_contract" }, { "id": "y6", "label": "Tombstone Linked", "kind": "app", "sub": "root retained", "icon": "fa5_archive" }, { "id": "y7", "label": "Bytes Removed", "kind": "store", "sub": "lifecycle, lock expired", "icon": "fa5_trash_alt" } ], "ringLabels": [ "policy, not code", "nightly scan", "no hold applies", "record unreadable", "written back in", "range still verifies", "class reassigned on renewal" ], "note": "Question 5 answered as a sequence rather than a choice. Crypto-shredding makes the record unreadable the moment the key dies, while the ciphertext waits for the retention lock to expire before the lifecycle rule can remove it — so the erasure is instant and the deletion is still real.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "26-legal-hold", "title": "Legal Hold — Suspending Disposal Without Touching a Record", "layout": "flow", "chain": true, "align": "middle", "canvas": { "width": 1760 }, "nodeWidth": 215, "stages": [ { "title": "Matter", "nodes": [ { "id": "g-lit", "label": "Litigation Notice", "kind": "external", "sub": "counsel", "icon": "fa5_gavel" }, { "id": "g-reg", "label": "Regulator Request", "kind": "external", "sub": "supervisory" } ] }, { "title": "Scope", "nodes": [ { "id": "g-sc", "label": "Scope Declared", "kind": "app", "sub": "tenant · actor · resource" }, { "id": "g-rng", "label": "Time Range", "kind": "app", "sub": "open-ended permitted" }, { "id": "g-ap", "label": "Two-Person Approval", "kind": "decision", "sub": "legal + compliance", "icon": "fa5_user_check" } ] }, { "title": "Apply", "nodes": [ { "id": "g-reg2", "label": "Hold Registry", "kind": "store", "sub": "legal role writes", "icon": "postgresql" }, { "id": "g-ext", "label": "retention_until Extended", "kind": "app", "sub": "extend only" }, { "id": "g-blk", "label": "Disposal Suspended", "kind": "security", "sub": "indefinitely" } ] }, { "title": "Effect", "nodes": [ { "id": "g-seg", "label": "Segments Excluded", "kind": "store", "sub": "from every sweep" }, { "id": "g-tier", "label": "Tiering Continues", "kind": "app", "sub": "cost still falls" }, { "id": "g-idx", "label": "Index Unaffected", "kind": "store", "sub": "queries unchanged" } ] }, { "title": "Lift", "nodes": [ { "id": "g-cls", "label": "Matter Closed", "kind": "app", "sub": "same two roles" }, { "id": "g-rc", "label": "Expiry Recomputed", "kind": "app", "sub": "never backdated" } ] }, { "title": "Evidence", "nodes": [ { "id": "g-ev1", "label": "Hold Is a Record", "kind": "opportunity", "sub": "placed and lifted" }, { "id": "g-ev2", "label": "Certificate Cites It", "kind": "opportunity", "sub": "why it waited" } ] } ], "edges": [ { "from": "g-blk", "to": "g-ev1", "label": "audited", "kind": "async", "route": "gutter" } ], "note": "A hold changes when a record may be disposed of and nothing else. It does not hide the record, does not move it, does not alter its class and cannot be applied by anyone who can read the records it protects.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "27-fail-open-or-closed", "title": "When the Audit Plane Is Unavailable — Declared per Event Class", "layout": "swimlane", "canvas": { "width": 1790 }, "laneHeaderWidth": 185, "stages": [ "Declare", "Emit normally", "Audit plane down", "Spool exhausted", "Recover" ], "lanes": [ { "title": "Critical class — role.grant · key.use · policy.change · record.export", "cells": [ [ { "label": "fail_mode = closed", "kind": "decision" } ], [ { "label": "Synchronous ack required", "kind": "app" } ], [ { "label": "Retry 3× with backoff", "kind": "app" } ], [ { "label": "Business action refused", "kind": "risk" } ], [ { "label": "No gap to explain", "kind": "opportunity" } ] ] }, { "title": "Standard class — record.read · session.start · config.view", "cells": [ [ { "label": "fail_mode = open", "kind": "decision" } ], [ { "label": "Async, receipt later", "kind": "app" } ], [ { "label": "Spool locally ≥ 1 h", "kind": "store" } ], [ { "label": "Gap marker emitted", "kind": "app" } ], [ { "label": "Replay, original occurred_at", "kind": "app" } ] ] }, { "title": "What the service does", "cells": [ [ { "label": "Class fixed in the registry", "kind": "store", "icon": "fa5_file_code" } ], [ { "label": "Dedup on idempotency key", "kind": "app" } ], [ { "label": "Buffer holds 72 h", "kind": "queue" } ], [ { "label": "Counts dropped, per tenant", "kind": "app" } ], [ { "label": "Flags skew beyond 5 min", "kind": "app" } ] ] }, { "title": "What an auditor later sees", "cells": [ [ { "label": "Policy, per class, dated", "kind": "opportunity" } ], [], [], [ { "label": "The gap marker is a record", "kind": "opportunity" } ], [ { "label": "Late arrival is visible", "kind": "opportunity" } ] ] } ], "note": "Question 2 answered. Fail-open everywhere buys availability and quietly sells the record; fail-closed everywhere makes this service a first-order dependency of 410 others at a 99.99% target its callers do not themselves meet. The class declares it, the producer cannot change it at runtime, and the gap marker means an open failure is still an event rather than a silence.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "28-index-loss-and-rebuild", "title": "Losing the Index — and Why It Loses No Record", "layout": "flow", "chain": true, "align": "middle", "canvas": { "width": 1780 }, "nodeWidth": 215, "stages": [ { "title": "Detect", "nodes": [ { "id": "x-div", "label": "Reconciler Divergence", "kind": "app", "sub": "count or root mismatch" }, { "id": "x-loss", "label": "Node or Disk Loss", "kind": "risk", "sub": "shard unavailable" } ] }, { "title": "Decide", "nodes": [ { "id": "x-part", "label": "Partition Repair", "kind": "decision", "sub": "minutes" }, { "id": "x-full", "label": "Full Rebuild", "kind": "decision", "sub": "90 d ≈ 14 h" }, { "id": "x-ord", "label": "Tenant Priority", "kind": "app", "sub": "largest and regulated first" } ] }, { "title": "Degrade", "nodes": [ { "id": "x-tr", "label": "Trino Over Segments", "kind": "app", "sub": "slower, still correct", "icon": "trino" }, { "id": "x-slo", "label": "Query SLO Suspended", "kind": "app", "sub": "stated, not hidden" }, { "id": "x-ing", "label": "Ingest Untouched", "kind": "opportunity", "sub": "still 99.99%" } ] }, { "title": "Rebuild", "nodes": [ { "id": "x-read", "label": "Read Segments", "kind": "store", "sub": "650 GB/day compressed" }, { "id": "x-ins", "label": "Parallel Insert", "kind": "app", "sub": "24 indexer pods" }, { "id": "x-abs", "label": "Absence Filters", "kind": "store", "sub": "rebuilt with it", "icon": "fa5_filter" } ] }, { "title": "Reconcile", "nodes": [ { "id": "x-cnt", "label": "Count per Segment", "kind": "decision", "sub": "index vs registry" }, { "id": "x-root", "label": "Root per Segment", "kind": "decision", "sub": "index vs chain" } ] }, { "title": "Restore", "nodes": [ { "id": "x-cb", "label": "Completeness Bound", "kind": "app", "sub": "advances to now", "icon": "fa5_ruler_horizontal" }, { "id": "x-res", "label": "SLO Resumed", "kind": "opportunity", "sub": "p95 2 s again" } ] } ], "edges": [ { "from": "x-root", "to": "x-read", "label": "mismatch, re-read", "kind": "error", "route": "gutter" } ], "note": "Question 4 answered by making divergence measurable rather than impossible. Two stores are kept, and every query result states the sealed segment it is complete to — so the index is never quietly treated as authoritative, and a record that exists but cannot yet be found is visible as such.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "29-deployment-architecture", "title": "Deployment — One Region, Three Availability Zones", "layout": "nested", "canvas": { "width": 1790 }, "boxes": [ { "title": "Azure · West Europe", "kind": "cloud", "dir": "row", "children": [ { "title": "Availability zone 1", "kind": "boundary", "nodes": [ { "id": "z1a", "label": "AKS Ingest Pool", "kind": "platform", "sub": "3 × D16s v5", "icon": "kubernetes" }, { "id": "z1b", "label": "Kafka Brokers", "kind": "queue", "sub": "4 of 12", "icon": "kafka" }, { "id": "z1c", "label": "ClickHouse Shards", "kind": "store", "sub": "replica set A", "icon": "clickhouse" }, { "id": "z1d", "label": "Custody Pool", "kind": "platform", "sub": "sealer, chain", "icon": "kubernetes" } ] }, { "title": "Availability zone 2", "kind": "boundary", "nodes": [ { "id": "z2a", "label": "AKS Ingest Pool", "kind": "platform", "sub": "3 × D16s v5", "icon": "kubernetes" }, { "id": "z2b", "label": "Kafka Brokers", "kind": "queue", "sub": "4 of 12", "icon": "kafka" }, { "id": "z2c", "label": "ClickHouse Shards", "kind": "store", "sub": "replica set B", "icon": "clickhouse" }, { "id": "z2d", "label": "Custody Pool", "kind": "platform", "sub": "sealer, chain", "icon": "kubernetes" } ] }, { "title": "Availability zone 3", "kind": "boundary", "nodes": [ { "id": "z3a", "label": "AKS Ingest Pool", "kind": "platform", "sub": "3 × D16s v5", "icon": "kubernetes" }, { "id": "z3b", "label": "Kafka Brokers", "kind": "queue", "sub": "4 of 12", "icon": "kafka" }, { "id": "z3c", "label": "ClickHouse Shards", "kind": "store", "sub": "replica set C", "icon": "clickhouse" }, { "id": "z3d", "label": "Custody Pool", "kind": "platform", "sub": "sealer, chain", "icon": "kubernetes" } ] } ] }, { "title": "Zone-redundant by construction — no single zone holds the only copy", "kind": "boundary", "nodes": [ { "id": "zr1", "label": "Blob Storage ZRS", "kind": "store", "sub": "immutable containers", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "zr2", "label": "PostgreSQL Flexible", "kind": "store", "sub": "zone-redundant HA", "icon": "postgresql" }, { "id": "zr3", "label": "Managed HSM", "kind": "security", "sub": "3-zone pool", "icon": "azure_AzureKeyVault__Security" }, { "id": "zr4", "label": "Front Door + WAF", "kind": "integration", "sub": "global anycast", "icon": "azure_AzureFrontDoor__Networking" }, { "id": "zr5", "label": "Private Endpoints", "kind": "security", "sub": "in-estate producers", "icon": "azure_private-endpoints" } ] }, { "title": "Separate subscription, separate administrators, separate break-glass", "kind": "trust", "nodes": [ { "id": "sp1", "label": "Confidential Ledger", "kind": "store", "sub": "chain heads", "icon": "azure_confidential-ledgers" }, { "id": "sp2", "label": "Checkpoint Signing Key", "kind": "security", "sub": "own HSM pool", "icon": "fa5_key" }, { "id": "sp3", "label": "Ledger Admin Group", "kind": "security", "sub": "no segment access" } ] } ], "outside": [ { "id": "ot1", "label": "Microsoft Entra ID", "kind": "security", "sub": "global tenant", "icon": "azure_AzureActiveDirectory__Identity" }, { "id": "ot2", "label": "Timestamp Authority", "kind": "external", "sub": "third party", "icon": "fa5_clock" } ], "edges": [ { "from": "z1b", "to": "zr1", "label": "seal", "kind": "async" }, { "from": "sp2", "to": "sp1", "label": "signs heads", "kind": "sync" } ], "note": "Loss of one zone removes a third of ingest capacity and no data: Kafka keeps isr ≥ 2, ClickHouse keeps two replicas, and the segment store is zone-redundant beneath the immutability policy. Producers see nothing but a brief drop in a connection pool.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "30-multi-region-and-anchoring", "title": "Two Regions, and a Custody Chain That Belongs to Neither", "layout": "nested", "canvas": { "width": 1780 }, "boxes": [ { "title": "West Europe — active", "kind": "cloud", "nodes": [ { "id": "m-ing", "label": "Ingest Plane", "kind": "app", "sub": "25 k/s steady", "icon": "rust" }, { "id": "m-cus", "label": "Custody Plane", "kind": "app", "sub": "sealing, chaining" }, { "id": "m-idx", "label": "Query Plane", "kind": "app", "sub": "full index", "icon": "fa5_search" }, { "id": "m-blob", "label": "Segment Store", "kind": "store", "sub": "locked, ZRS", "icon": "azure_AzureBlobStorage__Storage" } ] }, { "title": "North Europe — warm standby", "kind": "cloud", "nodes": [ { "id": "s-ing", "label": "Ingest Plane", "kind": "app", "sub": "20% warm · RTO 15 min", "icon": "rust" }, { "id": "s-cus", "label": "Custody Plane", "kind": "app", "sub": "idle, resumes chain" }, { "id": "s-idx", "label": "Query Plane", "kind": "app", "sub": "lagging · RTO 4 h", "icon": "fa5_search" }, { "id": "s-blob", "label": "Segment Replica", "kind": "store", "sub": "locked at destination", "icon": "azure_AzureBlobStorage__Storage" } ] }, { "title": "Continuous replication — one way, no delete right at the destination", "kind": "boundary", "nodes": [ { "id": "r-obj", "label": "Object Replication", "kind": "app", "sub": "segments · RPO ≈ 0", "icon": "fa5_sync_alt" }, { "id": "r-mm", "label": "MirrorMaker 2", "kind": "queue", "sub": "buffer · 72 h", "icon": "kafka" }, { "id": "r-ch", "label": "ClickHouse Replica", "kind": "store", "sub": "rebuildable anyway", "icon": "clickhouse" }, { "id": "r-pg", "label": "PostgreSQL Replica", "kind": "store", "sub": "registry, holds", "icon": "postgresql" } ] }, { "title": "Custody independent of both regions", "kind": "trust", "nodes": [ { "id": "a-led", "label": "Confidential Ledger", "kind": "store", "sub": "own subscription", "icon": "azure_confidential-ledgers" }, { "id": "a-tsa", "label": "Timestamp Authority", "kind": "external", "sub": "outside the estate", "icon": "fa5_clock" }, { "id": "a-pub", "label": "Published Public Key", "kind": "security", "sub": "verifier needs no API", "icon": "fa5_key" }, { "id": "a-wit", "label": "Public Witness Log", "kind": "external", "sub": "Rekor · phase 3" } ] } ], "edges": [ { "from": "m-blob", "to": "s-blob", "label": "locked on arrival", "kind": "async" }, { "from": "m-cus", "to": "a-led", "label": "signed heads", "kind": "sync" } ], "note": "A region failover changes which region seals; it changes nothing about what has already been sealed. The chain continues in the second region from the last checkpoint the ledger holds, and the ledger is in neither region's subscription — which is what stops a regional administrator from being able to rewrite both sides of the evidence.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "31-cicd-and-change-control", "title": "Shipping the Service, and Changing the Rules It Enforces", "layout": "flow", "chain": true, "align": "middle", "canvas": { "width": 1790 }, "nodeWidth": 215, "stages": [ { "title": "Source", "nodes": [ { "id": "p-git", "label": "GitHub Repository", "kind": "external", "sub": "service + IaC + policy", "icon": "github" }, { "id": "p-sch", "label": "Schema Pull Request", "kind": "app", "sub": "vocabulary review", "icon": "fa5_file_code" } ] }, { "title": "Build", "nodes": [ { "id": "p-img", "label": "Container Build", "kind": "app", "sub": "reproducible", "icon": "docker" }, { "id": "p-sbom", "label": "SBOM + Signature", "kind": "security", "sub": "Syft · Cosign", "icon": "fa5_clipboard_list" }, { "id": "p-test", "label": "Property Tests", "kind": "app", "sub": "chain invariants" } ] }, { "title": "Gate", "nodes": [ { "id": "p-nod", "label": "No-Delete-Verb Check", "kind": "decision", "sub": "custody data layer" }, { "id": "p-comp", "label": "Schema Compatibility", "kind": "decision", "sub": "backward only" }, { "id": "p-iac", "label": "Retention Lock Scan", "kind": "decision", "sub": "IaC policy" } ] }, { "title": "Approve", "nodes": [ { "id": "p-mp", "label": "Multi-Party Approval", "kind": "security", "sub": "retention · keys · access", "icon": "fa5_user_check" }, { "id": "p-pim", "label": "Entra PIM", "kind": "security", "sub": "time-boxed elevation", "icon": "azure_AzureActiveDirectory__Identity" }, { "id": "p-aud", "label": "Approval Is a Record", "kind": "opportunity", "sub": "each approver" } ] }, { "title": "Deploy", "nodes": [ { "id": "p-argo", "label": "Argo CD", "kind": "platform", "sub": "one app per plane", "icon": "argocd" }, { "id": "p-bg", "label": "Blue / Green Ingest", "kind": "app", "sub": "no dropped ack" }, { "id": "p-can", "label": "Canary Custody", "kind": "app", "sub": "one shard first" } ] }, { "title": "Verify", "nodes": [ { "id": "p-syn", "label": "Synthetic Probe", "kind": "app", "sub": "ingest → seal → verify", "icon": "fa5_heartbeat" }, { "id": "p-rb", "label": "Automatic Rollback", "kind": "app", "sub": "on chain conformance" } ] } ], "edges": [ { "from": "p-rb", "to": "p-argo", "label": "revert revision", "kind": "error", "route": "gutter" } ], "note": "The no-delete-verb gate is a static check over the custody plane's data-access layer: any UPDATE, DELETE or blob-delete call reaching that package fails the build. Principle 1 of this architecture is enforced by CI, not by a code-review convention.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "32-observability", "title": "Observability — Signal Type by Pipeline Stage", "layout": "grid", "canvas": { "width": 1790 }, "laneHeaderWidth": 180, "columns": [ "Ingest", "Buffer", "Seal and chain", "Custody", "Query", "Evidence" ], "lanes": [ { "title": "Metrics · Prometheus", "cells": [ [ { "label": "ack p50 / p99", "kind": "platform", "sub": "reject rate" } ], [ { "label": "isr, lag, disk", "kind": "platform", "sub": "per partition" } ], [ { "label": "Seal age", "kind": "platform", "sub": "unsigned interval" } ], [ { "label": "Objects locked", "kind": "platform", "sub": "tier mix" } ], [ { "label": "p95 by range", "kind": "platform", "sub": "rows scanned" } ], [ { "label": "Verify duration", "kind": "platform", "sub": "divergence count" } ] ] }, { "title": "Logs · Loki", "cells": [ [ { "label": "Rejection reasons", "kind": "app", "sub": "never the payload" } ], [ { "label": "Broker events", "kind": "app" } ], [ { "label": "Seal gaps", "kind": "app", "sub": "resume points" } ], [ { "label": "Lifecycle actions", "kind": "app" } ], [ { "label": "Query hash only", "kind": "app", "sub": "text is in the log" } ], [ { "label": "Verdict lines", "kind": "app" } ] ] }, { "title": "Traces · OpenTelemetry", "cells": [ [ { "label": "Gateway → produce", "kind": "app" } ], [], [ { "label": "Seal → sign", "kind": "app" } ], [], [ { "label": "Query → fetch", "kind": "app" } ], [ { "label": "Bundle assembly", "kind": "app" } ] ] }, { "title": "Alerts · Alertmanager", "cells": [ [ { "label": "p99 > 150 ms", "kind": "risk", "sub": "5 min window" } ], [ { "label": "isr < 2", "kind": "risk", "sub": "page immediately" } ], [ { "label": "Seal age > 120 s", "kind": "risk" } ], [ { "label": "Delete attempted", "kind": "risk", "sub": "always an incident" } ], [ { "label": "Export anomaly", "kind": "risk", "sub": "volume, hour, scope" } ], [ { "label": "Divergence", "kind": "risk", "sub": "within 15 min" } ] ] }, { "title": "Into the log itself", "cells": [ [ { "label": "Quota denial", "kind": "opportunity" } ], [ { "label": "Topic config change", "kind": "opportunity" } ], [ { "label": "Key use", "kind": "opportunity" } ], [ { "label": "Disposal certificate", "kind": "opportunity", "icon": "fa5_file_contract" } ], [ { "label": "record.read", "kind": "opportunity" } ], [ { "label": "record.export", "kind": "opportunity" } ] ] } ], "note": "The monitoring stack never receives record content. Metrics carry counts and latencies, logs carry reasons and hashes, and anything that would tell an operator what a record said belongs in the log rather than in Grafana.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "33-cost-model", "title": "Cost — Where the Money Goes, and Where the Target Does Not Reach", "layout": "flow", "chain": false, "align": "top", "canvas": { "width": 1790 }, "nodeWidth": 220, "stages": [ { "title": "Capture", "nodes": [ { "id": "k-aks", "label": "AKS Ingest", "kind": "app", "sub": "$5.0 k / month", "icon": "kubernetes" }, { "id": "k-kaf", "label": "Kafka + Tiering", "kind": "queue", "sub": "$7.5 k / month", "icon": "kafka" } ] }, { "title": "Custody", "nodes": [ { "id": "k-blob", "label": "Segments, Primary", "kind": "store", "sub": "$4.1 k · 257 TB", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "k-rep", "label": "Replication + DR", "kind": "store", "sub": "$4.0 k / month", "icon": "fa5_sync_alt" }, { "id": "k-hsm", "label": "Managed HSM × 2", "kind": "security", "sub": "$4.6 k · fixed", "icon": "azure_AzureKeyVault__Security" }, { "id": "k-led", "label": "Confidential Ledger", "kind": "store", "sub": "$2.0 k · fixed", "icon": "azure_confidential-ledgers" } ] }, { "title": "Index — the dominant marginal cost", "nodes": [ { "id": "k-ch90", "label": "ClickHouse · 90 days", "kind": "store", "sub": "$5.5 k / month", "icon": "clickhouse" }, { "id": "k-ch13", "label": "Extend to 13 months", "kind": "store", "sub": "+ $6.5 k / month", "icon": "clickhouse" } ] }, { "title": "Serve and prove", "nodes": [ { "id": "k-tri", "label": "Trino Archive", "kind": "app", "sub": "$1.5 k · on demand", "icon": "trino" }, { "id": "k-ev", "label": "Verify + Export", "kind": "app", "sub": "$1.5 k / month" }, { "id": "k-sh", "label": "Shared Services", "kind": "platform", "sub": "$3.0 k / month", "icon": "fa5_cogs" } ] }, { "title": "Per million events retained 13 months", "nodes": [ { "id": "k-ach", "label": "Achieved", "kind": "opportunity", "sub": "$0.65 per million", "icon": "fa5_money_check_alt" }, { "id": "k-floor", "label": "Fixed Floor", "kind": "decision", "sub": "$0.24 per million", "icon": "fa5_money_check_alt" }, { "id": "k-tgt", "label": "Target $0.09", "kind": "risk", "sub": "7× short — see cards" } ] } ], "note": "66 billion events a month at $42.7 k gives $0.65 per million. The stated $0.09 target implies $5.9 k a month, which the segment store alone consumes before a single core is switched on. The number to renegotiate is the target, and the cards say what could move it.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "34-operations-loop", "title": "The Operator's Loop — Running a System You Cannot Read", "layout": "cycle", "canvas": { "width": 1500 }, "centre": { "label": "Operate", "sub": "no record access" }, "nodes": [ { "id": "o1", "label": "Watch", "kind": "platform", "sub": "seal age, isr, divergence", "icon": "fa5_eye" }, { "id": "o2", "label": "Detect", "kind": "decision", "sub": "alert names a runbook" }, { "id": "o3", "label": "Degrade", "kind": "app", "sub": "in the declared order" }, { "id": "o4", "label": "Repair", "kind": "app", "sub": "index from segments" }, { "id": "o5", "label": "Evidence", "kind": "opportunity", "sub": "the action is a record" }, { "id": "o6", "label": "Review", "kind": "app", "sub": "weekly sweep, exports", "icon": "fa5_search" } ], "ringLabels": [ "signal, not content", "buffer first, index last", "never the reverse", "including break-glass", "what the sweep found", "tune the thresholds" ], "note": "Degradation has a declared order and the operator does not get to choose it: accept and durably buffer first, seal and verify second, index for query third. Query freshness is spent before durability, never the reverse.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "35-security-trust-zones", "title": "Trust Zones — Every Crossing Authenticated, Every Zone Narrower", "layout": "zones", "canvas": { "width": 1780 }, "zones": [ { "title": "Internet — assumed hostile", "kind": "trust", "nodes": [ { "id": "s-off", "label": "Off-Estate Producer", "kind": "external", "sub": "partner systems" }, { "id": "s-aud", "label": "Auditor Workstation", "kind": "external", "sub": "offline verification", "icon": "fa5_laptop" }, { "id": "s-pub", "label": "Published Key + Algorithm", "kind": "security", "sub": "deliberately public", "icon": "fa5_key" } ] }, { "title": "Perimeter — holds no key", "kind": "trust", "nodes": [ { "id": "s-fd", "label": "Front Door + WAF", "kind": "integration", "sub": "rate limit, TLS 1.3", "icon": "azure_AzureFrontDoor__Networking" }, { "id": "s-pe", "label": "Private Endpoint", "kind": "security", "sub": "in-estate default path", "icon": "azure_private-endpoints" }, { "id": "s-env", "label": "Envoy mTLS", "kind": "security", "sub": "SVID or Entra token", "icon": "envoy" } ] }, { "title": "Capture — write only", "kind": "trust", "nodes": [ { "id": "s-ing", "label": "Ingest API", "kind": "app", "sub": "ingest-mi", "icon": "rust" }, { "id": "s-val", "label": "Validator + Detector", "kind": "app", "sub": "reject or accept", "icon": "fa5_clipboard_check" }, { "id": "s-kaf", "label": "Kafka Buffer", "kind": "queue", "sub": "no delete API exposed", "icon": "kafka" } ] }, { "title": "Custody — no standing access", "kind": "trust", "nodes": [ { "id": "s-seal", "label": "Sealer + Chain", "kind": "app", "sub": "custody-mi", "icon": "fa5_stamp" }, { "id": "s-seg", "label": "Segment Store", "kind": "store", "sub": "create-only role", "icon": "azure_AzureBlobStorage__Storage" }, { "id": "s-ret", "label": "Retention Engine", "kind": "app", "sub": "extend only", "icon": "fa5_hourglass_half" } ] }, { "title": "Evidence and query", "kind": "trust", "nodes": [ { "id": "s-q", "label": "Query API", "kind": "app", "sub": "query-mi, tenant scoped", "icon": "fa5_search" }, { "id": "s-ver", "label": "Verifier", "kind": "app", "sub": "read-only on segments", "icon": "fa5_check_double" }, { "id": "s-exp", "label": "Export Service", "kind": "app", "sub": "capped, watermarked", "icon": "fa5_file_export" } ] }, { "title": "Key and checkpoint custody", "kind": "trust", "nodes": [ { "id": "s-hsm", "label": "Managed HSM", "kind": "security", "sub": "non-exportable", "icon": "azure_AzureKeyVault__Security" }, { "id": "s-led", "label": "Confidential Ledger", "kind": "store", "sub": "append only", "icon": "azure_confidential-ledgers" }, { "id": "s-adm", "label": "Custody Admin Group", "kind": "security", "sub": "disjoint membership" } ] } ], "edges": [ { "from": "s-off", "to": "s-fd", "label": "TLS 1.3 + workload token", "kind": "sync" }, { "from": "s-env", "to": "s-ing", "label": "principal, tenant, write scope", "kind": "sync" }, { "from": "s-kaf", "to": "s-seal", "label": "consume, in-cluster mTLS", "kind": "async" }, { "from": "s-seg", "to": "s-ver", "label": "read-only SAS-free RBAC", "kind": "sync" }, { "from": "s-seal", "to": "s-hsm", "label": "sign · Crypto User only", "kind": "sync" }, { "from": "s-exp", "to": "s-aud", "label": "signed bundle, out of band", "kind": "async" } ], "note": "Zones narrow left to right and every crossing is mutually authenticated, in-cluster included. The perimeter can see a record in flight and can read none at rest; the custody zone has no standing human access at all; and the last edge leaves unauthenticated on purpose, because the bundle is built to be verified by someone holding no credential of ours.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "36-identity-and-credentials", "title": "Identity — Short-Lived Everywhere, No Shared Secret Anywhere", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "wl", "label": "Producer Workload", "kind": "external", "icon": "fa5_fingerprint" }, { "id": "spire", "label": "SPIRE Server", "kind": "security", "icon": "fa5_fingerprint" }, { "id": "entra", "label": "Microsoft Entra ID", "kind": "security", "icon": "azure_AzureActiveDirectory__Identity" }, { "id": "env", "label": "Envoy Gateway", "kind": "integration", "icon": "envoy" }, { "id": "api", "label": "Ingest API", "kind": "app", "icon": "rust" }, { "id": "cust", "label": "Custody Pod", "kind": "app", "icon": "kubernetes" }, { "id": "hsm", "label": "Managed HSM", "kind": "security", "icon": "azure_AzureKeyVault__Security" } ], "messages": [ { "from": "wl", "to": "spire", "label": "attest node and workload selectors", "kind": "call" }, { "from": "spire", "to": "wl", "label": "SVID · TTL 1 h · no secret on disk", "kind": "return" }, { "from": "wl", "to": "entra", "label": "federated credential exchange", "kind": "call" }, { "from": "entra", "to": "wl", "label": "token · 60 min · aud audit-ingest", "kind": "return" }, { "from": "wl", "to": "env", "label": "mTLS with SVID + bearer token", "kind": "call" }, { "from": "env", "to": "entra", "label": "validate signature, audience, tenant claim", "kind": "call" }, { "from": "env", "to": "api", "label": "observed principal · scope = write", "kind": "call" }, { "from": "api", "to": "api", "label": "no read scope exists for this principal", "kind": "self" }, { "from": "cust", "to": "entra", "label": "managed identity custody-mi", "kind": "call" }, { "from": "cust", "to": "hsm", "label": "sign checkpoint · role Crypto User", "kind": "call" }, { "from": "hsm", "to": "cust", "label": "signature — key cannot be exported", "kind": "return" }, { "from": "hsm", "to": "api", "label": "key-use event, into this log", "kind": "async" } ], "note": "The identity the service records on an event is the one it observed at message 7, never the one the payload claims. A producer that lies about its own actor field is contradicted by the record beside it, which is the point of recording both.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "37-roles-and-separation-of-duties", "title": "Who May Do What — Blanks Are Denials, Not Omissions", "layout": "grid", "canvas": { "width": 1790 }, "laneHeaderWidth": 190, "columns": [ "Write a record", "Read records", "Export evidence", "Legal hold", "Retention policy", "Verify", "Operate" ], "lanes": [ { "title": "Producer", "cells": [ [ { "label": "Own tenant only", "kind": "opportunity" } ], [], [], [], [], [], [] ] }, { "title": "Investigator", "cells": [ [], [ { "label": "Scoped + justified", "kind": "opportunity" } ], [], [], [], [ { "label": "Own result range", "kind": "opportunity" } ], [] ] }, { "title": "Compliance", "cells": [ [], [ { "label": "Tenant-wide", "kind": "opportunity" } ], [ { "label": "Budgeted, watermarked", "kind": "opportunity" } ], [ { "label": "Propose only", "kind": "decision" } ], [], [ { "label": "Any range", "kind": "opportunity" } ], [] ] }, { "title": "Legal Hold Officer", "cells": [ [], [], [], [ { "label": "Place and lift", "kind": "opportunity" } ], [], [], [] ] }, { "title": "Operator", "cells": [ [], [], [], [], [], [], [ { "label": "No record access", "kind": "opportunity" } ] ] }, { "title": "Verifier", "cells": [ [], [], [], [], [], [ { "label": "Checkpoints only", "kind": "opportunity" } ], [] ] }, { "title": "Retention Board", "cells": [ [], [], [], [], [ { "label": "Two approvals", "kind": "decision" } ], [], [] ] }, { "title": "Break-glass", "cells": [ [], [], [], [], [], [], [ { "label": "PIM, time-boxed", "kind": "risk", "sub": "every action recorded" } ] ] } ], "note": "Two invariants shape the blanks. No role can both write a record and influence its retention or disposal; no role can both read records and administer the policy governing reading. The service's own operators appear here with a single capability and no read.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "38-key-custody", "title": "Key Custody — What Nobody Holds Is the Design", "layout": "nested", "canvas": { "width": 1770 }, "boxes": [ { "title": "Signing — one key makes a checkpoint believable", "kind": "trust", "nodes": [ { "id": "kk1", "label": "Managed HSM Pool", "kind": "security", "sub": "own subscription", "icon": "azure_AzureKeyVault__Security" }, { "id": "kk2", "label": "EC P-384, Non-Exportable", "kind": "security", "sub": "FIPS 140-3 L3", "icon": "fa5_key" }, { "id": "kk3", "label": "Crypto User: Signer Only", "kind": "security", "sub": "one workload identity", "icon": "fa5_user_lock" }, { "id": "kk4", "label": "Rotation with Overlap", "kind": "app", "sub": "old key still verifies", "icon": "fa5_sync_alt" } ] }, { "title": "Encryption — one key per tenant, one data key per segment", "kind": "trust", "nodes": [ { "id": "ke1", "label": "Tenant KEK", "kind": "security", "sub": "500 keys in HSM", "icon": "azure_AzureKeyVault__Security" }, { "id": "ke2", "label": "Segment DEK", "kind": "security", "sub": "wrapped, per object", "icon": "fa5_key" }, { "id": "ke3", "label": "Blob Encryption Scope", "kind": "security", "sub": "bound to the prefix", "icon": "fa5_lock" }, { "id": "ke4", "label": "Shred = Destroy DEK", "kind": "app", "sub": "instant, irreversible", "icon": "fa5_burn" } ] }, { "title": "Publication — what an outsider needs and gets", "kind": "boundary", "nodes": [ { "id": "kp1", "label": "Public Verification Key", "kind": "security", "sub": "published, versioned", "icon": "fa5_key" }, { "id": "kp2", "label": "Verification Algorithm", "kind": "app", "sub": "specified, open", "icon": "fa5_check_double" }, { "id": "kp3", "label": "TSA Certificate Chain", "kind": "external", "sub": "third-party root" }, { "id": "kp4", "label": "Witness Anchor", "kind": "external", "sub": "phase 3" } ] }, { "title": "What no principal holds, by construction", "kind": "plain", "nodes": [ { "id": "kn1", "label": "Exportable Private Key", "kind": "risk", "sub": "no export path" }, { "id": "kn2", "label": "Standing HSM Admin", "kind": "risk", "sub": "PIM, quorum, audited" }, { "id": "kn3", "label": "Sign and Delete", "kind": "risk", "sub": "no such principal" } ] } ], "edges": [ { "from": "ke1", "to": "ke2", "label": "wraps", "kind": "sync" }, { "from": "kk2", "to": "kp1", "label": "public half", "kind": "sync" } ], "note": "Every use of the signing key is itself an audit event in this log, which means the log records the operations that make the log believable. A rotation overlaps so old checkpoints keep verifying, and needs two approvals recorded as events before it can begin.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "39-insider-threat", "title": "The Operators Are In Scope — What Each Attempt Actually Meets", "layout": "swimlane", "canvas": { "width": 1790 }, "laneHeaderWidth": 190, "stages": [ "The attempt", "What stops it", "What it leaves", "Detected within", "Residual" ], "lanes": [ { "title": "Service operator", "cells": [ [ { "label": "Delete a segment", "kind": "risk" } ], [ { "label": "Locked retention policy", "kind": "security" } ], [ { "label": "Refused delete, logged", "kind": "opportunity" } ], [ { "label": "Immediate alert", "kind": "app", "icon": "fa5_bell" } ], [ { "label": "None", "kind": "opportunity" } ] ] }, { "title": "Subscription owner", "cells": [ [ { "label": "Delete the container", "kind": "risk" } ], [ { "label": "Policy outlives the owner", "kind": "security" } ], [ { "label": "Activity Log + our log", "kind": "opportunity" } ], [ { "label": "≤ 15 minutes", "kind": "app" } ], [ { "label": "Index can be destroyed", "kind": "decision", "sub": "rebuildable" } ] ] }, { "title": "Custody engineer", "cells": [ [ { "label": "Rewrite a segment", "kind": "risk" } ], [ { "label": "Create-only permission", "kind": "security" } ], [ { "label": "Chain diverges", "kind": "opportunity" } ], [ { "label": "Next sweep or export", "kind": "app" } ], [ { "label": "Detection lag ≤ 7 d", "kind": "decision" } ] ] }, { "title": "Checkpoint administrator", "cells": [ [ { "label": "Rewrite chain heads", "kind": "risk" } ], [ { "label": "No segment access", "kind": "security" } ], [ { "label": "Heads vs segments differ", "kind": "opportunity" } ], [ { "label": "On verification", "kind": "app" } ], [ { "label": "Needs the other admin", "kind": "decision" } ] ] }, { "title": "Both administrators colluding", "cells": [ [ { "label": "Rewrite both sides", "kind": "risk" } ], [ { "label": "External TSA token", "kind": "security" } ], [ { "label": "Token time mismatch", "kind": "opportunity" } ], [ { "label": "On offline verify", "kind": "app" } ], [ { "label": "Phase 3 anchor closes it", "kind": "risk" } ] ] }, { "title": "Authorised investigator", "cells": [ [ { "label": "Bulk-export a tenant", "kind": "risk" } ], [ { "label": "Per-role export cap", "kind": "security" } ], [ { "label": "Watermarked export record", "kind": "opportunity" } ], [ { "label": "Anomaly alert", "kind": "app", "icon": "fa5_bell" } ], [ { "label": "Read cannot be undone", "kind": "risk" } ] ] } ], "note": "Bulk export is the realistic attack on a store that is otherwise hard to touch, and it is the one row where the residual risk is genuine: prevention caps the volume and detection names the exporter, but nothing puts read data back.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } }, { "id": "40-failure-modes", "title": "Named Failure Modes and Their Required Handling", "layout": "nested", "canvas": { "width": 1790 }, "boxes": [ { "title": "Absorbed — the producer sees nothing", "kind": "boundary", "nodes": [ { "id": "w1", "label": "Ingest Timeout", "kind": "app", "sub": "retry + idempotency key" }, { "id": "w2", "label": "Duplicate Delivery", "kind": "app", "sub": "one record, 24 h window" }, { "id": "w3", "label": "Zone Loss", "kind": "app", "sub": "isr ≥ 2, transparent" }, { "id": "w4", "label": "Broker Restart", "kind": "app", "sub": "producer backoff" } ] }, { "title": "Degraded — in the declared order, never the reverse", "kind": "boundary", "nodes": [ { "id": "w5", "label": "Key Service Down", "kind": "app", "sub": "ingest on, sealing pauses" }, { "id": "w6", "label": "Index Loss", "kind": "app", "sub": "Trino path, SLO suspended" }, { "id": "w7", "label": "Region Loss", "kind": "app", "sub": "RTO 15 min / 4 h" }, { "id": "w8", "label": "Sealer Backlog", "kind": "app", "sub": "buffer absorbs 72 h" } ] }, { "title": "Contained and made visible — the gap is itself a record", "kind": "boundary", "nodes": [ { "id": "w9", "label": "Producer Outage", "kind": "app", "sub": "spool 1 h, gap marker" }, { "id": "w10", "label": "Poison Record", "kind": "app", "sub": "quarantine, no head-of-line" }, { "id": "w11", "label": "Clock Skew", "kind": "app", "sub": "flagged beyond 5 min" }, { "id": "w12", "label": "Seal Gap", "kind": "app", "sub": "re-sealed, never discarded" } ] }, { "title": "Incident — never handled automatically", "kind": "trust", "nodes": [ { "id": "w13", "label": "Insider Deletion Attempt", "kind": "risk", "sub": "prevented, alerted" }, { "id": "w14", "label": "Segment Divergence", "kind": "risk", "sub": "Sev-1, no auto-repair" }, { "id": "w15", "label": "Acknowledged, Never Sealed", "kind": "risk", "sub": "Sev-1 defect, RPO 0" }, { "id": "w16", "label": "Checkpoint Signature Fails", "kind": "risk", "sub": "key or ledger compromise" } ] } ], "edges": [ { "from": "w6", "to": "w14", "label": "unless roots differ", "kind": "error" } ], "note": "Index divergence is repaired from segments and is routine; segment divergence is repaired by nobody and is an incident. The distinction is the whole architecture in one line: the derived thing may be rebuilt, and the thing it was derived from may not be touched.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-09" } } ]