[ { "id": "01-system-context", "title": "Embedding Pipeline Service — System Context", "layout": "context", "canvas": { "width": 1700 }, "colWidth": 250, "system": { "label": "Embedding Pipeline Service", "sub": "corpus in, retrieval out" }, "groups": [ { "side": "left", "title": "Consuming product surfaces", "nodes": [ { "id": "search", "label": "Workspace search", "kind": "external", "sub": "12k queries/s", "rel": "retrieval", "dir": "in" }, { "id": "assistant", "label": "Ask-your-docs assistant", "kind": "external", "sub": "answer layer", "rel": "top-k", "dir": "in" }, { "id": "similar", "label": "Similar documents", "kind": "external", "rel": "neighbours", "dir": "in" }, { "id": "dedupe", "label": "Duplicate detection", "kind": "external", "rel": "scores", "dir": "in" } ] }, { "side": "right", "title": "Corpora (systems of record)", "nodes": [ { "id": "docs", "label": "Document service", "kind": "external", "sub": "400 M documents", "rel": "changes", "dir": "in", "kind2": "async" }, { "id": "files", "label": "File store", "kind": "external", "sub": "uploads, PDFs", "rel": "bytes", "dir": "in" }, { "id": "threads", "label": "Comment threads", "kind": "external", "rel": "changes", "dir": "in", "kind2": "async" }, { "id": "connectors", "label": "Connected SaaS", "kind": "external", "sub": "tenant-authorised", "rel": "webhooks", "dir": "in", "kind2": "async" } ] }, { "side": "top", "title": "People", "nodes": [ { "id": "worker", "label": "Knowledge worker", "kind": "actor", "rel": "searches", "dir": "in" }, { "id": "prodeng", "label": "Product engineer", "kind": "actor", "rel": "registers", "dir": "in" }, { "id": "plateng", "label": "Platform engineer", "kind": "actor", "rel": "migrates", "dir": "in" }, { "id": "admin", "label": "Tenant admin", "kind": "actor", "rel": "erases", "dir": "in" } ] }, { "side": "bottom", "title": "Platform dependencies", "nodes": [ { "id": "authz", "label": "Permission authority", "kind": "security", "sub": "query-time ACLs", "rel": "checks" }, { "id": "idp", "label": "Workload identity", "kind": "security", "sub": "SPIRE", "rel": "mTLS" }, { "id": "obs", "label": "Observability platform", "kind": "platform", "sub": "Prometheus", "rel": "OTLP", "dir": "in" } ] } ], "note": "Out of scope: the generative answer layer, the document stores themselves, and the product user interfaces. The nightly warehouse export is omitted here and appears on view 9.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "02-high-level-architecture", "title": "Embedding Pipeline Service — High-Level Architecture", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "align": "middle", "nodeWidth": 215, "stages": [ { "title": "Capture", "nodes": [ { "id": "feed", "label": "Change capture", "kind": "integration", "sub": "feeds + sweep" }, { "id": "log", "label": "Change log", "kind": "queue", "sub": "Kafka, replayable" } ] }, { "title": "Prepare", "nodes": [ { "id": "extract", "label": "Extract + normalise", "kind": "app", "sub": "Tika, Tesseract" }, { "id": "text", "label": "Normalised text", "kind": "store", "sub": "MinIO, 30 days" }, { "id": "chunk", "label": "Versioned chunker", "kind": "app", "sub": "content hashing" } ] }, { "title": "Decide what changed", "nodes": [ { "id": "ledger", "label": "Chunk ledger", "kind": "store", "sub": "PostgreSQL" }, { "id": "diff", "label": "Ledger diff", "kind": "decision", "sub": "75% reuse" } ] }, { "title": "Embed", "nodes": [ { "id": "cache", "label": "Vector cache", "kind": "store", "sub": "Redis" }, { "id": "gpu", "label": "Embedding fleet", "kind": "app", "sub": "Ray Serve, GPU" } ] }, { "title": "Index", "nodes": [ { "id": "builder", "label": "Index builder", "kind": "app", "sub": "per contract" }, { "id": "ann", "label": "Vector index", "kind": "store", "sub": "Qdrant, immutable" }, { "id": "alias", "label": "Alias", "kind": "platform", "sub": "etcd pointer" } ] }, { "title": "Serve", "nodes": [ { "id": "api", "label": "Retrieval API", "kind": "integration", "sub": "p99 180 ms" }, { "id": "lex", "label": "Lexical index", "kind": "store", "sub": "OpenSearch" }, { "id": "acl", "label": "ACL filter", "kind": "security", "sub": "fails closed" } ] } ], "edges": [ { "from": "cache", "to": "gpu", "label": "cache miss", "kind": "sync" }, { "from": "alias", "to": "api", "label": "alias", "kind": "sync" }, { "from": "api", "to": "lex", "label": "fallback", "kind": "error" } ], "note": "The chain is the build path. One contract change rebuilds from the ledger and the retained text, never from the source systems.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "03-actors-and-journeys", "title": "Who the Platform Is For, and What They Get to Do", "layout": "actors", "canvas": { "width": 1740 }, "cardWidth": 268, "groups": [ { "title": "The people the product exists for", "kind": "boundary", "actors": [ { "id": "worker", "label": "Knowledge worker", "sub": "3 M monthly", "goal": "I edited that page an hour ago. When I search for it, I want today's version — not the one from last week.", "journeys": [ { "id": "j-find", "label": "Find what I just wrote", "sub": "most-run journey" }, { "label": "Ask a question of the workspace" }, { "label": "Find documents like this one" } ] }, { "id": "admin", "label": "Tenant admin", "sub": "25,000 organisations", "goal": "When I revoke access to a document, I need it gone from search immediately — not after whatever batch job runs next.", "journeys": [ { "label": "Revoke access to a document" }, { "label": "Erase a departing employee's content" }, { "label": "Prove what the platform holds" } ] } ] }, { "title": "The people who build on the platform", "kind": "boundary", "actors": [ { "id": "prodeng", "label": "Product engineer", "sub": "14 consuming teams", "goal": "I want retrieval over my corpus without learning what a chunker is or owning a GPU node pool.", "journeys": [ { "id": "j-ship", "label": "Ship a new corpus", "sub": "onboarding journey" }, { "label": "Measure my retrieval quality" }, { "label": "Choose a freshness tier" } ] }, { "id": "plateng", "label": "Platform engineer", "sub": "on call for the fleet", "goal": "A better embedding model shipped this week. I want to adopt it without a quarter-long project or a quality regression nobody notices.", "journeys": [ { "id": "j-migrate", "label": "Upgrade the embedding model", "sub": "the hard one" }, { "label": "Rebuild a corrupted index" }, { "label": "Drain a backlog after an outage" } ] }, { "id": "dpo", "label": "Privacy officer", "kind": "security", "sub": "one per region", "goal": "I need to show that an erasure actually happened, in every store, including the ones nobody remembers exist.", "journeys": [ { "label": "Audit an erasure end to end" }, { "label": "Check a vector export control" } ] } ] }, { "title": "Machines in the cast", "kind": "cloud", "actors": [ { "id": "feedbot", "label": "Corpus change feed", "kind": "external", "sub": "8 M versions/day", "goal": "Hand over every edit once, in order per document, and be told plainly when I am being throttled.", "journeys": [ { "label": "Deliver an edit" }, { "label": "Deliver a tombstone" } ] }, { "id": "sweeper", "label": "Reconciliation sweep", "kind": "platform", "sub": "nightly per corpus", "goal": "Find the documents the feed never mentioned, before a user notices they are missing from search.", "journeys": [ { "label": "Compare source to ledger" }, { "label": "Confirm a suspected deletion" } ] }, { "id": "evalbot", "label": "Quality harness", "kind": "platform", "sub": "per corpus, nightly", "goal": "Say whether retrieval got worse, and which of the three causes the evidence actually supports.", "journeys": [ { "label": "Score the frozen query set" }, { "label": "Gate a contract cutover" } ] } ] } ], "note": "Three journeys get their own map: find what I just wrote, ship a new corpus, and upgrade the embedding model.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "04-journey-find-what-i-wrote", "title": "Journey — Find What I Just Wrote", "layout": "journey", "canvas": { "width": 1760 }, "cellWidth": 215, "actor": { "label": "Knowledge worker", "sub": "3 M monthly active", "goal": "Get back to the page I edited an hour ago, by searching for what I remember writing in it", "trigger": "A half-remembered phrase, and no memory of which space the page is in", "success": "The page appears with today's wording, and the citation opens at the paragraph that matched" }, "phases": [ { "title": "Edit", "sub": "in the editor" }, { "title": "Wait", "sub": "unknowingly" }, { "title": "Search", "moment": true }, { "title": "Judge", "moment": true }, { "title": "Open" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Rewrites a section" } ], [ { "label": "Closes the tab" } ], [ { "label": "Types the phrase" } ], [ { "label": "Scans the snippets" } ], [ { "label": "Clicks a citation" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "Change event accepted" }, { "label": "Interactive lane" } ], [ { "label": "Chunk diff: 3 of 41" }, { "label": "Embed + index" } ], [ { "label": "Query embedded" }, { "label": "Hybrid top-50" } ], [ { "label": "ACL filter applied" }, { "label": "Staleness returned" } ], [ { "label": "Offset resolved" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Trusts it", "Unsure", "Gives up" ], "points": [ 0, 1, 1, 2, 0 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "No signal it is indexing" } ], [ { "label": "Old wording still matches" } ], [ { "label": "Snippet is a fragment with no context" } ], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "30 s p50 interactive lane" } ], [ { "label": "Atomic version visibility" } ], [ { "label": "Content-hash diff: only 3 chunks re-embedded" } ], [ { "label": "Heading path carried on the chunk" } ], [ { "label": "Offsets retained at extraction" } ] ] } ], "chain": true, "note": "The trough is phase 4: a correct result the reader cannot interpret. Both answers to it are structural, decided at extraction and chunking.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "05-journey-ship-a-corpus", "title": "Journey — Ship a New Corpus", "layout": "journey", "canvas": { "width": 1760 }, "cellWidth": 215, "actor": { "label": "Product engineer", "sub": "owns one product surface", "goal": "Get retrieval over my corpus into production without owning GPUs, a chunker or an index", "trigger": "A product commitment to ship search over a new content type this quarter", "success": "Retrieval live, quality measured against a query set I wrote, and a cost figure I can defend" }, "phases": [ { "title": "Register", "sub": "self-service" }, { "title": "Backfill", "sub": "bulk lane" }, { "title": "Measure", "moment": true }, { "title": "Launch", "moment": true }, { "title": "Own it" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Declares the corpus" }, { "label": "Picks a freshness tier" } ], [ { "label": "Waits on the backfill" } ], [ { "label": "Writes 80 eval queries" } ], [ { "label": "Switches traffic on" } ], [ { "label": "Watches the dashboard" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "Contract assigned" }, { "label": "Index provisioned" } ], [ { "label": "Bulk lane, preemptible" }, { "label": "Progress + ETA" } ], [ { "label": "Scores recall@10" } ], [ { "label": "Alias published" } ], [ { "label": "Cost + freshness showback" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Uneasy", "Blocked" ], "points": [ 0, 1, 2, 0, 0 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "No ETA, unclear if it is stuck" } ], [ { "label": "Nobody has labelled relevance" } ], [], [ { "label": "Bill arrives without a cause" } ] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Chunker chosen by corpus type" } ], [ { "label": "Oldest-unembedded age as the ETA" } ], [ { "label": "Harness scores an unlabelled set" } ], [ { "label": "Recall gate before the alias flips" } ], [ { "label": "Reuse rate + cache hit as the two cost ratios" } ] ] } ], "chain": true, "note": "The trough is measurement: nobody has ground truth on day one. The platform's answer is a frozen query set with reusable judgements, not a label budget.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "06-journey-upgrade-the-model", "title": "Journey — Upgrade the Embedding Model", "layout": "journey", "canvas": { "width": 1760 }, "cellWidth": 215, "actor": { "label": "Platform engineer", "sub": "on call for the fleet", "goal": "Adopt a better embedding model across 4.8 billion chunks without a quality regression or a quarter-long project", "trigger": "A new open-weights model benchmarks 6 points better on the corpus's own query set", "success": "Every tenant served by the new contract, the old index retired, and a rollback that was never needed but stayed possible" }, "phases": [ { "title": "Evaluate", "sub": "on a sample" }, { "title": "Build beside", "sub": "dual-write" }, { "title": "Gate", "moment": true }, { "title": "Cut over", "moment": true }, { "title": "Retire" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Pins a model digest" }, { "label": "Prices the migration" } ], [ { "label": "Starts the shadow build" } ], [ { "label": "Reads the recall diff" } ], [ { "label": "Flips tenants in waves" } ], [ { "label": "Reclaims the old index" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "New contract registered" }, { "label": "GPU hours estimated" } ], [ { "label": "Both indexes live-fed" }, { "label": "Oldest unmigrated chunk" } ], [ { "label": "Recall@10 vs outgoing" } ], [ { "label": "Alias write per tenant" } ], [ { "label": "21-day rollback window" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "In control", "Exposed", "Cornered" ], "points": [ 0, 1, 1, 2, 0 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Storage doubled for 3 weeks" } ], [ { "label": "Scores not comparable across models" } ], [ { "label": "A half-flipped estate during the wave" } ], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Contract is identity, so the build is separate" } ], [ { "label": "Dual-index overhead is a budgeted capacity state" } ], [ { "label": "Gate on the frozen set, not on raw scores" } ], [ { "label": "No query ever spans two contracts" } ], [ { "label": "Rollback is an alias write, not a rebuild" } ] ] } ], "chain": true, "note": "The trough is cut-over, and it is where the critical design decision earns itself: a per-tenant alias flip is safe only because a query never spans contracts.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "07-layered-architecture", "title": "Embedding Pipeline Service — Layered Architecture", "layout": "bands", "canvas": { "width": 1740 }, "layerHeaderWidth": 158, "bands": [ { "name": "Consumers", "nodes": [ { "id": "l-search", "label": "Workspace search", "kind": "external" }, { "id": "l-assist", "label": "Ask-your-docs", "kind": "external" }, { "id": "l-similar", "label": "Similar documents", "kind": "external" }, { "id": "l-dedupe", "label": "Duplicate detection", "kind": "external" } ] }, { "name": "Serve", "nodes": [ { "id": "l-api", "label": "Retrieval API", "kind": "integration", "sub": "p99 180 ms" }, { "id": "l-qembed", "label": "Query embedding", "kind": "app" }, { "id": "l-hybrid", "label": "Hybrid fusion", "kind": "app", "sub": "vector + lexical" }, { "id": "l-aclf", "label": "ACL filter", "kind": "security", "sub": "fails closed" }, { "id": "l-supp", "label": "Suppression list", "kind": "security", "sub": "5 s revocation" } ] }, { "name": "Index", "nodes": [ { "id": "l-build", "label": "Index builder", "kind": "app" }, { "id": "l-ann", "label": "Vector index", "kind": "store", "sub": "Qdrant" }, { "id": "l-lex", "label": "Lexical index", "kind": "store", "sub": "OpenSearch" }, { "id": "l-alias", "label": "Alias resolver", "kind": "platform", "sub": "etcd" } ] }, { "name": "Embed", "nodes": [ { "id": "l-batch", "label": "Batcher", "kind": "app", "sub": "lane-aware" }, { "id": "l-fleet", "label": "Embedding fleet", "kind": "app", "sub": "Ray Serve, TEI" }, { "id": "l-cache", "label": "Vector cache", "kind": "store", "sub": "Redis" }, { "id": "l-probe", "label": "Reference probe", "kind": "platform", "sub": "frozen set" } ] }, { "name": "Prepare", "nodes": [ { "id": "l-extract", "label": "Extract + OCR", "kind": "app", "sub": "Tika, Tesseract" }, { "id": "l-norm", "label": "Normaliser", "kind": "app", "sub": "versioned" }, { "id": "l-chunk", "label": "Chunker", "kind": "app", "sub": "versioned" }, { "id": "l-ledger", "label": "Chunk ledger", "kind": "store", "sub": "PostgreSQL" }, { "id": "l-text", "label": "Normalised text", "kind": "store", "sub": "MinIO, 30 d" } ] }, { "name": "Capture", "nodes": [ { "id": "l-feed", "label": "Connector workers", "kind": "integration" }, { "id": "l-log", "label": "Change log", "kind": "queue", "sub": "Kafka" }, { "id": "l-sweep", "label": "Reconciliation sweep", "kind": "platform" }, { "id": "l-lane", "label": "Lane admission", "kind": "decision", "sub": "3 lanes" } ] }, { "name": "Corpora", "nodes": [ { "id": "l-docs", "label": "Document service", "kind": "external" }, { "id": "l-files", "label": "File store", "kind": "external" }, { "id": "l-threads", "label": "Comment threads", "kind": "external" }, { "id": "l-saas", "label": "Connected SaaS", "kind": "external" } ] }, { "name": "Control + ops", "accent": "#e1d5e7", "nodes": [ { "id": "l-contract", "label": "Contract registry", "kind": "platform" }, { "id": "l-cat", "label": "Index catalogue", "kind": "platform" }, { "id": "l-qual", "label": "Quality harness", "kind": "platform", "sub": "ClickHouse" }, { "id": "l-id", "label": "Workload identity", "kind": "security", "sub": "SPIRE" }, { "id": "l-obsv", "label": "Observability", "kind": "platform", "sub": "Prometheus" } ] } ], "edges": [ { "from": "l-api", "to": "l-alias", "label": "which index", "kind": "sync" }, { "from": "l-chunk", "to": "l-batch", "label": "changed chunks", "kind": "sync" }, { "from": "l-fleet", "to": "l-build", "label": "vectors", "kind": "sync" }, { "from": "l-log", "to": "l-extract", "label": "accepted change", "kind": "async" } ], "note": "Every layer below Serve may be down without retrieval failing: an index already built keeps serving. That asymmetry is why the control plane carries a lower availability target.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "08-platform-components", "title": "Platform Components — Container View", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Capture and preparation", "kind": "boundary", "dir": "row", "children": [ { "title": "Ingest", "kind": "lane", "nodes": [ { "id": "c-conn", "label": "Connector workers", "kind": "integration", "sub": "per source type" }, { "id": "c-adm", "label": "Admission", "kind": "app", "sub": "quotas, lanes" }, { "id": "c-log", "label": "Change log", "kind": "queue", "sub": "Kafka, 13 mo" }, { "id": "c-sweep", "label": "Reconciliation sweep", "kind": "platform", "sub": "Argo CronWorkflow" } ] }, { "title": "Prepare", "kind": "lane", "nodes": [ { "id": "c-ext", "label": "Extraction workers", "kind": "app", "sub": "sandboxed, bounded" }, { "id": "c-chunk", "label": "Chunker", "kind": "app", "sub": "versioned" }, { "id": "c-text", "label": "Normalised text", "kind": "store", "sub": "MinIO" }, { "id": "c-ledger", "label": "Chunk ledger", "kind": "store", "sub": "PostgreSQL, Patroni" } ] } ] }, { "title": "Embedding and index build", "kind": "boundary", "dir": "row", "children": [ { "title": "Inference", "kind": "lane", "nodes": [ { "id": "c-batch", "label": "Batcher", "kind": "app", "sub": "3 lanes" }, { "id": "c-fleet", "label": "Embedding fleet", "kind": "app", "sub": "KubeRay + TEI" }, { "id": "c-cache", "label": "Vector cache", "kind": "store", "sub": "Redis" }, { "id": "c-probe", "label": "Reference probe", "kind": "platform", "sub": "rollout gate" } ] }, { "title": "Index", "kind": "lane", "nodes": [ { "id": "c-build", "label": "Index builder", "kind": "app", "sub": "per contract" }, { "id": "c-ann", "label": "Vector index", "kind": "store", "sub": "Qdrant" }, { "id": "c-lex", "label": "Lexical index", "kind": "store", "sub": "OpenSearch" }, { "id": "c-snap", "label": "Snapshots", "kind": "store", "sub": "MinIO, RTO 4 h" } ] } ] }, { "title": "Serving", "kind": "boundary", "dir": "row", "children": [ { "title": "Query path", "kind": "lane", "nodes": [ { "id": "c-gw", "label": "Retrieval gateway", "kind": "integration", "sub": "Envoy" }, { "id": "c-qembed", "label": "Query embedder", "kind": "app", "sub": "same contract" }, { "id": "c-fuse", "label": "Hybrid fusion", "kind": "app" }, { "id": "c-cite", "label": "Citation resolver", "kind": "app", "sub": "offsets" } ] }, { "title": "Access control", "kind": "trust", "nodes": [ { "id": "c-acl", "label": "ACL filter", "kind": "security", "sub": "fails closed" }, { "id": "c-supp", "label": "Suppression list", "kind": "security", "sub": "5 s p99" } ] } ] }, { "title": "Control plane", "kind": "trust", "dir": "row", "children": [ { "title": "Authority", "kind": "lane", "nodes": [ { "id": "c-contract", "label": "Contract registry", "kind": "platform", "sub": "PostgreSQL" }, { "id": "c-cat", "label": "Index catalogue", "kind": "platform", "sub": "aliases in etcd" }, { "id": "c-mig", "label": "Migration orchestrator", "kind": "platform", "sub": "Argo Workflows" } ] }, { "title": "Quality and cost", "kind": "lane", "nodes": [ { "id": "c-qual", "label": "Quality harness", "kind": "platform", "sub": "ClickHouse" }, { "id": "c-drift", "label": "Drift monitors", "kind": "platform" }, { "id": "c-cost", "label": "Cost attribution", "kind": "platform", "sub": "showback" } ] } ] } ], "outside": [ { "id": "c-src", "label": "Corpus sources", "kind": "external", "sub": "4 kinds" }, { "id": "c-authz", "label": "Permission authority", "kind": "external", "sub": "query-time" }, { "id": "c-cons", "label": "Product surfaces", "kind": "external", "sub": "14 teams" } ], "edges": [ { "from": "c-src", "to": "c-conn", "label": "change feed", "kind": "async" }, { "from": "c-cons", "to": "c-gw", "label": "retrieval", "kind": "sync" }, { "from": "c-acl", "to": "c-authz", "label": "can read?", "kind": "sync" }, { "from": "c-cat", "to": "c-gw", "label": "alias", "kind": "sync" } ], "note": "Omitted for clarity: the observability stack, secrets distribution, and the warehouse export. Each appears on its own view.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "09-integration-surface", "title": "Integration Surface — Consumers, Platform, Dependencies", "layout": "hub", "canvas": { "width": 1740 }, "left": { "title": "Consumers and producers", "nodes": [ { "id": "h-search", "label": "Workspace search", "kind": "external", "rel": "retrieve" }, { "id": "h-assist", "label": "Ask-your-docs", "kind": "external", "rel": "retrieve" }, { "id": "h-reg", "label": "Product engineer", "kind": "actor", "rel": "register" }, { "id": "h-feed", "label": "Corpus change feed", "kind": "external", "rel": "events", "kind2": "async" } ] }, "centre": { "title": "Embedding Pipeline Service", "nodes": [ { "id": "h-gw", "label": "Retrieval gateway", "kind": "integration", "sub": "Envoy, mTLS" }, { "id": "h-ingest", "label": "Ingest API", "kind": "integration", "sub": "idempotent" }, { "id": "h-ctl", "label": "Control API", "kind": "platform", "sub": "contracts, corpora" } ] }, "right": { "title": "Dependencies", "nodes": [ { "id": "h-authz", "label": "Permission authority", "kind": "external", "rel": "ACL check", "dir": "out" }, { "id": "h-src", "label": "Corpus fetch APIs", "kind": "external", "rel": "fetch", "dir": "out" }, { "id": "h-spire", "label": "SPIRE", "kind": "security", "rel": "SVID", "dir": "out" } ] }, "note": "Three surfaces, three availability targets: retrieval 99.95%, ingest 99.9%, control 99.5%. A consumer never calls the control plane on the hot path. Omitted: OpenBao, the admin console, the observability sink and the nightly warehouse export.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "10-embedding-data-flow", "title": "Data Flow — Edit to Retrievable", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "align": "top", "nodeWidth": 208, "stages": [ { "title": "Source", "nodes": [ { "id": "f-edit", "label": "Document edited", "kind": "external", "sub": "monotonic version" }, { "id": "f-tomb", "label": "Tombstone", "kind": "external", "sub": "deletion" } ] }, { "title": "Landing", "nodes": [ { "id": "f-adm", "label": "Admission", "kind": "decision", "sub": "lane + quota" }, { "id": "f-log", "label": "Change log", "kind": "queue", "sub": "Kafka, keyed by document" } ] }, { "title": "Preparation", "nodes": [ { "id": "f-fetch", "label": "Fetch + extract", "kind": "app", "sub": "bounded sandbox" }, { "id": "f-text", "label": "Normalised text", "kind": "store", "sub": "MinIO, 30 d" }, { "id": "f-chunk", "label": "Chunk + hash", "kind": "app", "sub": "versioned chunker" } ] }, { "title": "Diff and vectorise", "nodes": [ { "id": "f-ledger", "label": "Chunk ledger", "kind": "store", "sub": "PostgreSQL" }, { "id": "f-new", "label": "Changed only", "kind": "decision", "sub": "3 of 41 typical" }, { "id": "f-cache", "label": "Vector cache", "kind": "store", "sub": "hit costs nothing" }, { "id": "f-embed", "label": "Embedding fleet", "kind": "app", "sub": "3,000 chunks/s" } ] }, { "title": "Index", "nodes": [ { "id": "f-up", "label": "Upsert", "kind": "app", "sub": "contract-checked" }, { "id": "f-ann", "label": "Vector index", "kind": "store", "sub": "Qdrant" }, { "id": "f-lex", "label": "Lexical index", "kind": "store", "sub": "OpenSearch" }, { "id": "f-vis", "label": "Version visible", "kind": "decision", "sub": "atomic" } ] }, { "title": "Serve", "nodes": [ { "id": "f-q", "label": "Retrieval", "kind": "integration", "sub": "hybrid" }, { "id": "f-supp", "label": "Suppression", "kind": "security", "sub": "5 s revocation" } ] } ], "edges": [ { "from": "f-tomb", "to": "f-supp", "label": "seconds", "kind": "error", "route": "gutter" }, { "from": "f-text", "to": "f-chunk", "label": "re-chunk", "kind": "sync" } ], "note": "Deletion takes the red path and reaches the read path in seconds; indexing takes the chain and reaches it in minutes. They are deliberately different paths.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "11-storage-zones", "title": "Storage Zones — Ownership and Rebuildability", "layout": "nested", "canvas": { "width": 1740 }, "boxes": [ { "title": "Not ours — systems of record for content and permissions", "kind": "onprem", "dir": "row", "children": [ { "title": "Corpus owners", "kind": "plain", "nodes": [ { "id": "z-doc", "label": "Document service", "kind": "external", "sub": "content + ACLs" }, { "id": "z-file", "label": "File store", "kind": "external", "sub": "bytes" }, { "id": "z-saas", "label": "Connected SaaS", "kind": "external", "sub": "tenant-authorised" } ] } ] }, { "title": "Ours and authoritative — must not lose a write", "kind": "trust", "dir": "row", "children": [ { "title": "Chunk ledger", "kind": "lane", "nodes": [ { "id": "z-ledger", "label": "Chunk ledger", "kind": "store", "sub": "RPO 0, RTO 15 min" } ] }, { "title": "Control", "kind": "lane", "nodes": [ { "id": "z-contract", "label": "Contract registry", "kind": "store", "sub": "RPO 0" }, { "id": "z-cat", "label": "Index catalogue", "kind": "store", "sub": "aliases" } ] }, { "title": "Evidence", "kind": "lane", "nodes": [ { "id": "z-del", "label": "Change + deletion log", "kind": "store", "sub": "append-only, 13 mo" }, { "id": "z-audit", "label": "Audit log", "kind": "store", "sub": "immutable" } ] } ] }, { "title": "Ours and recoverable — re-derivable at a cost", "kind": "boundary", "dir": "row", "children": [ { "title": "Retained input", "kind": "lane", "nodes": [ { "id": "z-text", "label": "Normalised text", "kind": "store", "sub": "30 d, RPO 1 h" } ] }, { "title": "Quality record", "kind": "lane", "nodes": [ { "id": "z-eval", "label": "Evaluation + drift", "kind": "store", "sub": "13 months" } ] } ] }, { "title": "Ours and disposable — rebuildable projections, never backed up as truth", "kind": "cloud", "dir": "row", "children": [ { "title": "Serving indexes", "kind": "lane", "nodes": [ { "id": "z-ann", "label": "Vector indexes", "kind": "store", "sub": "one per contract" }, { "id": "z-lex", "label": "Lexical indexes", "kind": "store" }, { "id": "z-snap", "label": "Index snapshots", "kind": "store", "sub": "RTO 4 h, not truth" } ] }, { "title": "Caches", "kind": "lane", "nodes": [ { "id": "z-vcache", "label": "Vector cache", "kind": "store", "sub": "cost, not correctness" }, { "id": "z-qcache", "label": "Query cache", "kind": "store" } ] } ] } ], "edges": [ { "from": "z-ledger", "to": "z-ann", "label": "rebuilds", "kind": "batch" }, { "from": "z-text", "to": "z-ledger", "label": "re-chunk input", "kind": "batch" }, { "from": "z-del", "to": "z-snap", "label": "replay before serving", "kind": "error" } ], "note": "Snapshots of a rebuildable store exist for recovery time, not durability: a from-scratch rebuild is 3 to 14 days and satisfies no RTO.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "12-data-model", "title": "Data Model — Contract, Document, Chunk, Vector", "layout": "er", "canvas": { "width": 1780, "cols": 5 }, "rowGap": 235, "entities": [ { "id": "tenant", "name": "tenant", "kind": "store", "row": 0, "col": 0, "attrs": [ "tenant_id PK", "residency_region", "quota_chunks", "quota_queries_s" ] }, { "id": "corpus", "name": "corpus", "kind": "store", "row": 0, "col": 1, "attrs": [ "corpus_id PK", "tenant_id FK", "source_kind", "freshness_tier" ] }, { "id": "contract", "name": "embedding_contract", "kind": "store", "row": 0, "col": 2, "attrs": [ "contract_id PK", "normaliser_version", "chunker_version", "model_digest", "pooling_rule", "dimensions" ] }, { "id": "index", "name": "vector_index", "kind": "store", "row": 0, "col": 3, "attrs": [ "index_id PK", "contract_id FK", "tenant_id FK", "state", "built_at" ] }, { "id": "alias", "name": "index_alias", "kind": "integration", "row": 0, "col": 4, "attrs": [ "alias PK", "index_id FK", "previous_index_id", "flipped_at" ] }, { "id": "document", "name": "document", "kind": "store", "row": 1, "col": 0, "attrs": [ "document_id PK", "tenant_id FK", "corpus_id FK", "acl_ref", "deleted_at" ] }, { "id": "docver", "name": "document_version", "kind": "store", "row": 1, "col": 1, "attrs": [ "document_id PK FK", "version PK", "source_version", "visible_at" ] }, { "id": "chunk", "name": "chunk", "kind": "store", "row": 1, "col": 2, "attrs": [ "chunk_hash PK", "document_id FK", "version FK", "ordinal", "offset_start", "heading_path" ] }, { "id": "vector", "name": "vector", "kind": "store", "row": 1, "col": 3, "attrs": [ "chunk_hash PK FK", "contract_id PK FK", "index_id FK", "status" ] }, { "id": "migration", "name": "migration", "kind": "platform", "row": 1, "col": 4, "attrs": [ "migration_id PK", "from_contract FK", "to_contract FK", "oldest_unmigrated", "state" ] }, { "id": "suppress", "name": "suppression", "kind": "security", "row": 2, "col": 0, "attrs": [ "document_id PK FK", "reason", "effective_at", "erasure_confirmed_at" ] }, { "id": "text", "name": "normalised_text", "kind": "store", "row": 2, "col": 1, "attrs": [ "document_id PK FK", "version PK FK", "normaliser_version", "text_uri", "expires_at" ] }, { "id": "quar", "name": "quarantine", "kind": "risk", "row": 2, "col": 2, "attrs": [ "chunk_hash PK FK", "reason", "attempts", "first_seen" ] }, { "id": "evalrun", "name": "evaluation_run", "kind": "store", "row": 2, "col": 4, "attrs": [ "run_id PK", "contract_id FK", "corpus_id FK", "recall_at_10", "mrr" ] } ], "relations": [ { "from": "tenant", "to": "corpus", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "corpus", "to": "contract", "label": "N : 1", "from_side": "e", "to_side": "w" }, { "from": "contract", "to": "index", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "index", "to": "alias", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "tenant", "to": "document", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "document", "to": "docver", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "docver", "to": "chunk", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "chunk", "to": "vector", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "contract", "to": "chunk", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "index", "to": "vector", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "alias", "to": "migration", "label": "N : 1", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "document", "to": "suppress", "label": "1 : 1", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "docver", "to": "text", "label": "1 : 1", "from_side": "s", "to_side": "n" }, { "from": "chunk", "to": "quar", "label": "1 : 1", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "migration", "to": "evalrun", "label": "N : 1", "from_side": "s", "to_side": "n" } ], "note": "A vector's primary key is (chunk_hash, contract_id). That compound key is the critical design decision written as a constraint.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "13-edit-to-retrievable-sequence", "title": "Critical Flow — One Edit Becomes Retrievable", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "src", "label": "Document service", "kind": "external" }, { "id": "ing", "label": "Ingest + admission", "kind": "integration" }, { "id": "log", "label": "Change log", "kind": "queue" }, { "id": "prep", "label": "Extract + chunk", "kind": "app" }, { "id": "led", "label": "Chunk ledger", "kind": "store" }, { "id": "emb", "label": "Embedding fleet", "kind": "app" }, { "id": "idx", "label": "Vector index", "kind": "store" }, { "id": "api", "label": "Retrieval API", "kind": "integration" } ], "messages": [ { "from": "src", "to": "ing", "label": "changed(doc, v41)", "kind": "async" }, { "from": "ing", "to": "ing", "label": "quota + lane: interactive", "kind": "self" }, { "from": "ing", "to": "log", "label": "append, key = doc", "kind": "call" }, { "from": "log", "to": "ing", "label": "durable", "kind": "return" }, { "from": "ing", "to": "src", "label": "202 accepted", "kind": "return" }, { "from": "log", "to": "prep", "label": "deliver, in order per doc", "kind": "async" }, { "from": "prep", "to": "led", "label": "version 40 known?", "kind": "call" }, { "from": "led", "to": "prep", "label": "41 chunk hashes", "kind": "return" }, { "from": "prep", "to": "prep", "label": "extract, normalise, chunk", "kind": "self" }, { "from": "prep", "to": "led", "label": "diff: 3 new, 38 reused", "kind": "call" }, { "from": "prep", "to": "emb", "label": "embed 3 chunks", "kind": "call" }, { "from": "emb", "to": "emb", "label": "cache miss, batch, infer", "kind": "self" }, { "from": "emb", "to": "idx", "label": "upsert 3 vectors", "kind": "call" }, { "from": "idx", "to": "led", "label": "mark v41 complete", "kind": "call" }, { "from": "led", "to": "idx", "label": "flip visible version", "kind": "return" }, { "from": "api", "to": "idx", "label": "search (contract-scoped)", "kind": "call" }, { "from": "idx", "to": "api", "label": "v41 chunks + staleness", "kind": "return" } ], "note": "Message 15 is the atomicity point: until all three vectors land, retrieval keeps returning version 40 rather than a half-updated document.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "14-embedding-pipeline", "title": "Embedding Pipeline — Lanes, Batching and GPU Economics", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "top", "nodeWidth": 212, "stages": [ { "title": "Work arrives", "nodes": [ { "id": "p-int", "label": "Interactive lane", "kind": "queue", "sub": "p50 30 s" }, { "id": "p-std", "label": "Standard lane", "kind": "queue", "sub": "p95 30 min" }, { "id": "p-bulk", "label": "Bulk lane", "kind": "queue", "sub": "no SLO" } ] }, { "title": "Deduplicate", "nodes": [ { "id": "p-cache", "label": "Vector cache", "kind": "store", "sub": "hash x contract" }, { "id": "p-hit", "label": "Hit: no GPU", "kind": "opportunity", "sub": "reverted edits, templates" } ] }, { "title": "Batch", "nodes": [ { "id": "p-batch", "label": "Batcher", "kind": "app", "sub": "size + max wait" }, { "id": "p-trade", "label": "Wait vs cost", "kind": "decision", "sub": "declared per lane" } ] }, { "title": "Infer", "nodes": [ { "id": "p-ray", "label": "Ray Serve", "kind": "app", "sub": "autoscaled replicas" }, { "id": "p-tei", "label": "Model replicas", "kind": "app", "sub": "pinned by digest" }, { "id": "p-gpu", "label": "GPU node pool", "kind": "platform", "sub": "preemptible for bulk" } ] }, { "title": "Guard", "nodes": [ { "id": "p-probe", "label": "Reference probe", "kind": "platform", "sub": "frozen chunk set" }, { "id": "p-trunc", "label": "Over-length rule", "kind": "decision", "sub": "split, recorded" }, { "id": "p-quar", "label": "Quarantine", "kind": "risk", "sub": "typed reason" } ] }, { "title": "Persist", "nodes": [ { "id": "p-write", "label": "Cache write", "kind": "store" }, { "id": "p-idx", "label": "Index upsert", "kind": "store", "sub": "contract-checked" } ] } ], "edges": [ { "from": "p-int", "to": "p-cache", "label": "priority", "kind": "sync" }, { "from": "p-bulk", "to": "p-gpu", "label": "preempted first", "kind": "error", "route": "gutter" }, { "from": "p-tei", "to": "p-quar", "label": "repeat failure", "kind": "error" }, { "from": "p-probe", "to": "p-tei", "label": "gates rollout", "kind": "sync" } ], "note": "Background re-embedding runs at 4,000 chunks/s for a 14-day rebuild; surge runs at 18,500 for 72 hours at about 4.5x the hourly cost.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "15-retrieval-paths", "title": "Retrieval Paths by Consumer and Degradation State", "layout": "swimlane", "canvas": { "width": 1780 }, "laneHeaderWidth": 172, "stages": [ "Resolve", "Embed query", "Candidate search", "Filter", "Return" ], "lanes": [ { "title": "Workspace search", "cells": [ [ { "label": "Alias -> index", "kind": "platform" } ], [ { "label": "Same contract as index", "kind": "app" } ], [ { "label": "Vector top-200", "kind": "store" }, { "label": "Lexical top-200", "kind": "store" } ], [ { "label": "ACL filter", "kind": "security" }, { "label": "Suppression", "kind": "security" } ], [ { "label": "Top-50 + staleness", "kind": "integration" } ] ] }, { "title": "Ask-your-docs", "cells": [ [ { "label": "Alias -> index", "kind": "platform" } ], [ { "label": "Query embedded", "kind": "app" } ], [ { "label": "Vector top-200", "kind": "store" } ], [ { "label": "ACL filter", "kind": "security" } ], [ { "label": "Top-10 + offsets", "kind": "integration" } ] ] }, { "title": "Similar documents", "cells": [ [ { "label": "Alias -> index", "kind": "platform" } ], [ { "label": "Reuse stored vector", "kind": "opportunity" } ], [ { "label": "Neighbours, self excluded", "kind": "store" } ], [ { "label": "ACL filter", "kind": "security" } ], [ { "label": "Ranked documents", "kind": "integration" } ] ] }, { "title": "Vector tier degraded", "cells": [ [ { "label": "Alias still resolves", "kind": "platform" } ], [ { "label": "Skipped", "kind": "risk" } ], [ { "label": "Lexical only", "kind": "store" } ], [ { "label": "ACL filter unchanged", "kind": "security" } ], [ { "label": "degraded = true", "kind": "risk" } ] ] }, { "title": "ACL authority down", "cells": [ [ { "label": "Alias resolves", "kind": "platform" } ], [ { "label": "Query embedded", "kind": "app" } ], [ { "label": "Candidates found", "kind": "store" } ], [ { "label": "Fail closed", "kind": "risk" } ], [ { "label": "withheld = N", "kind": "risk" } ] ] } ], "note": "Four degradations are available before an error: no re-rank, no vectors, no cache, sampled telemetry. Only a permission failure withholds results.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "16-contract-migration", "title": "Critical Flow — Contract Migration and Alias Cutover", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "eng", "label": "Platform engineer", "kind": "actor" }, { "id": "reg", "label": "Contract registry", "kind": "platform" }, { "id": "orch", "label": "Migration orchestrator", "kind": "platform" }, { "id": "emb", "label": "Embedding fleet", "kind": "app" }, { "id": "new", "label": "Shadow index", "kind": "store" }, { "id": "old", "label": "Serving index", "kind": "store" }, { "id": "cat", "label": "Index catalogue", "kind": "platform" }, { "id": "qual", "label": "Quality harness", "kind": "platform" } ], "messages": [ { "from": "eng", "to": "reg", "label": "register contract C2 (digest pinned)", "kind": "call" }, { "from": "reg", "to": "orch", "label": "price: chunks, GPU hours, storage", "kind": "call" }, { "from": "orch", "to": "eng", "label": "estimate: 14 d / 72 h, +60% storage", "kind": "return" }, { "from": "eng", "to": "orch", "label": "accept, background rate", "kind": "call" }, { "from": "orch", "to": "new", "label": "provision index under C2", "kind": "call" }, { "from": "orch", "to": "orch", "label": "enable dual-write", "kind": "self" }, { "from": "orch", "to": "emb", "label": "re-embed ledger under C2", "kind": "async" }, { "from": "emb", "to": "new", "label": "vectors (C2 only)", "kind": "async" }, { "from": "orch", "to": "old", "label": "live edits continue", "kind": "async" }, { "from": "orch", "to": "eng", "label": "oldest unmigrated chunk: 4 h", "kind": "return" }, { "from": "orch", "to": "qual", "label": "score frozen set on C2", "kind": "call" }, { "from": "qual", "to": "orch", "label": "recall@10 +4.1 pts: gate passed", "kind": "return" }, { "from": "orch", "to": "cat", "label": "flip alias, tenant wave 1", "kind": "call" }, { "from": "cat", "to": "orch", "label": "serving C2", "kind": "return" }, { "from": "qual", "to": "eng", "label": "regression on wave 1?", "kind": "call" }, { "from": "eng", "to": "cat", "label": "flip back (alias write)", "kind": "error" }, { "from": "orch", "to": "old", "label": "retire after 21 days", "kind": "batch" } ], "note": "Message 16 is the whole payoff: rollback is one alias write against an index that is still built, still fed and still measured.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "17-deployment-architecture", "title": "Deployment — Kubernetes, Node Pools and Failure Domains", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Region eu-west — primary write region", "kind": "cloud", "dir": "row", "children": [ { "title": "Zone A", "kind": "boundary", "nodes": [ { "id": "d-api-a", "label": "Retrieval API", "kind": "app", "sub": "HPA on p99" }, { "id": "d-prep-a", "label": "Extract + chunk", "kind": "app", "sub": "CPU pool" }, { "id": "d-qd-a", "label": "Qdrant shard", "kind": "store", "sub": "replica 1" }, { "id": "d-kafka-a", "label": "Kafka broker", "kind": "queue" } ] }, { "title": "Zone B", "kind": "boundary", "nodes": [ { "id": "d-api-b", "label": "Retrieval API", "kind": "app" }, { "id": "d-gpu-b", "label": "GPU node pool", "kind": "platform", "sub": "KubeRay, on-demand" }, { "id": "d-qd-b", "label": "Qdrant shard", "kind": "store", "sub": "replica 2" }, { "id": "d-pg-b", "label": "PostgreSQL primary", "kind": "store", "sub": "Patroni" } ] }, { "title": "Zone C", "kind": "boundary", "nodes": [ { "id": "d-api-c", "label": "Retrieval API", "kind": "app" }, { "id": "d-gpu-c", "label": "GPU spot pool", "kind": "platform", "sub": "bulk lane only" }, { "id": "d-os-c", "label": "OpenSearch node", "kind": "store" }, { "id": "d-pg-c", "label": "PostgreSQL standby", "kind": "store", "sub": "synchronous" } ] } ] }, { "title": "Shared regional services", "kind": "boundary", "dir": "row", "children": [ { "title": "Object storage", "kind": "lane", "nodes": [ { "id": "d-minio", "label": "MinIO", "kind": "store", "sub": "erasure-coded" }, { "id": "d-snap", "label": "Index snapshots", "kind": "store", "sub": "RTO 4 h" } ] }, { "title": "Control plane", "kind": "trust", "nodes": [ { "id": "d-etcd", "label": "etcd", "kind": "platform", "sub": "aliases" }, { "id": "d-argo", "label": "Argo Workflows", "kind": "platform", "sub": "rebuilds" }, { "id": "d-bao", "label": "OpenBao", "kind": "security" }, { "id": "d-spire", "label": "SPIRE server", "kind": "security" } ] }, { "title": "Telemetry", "kind": "lane", "nodes": [ { "id": "d-prom", "label": "Prometheus + Mimir", "kind": "platform" }, { "id": "d-ch", "label": "ClickHouse", "kind": "store", "sub": "quality + drift" } ] } ] }, { "title": "Region us-east — read standby (Phase 3)", "kind": "cloud", "dir": "row", "children": [ { "title": "Read serving", "kind": "boundary", "nodes": [ { "id": "d-api-r", "label": "Retrieval API", "kind": "app", "sub": "read only" }, { "id": "d-qd-r", "label": "Qdrant replica", "kind": "store", "sub": "snapshot shipped" }, { "id": "d-pg-r", "label": "PostgreSQL replica", "kind": "store", "sub": "async" } ] } ] } ], "outside": [ { "id": "d-lb", "label": "Global load balancer", "kind": "integration", "sub": "latency routing" } ], "edges": [ { "from": "d-lb", "to": "d-api-a", "label": "retrieval", "kind": "sync" }, { "from": "d-minio", "to": "d-snap", "label": "snapshots", "kind": "batch" }, { "from": "d-gpu-c", "to": "d-gpu-b", "label": "spills on preemption", "kind": "error" } ], "note": "Corpus sources reach Kafka through the ingest API on view 9. The bulk lane runs only on spot GPU capacity: a cheaper rebuild is a longer dual-write window.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "18-observability", "title": "Observability — Signals by Pipeline Stage", "layout": "grid", "canvas": { "width": 1780 }, "laneHeaderWidth": 176, "stages": [ "Capture", "Prepare", "Embed", "Index", "Serve", "Contract" ], "lanes": [ { "title": "The SLO signal", "cells": [ [ { "label": "Feed lag per corpus", "kind": "platform" } ], [ { "label": "Oldest unprepared age", "kind": "platform" } ], [ { "label": "Oldest unembedded age", "kind": "platform" } ], [ { "label": "Version visibility lag", "kind": "platform" } ], [ { "label": "Retrieval p99", "kind": "platform" } ], [ { "label": "Oldest unmigrated chunk", "kind": "platform" } ] ] }, { "title": "Cost ratios", "cells": [ [ { "label": "Events accepted / dropped", "kind": "app" } ], [ { "label": "Chunk reuse rate", "kind": "opportunity" } ], [ { "label": "Cache hit rate", "kind": "opportunity" }, { "label": "GPU batch efficiency", "kind": "app" } ], [ { "label": "Bytes per million chunks", "kind": "app" } ], [ { "label": "Cost per million queries", "kind": "app" } ], [ { "label": "Migration burn vs estimate", "kind": "app" } ] ] }, { "title": "Correctness", "cells": [ [ { "label": "Sweep discrepancies", "kind": "risk" } ], [ { "label": "Unextractable rate", "kind": "risk" } ], [ { "label": "Quarantined chunks", "kind": "risk" } ], [ { "label": "Orphaned vectors", "kind": "risk" } ], [ { "label": "Results withheld", "kind": "risk" } ], [ { "label": "Contract mismatch rejects", "kind": "risk" } ] ] }, { "title": "Quality", "cells": [ [], [ { "label": "Chunks per document", "kind": "store" } ], [ { "label": "Vector norm + centroid drift", "kind": "store" } ], [ { "label": "Index recall vs brute force", "kind": "store" } ], [ { "label": "Query distribution drift", "kind": "store" }, { "label": "Answer acceptance", "kind": "store" } ], [ { "label": "Recall@10 by contract", "kind": "store" } ] ] }, { "title": "Compliance", "cells": [ [ { "label": "Tombstones received", "kind": "security" } ], [ { "label": "Text retention age", "kind": "security" } ], [], [ { "label": "Erasure unconfirmed", "kind": "security" } ], [ { "label": "Operator text access", "kind": "security" } ], [ { "label": "Alias flips audited", "kind": "security" } ] ] } ], "note": "Oldest-unembedded age is both the freshness SLI and the autoscaling trigger. A queue depth would not distinguish a big backlog from a stuck one.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "19-contract-lifecycle", "title": "Contract Lifecycle — The Loop That Keeps Retrieval Current", "layout": "cycle", "canvas": { "width": 1600 }, "centre": { "label": "Embedding contract" }, "nodes": [ { "id": "y-prop", "label": "Proposed", "kind": "decision", "sub": "digest pinned, priced" }, { "id": "y-build", "label": "Building", "kind": "app", "sub": "shadow index, dual-write" }, { "id": "y-gate", "label": "Gated", "kind": "platform", "sub": "recall@10 vs outgoing" }, { "id": "y-serve", "label": "Serving", "kind": "store", "sub": "alias points here" }, { "id": "y-super", "label": "Superseded", "kind": "platform", "sub": "21-day rollback window" }, { "id": "y-ret", "label": "Retired", "kind": "external", "sub": "storage reclaimed in 7 d" }, { "id": "y-drift", "label": "Under watch", "kind": "risk", "sub": "drift + query shift" } ], "ringLabels": [ "accepted estimate", "completeness + oldest chunk", "alias flip, per tenant", "a better model, or drift", "no rollback needed", "evidence kept 13 months", "proposes the next contract" ], "rx": 430, "ry": 215, "note": "The loop closes through drift: the signal that retires a contract is the same signal that justifies proposing its replacement.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "20-security-trust-zones", "title": "Security — Trust Zones and What Crosses Them", "layout": "zones", "canvas": { "width": 1740 }, "zones": [ { "title": "Untrusted — tenant content, before any parsing", "kind": "trust", "nodes": [ { "id": "s-bytes", "label": "Uploaded bytes", "kind": "risk", "sub": "PDFs, archives, images" }, { "id": "s-saas", "label": "Connected SaaS payloads", "kind": "risk" }, { "id": "s-q", "label": "Consumer queries", "kind": "external" } ] }, { "title": "Sandbox — least-trusted compute in the system", "kind": "trust", "nodes": [ { "id": "s-ext", "label": "Extraction workers", "kind": "app", "sub": "no egress, bounded" }, { "id": "s-ocr", "label": "OCR workers", "kind": "app", "sub": "time + memory capped" }, { "id": "s-quar", "label": "Quarantine", "kind": "risk", "sub": "typed failure" } ] }, { "title": "Platform — mTLS between every workload", "kind": "trust", "nodes": [ { "id": "s-gw", "label": "Retrieval gateway", "kind": "integration", "sub": "Envoy, SVID" }, { "id": "s-chunk", "label": "Chunker", "kind": "app" }, { "id": "s-emb", "label": "Embedding fleet", "kind": "app" }, { "id": "s-spire", "label": "SPIRE", "kind": "security" }, { "id": "s-bao", "label": "OpenBao", "kind": "security", "sub": "source credentials" } ] }, { "title": "Data — tenant-scoped keys, no operator read path", "kind": "trust", "nodes": [ { "id": "s-text", "label": "Normalised text", "kind": "store", "sub": "tenant key" }, { "id": "s-vec", "label": "Vectors + indexes", "kind": "store", "sub": "tenant key" }, { "id": "s-led", "label": "Chunk ledger", "kind": "store" }, { "id": "s-aud", "label": "Audit log", "kind": "store", "sub": "immutable" } ] }, { "title": "Authority — never inside our trust boundary", "kind": "trust", "nodes": [ { "id": "s-acl", "label": "Permission authority", "kind": "external", "sub": "query-time ACL" }, { "id": "s-kc", "label": "Keycloak", "kind": "security", "sub": "human identity" } ] } ], "edges": [ { "from": "s-bytes", "to": "s-ext", "label": "fetched bytes only", "kind": "sync" }, { "from": "s-ext", "to": "s-quar", "label": "bound exceeded", "kind": "error" }, { "from": "s-ext", "to": "s-text", "label": "normalised text", "kind": "sync" }, { "from": "s-gw", "to": "s-acl", "label": "ACL check", "kind": "sync" }, { "from": "s-gw", "to": "s-vec", "label": "tenant partition", "kind": "sync" }, { "from": "s-bao", "to": "s-chunk", "label": "short-lived cred", "kind": "sync" } ], "note": "A vector is recoverable information about its source text, so it sits in the data zone under the same controls as the document, not in a cache tier.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "21-retrieval-authorisation", "title": "Identity and Access — Authorising One Retrieval", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "user", "label": "Knowledge worker", "kind": "actor" }, { "id": "prod", "label": "Product surface", "kind": "external" }, { "id": "gw", "label": "Retrieval gateway", "kind": "integration" }, { "id": "spire", "label": "SPIRE", "kind": "security" }, { "id": "idx", "label": "Vector index", "kind": "store" }, { "id": "supp", "label": "Suppression list", "kind": "security" }, { "id": "acl", "label": "Permission authority", "kind": "external" }, { "id": "aud", "label": "Audit log", "kind": "store" } ], "messages": [ { "from": "user", "to": "prod", "label": "search \"renewal terms\"", "kind": "call" }, { "from": "prod", "to": "gw", "label": "retrieve(tenant, subject, query)", "kind": "call" }, { "from": "gw", "to": "spire", "label": "verify caller SVID", "kind": "call" }, { "from": "spire", "to": "gw", "label": "workload = search-api", "kind": "return" }, { "from": "gw", "to": "gw", "label": "tenant claim -> partition", "kind": "self" }, { "from": "gw", "to": "idx", "label": "search in tenant partition", "kind": "call" }, { "from": "idx", "to": "gw", "label": "200 candidate chunks", "kind": "return" }, { "from": "gw", "to": "supp", "label": "any suppressed documents?", "kind": "call" }, { "from": "supp", "to": "gw", "label": "drop 3 (revoked 4 s ago)", "kind": "return" }, { "from": "gw", "to": "acl", "label": "batch check 90 documents", "kind": "call" }, { "from": "acl", "to": "gw", "label": "readable: 41", "kind": "return" }, { "from": "acl", "to": "gw", "label": "timeout -> withhold all", "kind": "error" }, { "from": "gw", "to": "aud", "label": "withheld count, not content", "kind": "async" }, { "from": "gw", "to": "prod", "label": "top-50 of 41 + withheld = N", "kind": "return" }, { "from": "prod", "to": "user", "label": "results with citations", "kind": "return" } ], "note": "Message 12 is the fail-closed branch: an unreachable authority withholds everything and says so, rather than returning what the index happens to hold.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } }, { "id": "22-failure-classes", "title": "Assurance — Failure Classes and Their Handling", "layout": "grid", "canvas": { "width": 1780 }, "laneHeaderWidth": 176, "stages": [ "What fails", "How it shows", "Immediate handling", "Recovery", "Residual risk" ], "lanes": [ { "title": "Model endpoint drift", "cells": [ [ { "label": "Same name, new weights", "kind": "risk" } ], [ { "label": "Reference probe diverges", "kind": "platform" } ], [ { "label": "Rollout failed, not pipeline", "kind": "decision" } ], [ { "label": "Pinned digest re-pulled", "kind": "app" } ], [ { "label": "Undetected if probe set is unrepresentative", "kind": "risk" } ] ] }, { "title": "Partial document", "cells": [ [ { "label": "Some chunks fail", "kind": "risk" } ], [ { "label": "Version never marked visible", "kind": "platform" } ], [ { "label": "Previous version serves", "kind": "decision" } ], [ { "label": "Retry, then quarantine", "kind": "app" } ], [ { "label": "A document stuck one version behind", "kind": "risk" } ] ] }, { "title": "Change-feed gap", "cells": [ [ { "label": "Events never emitted", "kind": "risk" } ], [ { "label": "Sweep finds discrepancy", "kind": "platform" } ], [ { "label": "Re-ingest from source", "kind": "decision" } ], [ { "label": "Ledger reconciled", "kind": "app" } ], [ { "label": "Up to one sweep interval blind", "kind": "risk" } ] ] }, { "title": "Bad index build", "cells": [ [ { "label": "Low recall or corrupt", "kind": "risk" } ], [ { "label": "Recall gate fails", "kind": "platform" } ], [ { "label": "Alias never flips", "kind": "decision" } ], [ { "label": "Rebuild or roll back", "kind": "app" } ], [ { "label": "Gate blind to a shift the frozen set misses", "kind": "risk" } ] ] }, { "title": "Vector tier down", "cells": [ [ { "label": "Qdrant unavailable", "kind": "risk" } ], [ { "label": "Retrieval errors spike", "kind": "platform" } ], [ { "label": "Lexical fallback, flagged", "kind": "decision" } ], [ { "label": "Snapshot restore, RTO 4 h", "kind": "app" } ], [ { "label": "Semantic recall lost while degraded", "kind": "risk" } ] ] }, { "title": "Erasure not confirmed", "cells": [ [ { "label": "A store does not ack", "kind": "risk" } ], [ { "label": "Unconfirmed erasure alert", "kind": "platform" } ], [ { "label": "Suppressed on read path", "kind": "decision" } ], [ { "label": "Retry to completion", "kind": "app" } ], [ { "label": "Bytes present though unreachable", "kind": "risk" } ] ] }, { "title": "Backlog after an outage", "cells": [ [ { "label": "Hours of edits queued", "kind": "risk" } ], [ { "label": "Oldest-unembedded age climbs", "kind": "platform" } ], [ { "label": "Lane priority, bulk shed", "kind": "decision" } ], [ { "label": "Surge GPU capacity", "kind": "app" } ], [ { "label": "Standard lane breaches while draining", "kind": "risk" } ] ] } ], "note": "Six more classes are named in the requirement and handled the same way. These seven are the ones that have changed a design decision.", "meta": { "v": "1.0", "owner": "Data & AI Platform Architecture", "date": "2026-10" } } ]