[ { "id": "01-system-context", "title": "Enterprise Metadata Platform — System Context", "layout": "context", "canvas": { "width": 1620 }, "colWidth": 260, "system": { "label": "Enterprise Metadata Platform", "sub": "Active metadata hub" }, "groups": [ { "side": "top", "title": "Metadata sources — 200 registered systems", "nodes": [ { "id": "c-wh", "label": "Warehouses & Lakehouses", "sub": "Snowflake · Databricks", "kind": "external", "rel": "schema · usage", "dir": "in", "kind2": "batch" }, { "id": "c-bi", "label": "BI & Reporting", "sub": "Power BI · Tableau", "kind": "external", "rel": "reports · fields", "dir": "in", "kind2": "batch", "icon": "powerbi" }, { "id": "c-etl", "label": "ETL / Orchestration", "sub": "dbt · Airflow · Spark", "kind": "external", "rel": "run lineage", "dir": "in", "kind2": "async" }, { "id": "c-app", "label": "SaaS & Operational Apps", "sub": "SAP · Salesforce", "kind": "external", "rel": "object metadata", "dir": "in", "kind2": "batch" } ] }, { "side": "left", "title": "People", "nodes": [ { "id": "c-con", "label": "Data Consumer", "sub": "analyst · scientist", "kind": "actor", "rel": "find and trust data", "dir": "in" }, { "id": "c-ste", "label": "Data Steward", "sub": "curates a domain", "kind": "actor", "rel": "curate · classify", "dir": "in" }, { "id": "c-own", "label": "Domain Owner", "sub": "accountable", "kind": "actor", "rel": "certify · approve", "dir": "in" }, { "id": "c-gov", "label": "Governance & Security", "sub": "CDO · CISO office", "kind": "actor", "rel": "policy · audit", "dir": "in" }, { "id": "c-eng", "label": "Data Engineer", "sub": "platform team", "kind": "actor", "rel": "integrate · automate", "dir": "in" } ] }, { "side": "right", "title": "Metadata consumers", "nodes": [ { "id": "c-pep", "label": "Policy Enforcement", "sub": "Unity Catalog · Ranger", "kind": "external", "rel": "labels · tags", "kind2": "sync" }, { "id": "c-dq", "label": "Data Quality Platform", "sub": "Soda · Great Expectations", "kind": "external", "rel": "scope · results", "kind2": "bidirectional" }, { "id": "c-itsm", "label": "ITSM & Incident", "sub": "ServiceNow", "kind": "external", "rel": "impact tickets", "kind2": "async" }, { "id": "c-ide", "label": "Notebooks & IDEs", "sub": "embedded lookup", "kind": "external", "rel": "GraphQL", "kind2": "sync" } ] }, { "side": "bottom", "title": "Enterprise platforms", "nodes": [ { "id": "c-idp", "label": "Identity Provider", "sub": "Entra ID · SCIM", "kind": "security", "rel": "OIDC · groups", "dir": "in" }, { "id": "c-not", "label": "Notification", "sub": "Teams · email", "kind": "external", "rel": "tasks · alerts", "kind2": "async" }, { "id": "c-obs", "label": "Observability", "sub": "Prometheus · Grafana", "kind": "platform", "rel": "metrics · traces", "kind2": "async" } ] } ], "note": "Out of scope: running quality tests, enforcing access at query time, and storing data values.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "02-high-level-architecture", "title": "High-Level Architecture — Source to Consumption", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "align": "top", "stages": [ { "title": "Data & tool estate", "nodes": [ { "id": "h-wh", "label": "Warehouses & lakes", "kind": "external" }, { "id": "h-bi", "label": "BI & reporting", "kind": "external" }, { "id": "h-pipe", "label": "Pipelines & jobs", "kind": "external" }, { "id": "h-saas", "label": "SaaS & apps", "kind": "external" } ] }, { "title": "Harvest", "nodes": [ { "id": "h-conn", "label": "Connector Fleet", "sub": "40 types", "kind": "integration" }, { "id": "h-agt", "label": "Ingestion Agents", "sub": "run in-VPC", "kind": "integration" }, { "id": "h-evt", "label": "Event Listeners", "sub": "webhook · CDC", "kind": "queue" }, { "id": "h-orc", "label": "Harvest Orchestrator", "sub": "watermarks", "kind": "app" } ] }, { "title": "Process & enrich", "nodes": [ { "id": "h-nrm", "label": "Canonical Mapper", "kind": "app" }, { "id": "h-cls", "label": "Classify & Profile", "sub": "rules + ML", "kind": "app" }, { "id": "h-lin", "label": "Lineage Parser", "sub": "column level", "kind": "app" }, { "id": "h-mrg", "label": "Merge & Precedence", "sub": "curated wins", "kind": "app" } ] }, { "title": "Metadata repository", "nodes": [ { "id": "h-asp", "label": "Aspect Store", "sub": "PostgreSQL", "kind": "store" }, { "id": "h-kg", "label": "Knowledge Graph", "sub": "Neo4j", "kind": "store" }, { "id": "h-idx", "label": "Search Index", "sub": "OpenSearch", "kind": "store" }, { "id": "h-raw", "label": "Payload Archive", "sub": "S3 · replay", "kind": "store" } ] }, { "title": "Access", "nodes": [ { "id": "h-gql", "label": "GraphQL API", "sub": "primary read", "kind": "integration" }, { "id": "h-rst", "label": "REST API", "sub": "write · bulk", "kind": "integration" }, { "id": "h-out", "label": "Event Egress", "sub": "Kafka · webhook", "kind": "queue" }, { "id": "h-sync", "label": "Policy Sync", "sub": "outbound tags", "kind": "integration" } ] }, { "title": "Experience", "nodes": [ { "id": "h-cat", "label": "Catalog & Search", "kind": "app" }, { "id": "h-gls", "label": "Glossary Workbench", "kind": "app" }, { "id": "h-lex", "label": "Lineage Explorer", "kind": "app" }, { "id": "h-gcn", "label": "Governance Console", "kind": "app" } ] } ], "note": "Identity, workflow, audit, notification and observability are cross-cutting; see views 03 and 04.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "03-layered-architecture", "title": "Layered Architecture", "layout": "bands", "canvas": { "width": 1700 }, "layerHeaderWidth": 170, "bands": [ { "name": "Experience", "nodes": [ { "label": "Catalog & Search UI", "kind": "app" }, { "label": "Asset Profile", "kind": "app" }, { "label": "Glossary Workbench", "kind": "app" }, { "label": "Lineage Explorer", "kind": "app" }, { "label": "Stewardship Inbox", "kind": "app" }, { "label": "Governance Console", "kind": "app" } ] }, { "name": "Access", "nodes": [ { "label": "GraphQL API", "kind": "integration" }, { "label": "REST API", "kind": "integration" }, { "label": "Event Egress", "kind": "queue", "icon": "kafka" }, { "label": "Bulk Import / Export", "kind": "integration" }, { "label": "SDK & CLI", "kind": "integration" } ] }, { "name": "Metadata services", "nodes": [ { "label": "Search Service", "kind": "app", "icon": "fa5_search" }, { "label": "Lineage Service", "kind": "app" }, { "label": "Glossary Service", "kind": "app" }, { "label": "Classification Service", "kind": "app" }, { "label": "Quality Metadata", "kind": "app" }, { "label": "Workflow Engine", "kind": "app" } ] }, { "name": "Core platform", "nodes": [ { "label": "Model & Schema Registry", "kind": "app" }, { "label": "Merge & Precedence", "kind": "app" }, { "label": "Versioning & History", "kind": "app" }, { "label": "Entitlement Filter", "kind": "security" } ] }, { "name": "Persistence", "nodes": [ { "label": "Aspect Store", "kind": "store" }, { "label": "Knowledge Graph", "kind": "store" }, { "label": "Search Index", "kind": "store" }, { "label": "Audit Log", "kind": "security" }, { "label": "Payload Archive", "kind": "store" } ] }, { "name": "Ingestion", "nodes": [ { "label": "Connector Framework", "kind": "integration" }, { "label": "Harvest Orchestrator", "kind": "app" }, { "label": "Metadata Event Bus", "kind": "queue" }, { "label": "Enrichment Workers", "kind": "app" }, { "label": "Drift Detector", "kind": "app" } ] }, { "name": "Cross-cutting", "nodes": [ { "label": "Identity & Access", "kind": "security", "icon": "fa5_shield_alt" }, { "label": "Secrets & Keys", "kind": "security" }, { "label": "Notification", "kind": "app" }, { "label": "Observability", "kind": "platform" }, { "label": "Backup & DR", "kind": "platform" } ] } ], "note": "A layer calls only the layer below it. Ingestion reaches persistence through the core platform, never directly.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "04-container-architecture", "title": "Container Architecture (C4 Level 2)", "layout": "nested", "canvas": { "width": 1740 }, "boxes": [ { "title": "Enterprise Metadata Platform · Kubernetes", "kind": "cloud", "dir": "col", "children": [ { "title": "Experience", "kind": "boundary", "nodes": [ { "id": "n-ui", "label": "Catalog Web App", "sub": "React SPA", "kind": "app" }, { "id": "n-gov", "label": "Governance Console", "sub": "React SPA", "kind": "app" } ] }, { "title": "Access", "kind": "boundary", "nodes": [ { "id": "n-gw", "label": "API Gateway", "sub": "OIDC · rate limit", "kind": "integration" }, { "id": "n-gql", "label": "GraphQL Service", "sub": "graph-shaped read", "kind": "integration" }, { "id": "n-rst", "label": "REST Service", "sub": "write · bulk", "kind": "integration" }, { "id": "n-egr", "label": "Event Egress", "sub": "Kafka · webhook", "kind": "queue" } ] }, { "title": "Discovery services", "kind": "boundary", "nodes": [ { "id": "n-cat", "label": "Catalog Service", "kind": "app", "icon": "fa5_book" }, { "id": "n-srh", "label": "Search Service", "kind": "app", "icon": "fa5_search" }, { "id": "n-lin", "label": "Lineage Service", "kind": "app" }, { "id": "n-gls", "label": "Glossary Service", "kind": "app" } ] }, { "title": "Governance services", "kind": "boundary", "nodes": [ { "id": "n-cls", "label": "Classification Service", "kind": "app" }, { "id": "n-dq", "label": "Quality Metadata", "kind": "app" }, { "id": "n-wf", "label": "Workflow Engine", "sub": "Temporal", "kind": "app" }, { "id": "n-not", "label": "Notification Service", "kind": "app", "icon": "fa5_bell" }, { "id": "n-ent", "label": "Entitlement Filter", "sub": "ABAC", "kind": "security" } ] }, { "title": "Ingestion", "kind": "boundary", "nodes": [ { "id": "n-orc", "label": "Harvest Orchestrator", "sub": "Temporal", "kind": "app" }, { "id": "n-crt", "label": "Connector Runtime", "sub": "plugin SDK", "kind": "integration" }, { "id": "n-bus", "label": "Metadata Event Bus", "sub": "Kafka · MCP/MCL", "kind": "queue" }, { "id": "n-enr", "label": "Enrichment Workers", "sub": "profile · classify", "kind": "app" }, { "id": "n-mrg", "label": "Merge & Precedence", "kind": "app" } ] }, { "title": "Persistence", "kind": "boundary", "nodes": [ { "id": "n-asp", "label": "Aspect Store", "sub": "PostgreSQL", "kind": "store" }, { "id": "n-kg", "label": "Knowledge Graph", "sub": "Neo4j", "kind": "store" }, { "id": "n-idx", "label": "Search Index", "sub": "OpenSearch", "kind": "store" }, { "id": "n-raw", "label": "Payload Archive", "sub": "S3", "kind": "store" }, { "id": "n-aud", "label": "Audit Log", "sub": "append-only", "kind": "security" } ] } ] } ], "outside": [ { "id": "n-src", "label": "Source systems", "sub": "200 registered", "kind": "external" }, { "id": "n-idp", "label": "Entra ID", "sub": "OIDC · SCIM", "kind": "security" }, { "id": "n-pep", "label": "Policy enforcement", "kind": "external" }, { "id": "n-dqe", "label": "Quality engines", "kind": "external" } ], "edges": [ { "from": "n-ui", "to": "n-gw", "label": "HTTPS" }, { "from": "n-gw", "to": "n-gql", "label": "read" }, { "from": "n-gw", "to": "n-rst", "label": "write" }, { "from": "n-gql", "to": "n-srh", "label": "search" }, { "from": "n-srh", "to": "n-idx", "label": "query" }, { "from": "n-mrg", "to": "n-asp", "label": "commit", "route": "gutter" }, { "from": "n-bus", "to": "n-kg", "label": "project", "kind": "async" }, { "from": "n-bus", "to": "n-idx", "label": "", "kind": "async" }, { "from": "n-crt", "to": "n-src", "label": "harvest", "kind": "batch" }, { "from": "n-cls", "to": "n-pep", "label": "tags", "kind": "async", "route": "gutter" }, { "from": "n-dqe", "to": "n-dq", "label": "results", "kind": "async", "route": "gutter" } ], "note": "Eleven edges shown, others omitted for legibility. Identity is validated at the gateway on every call; the authoritative call graph is the service contract register.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "05-integration-architecture", "title": "Integration Architecture — Producers, Hub, Consumers", "layout": "hub", "canvas": { "width": 1660 }, "left": { "title": "Metadata producers — inbound", "nodes": [ { "id": "i-snow", "label": "Snowflake / BigQuery", "sub": "JDBC · ACCESS_HISTORY", "kind": "external", "rel": "schema · query log", "kind2": "batch" }, { "id": "i-dbx", "label": "Databricks Unity Catalog", "sub": "REST · system tables", "kind": "external", "rel": "schema · lineage", "kind2": "batch" }, { "id": "i-dbt", "label": "dbt / Airflow / Spark", "sub": "OpenLineage", "kind": "external", "rel": "run events", "kind2": "async" }, { "id": "i-bi", "label": "Power BI / Tableau", "sub": "REST · usage API", "kind": "external", "rel": "reports · usage", "kind2": "batch", "icon": "powerbi" }, { "id": "i-kaf", "label": "Kafka Schema Registry", "sub": "subjects", "kind": "external", "rel": "event schemas", "kind2": "async" }, { "id": "i-saas", "label": "SAP / Salesforce", "sub": "metadata API", "kind": "external", "rel": "objects · fields", "kind2": "batch" } ] }, "centre": { "title": "Enterprise Metadata Platform", "nodes": [ { "id": "i-hub", "label": "Metadata Hub", "sub": "canonical model · APIs", "kind": "app" }, { "id": "i-cf", "label": "Connector Framework", "sub": "pluggable SDK", "kind": "integration" }, { "id": "i-bus", "label": "Metadata Event Bus", "sub": "Kafka", "kind": "queue" }, { "id": "i-api", "label": "GraphQL · REST · Events", "kind": "integration" } ] }, "right": { "title": "Metadata consumers — outbound", "nodes": [ { "id": "o-pep", "label": "Policy Enforcement", "sub": "masking · row filters", "kind": "external", "rel": "labels · tags", "dir": "out", "kind2": "sync" }, { "id": "o-dq", "label": "Quality Platform", "sub": "Soda · GX", "kind": "external", "rel": "asset scope", "dir": "out", "kind2": "sync" }, { "id": "o-itsm", "label": "ITSM & Incident", "sub": "ServiceNow", "kind": "external", "rel": "impact tickets", "dir": "out", "kind2": "async" }, { "id": "o-ide", "label": "Notebooks & IDEs", "sub": "inline lookup", "kind": "external", "rel": "GraphQL", "dir": "out", "kind2": "sync" }, { "id": "o-srh", "label": "Enterprise Search", "sub": "M365 · Glean", "kind": "external", "rel": "asset index", "dir": "out", "kind2": "batch" }, { "id": "o-exp", "label": "Metadata Exchange", "sub": "OpenLineage · CSV", "kind": "external", "rel": "open export", "dir": "out", "kind2": "batch" } ] }, "note": "Every inbound connector is read-only. Outbound writes are limited to tags, labels and tickets — never to source data.", "meta": { "v": "1.0", "owner": "Integration Architecture", "date": "2026-08" } }, { "id": "06-connector-coverage", "title": "Connector Coverage by Source Class", "layout": "grid", "canvas": { "width": 1700 }, "columns": [ "Schema & structure", "Usage & popularity", "Lineage granularity", "Quality results", "Classification" ], "rows": [ { "title": "Cloud warehouse", "cells": [ [ { "id": "g11", "label": "Full", "sub": "incl. constraints", "kind": "app", "icon": false } ], [ { "id": "g12", "label": "Full", "sub": "access history", "kind": "app", "icon": false } ], [ { "id": "g13", "label": "Column level", "sub": "SQL parsed", "kind": "app", "icon": false } ], [ { "id": "g14", "label": "Via DQ platform", "kind": "integration", "icon": false } ], [ { "id": "g15", "label": "Auto + curated", "kind": "app", "icon": false } ] ] }, { "title": "Lakehouse", "cells": [ [ { "id": "g21", "label": "Full", "sub": "Unity Catalog", "kind": "app", "icon": false } ], [ { "id": "g22", "label": "Full", "kind": "app", "icon": false } ], [ { "id": "g23", "label": "Column level", "sub": "system tables", "kind": "app", "icon": false } ], [ { "id": "g24", "label": "Native + DQ", "kind": "app", "icon": false } ], [ { "id": "g25", "label": "Auto + curated", "kind": "app", "icon": false } ] ] }, { "title": "Relational OLTP", "cells": [ [ { "id": "g31", "label": "Full", "kind": "app", "icon": false } ], [ { "id": "g32", "label": "Partial", "sub": "sampled logs", "kind": "integration", "icon": false } ], [ { "id": "g33", "label": "Table level", "kind": "integration", "icon": false } ], [ { "id": "g34", "label": "Via DQ platform", "kind": "integration", "icon": false } ], [ { "id": "g35", "label": "Auto + curated", "kind": "app", "icon": false } ] ] }, { "title": "BI & reporting", "cells": [ [ { "id": "g41", "label": "Full", "sub": "datasets · fields", "kind": "app", "icon": false } ], [ { "id": "g42", "label": "Full", "sub": "views · viewers", "kind": "app", "icon": false } ], [ { "id": "g43", "label": "Field to column", "sub": "semantic model", "kind": "app", "icon": false } ], [ { "id": "g44", "label": "Not available", "kind": "risk", "icon": false } ], [ { "id": "g45", "label": "Inherited", "sub": "from upstream", "kind": "integration", "icon": false } ] ] }, { "title": "Orchestration & ELT", "cells": [ [ { "id": "g51", "label": "Jobs & tasks", "kind": "app", "icon": false } ], [ { "id": "g52", "label": "Run history", "kind": "app", "icon": false } ], [ { "id": "g53", "label": "Column level", "sub": "OpenLineage", "kind": "app", "icon": false } ], [ { "id": "g54", "label": "Test results", "sub": "dbt tests", "kind": "app", "icon": false } ], [ { "id": "g55", "label": "Not applicable", "kind": "external", "icon": false } ] ] }, { "title": "Streaming", "cells": [ [ { "id": "g61", "label": "Full", "sub": "Avro · Protobuf", "kind": "app", "icon": false } ], [ { "id": "g62", "label": "Partial", "sub": "consumer groups", "kind": "integration", "icon": false } ], [ { "id": "g63", "label": "Topic level", "kind": "integration", "icon": false } ], [ { "id": "g64", "label": "Not available", "kind": "risk", "icon": false } ], [ { "id": "g65", "label": "Auto + curated", "kind": "app", "icon": false } ] ] }, { "title": "SaaS & files", "cells": [ [ { "id": "g71", "label": "Objects & fields", "kind": "app", "icon": false } ], [ { "id": "g72", "label": "Not available", "kind": "risk", "icon": false } ], [ { "id": "g73", "label": "Manual only", "sub": "declared", "kind": "risk", "icon": false } ], [ { "id": "g74", "label": "Not available", "kind": "risk", "icon": false } ], [ { "id": "g75", "label": "Curated", "kind": "integration", "icon": false } ] ] } ], "note": "Red cells are declared gaps, not omissions. Lineage below column level is stated on each asset profile so trust is never assumed.", "meta": { "v": "1.0", "owner": "Integration Architecture", "date": "2026-08" } }, { "id": "07-canonical-metadata-model", "title": "Canonical Metadata Model", "layout": "er", "canvas": { "width": 1660, "cols": 4 }, "rowGap": 260, "entities": [ { "id": "e-term", "name": "business_term", "kind": "store", "row": 0, "col": 0, "attrs": [ "term_id PK", "name", "definition", "parent_term_id FK", "status draft|approved", "domain_id FK" ] }, { "id": "e-fld", "name": "field", "kind": "store", "row": 0, "col": 1, "attrs": [ "field_urn PK", "asset_urn FK", "name", "data_type", "ordinal", "nullable" ] }, { "id": "e-lin", "name": "lineage_edge", "kind": "store", "row": 0, "col": 2, "attrs": [ "edge_id PK", "upstream_urn FK", "downstream_urn FK", "granularity col|table", "evidence sql|event", "confidence" ] }, { "id": "e-job", "name": "job_run", "kind": "store", "row": 0, "col": 3, "attrs": [ "run_id PK", "job_urn", "platform", "started_at", "state" ] }, { "id": "e-dom", "name": "domain", "kind": "store", "row": 1, "col": 0, "attrs": [ "domain_id PK", "name", "parent_domain_id FK", "owner_group" ] }, { "id": "e-ast", "name": "asset", "kind": "store", "row": 1, "col": 1, "attrs": [ "asset_urn PK", "type table|report|job", "platform", "domain_id FK", "lifecycle draft|certified", "deprecated_at" ] }, { "id": "e-asp", "name": "aspect", "kind": "store", "row": 1, "col": 2, "attrs": [ "aspect_id PK", "entity_urn FK", "aspect_name", "version", "payload JSONB", "provenance curated|source", "created_at" ] }, { "id": "e-cls", "name": "classification", "kind": "store", "row": 1, "col": 3, "attrs": [ "class_id PK", "aspect_id FK", "label public..restricted", "regulation GDPR|PCI", "pii_flag", "confidence" ] }, { "id": "e-pol", "name": "policy", "kind": "store", "row": 2, "col": 0, "attrs": [ "policy_id PK", "name", "applies_to label|domain", "enforcement_ref", "status" ] }, { "id": "e-own", "name": "ownership", "kind": "store", "row": 2, "col": 1, "attrs": [ "own_id PK", "entity_urn FK", "principal", "role owner|steward", "valid_from" ] }, { "id": "e-rul", "name": "quality_rule", "kind": "store", "row": 2, "col": 2, "attrs": [ "rule_id PK", "aspect_id FK", "dimension", "expression", "engine soda|gx|dbt" ] }, { "id": "e-res", "name": "quality_result", "kind": "store", "row": 2, "col": 3, "attrs": [ "result_id PK", "rule_id FK", "score", "passed", "run_at" ] } ], "relations": [ { "from": "e-dom", "to": "e-ast", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-dom", "to": "e-term", "label": "1 : N", "from_side": "n", "to_side": "s" }, { "from": "e-ast", "to": "e-fld", "label": "1 : N", "from_side": "n", "to_side": "s" }, { "from": "e-fld", "to": "e-lin", "label": "N : M", "from_side": "e", "to_side": "w" }, { "from": "e-lin", "to": "e-job", "label": "N : 1", "from_side": "e", "to_side": "w" }, { "from": "e-ast", "to": "e-asp", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-asp", "to": "e-cls", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-ast", "to": "e-own", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "e-asp", "to": "e-rul", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "e-rul", "to": "e-res", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "e-dom", "to": "e-pol", "label": "N : M", "from_side": "s", "to_side": "n" }, { "from": "e-term", "to": "e-fld", "label": "N : M", "from_side": "e", "to_side": "w" } ], "note": "Every entity is addressed by URN. Classification and quality rules are stored as aspects, so a new metadata type needs no schema change. workflow_task and audit_event are in views 13 and 21.", "meta": { "v": "1.0", "owner": "Data Architecture", "date": "2026-08" } }, { "id": "08-metadata-storage-architecture", "title": "Storage Architecture — System of Record and Projections", "layout": "nested", "canvas": { "width": 1700 }, "boxes": [ { "title": "Metadata Repository", "kind": "cloud", "dir": "col", "children": [ { "title": "System of record — durable, versioned, backed up", "kind": "boundary", "nodes": [ { "id": "s-asp", "label": "Aspect Store", "sub": "PostgreSQL · partitioned", "kind": "store" }, { "id": "s-hist", "label": "Version History", "sub": "24 months online", "kind": "store" }, { "id": "s-aud", "label": "Audit Log", "sub": "append-only · 7 y", "kind": "security" }, { "id": "s-raw", "label": "Payload Archive", "sub": "S3 · object lock", "kind": "store" } ] }, { "title": "Projections — derived, disposable, rebuilt from the change log", "kind": "boundary", "nodes": [ { "id": "s-kg", "label": "Knowledge Graph", "sub": "Neo4j · 50M edges", "kind": "store" }, { "id": "s-idx", "label": "Search Index", "sub": "OpenSearch · 5M docs", "kind": "store" }, { "id": "s-cch", "label": "Read Cache", "sub": "Redis · 15 min TTL", "kind": "store" } ] }, { "title": "Operational state", "kind": "boundary", "nodes": [ { "id": "s-wf", "label": "Workflow State", "sub": "Temporal · Postgres", "kind": "store" }, { "id": "s-wm", "label": "Watermarks & Jobs", "sub": "per connector", "kind": "store" }, { "id": "s-sec", "label": "Connector Secrets", "sub": "KMS-backed vault", "kind": "security", "icon": "fa5_key" } ] } ] } ], "outside": [ { "id": "s-bus", "label": "Metadata Change Log", "sub": "Kafka · 7-day retention", "kind": "queue" }, { "id": "s-bak", "label": "Backup Vault", "sub": "PITR 35 days", "kind": "store", "icon": "aws_aws-backup" }, { "id": "s-src", "label": "Source systems", "sub": "own their technical truth", "kind": "external" } ], "edges": [ { "from": "s-asp", "to": "s-bus", "label": "emit on commit", "kind": "async" }, { "from": "s-bus", "to": "s-kg", "label": "project", "kind": "async" }, { "from": "s-bus", "to": "s-idx", "label": "", "kind": "async" }, { "from": "s-asp", "to": "s-bak", "label": "continuous backup", "kind": "batch" }, { "from": "s-raw", "to": "s-bus", "label": "replay on rebuild", "kind": "batch" }, { "from": "s-src", "to": "s-raw", "label": "harvest payload", "kind": "batch" } ], "note": "Losing a projection costs a rebuild, not data. Losing the aspect store costs a restore — which is why only it is on the RPO 5 min path.", "meta": { "v": "1.0", "owner": "Data Architecture", "date": "2026-08" } }, { "id": "09-metadata-flow-and-precedence", "title": "Metadata Flow — Capture to Activation", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "top", "stages": [ { "title": "Capture", "nodes": [ { "id": "f-sch", "label": "Scheduled harvest", "sub": "hourly incremental", "kind": "integration" }, { "id": "f-psh", "label": "Event push", "sub": "webhook · CDC", "kind": "queue" }, { "id": "f-man", "label": "Manual & bulk", "sub": "UI · CSV · API", "kind": "integration" } ] }, { "title": "Normalise", "nodes": [ { "id": "f-adp", "label": "Source adapter", "kind": "app" }, { "id": "f-map", "label": "Canonical mapper", "sub": "to URN + aspects", "kind": "app" }, { "id": "f-val", "label": "Schema valid?", "sub": "aspect registry", "kind": "decision" }, { "id": "f-dlq", "label": "Dead letter", "sub": "quarantine", "kind": "risk" } ] }, { "title": "Enrich", "nodes": [ { "id": "f-prf", "label": "Profiler", "sub": "sampled stats", "kind": "app", "icon": "fa5_chart_bar" }, { "id": "f-pii", "label": "PII classifier", "sub": "rules + ML", "kind": "app" }, { "id": "f-sug", "label": "Term suggester", "sub": "advisory only", "kind": "app" }, { "id": "f-drf", "label": "Drift detector", "sub": "diff vs last", "kind": "app" } ] }, { "title": "Resolve", "nodes": [ { "id": "f-prc", "label": "Precedence engine", "sub": "curated wins", "kind": "decision" }, { "id": "f-cnf", "label": "Conflict", "sub": "two sources disagree", "kind": "risk" }, { "id": "f-tsk", "label": "Stewardship task", "sub": "routed by domain", "kind": "app" } ] }, { "title": "Commit", "nodes": [ { "id": "f-asp", "label": "Aspect Store", "sub": "version + 1", "kind": "store" }, { "id": "f-mcl", "label": "Change log", "sub": "Kafka MCL", "kind": "queue" }, { "id": "f-aud", "label": "Audit Log", "sub": "who · what · before", "kind": "security" } ] }, { "title": "Activate", "nodes": [ { "id": "f-kg", "label": "Knowledge Graph", "kind": "store" }, { "id": "f-idx", "label": "Search Index", "kind": "store" }, { "id": "f-pep", "label": "Policy Sync", "sub": "labels outbound", "kind": "integration" }, { "id": "f-not", "label": "Notify owners", "sub": "Teams · email", "kind": "integration" } ] } ], "edges": [ { "from": "f-val", "to": "f-dlq", "label": "reject", "kind": "error" }, { "from": "f-prc", "to": "f-cnf", "label": "unresolved", "kind": "error" }, { "from": "f-cnf", "to": "f-tsk", "label": "raise" } ], "note": "Precedence: certified curation > curation > source-declared > inferred. Inference never overwrites a human value; it only proposes.", "meta": { "v": "1.0", "owner": "Data Architecture", "date": "2026-08" } }, { "id": "10-harvest-and-merge", "title": "Incremental Harvest and Conflict Resolution", "layout": "sequence", "canvas": { "width": 1620 }, "lifelines": [ { "id": "q-orc", "label": "Harvest Orchestrator", "kind": "app" }, { "id": "q-cn", "label": "Connector", "kind": "integration", "icon": "fa5_plug" }, { "id": "q-src", "label": "Source System", "kind": "external" }, { "id": "q-nrm", "label": "Canonical Mapper", "kind": "app" }, { "id": "q-prc", "label": "Precedence Engine", "kind": "app" }, { "id": "q-asp", "label": "Aspect Store", "kind": "store" }, { "id": "q-bus", "label": "Change Log", "kind": "queue" } ], "messages": [ { "from": "q-orc", "to": "q-cn", "label": "run since watermark", "kind": "call" }, { "from": "q-cn", "to": "q-src", "label": "read metadata only", "kind": "call" }, { "from": "q-src", "to": "q-cn", "label": "schemas, stats, logs", "kind": "return" }, { "from": "q-cn", "to": "q-src", "label": "429 rate limited", "kind": "error" }, { "from": "q-cn", "to": "q-cn", "label": "backoff and resume", "kind": "self" }, { "from": "q-cn", "to": "q-nrm", "label": "raw payload", "kind": "call" }, { "from": "q-nrm", "to": "q-nrm", "label": "map to URN + aspects", "kind": "self" }, { "from": "q-nrm", "to": "q-prc", "label": "change proposal", "kind": "call" }, { "from": "q-prc", "to": "q-asp", "label": "read current version", "kind": "call" }, { "from": "q-asp", "to": "q-prc", "label": "curated description", "kind": "return" }, { "from": "q-prc", "to": "q-prc", "label": "curated outranks source", "kind": "self" }, { "from": "q-prc", "to": "q-asp", "label": "write version + 1", "kind": "call" }, { "from": "q-prc", "to": "q-bus", "label": "conflict to steward", "kind": "error" }, { "from": "q-asp", "to": "q-bus", "label": "change log event", "kind": "async" }, { "from": "q-orc", "to": "q-orc", "label": "advance watermark", "kind": "self" } ], "note": "A harvest never rewrites a curated field. When source and curation disagree the source value is retained as evidence and a task is raised.", "meta": { "v": "1.0", "owner": "Platform Engineering", "date": "2026-08" } }, { "id": "11-lineage-construction", "title": "Lineage Construction — Evidence to Column Edges", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "align": "top", "stages": [ { "title": "Evidence", "nodes": [ { "id": "l-log", "label": "Query logs", "sub": "Snowflake · BigQuery", "kind": "external" }, { "id": "l-dbt", "label": "dbt manifests", "sub": "model graph", "kind": "external" }, { "id": "l-ol", "label": "OpenLineage events", "sub": "Airflow · Spark", "kind": "queue" }, { "id": "l-bi", "label": "BI semantic models", "sub": "Power BI · Tableau", "kind": "external" } ] }, { "title": "Parse", "nodes": [ { "id": "l-sql", "label": "SQL parser", "sub": "sqlglot · 12 dialects", "kind": "app" }, { "id": "l-mfd", "label": "Manifest reader", "kind": "app" }, { "id": "l-evt", "label": "Event consumer", "kind": "app" }, { "id": "l-rpt", "label": "Report binder", "sub": "field to column", "kind": "app" } ] }, { "title": "Resolve", "nodes": [ { "id": "l-urn", "label": "URN resolver", "sub": "platform · db · schema", "kind": "app" }, { "id": "l-col", "label": "Column mapper", "sub": "expression trace", "kind": "app" }, { "id": "l-cnf", "label": "Confidence scorer", "sub": "parsed vs declared", "kind": "app" } ] }, { "title": "Emit", "nodes": [ { "id": "l-cle", "label": "Column edges", "sub": "high confidence", "kind": "app" }, { "id": "l-tbl", "label": "Table edges", "sub": "fallback", "kind": "integration" }, { "id": "l-unr", "label": "Unresolved refs", "sub": "manual review", "kind": "risk" } ] }, { "title": "Serve", "nodes": [ { "id": "l-kg", "label": "Knowledge Graph", "sub": "Neo4j", "kind": "store" }, { "id": "l-api", "label": "Lineage API", "sub": "GraphQL n-hop", "kind": "integration" }, { "id": "l-ui", "label": "Lineage Explorer", "sub": "collapsed by default", "kind": "app" } ] } ], "edges": [ { "from": "l-cnf", "to": "l-unr", "label": "below threshold", "kind": "error" } ], "note": "Column-level lineage is produced only where an expression can be traced. Everything else degrades to table level and says so on the asset profile.", "meta": { "v": "1.0", "owner": "Platform Engineering", "date": "2026-08" } }, { "id": "12-search-and-discovery", "title": "Search and Discovery — Request Path", "layout": "sequence", "canvas": { "width": 1620 }, "lifelines": [ { "id": "d-usr", "label": "Data Consumer", "kind": "actor" }, { "id": "d-ui", "label": "Catalog UI", "kind": "app" }, { "id": "d-gw", "label": "API Gateway", "kind": "integration" }, { "id": "d-srh", "label": "Search Service", "kind": "app", "icon": "fa5_search" }, { "id": "d-ent", "label": "Entitlement Filter", "kind": "security" }, { "id": "d-idx", "label": "Search Index", "kind": "store" }, { "id": "d-kg", "label": "Knowledge Graph", "kind": "store" } ], "messages": [ { "from": "d-usr", "to": "d-ui", "label": "search \"net revenue\"", "kind": "call" }, { "from": "d-ui", "to": "d-gw", "label": "GraphQL search", "kind": "call" }, { "from": "d-gw", "to": "d-gw", "label": "verify OIDC token", "kind": "self" }, { "from": "d-gw", "to": "d-srh", "label": "query + principal", "kind": "call" }, { "from": "d-srh", "to": "d-ent", "label": "resolve visibility", "kind": "call" }, { "from": "d-ent", "to": "d-srh", "label": "domain + label filter", "kind": "return" }, { "from": "d-srh", "to": "d-idx", "label": "faceted query", "kind": "call" }, { "from": "d-idx", "to": "d-srh", "label": "ranked hits", "kind": "return" }, { "from": "d-srh", "to": "d-kg", "label": "related and certified", "kind": "call" }, { "from": "d-kg", "to": "d-srh", "label": "neighbours", "kind": "return" }, { "from": "d-srh", "to": "d-srh", "label": "boost certified assets", "kind": "self" }, { "from": "d-srh", "to": "d-ui", "label": "results + facets", "kind": "return" }, { "from": "d-ui", "to": "d-usr", "label": "profile: owner, quality", "kind": "return" }, { "from": "d-srh", "to": "d-idx", "label": "log for popularity", "kind": "async" } ], "note": "Entitlements are applied before ranking, so a restricted asset never appears in a count, a facet or a total.", "meta": { "v": "1.0", "owner": "Platform Engineering", "date": "2026-08" } }, { "id": "13-stewardship-workflows", "title": "Stewardship Workflows by Persona", "layout": "swimlane", "canvas": { "width": 1740 }, "laneHeaderWidth": 160, "stages": [ "Trigger", "Assign", "Curate", "Review", "Publish", "Monitor" ], "lanes": [ { "title": "Data consumer", "cells": [ [ { "id": "p11", "label": "Requests a definition", "kind": "app" } ], [], [ { "id": "p13", "label": "Suggests description", "kind": "app" } ], [], [ { "id": "p15", "label": "Sees certified badge", "kind": "app" } ], [ { "id": "p16", "label": "Reports an issue", "kind": "app" } ] ] }, { "title": "Data steward", "cells": [ [ { "id": "p21", "label": "Stewardship inbox", "kind": "app" } ], [ { "id": "p22", "label": "Claims task", "kind": "app" } ], [ { "id": "p23", "label": "Writes term", "sub": "definition · synonyms", "kind": "app" } ], [ { "id": "p24", "label": "Links term to columns", "kind": "app" } ], [ { "id": "p25", "label": "Submits for approval", "kind": "app" } ], [ { "id": "p26", "label": "Coverage scorecard", "kind": "platform" } ] ] }, { "title": "Domain owner", "cells": [ [ { "id": "p31", "label": "Drift alert", "kind": "queue" } ], [ { "id": "p32", "label": "Route by domain", "kind": "decision" } ], [], [ { "id": "p34", "label": "Approve or reject", "kind": "decision" } ], [ { "id": "p35", "label": "Certifies asset", "kind": "security" } ], [ { "id": "p36", "label": "Recertifies", "sub": "annual", "kind": "platform" } ] ] }, { "title": "Platform team", "cells": [ [ { "id": "p41", "label": "Harvest gap", "kind": "risk" } ], [ { "id": "p42", "label": "Auto-assign rule", "kind": "app" } ], [ { "id": "p43", "label": "Fixes connector", "kind": "app" } ], [], [ { "id": "p45", "label": "Backfills metadata", "kind": "app" } ], [ { "id": "p46", "label": "SLA dashboard", "kind": "platform" } ] ] } ], "note": "Workflows are configurable per domain and asset type. Unclaimed tasks escalate to the domain owner after 5 working days.", "meta": { "v": "1.0", "owner": "Data Governance", "date": "2026-08" } }, { "id": "14-drift-and-impact-analysis", "title": "Schema Drift and Impact Analysis", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "top", "stages": [ { "title": "Detect", "nodes": [ { "id": "x-dif", "label": "Schema diff", "sub": "vs last harvest", "kind": "app" }, { "id": "x-dep", "label": "Deprecation set", "sub": "by owner", "kind": "app" }, { "id": "x-brk", "label": "Quality breach", "sub": "from DQ platform", "kind": "queue" } ] }, { "title": "Classify", "nodes": [ { "id": "x-cls", "label": "Breaking change?", "sub": "drop · retype · rename", "kind": "decision" }, { "id": "x-add", "label": "Additive only", "sub": "log and move on", "kind": "app" } ] }, { "title": "Traverse", "nodes": [ { "id": "x-hop", "label": "Graph traversal", "sub": "n-hop downstream", "kind": "app" }, { "id": "x-blr", "label": "Blast radius", "sub": "assets · reports · jobs", "kind": "app" }, { "id": "x-crt", "label": "Certified assets hit", "kind": "risk" } ] }, { "title": "Notify", "nodes": [ { "id": "x-own", "label": "Owner notification", "sub": "Teams · email", "kind": "integration" }, { "id": "x-tkt", "label": "Change record", "sub": "ServiceNow", "kind": "external" }, { "id": "x-ban", "label": "Catalog banner", "sub": "on every consumer", "kind": "app" } ] }, { "title": "Act", "nodes": [ { "id": "x-gat", "label": "CI change gate", "sub": "block or warn", "kind": "decision" }, { "id": "x-win", "label": "Deprecation window", "sub": "90 days", "kind": "app" }, { "id": "x-ret", "label": "Retire asset", "sub": "lifecycle state", "kind": "app" } ] } ], "edges": [ { "from": "x-cls", "to": "x-add", "label": "no", "kind": "error" } ], "note": "Impact is answered from the graph, not from a spreadsheet. A change gate is advisory for uncertified assets and blocking for certified ones.", "meta": { "v": "1.0", "owner": "Data Governance", "date": "2026-08" } }, { "id": "15-quality-metadata", "title": "Data Quality Metadata — Ingest and Surface", "layout": "hub", "canvas": { "width": 1620 }, "left": { "title": "Quality engines — execution stays there", "nodes": [ { "id": "y-gx", "label": "Great Expectations", "sub": "suite results", "kind": "external", "rel": "results API", "kind2": "async" }, { "id": "y-sod", "label": "Soda Core", "sub": "scan results", "kind": "external", "rel": "results API", "kind2": "async" }, { "id": "y-mc", "label": "Monte Carlo", "sub": "anomaly alerts", "kind": "external", "rel": "incidents", "kind2": "async" }, { "id": "y-dbt", "label": "dbt tests", "sub": "run_results.json", "kind": "external", "rel": "test outcomes", "kind2": "batch" }, { "id": "y-nat", "label": "Native platform checks", "sub": "constraints · freshness", "kind": "external", "rel": "check state", "kind2": "batch" } ] }, "centre": { "title": "Quality Metadata Service", "nodes": [ { "id": "y-svc", "label": "Quality Metadata Service", "sub": "dimensions · scores", "kind": "app" }, { "id": "y-reg", "label": "Rule Registry", "sub": "stored as aspects", "kind": "app" }, { "id": "y-agg", "label": "Score Aggregator", "sub": "asset · domain roll-up", "kind": "app" }, { "id": "y-iss", "label": "Issue Linker", "sub": "to asset and owner", "kind": "app" } ] }, "right": { "title": "Where quality becomes visible", "nodes": [ { "id": "y-prf", "label": "Asset profile badge", "sub": "score + last run", "kind": "app", "rel": "quality panel", "dir": "out", "kind2": "sync" }, { "id": "y-rnk", "label": "Search ranking signal", "kind": "app", "rel": "trust boost", "dir": "out", "kind2": "sync" }, { "id": "y-alr", "label": "Owner alert", "sub": "breach on owned asset", "kind": "integration", "rel": "notify", "dir": "out", "kind2": "async" }, { "id": "y-cer", "label": "Certification gate", "sub": "no cert while failing", "kind": "decision", "rel": "gate", "dir": "out", "kind2": "sync" }, { "id": "y-sla", "label": "Domain SLA report", "kind": "platform", "rel": "monthly", "dir": "out", "kind2": "batch" } ] }, "note": "The platform stores quality dimensions, rules, scores and history. It deliberately runs no tests — rebuilding a quality engine would create the silo this system exists to remove.", "meta": { "v": "1.0", "owner": "Data Governance", "date": "2026-08" } }, { "id": "16-deployment-architecture", "title": "Deployment and Infrastructure", "layout": "nested", "canvas": { "width": 1740 }, "boxes": [ { "title": "AWS eu-west-1 · primary · active", "kind": "cloud", "dir": "row", "children": [ { "title": "AZ-a", "kind": "boundary", "nodes": [ { "id": "k-n1", "label": "EKS node group", "sub": "services · workers", "kind": "app", "icon": "aws_amazon-elastic-kubernetes-service" }, { "id": "k-db1", "label": "Aurora writer", "sub": "PostgreSQL 16", "kind": "store", "icon": "aws_amazon-aurora" }, { "id": "k-os1", "label": "OpenSearch data", "kind": "store", "icon": "opensearch" } ] }, { "title": "AZ-b", "kind": "boundary", "nodes": [ { "id": "k-n2", "label": "EKS node group", "kind": "app", "icon": "aws_amazon-elastic-kubernetes-service" }, { "id": "k-db2", "label": "Aurora reader", "sub": "failover target", "kind": "store", "icon": "aws_amazon-aurora" }, { "id": "k-os2", "label": "OpenSearch data", "kind": "store", "icon": "opensearch" } ] }, { "title": "AZ-c", "kind": "boundary", "nodes": [ { "id": "k-n3", "label": "EKS node group", "kind": "app", "icon": "aws_amazon-elastic-kubernetes-service" }, { "id": "k-kg", "label": "Neo4j cluster", "sub": "3 core members", "kind": "store" }, { "id": "k-msk", "label": "MSK brokers", "sub": "Kafka", "kind": "queue", "icon": "aws_amazon-managed-streaming-for-apache-kafka" } ] } ] }, { "title": "AWS eu-central-1 · warm standby", "kind": "cloud", "dir": "row", "children": [ { "title": "Standby estate", "kind": "boundary", "nodes": [ { "id": "k-sk", "label": "EKS scaled to zero", "sub": "IaC identical", "kind": "app", "icon": "aws_amazon-elastic-kubernetes-service" }, { "id": "k-sg", "label": "Aurora global replica", "sub": "RPO under 1 min", "kind": "store", "icon": "aws_amazon-aurora" }, { "id": "k-s3", "label": "S3 cross-region copy", "kind": "store", "icon": "aws_amazon-simple-storage-service" } ] } ] }, { "title": "Network-isolated estates", "kind": "onprem", "dir": "row", "children": [ { "title": "Restricted VPC or data centre", "kind": "boundary", "nodes": [ { "id": "k-agt", "label": "Ingestion agent", "sub": "outbound 443 only", "kind": "integration", "icon": "server" }, { "id": "k-src", "label": "Private sources", "sub": "no inbound route", "kind": "external" } ] } ] } ], "outside": [ { "id": "k-r53", "label": "Route 53 + WAF", "sub": "health-checked", "kind": "security", "icon": "aws_amazon-route-53" }, { "id": "k-idp", "label": "Entra ID", "kind": "security" }, { "id": "k-bak", "label": "Backup vault", "sub": "PITR 35 days", "kind": "store", "icon": "aws_aws-backup" } ], "edges": [ { "from": "k-r53", "to": "k-n1", "label": "HTTPS 443" }, { "from": "k-db1", "to": "k-sg", "label": "global replication", "kind": "async" }, { "from": "k-agt", "to": "k-msk", "label": "mTLS outbound", "kind": "async" }, { "from": "k-agt", "to": "k-src", "label": "read-only", "kind": "batch" }, { "from": "k-db1", "to": "k-bak", "label": "continuous backup", "kind": "batch" } ], "note": "Warm standby, not active-active: metadata reads tolerate a 30-minute RTO, and dual-region write consistency for a versioned store is not worth its cost here.", "meta": { "v": "1.0", "owner": "Platform Engineering", "date": "2026-08" }, "iconset": "aws" }, { "id": "17-cicd-and-extensibility", "title": "Delivery Pipeline and Extensibility", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "top", "stages": [ { "title": "Source", "nodes": [ { "id": "b-plt", "label": "Platform services", "sub": "monorepo", "kind": "app", "icon": "fa5_cube" }, { "id": "b-con", "label": "Connector plugins", "sub": "versioned separately", "kind": "app" }, { "id": "b-mdl", "label": "Aspect schemas", "sub": "the metadata model", "kind": "app" } ] }, { "title": "Build & test", "nodes": [ { "id": "b-unt", "label": "Unit + contract tests", "kind": "app" }, { "id": "b-cnf", "label": "Connector conformance", "sub": "golden payloads", "kind": "app" }, { "id": "b-cmp", "label": "Schema compatible?", "sub": "backward only", "kind": "decision" } ] }, { "title": "Gate", "nodes": [ { "id": "b-sec", "label": "SAST · SCA · secrets", "kind": "security" }, { "id": "b-mig", "label": "Migration dry run", "sub": "on prod snapshot", "kind": "app" }, { "id": "b-cab", "label": "Model change board", "sub": "new asset types only", "kind": "decision" }, { "id": "b-rej", "label": "Blocked", "sub": "breaking change", "kind": "risk" } ] }, { "title": "Deploy", "nodes": [ { "id": "b-dev", "label": "Dev", "sub": "synthetic estate", "kind": "app" }, { "id": "b-stg", "label": "Staging", "sub": "prod metadata subset", "kind": "app", "icon": "fa5_cube" }, { "id": "b-prd", "label": "Production", "sub": "blue/green", "kind": "app", "icon": "fa5_cube" } ] }, { "title": "Verify", "nodes": [ { "id": "b-smk", "label": "Smoke harvest", "sub": "one asset per class", "kind": "app" }, { "id": "b-lag", "label": "Projection lag check", "sub": "under 60 s", "kind": "platform" }, { "id": "b-rbk", "label": "Rollback", "sub": "one release back", "kind": "risk" } ] } ], "edges": [ { "from": "b-cmp", "to": "b-rej", "label": "breaking", "kind": "error" }, { "from": "b-lag", "to": "b-rbk", "label": "on failure", "kind": "error" } ], "note": "A new source type ships as a plugin and a new metadata attribute ships as an aspect schema. Neither requires a core platform release.", "meta": { "v": "1.0", "owner": "Platform Engineering", "date": "2026-08" } }, { "id": "18-observability", "title": "Observability and Operations", "layout": "grid", "canvas": { "width": 1720 }, "columns": [ "Emit", "Collect", "Store", "Consume", "Act" ], "rows": [ { "title": "Service health", "cells": [ [ { "id": "v11", "label": "OpenTelemetry SDK", "kind": "app" } ], [ { "id": "v12", "label": "OTel Collector", "kind": "integration" } ], [ { "id": "v13", "label": "Prometheus", "kind": "store" } ], [ { "id": "v14", "label": "SLO dashboard", "sub": "99.9% read path", "kind": "platform" } ], [ { "id": "v15", "label": "Page on burn rate", "kind": "security" } ] ] }, { "title": "Harvest health", "cells": [ [ { "id": "v21", "label": "Connector metrics", "sub": "rows · duration", "kind": "app" } ], [ { "id": "v22", "label": "OTel Collector", "kind": "integration" } ], [ { "id": "v23", "label": "Prometheus", "kind": "store" } ], [ { "id": "v24", "label": "Freshness by source", "sub": "age of last run", "kind": "platform" } ], [ { "id": "v25", "label": "Retry, then quarantine", "kind": "security" } ] ] }, { "title": "Projection lag", "cells": [ [ { "id": "v31", "label": "Change log offsets", "kind": "queue" } ], [ { "id": "v32", "label": "Lag exporter", "kind": "integration" } ], [ { "id": "v33", "label": "Prometheus", "kind": "store" } ], [ { "id": "v34", "label": "Index vs store lag", "kind": "platform" } ], [ { "id": "v35", "label": "Show staleness banner", "kind": "security" } ] ] }, { "title": "Metadata coverage", "cells": [ [ { "id": "v41", "label": "Nightly coverage job", "kind": "app" } ], [ { "id": "v42", "label": "Batch export", "kind": "integration" } ], [ { "id": "v43", "label": "Aspect Store", "kind": "store" } ], [ { "id": "v44", "label": "Domain scorecard", "sub": "owned · described", "kind": "platform" } ], [ { "id": "v45", "label": "Steward campaign", "kind": "security" } ] ] }, { "title": "Access & audit", "cells": [ [ { "id": "v51", "label": "Audit events", "kind": "security" } ], [ { "id": "v52", "label": "Log shipper", "kind": "integration" } ], [ { "id": "v53", "label": "OpenSearch", "sub": "7-year archive", "kind": "store" } ], [ { "id": "v54", "label": "Access review", "sub": "quarterly", "kind": "platform" } ], [ { "id": "v55", "label": "Revoke and investigate", "kind": "security" } ] ] } ], "note": "Coverage is treated as a production signal, not a report: a domain whose ownership drops below target raises work, not just a number.", "meta": { "v": "1.0", "owner": "SRE", "date": "2026-08" } }, { "id": "19-active-metadata-loop", "title": "The Active Metadata Loop", "layout": "cycle", "canvas": { "width": 1320 }, "rx": 430, "ry": 215, "centre": { "label": "Active Metadata" }, "nodes": [ { "id": "z1", "label": "Harvest", "sub": "200 systems", "kind": "integration" }, { "id": "z2", "label": "Enrich", "sub": "classify · profile", "kind": "app" }, { "id": "z3", "label": "Curate", "sub": "steward · glossary", "kind": "app" }, { "id": "z4", "label": "Certify", "sub": "domain owner", "kind": "security" }, { "id": "z5", "label": "Activate", "sub": "push to enforcement", "kind": "integration", "icon": "fa5_exchange_alt" }, { "id": "z6", "label": "Observe", "sub": "usage · quality · gaps", "kind": "platform" } ], "ringLabels": [ "raw technical metadata", "candidate labels", "approved definitions", "trusted asset", "policy applied at source", "gaps become tasks" ], "note": "If the loop stops at Curate the system is a catalog. Activate and Observe are what make it a hub.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "20-security-trust-zones", "title": "Security Architecture — Trust Zones", "layout": "zones", "canvas": { "width": 1740 }, "zones": [ { "title": "Untrusted · internet", "kind": "trust", "nodes": [ { "id": "t-usr", "label": "Catalog user", "sub": "managed device", "kind": "actor" }, { "id": "t-bot", "label": "Automation client", "sub": "service principal", "kind": "external" }, { "id": "t-atk", "label": "Metadata scraper", "sub": "column-name harvest", "kind": "risk" } ] }, { "title": "Perimeter · DMZ", "kind": "trust", "nodes": [ { "id": "t-waf", "label": "WAF + DDoS", "kind": "security" }, { "id": "t-alb", "label": "Load balancer", "sub": "TLS 1.3", "kind": "integration" }, { "id": "t-gw", "label": "API Gateway", "sub": "OIDC · quota", "kind": "integration" } ] }, { "title": "Application · private subnets", "kind": "trust", "nodes": [ { "id": "t-svc", "label": "Metadata services", "sub": "mTLS mesh", "kind": "app" }, { "id": "t-ent", "label": "Policy decision point", "sub": "ABAC", "kind": "security" }, { "id": "t-idp", "label": "Entra ID", "sub": "MFA · SCIM", "kind": "security" } ] }, { "title": "Metadata · restricted", "kind": "trust", "nodes": [ { "id": "t-asp", "label": "Aspect Store", "sub": "KMS at rest", "kind": "store" }, { "id": "t-kg", "label": "Knowledge Graph", "kind": "store" }, { "id": "t-idx", "label": "Search Index", "sub": "per-domain aliases", "kind": "store" }, { "id": "t-aud", "label": "Audit Log", "sub": "write-once · 7 y", "kind": "security" } ] }, { "title": "Source estate · least privilege", "kind": "trust", "nodes": [ { "id": "t-agt", "label": "Ingestion agent", "sub": "outbound 443 only", "kind": "integration", "icon": "server" }, { "id": "t-rol", "label": "Read-only role", "sub": "catalog views only", "kind": "security" }, { "id": "t-src", "label": "Source systems", "kind": "external" } ] } ], "edges": [ { "from": "t-usr", "to": "t-waf", "label": "HTTPS 443" }, { "from": "t-bot", "to": "t-waf", "label": "client credentials" }, { "from": "t-atk", "to": "t-waf", "label": "rate limited", "kind": "error" }, { "from": "t-gw", "to": "t-svc", "label": "mTLS + token" }, { "from": "t-svc", "to": "t-ent", "label": "authorise" }, { "from": "t-svc", "to": "t-asp", "label": "TLS · scoped role" }, { "from": "t-svc", "to": "t-aud", "label": "every action", "kind": "async" }, { "from": "t-agt", "to": "t-svc", "label": "mTLS outbound", "kind": "async" }, { "from": "t-agt", "to": "t-src", "label": "read-only metadata", "kind": "batch" } ], "note": "Metadata leaks too. No data values are persisted; profile statistics are bounded, and sample values are never stored for confidential or restricted assets.", "meta": { "v": "1.0", "owner": "Security Architecture", "date": "2026-08" } }, { "id": "21-access-and-audit", "title": "Authorisation and Audit — Changing a Classification", "layout": "sequence", "canvas": { "width": 1620 }, "lifelines": [ { "id": "a-ste", "label": "Data Steward", "kind": "actor" }, { "id": "a-ui", "label": "Governance Console", "kind": "app" }, { "id": "a-gw", "label": "API Gateway", "kind": "integration" }, { "id": "a-pdp", "label": "Policy Decision Point", "kind": "security" }, { "id": "a-cls", "label": "Classification Service", "kind": "app" }, { "id": "a-asp", "label": "Aspect Store", "kind": "store" }, { "id": "a-aud", "label": "Audit Log", "kind": "security" }, { "id": "a-pep", "label": "Policy Enforcement", "kind": "external" } ], "messages": [ { "from": "a-ste", "to": "a-ui", "label": "set label Restricted", "kind": "call" }, { "from": "a-ui", "to": "a-gw", "label": "PATCH classification", "kind": "call" }, { "from": "a-gw", "to": "a-pdp", "label": "authorise", "kind": "call" }, { "from": "a-pdp", "to": "a-pdp", "label": "role + domain + label", "kind": "self" }, { "from": "a-pdp", "to": "a-gw", "label": "deny, other domain", "kind": "error" }, { "from": "a-pdp", "to": "a-gw", "label": "permit", "kind": "return" }, { "from": "a-gw", "to": "a-cls", "label": "apply label", "kind": "call" }, { "from": "a-cls", "to": "a-asp", "label": "write version + 1", "kind": "call" }, { "from": "a-asp", "to": "a-aud", "label": "actor, before, after", "kind": "async" }, { "from": "a-cls", "to": "a-pep", "label": "sync tag", "kind": "async" }, { "from": "a-pep", "to": "a-cls", "label": "applied at source", "kind": "return" }, { "from": "a-gw", "to": "a-aud", "label": "denied attempt", "kind": "async" }, { "from": "a-gw", "to": "a-ui", "label": "204 with new version", "kind": "return" } ], "note": "Both outcomes are audited. The audit record is append-only and keeps the prior value, so a classification can always be explained after the fact.", "meta": { "v": "1.0", "owner": "Security Architecture", "date": "2026-08" } }, { "id": "22-failure-modes-and-recovery", "title": "Failure Modes, Tiers and Recovery", "layout": "nested", "canvas": { "width": 1720 }, "boxes": [ { "title": "Tier 1 · read path · 99.9% · RTO 30 min · must not fail", "kind": "boundary", "nodes": [ { "id": "r-api", "label": "Search & read API", "sub": "3 AZ · autoscaled", "kind": "app" }, { "id": "r-idx", "label": "Search Index", "sub": "3 replicas", "kind": "store" }, { "id": "r-cch", "label": "Read Cache", "sub": "serves on index loss", "kind": "store" }, { "id": "r-stl", "label": "Stale-read banner", "sub": "shows lag age", "kind": "risk" } ] }, { "title": "Tier 2 · write and ingest · may lag, must not lose", "kind": "boundary", "nodes": [ { "id": "r-bus", "label": "Metadata Event Bus", "sub": "7-day retention", "kind": "queue" }, { "id": "r-asp", "label": "Aspect Store", "sub": "RPO 5 min", "kind": "store" }, { "id": "r-cnf", "label": "Connector failure", "sub": "quarantine + alert", "kind": "risk" }, { "id": "r-poi", "label": "Poison payload", "sub": "dead letter, skip", "kind": "risk" } ] }, { "title": "Tier 3 · projections · disposable", "kind": "boundary", "nodes": [ { "id": "r-kg", "label": "Knowledge Graph", "sub": "rebuild under 4 h", "kind": "store" }, { "id": "r-rbi", "label": "Index rebuild", "sub": "replay change log", "kind": "app" }, { "id": "r-raw", "label": "Payload Archive", "sub": "replay beyond 7 days", "kind": "store" } ] }, { "title": "Standing risks · owned, not hidden", "kind": "boundary", "nodes": [ { "id": "r-hot", "label": "Graph hot spot", "sub": "wide lineage fan-out", "kind": "risk" }, { "id": "r-lim", "label": "Source rate limits", "sub": "harvest falls behind", "kind": "risk" }, { "id": "r-rot", "label": "Stale curation", "sub": "recertification overdue", "kind": "risk" }, { "id": "r-sil", "label": "Shadow catalogs", "sub": "the silo returns", "kind": "risk" } ] } ], "outside": [ { "id": "r-bak", "label": "Backup vault", "sub": "PITR 35 days", "kind": "store", "icon": "aws_aws-backup" }, { "id": "r-std", "label": "Warm standby region", "sub": "RTO 30 min", "kind": "external" } ], "edges": [ { "from": "r-idx", "to": "r-cch", "label": "fall back to cache", "kind": "error" }, { "from": "r-bus", "to": "r-rbi", "label": "replay", "kind": "batch" }, { "from": "r-raw", "to": "r-rbi", "label": "deep replay", "kind": "batch" }, { "from": "r-asp", "to": "r-bak", "label": "continuous backup", "kind": "batch" }, { "from": "r-asp", "to": "r-std", "label": "global replication", "kind": "async" } ], "note": "Adoption, not infrastructure, is the largest risk: a catalog nobody trusts is replaced by spreadsheets, which is why coverage and certification are on the operations dashboard in view 18.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } } ]