[ { "id": "01-system-context", "title": "Customer 360 & Real-Time Risk Intelligence — System Context", "layout": "context", "canvas": { "width": 1700 }, "colWidth": 250, "system": { "label": "Customer 360 & Risk Intelligence Platform", "sub": "Streaming lakehouse" }, "groups": [ { "side": "top", "title": "Systems of record", "nodes": [ { "id": "core", "label": "Core Banking", "sub": "T24 / FIS", "kind": "external", "rel": "CDC · accounts", "dir": "in", "kind2": "async" }, { "id": "cards", "label": "Card Platform", "sub": "Auth & settlement", "kind": "external", "rel": "auth events", "dir": "in", "kind2": "async" }, { "id": "crm", "label": "CRM", "sub": "Salesforce", "kind": "external", "rel": "party, cases", "dir": "in", "kind2": "async" }, { "id": "loans", "label": "Loan & Mortgage", "sub": "Exposure master", "kind": "external", "rel": "CDC · exposures", "dir": "in", "kind2": "batch" } ] }, { "side": "left", "title": "People", "nodes": [ { "id": "risk", "label": "Risk & Fraud Analyst", "kind": "actor", "rel": "investigates cases", "dir": "in" }, { "id": "cx", "label": "Marketing & CX Analyst", "kind": "actor", "rel": "segment queries", "dir": "in" }, { "id": "steward", "label": "Data Steward", "kind": "actor", "rel": "policy & glossary", "dir": "in" } ] }, { "side": "right", "title": "Digital & external sources", "nodes": [ { "id": "digital", "label": "Mobile & Web Apps", "sub": "Clickstream, logins", "kind": "external", "rel": "interaction events", "dir": "in", "kind2": "async" }, { "id": "pay", "label": "Payment Rails", "sub": "SWIFT / ACH / RTP", "kind": "external", "rel": "payment events", "dir": "in", "kind2": "async" }, { "id": "bureau", "label": "Bureau & Sanctions", "sub": "Credit, watchlists", "kind": "external", "rel": "reference feeds", "dir": "in", "kind2": "batch" } ] }, { "side": "bottom", "title": "Downstream consumers", "nodes": [ { "id": "fraud", "label": "Fraud & AML Decisioning", "sub": "< 2 s decision", "kind": "external", "rel": "enriched events", "kind2": "async" }, { "id": "bi", "label": "BI & Reporting", "sub": "Power BI / Tableau", "kind": "external", "rel": "curated marts" }, { "id": "ml", "label": "ML Feature Store", "sub": "Model training", "kind": "external", "rel": "features & history", "kind2": "batch" }, { "id": "reg", "label": "Regulatory Reporting", "sub": "BCBS 239 / AML", "kind": "external", "rel": "audited extracts", "kind2": "batch" } ] } ], "meta": { "v": "1.0", "owner": "Enterprise Data Architecture", "date": "2026-08" } }, { "id": "02-high-level-architecture", "title": "High-Level Platform Architecture", "layout": "flow", "canvas": { "width": 1720 }, "chain": true, "align": "top", "stages": [ { "title": "Sources", "nodes": [ { "id": "s-oltp", "label": "Core / Cards / Loans", "sub": "Oracle, DB2, Postgres", "kind": "external" }, { "id": "s-dig", "label": "Digital Channels", "sub": "Mobile, web, IVR", "kind": "external" }, { "id": "s-ext", "label": "External Feeds", "sub": "Bureau, sanctions", "kind": "external" } ] }, { "title": "Ingestion", "nodes": [ { "id": "i-cdc", "label": "CDC Connectors", "sub": "Debezium", "kind": "integration" }, { "id": "i-kafka", "label": "Kafka Event Backbone", "sub": "3 AZ, tiered storage", "kind": "queue" }, { "id": "i-batch", "label": "Batch & File Landing", "sub": "SFTP, API pull", "kind": "integration" } ] }, { "title": "Processing", "nodes": [ { "id": "p-flink", "label": "Stream Processing", "sub": "Flink, enrich & dedup", "kind": "app" }, { "id": "p-spark", "label": "Lakehouse Processing", "sub": "Spark streaming", "kind": "app" }, { "id": "p-dq", "label": "Data Quality Gates", "sub": "Expectations engine", "kind": "decision" } ] }, { "title": "Lakehouse", "nodes": [ { "id": "l-bronze", "label": "Bronze — Raw", "sub": "Immutable, replayable", "kind": "store" }, { "id": "l-silver", "label": "Silver — Conformed", "sub": "Deduped, SCD2", "kind": "store" }, { "id": "l-gold", "label": "Gold — Customer 360", "sub": "Marts & features", "kind": "store" } ] }, { "title": "Serving", "nodes": [ { "id": "sv-sql", "label": "SQL Warehouse", "sub": "Serverless", "kind": "app" }, { "id": "sv-api", "label": "Profile API", "sub": "p99 under 50 ms", "kind": "app" }, { "id": "sv-topic", "label": "Gold Event Topics", "sub": "Enriched, scored", "kind": "queue" } ] }, { "title": "Consumers", "nodes": [ { "id": "c-fraud", "label": "Fraud & AML", "kind": "external" }, { "id": "c-bi", "label": "BI & Analytics", "kind": "external" }, { "id": "c-ml", "label": "ML & Data Science", "kind": "external" } ] } ], "edges": [ { "from": "i-kafka", "to": "p-flink", "label": "sub-second", "kind": "async" }, { "from": "p-flink", "to": "sv-topic", "label": "enriched", "kind": "async" }, { "from": "sv-topic", "to": "c-fraud", "label": "under 2 s", "kind": "async" }, { "from": "l-gold", "to": "sv-api", "label": "profile cache" } ], "note": "Airflow orchestrates every batch and quality workflow; governance and security span all stages.", "meta": { "v": "1.0", "owner": "Enterprise Data Architecture", "date": "2026-08" } }, { "id": "03-layered-architecture", "title": "Layered Architecture and Dependency Direction", "layout": "bands", "canvas": { "width": 1620 }, "layerHeaderWidth": 160, "bands": [ { "name": "Consumption", "nodes": [ { "id": "b-bi", "label": "BI Dashboards", "sub": "Power BI", "kind": "external" }, { "id": "b-sqled", "label": "Ad-hoc SQL", "sub": "Analyst workbench", "kind": "external" }, { "id": "b-fraudc", "label": "Fraud Decisioning", "sub": "Event consumer", "kind": "external" }, { "id": "b-mlc", "label": "ML Training & Scoring", "kind": "external" } ] }, { "name": "Serving", "nodes": [ { "id": "b-wh", "label": "SQL Warehouse", "sub": "Serverless compute", "kind": "app" }, { "id": "b-api", "label": "Customer Profile API", "sub": "REST / gRPC", "kind": "app" }, { "id": "b-gtopic", "label": "Gold Kafka Topics", "kind": "queue" }, { "id": "b-fs", "label": "Feature Store", "sub": "Online + offline", "kind": "app" } ] }, { "name": "Curation", "nodes": [ { "id": "b-c360", "label": "Customer 360 Model", "sub": "Golden record", "kind": "app" }, { "id": "b-marts", "label": "Business Marts", "sub": "Txn, product, risk", "kind": "app" }, { "id": "b-agg", "label": "Aggregates & KPIs", "kind": "app" } ] }, { "name": "Processing", "nodes": [ { "id": "b-stream", "label": "Stream Processing", "sub": "Flink jobs", "kind": "app" }, { "id": "b-batch", "label": "Batch Processing", "sub": "Spark on lakehouse", "kind": "app" }, { "id": "b-dq", "label": "Quality & Dedup", "sub": "Rules + expectations", "kind": "app" }, { "id": "b-orch", "label": "Orchestration", "sub": "Airflow DAGs", "kind": "app" } ] }, { "name": "Storage", "nodes": [ { "id": "b-bz", "label": "Bronze", "sub": "Raw immutable", "kind": "store" }, { "id": "b-sv", "label": "Silver", "sub": "Conformed", "kind": "store" }, { "id": "b-gd", "label": "Gold", "sub": "Business ready", "kind": "store" }, { "id": "b-obj", "label": "Object Store", "sub": "S3 + Delta Lake", "kind": "store" } ] }, { "name": "Ingestion", "nodes": [ { "id": "b-kf", "label": "Kafka Backbone", "kind": "queue" }, { "id": "b-cdc", "label": "CDC Connectors", "kind": "integration" }, { "id": "b-file", "label": "File & API Ingest", "kind": "integration" }, { "id": "b-sr", "label": "Schema Registry", "sub": "Avro contracts", "kind": "integration" } ] }, { "name": "Platform", "nodes": [ { "id": "b-sec", "label": "Identity & Secrets", "sub": "Entra ID + Vault", "kind": "security" }, { "id": "b-gov", "label": "Catalog & Lineage", "sub": "Unity Catalog", "kind": "security" }, { "id": "b-obs", "label": "Observability", "sub": "Grafana + OTel", "kind": "security" }, { "id": "b-fin", "label": "FinOps & Cost", "kind": "security" } ] } ], "edges": [ { "from": "b-bi", "to": "b-wh", "label": "reads" }, { "from": "b-wh", "to": "b-c360", "label": "depends on" }, { "from": "b-c360", "to": "b-batch", "label": "built by" }, { "from": "b-batch", "to": "b-sv", "label": "reads / writes" }, { "from": "b-bz", "to": "b-kf", "label": "sourced from", "kind": "async" } ], "note": "Dependencies point downward only. Platform services are consumed by every layer above.", "meta": { "v": "1.0", "owner": "Enterprise Data Architecture", "date": "2026-08" } }, { "id": "04-container-architecture", "title": "Container Architecture (C4 Level 2)", "layout": "nested", "canvas": { "width": 1720 }, "boxes": [ { "title": "Customer 360 Platform — Cloud Landing Zone", "kind": "cloud", "dir": "col", "children": [ { "title": "Ingestion Segment", "kind": "boundary", "nodes": [ { "id": "k-connect", "label": "Kafka Connect Cluster", "sub": "Debezium, S3 sink", "kind": "integration" }, { "id": "k-broker", "label": "Kafka Brokers", "sub": "36 brokers, RF 3", "kind": "queue" }, { "id": "k-sr", "label": "Schema Registry", "sub": "Avro, BACKWARD", "kind": "integration" }, { "id": "k-gw", "label": "Event Gateway", "sub": "REST proxy, mTLS", "kind": "integration" } ] }, { "title": "Processing Segment", "kind": "boundary", "nodes": [ { "id": "flink", "label": "Flink Job Manager", "sub": "Enrichment topology", "kind": "app" }, { "id": "spark-str", "label": "Spark Streaming Jobs", "sub": "Bronze & silver writer", "kind": "app" }, { "id": "airflow", "label": "Airflow Scheduler", "sub": "Celery workers", "kind": "app" }, { "id": "spark-batch", "label": "Spark Batch Jobs", "sub": "Gold builds, backfill", "kind": "app" } ] }, { "title": "Storage & Serving Segment", "kind": "boundary", "nodes": [ { "id": "lake", "label": "Object Store Buckets", "sub": "Bronze / silver / gold", "kind": "store" }, { "id": "delta", "label": "Delta Tables", "sub": "ACID, time travel", "kind": "store" }, { "id": "wh", "label": "SQL Warehouse", "sub": "Serverless endpoints", "kind": "app" }, { "id": "papi", "label": "Profile API Service", "sub": "Containers, autoscaled", "kind": "app" }, { "id": "kv", "label": "Profile Store", "sub": "Low-latency key value", "kind": "store" } ] }, { "title": "Governance & Platform Segment", "kind": "boundary", "nodes": [ { "id": "uc", "label": "Unity Catalog", "sub": "Grants, lineage", "kind": "security" }, { "id": "gloss", "label": "Business Glossary", "sub": "Terms, ownership", "kind": "security" }, { "id": "vault", "label": "Tokenisation Vault", "sub": "PII surrogates", "kind": "security" }, { "id": "obs", "label": "Observability Stack", "sub": "Metrics, logs, traces", "kind": "security" } ] } ] } ], "outside": [ { "id": "src", "label": "Source Systems", "sub": "Core, cards, CRM, digital", "kind": "external" }, { "id": "cons", "label": "Consumers", "sub": "Fraud, BI, ML, regulators", "kind": "external" } ], "edges": [ { "from": "src", "to": "k-connect", "label": "CDC", "kind": "async" }, { "from": "k-connect", "to": "k-broker", "kind": "async" }, { "from": "k-broker", "to": "spark-str", "label": "consume", "kind": "async" }, { "from": "spark-str", "to": "delta", "label": "write bronze" }, { "from": "airflow", "to": "spark-batch", "kind": "batch" }, { "from": "papi", "to": "kv" }, { "from": "wh", "to": "cons", "label": "JDBC" } ], "note": "Cross-cutting grants, encryption and lineage apply to every segment; drawn once in the governance box.", "meta": { "v": "1.0", "owner": "Enterprise Data Architecture", "date": "2026-08" } }, { "id": "05-data-architecture-medallion", "title": "Data Architecture — Medallion Zones and Ownership", "layout": "nested", "canvas": { "width": 1720 }, "boxes": [ { "title": "Lakehouse — Delta Lake on Object Storage", "kind": "cloud", "dir": "col", "children": [ { "title": "Landing — as received, 7 day TTL", "kind": "boundary", "nodes": [ { "id": "z-land", "label": "Raw Files & Payloads", "sub": "Exactly as delivered", "kind": "store" }, { "id": "z-quar", "label": "Quarantine", "sub": "Rejected records", "kind": "risk" } ] }, { "title": "Bronze — immutable history, platform owned", "kind": "boundary", "nodes": [ { "id": "z-bcdc", "label": "CDC Change Log", "sub": "Before / after images", "kind": "store" }, { "id": "z-bevt", "label": "Event History", "sub": "Append only, partitioned", "kind": "store" }, { "id": "z-bref", "label": "Reference Snapshots", "sub": "Bureau, sanctions", "kind": "store" } ] }, { "title": "Silver — conformed, domain owned", "kind": "boundary", "nodes": [ { "id": "z-sparty", "label": "Party & Identity", "sub": "SCD2, matched keys", "kind": "store" }, { "id": "z-sacct", "label": "Account & Product", "sub": "Current + history", "kind": "store" }, { "id": "z-stxn", "label": "Transactions", "sub": "Deduped, standardised", "kind": "store" }, { "id": "z-sint", "label": "Interactions", "sub": "Sessions, channels", "kind": "store" } ] }, { "title": "Gold — business ready, product owned", "kind": "boundary", "nodes": [ { "id": "z-g360", "label": "Customer 360 Profile", "sub": "One row per customer", "kind": "store" }, { "id": "z-grisk", "label": "Risk & Exposure Mart", "kind": "store" }, { "id": "z-gfeat", "label": "Feature Tables", "sub": "Point in time correct", "kind": "store" }, { "id": "z-gkpi", "label": "KPI Aggregates", "sub": "Daily, monthly", "kind": "store" } ] } ] } ], "outside": [ { "id": "z-cat", "label": "Catalog & Retention Policy", "sub": "Class, owner, TTL, lineage", "kind": "security" } ], "edges": [ { "from": "z-land", "to": "z-bevt", "label": "ingest" }, { "from": "z-bevt", "to": "z-stxn", "label": "clean + dedup" }, { "from": "z-bcdc", "to": "z-sacct", "label": "merge SCD2" }, { "from": "z-stxn", "to": "z-g360", "label": "aggregate" }, { "from": "z-sparty", "to": "z-g360", "label": "golden record" }, { "from": "z-land", "to": "z-quar", "label": "schema fail", "kind": "error" } ], "note": "Bronze is never rewritten; every downstream zone is rebuildable from it by replay.", "meta": { "v": "1.0", "owner": "Enterprise Data Architecture", "date": "2026-08" } }, { "id": "06-streaming-ingestion-pipeline", "title": "Real-Time Event Ingestion Pipeline", "layout": "flow", "canvas": { "width": 1720 }, "chain": true, "align": "top", "stages": [ { "title": "Event producers", "nodes": [ { "id": "e-card", "label": "Card Authorisations", "sub": "40k events / sec peak", "kind": "external" }, { "id": "e-pay", "label": "Payments", "sub": "RTP, ACH, SWIFT", "kind": "external" }, { "id": "e-login", "label": "Logins & Sessions", "sub": "Mobile, web", "kind": "external" }, { "id": "e-acct", "label": "Account Activity", "sub": "CDC from core", "kind": "external" } ] }, { "title": "Publish", "nodes": [ { "id": "e-gw", "label": "Event Gateway", "sub": "mTLS, quota, authz", "kind": "integration" }, { "id": "e-sr", "label": "Schema Registry", "sub": "Avro, compatibility", "kind": "integration" }, { "id": "e-raw", "label": "Raw Topics", "sub": "Keyed by customer", "kind": "queue" } ] }, { "title": "Validate", "nodes": [ { "id": "e-val", "label": "Contract Validation", "sub": "Schema + mandatory", "kind": "decision" }, { "id": "e-dlq", "label": "Dead Letter Topic", "sub": "Replayable", "kind": "risk" } ] }, { "title": "Stream processing", "nodes": [ { "id": "e-dedup", "label": "Deduplication", "sub": "Event key, 24 h state", "kind": "app" }, { "id": "e-enrich", "label": "Enrichment", "sub": "Party, device, geo", "kind": "app" }, { "id": "e-win", "label": "Windowed Features", "sub": "1 m / 1 h / 24 h", "kind": "app" } ] }, { "title": "Persist & publish", "nodes": [ { "id": "e-bronze", "label": "Bronze Delta", "sub": "Exactly once sink", "kind": "store" }, { "id": "e-gold", "label": "Enriched Topic", "sub": "Gold event stream", "kind": "queue" }, { "id": "e-kv", "label": "Profile Store", "sub": "Upsert on key", "kind": "store" } ] }, { "title": "Consumers", "nodes": [ { "id": "e-fraud", "label": "Fraud Scoring", "sub": "Under 2 s budget", "kind": "external" }, { "id": "e-aml", "label": "AML Monitoring", "kind": "external" }, { "id": "e-alert", "label": "Customer Alerts", "kind": "external" } ] } ], "edges": [ { "from": "e-val", "to": "e-dlq", "label": "reject", "kind": "error" }, { "from": "e-val", "to": "e-dedup", "label": "valid events", "kind": "async" }, { "from": "e-gold", "to": "e-fraud", "label": "subscribe", "kind": "async" }, { "from": "e-win", "to": "e-kv", "label": "feature upsert" } ], "note": "Kafka delivers at least once; dedup makes it effectively exactly once. Tiered storage allows offset replay.", "meta": { "v": "1.0", "owner": "Streaming Platform", "date": "2026-08" } }, { "id": "07-cdc-batch-ingestion", "title": "CDC and Historical Batch Ingestion", "layout": "flow", "canvas": { "width": 1700 }, "chain": true, "align": "top", "stages": [ { "title": "Sources", "nodes": [ { "id": "c-ora", "label": "Core Banking DB", "sub": "Oracle redo logs", "kind": "external" }, { "id": "c-db2", "label": "Card Master", "sub": "DB2 journals", "kind": "external" }, { "id": "c-sf", "label": "CRM", "sub": "Salesforce bulk API", "kind": "external" }, { "id": "c-sftp", "label": "Partner Files", "sub": "SFTP, daily drops", "kind": "external" } ] }, { "title": "Capture", "nodes": [ { "id": "c-dbz", "label": "Debezium Connectors", "sub": "Log based, no query", "kind": "integration" }, { "id": "c-snap", "label": "Initial Snapshot", "sub": "Parallel, chunked", "kind": "integration" }, { "id": "c-land", "label": "File Landing Zone", "sub": "Checksum + manifest", "kind": "integration" } ] }, { "title": "Orchestration", "nodes": [ { "id": "c-af", "label": "Airflow DAGs", "sub": "Sensors, SLAs, retries", "kind": "app" }, { "id": "c-gate", "label": "Arrival Check", "sub": "Completeness, control totals", "kind": "decision" } ] }, { "title": "Land", "nodes": [ { "id": "c-bronze", "label": "Bronze Change Log", "sub": "Op, LSN, commit ts", "kind": "store" }, { "id": "c-late", "label": "Late & Out of Order", "sub": "Watermark buffer", "kind": "store" } ] }, { "title": "Apply", "nodes": [ { "id": "c-merge", "label": "Merge Into Silver", "sub": "Delta MERGE, idempotent", "kind": "app" }, { "id": "c-scd", "label": "SCD Type 2 Build", "sub": "Valid from / to", "kind": "app" }, { "id": "c-recon", "label": "Reconciliation", "sub": "Row counts vs source", "kind": "decision" } ] }, { "title": "Silver tables", "nodes": [ { "id": "c-party", "label": "Party & Account", "sub": "Current + history", "kind": "store" }, { "id": "c-prod", "label": "Product Holdings", "kind": "store" }, { "id": "c-hist", "label": "Historical Backfill", "sub": "10 years loaded", "kind": "store" } ] } ], "edges": [ { "from": "c-gate", "to": "c-af", "label": "hold on fail", "kind": "error" }, { "from": "c-snap", "to": "c-bronze", "label": "backfill", "kind": "batch", "route": "gutter" }, { "from": "c-recon", "to": "c-party", "label": "on pass" }, { "from": "c-late", "to": "c-merge", "label": "restated" } ], "note": "Snapshot and stream share one topic so cut-over from history to CDC needs no manual switch.", "meta": { "v": "1.0", "owner": "Data Integration", "date": "2026-08" } }, { "id": "08-source-onboarding-patterns", "title": "Source Onboarding Patterns by Data Class", "layout": "swimlane", "canvas": { "width": 1740 }, "laneHeaderWidth": 160, "stages": [ "Contract", "Capture", "Land in bronze", "Conform to silver", "Publish" ], "lanes": [ { "title": "Transactional events", "cells": [ [ { "id": "l1a", "label": "Avro schema + SLA", "kind": "integration" } ], [ { "id": "l1b", "label": "Produce to Kafka", "sub": "Self service topic", "kind": "queue" } ], [ { "id": "l1c", "label": "Streaming sink", "sub": "Exactly once", "kind": "app" } ], [ { "id": "l1d", "label": "Dedup + enrich", "kind": "app" } ], [ { "id": "l1e", "label": "Gold topic + mart", "kind": "store" } ] ] }, { "title": "Database CDC", "cells": [ [ { "id": "l2a", "label": "Table & key registry", "kind": "integration" } ], [ { "id": "l2b", "label": "Debezium connector", "sub": "Log based", "kind": "integration" } ], [ { "id": "l2c", "label": "Change log table", "kind": "store" } ], [ { "id": "l2d", "label": "MERGE + SCD2", "kind": "app" } ], [ { "id": "l2e", "label": "Dimension tables", "kind": "store" } ] ] }, { "title": "Files & partner feeds", "cells": [ [ { "id": "l3a", "label": "Layout + control totals", "kind": "integration" } ], [ { "id": "l3b", "label": "SFTP / API pull", "sub": "Airflow sensor", "kind": "integration" } ], [ { "id": "l3c", "label": "Raw file snapshot", "kind": "store" } ], [ { "id": "l3d", "label": "Parse + standardise", "kind": "app" } ], [ { "id": "l3e", "label": "Reference tables", "kind": "store" } ] ] }, { "title": "Semi-structured", "cells": [ [ { "id": "l4a", "label": "JSON contract", "sub": "Evolving fields", "kind": "integration" } ], [ { "id": "l4b", "label": "Gateway ingest", "kind": "integration" } ], [ { "id": "l4c", "label": "Variant column", "sub": "Schema on read", "kind": "store" } ], [ { "id": "l4d", "label": "Flatten known paths", "kind": "app" } ], [ { "id": "l4e", "label": "Interaction tables", "kind": "store" } ] ] } ], "note": "Each lane is an independent onboarding template; a new source reuses one without touching the others.", "meta": { "v": "1.0", "owner": "Data Integration", "date": "2026-08" } }, { "id": "09-data-quality-framework", "title": "Data Quality and Reconciliation Framework", "layout": "flow", "canvas": { "width": 1700 }, "chain": true, "align": "top", "stages": [ { "title": "Define", "nodes": [ { "id": "q-rules", "label": "Rule Library", "sub": "Versioned in Git", "kind": "app" }, { "id": "q-sla", "label": "Freshness SLA", "sub": "Per dataset", "kind": "app" }, { "id": "q-owner", "label": "Steward Ownership", "kind": "actor" } ] }, { "title": "Structural checks", "nodes": [ { "id": "q-schema", "label": "Schema Conformance", "sub": "Types, required", "kind": "decision" }, { "id": "q-dup", "label": "Duplicate Detection", "sub": "Business key hash", "kind": "decision" } ] }, { "title": "Semantic checks", "nodes": [ { "id": "q-ref", "label": "Referential Integrity", "sub": "Party, account keys", "kind": "decision" }, { "id": "q-range", "label": "Range & Domain", "sub": "Amounts, currency", "kind": "decision" }, { "id": "q-drift", "label": "Volume & Drift", "sub": "Vs 30 day baseline", "kind": "decision" } ] }, { "title": "Disposition", "nodes": [ { "id": "q-pass", "label": "Promote", "sub": "Publish to silver", "kind": "app" }, { "id": "q-quar", "label": "Quarantine Table", "sub": "Reason coded", "kind": "risk" }, { "id": "q-block", "label": "Block Downstream", "sub": "Circuit breaker", "kind": "risk" } ] }, { "title": "Act", "nodes": [ { "id": "q-score", "label": "Quality Scorecard", "sub": "Per domain, daily", "kind": "app" }, { "id": "q-alert", "label": "Alert & Ticket", "sub": "On call + ServiceNow", "kind": "app" }, { "id": "q-fix", "label": "Correct & Replay", "sub": "Reprocess window", "kind": "app" } ] } ], "edges": [ { "from": "q-dup", "to": "q-quar", "label": "fail", "kind": "error" }, { "from": "q-drift", "to": "q-block", "label": "breach", "kind": "error" }, { "from": "q-quar", "to": "q-fix", "label": "steward review" }, { "from": "q-fix", "to": "q-pass", "label": "re-run" } ], "note": "Checks run in-stream and post-load; the same rule definitions serve both engines.", "meta": { "v": "1.0", "owner": "Data Governance", "date": "2026-08" } }, { "id": "10-schema-evolution-lifecycle", "title": "Schema and Data Contract Lifecycle", "layout": "cycle", "canvas": { "width": 1500 }, "centre": { "label": "Data Contract Governance" }, "nodes": [ { "id": "sc-prop", "label": "Propose Change", "sub": "Producer team PR", "kind": "app" }, { "id": "sc-check", "label": "Compatibility Check", "sub": "BACKWARD in CI", "kind": "decision" }, { "id": "sc-reg", "label": "Register Version", "sub": "Schema Registry", "kind": "integration" }, { "id": "sc-roll", "label": "Roll Out", "sub": "Producer then consumer", "kind": "app" }, { "id": "sc-evolve", "label": "Evolve Tables", "sub": "Delta column add", "kind": "store" }, { "id": "sc-mon", "label": "Monitor Usage", "sub": "Version adoption", "kind": "security" }, { "id": "sc-dep", "label": "Deprecate Old", "sub": "After 90 days", "kind": "risk" } ], "ringLabels": [ "submitted", "approved", "published", "consumers upgraded", "columns added", "usage falls", "next change" ], "note": "Breaking changes require a new topic version, never an in-place overwrite.", "meta": { "v": "1.0", "owner": "Data Governance", "date": "2026-08" } }, { "id": "11-realtime-risk-enrichment", "title": "Real-Time Risk Enrichment and Decision Path", "layout": "flow", "canvas": { "width": 1700 }, "chain": true, "align": "top", "stages": [ { "title": "Trigger", "nodes": [ { "id": "r-auth", "label": "Card Authorisation", "sub": "ISO 8583 event", "kind": "external" }, { "id": "r-login", "label": "Digital Login", "sub": "Device, IP, geo", "kind": "external" }, { "id": "r-pmt", "label": "Payment Initiation", "kind": "external" } ] }, { "title": "Context lookup", "nodes": [ { "id": "r-prof", "label": "Customer Profile", "sub": "Gold, cached", "kind": "store" }, { "id": "r-feat", "label": "Online Features", "sub": "Velocity, ratios", "kind": "store" }, { "id": "r-watch", "label": "Watchlist & Sanctions", "sub": "Refreshed daily", "kind": "store" } ] }, { "title": "Stream enrichment", "nodes": [ { "id": "r-join", "label": "Stateful Join", "sub": "Flink keyed state", "kind": "app" }, { "id": "r-derive", "label": "Derive Signals", "sub": "Deviation, novelty", "kind": "app" }, { "id": "r-rule", "label": "Risk Rules", "sub": "Declarative, versioned", "kind": "decision" } ] }, { "title": "Publish", "nodes": [ { "id": "r-topic", "label": "Enriched Event Topic", "sub": "Contract v2", "kind": "queue" }, { "id": "r-audit", "label": "Decision Audit Log", "sub": "Immutable, bronze", "kind": "store" } ] }, { "title": "Decisioning", "nodes": [ { "id": "r-model", "label": "Fraud Model Service", "sub": "Owned by risk", "kind": "external" }, { "id": "r-aml", "label": "AML Scenario Engine", "kind": "external" }, { "id": "r-case", "label": "Case Management", "kind": "external" } ] }, { "title": "Outcome", "nodes": [ { "id": "r-block", "label": "Block or Challenge", "sub": "Step-up auth", "kind": "app" }, { "id": "r-alert", "label": "Analyst Alert", "kind": "app" }, { "id": "r-fb", "label": "Outcome Feedback", "sub": "Back to features", "kind": "app" } ] } ], "edges": [ { "from": "r-rule", "to": "r-topic", "label": "scored", "kind": "async" }, { "from": "r-topic", "to": "r-model", "label": "consume", "kind": "async" }, { "from": "r-fb", "to": "r-feat", "label": "label loop", "kind": "async" }, { "from": "r-join", "to": "r-audit", "label": "every decision" } ], "note": "The platform enriches and publishes; scoring and case handling stay owned by the risk domain.", "meta": { "v": "1.0", "owner": "Risk Data Engineering", "date": "2026-08" } }, { "id": "12-integration-architecture", "title": "Integration Architecture and Interface Catalogue", "layout": "hub", "canvas": { "width": 1620 }, "left": { "title": "Inbound sources", "nodes": [ { "id": "in-core", "label": "Core Banking", "sub": "Oracle CDC", "kind": "external", "rel": "CDC stream", "dir": "in", "kind2": "async" }, { "id": "in-card", "label": "Card Switch", "sub": "Auth stream", "kind": "external", "rel": "Kafka 40k/s", "dir": "in", "kind2": "async" }, { "id": "in-crm", "label": "CRM", "sub": "Salesforce", "kind": "external", "rel": "hourly API", "dir": "in", "kind2": "batch" }, { "id": "in-dig", "label": "Digital Channels", "sub": "Mobile & web SDK", "kind": "external", "rel": "HTTPS SDK", "dir": "in", "kind2": "async" }, { "id": "in-file", "label": "Partner Feeds", "sub": "Bureau, sanctions", "kind": "external", "rel": "SFTP daily", "dir": "in", "kind2": "batch" } ] }, "centre": { "title": "Platform", "nodes": [ { "id": "hub-core", "label": "Customer 360 Platform", "sub": "Kafka + lakehouse" } ] }, "right": { "title": "Outbound consumers", "nodes": [ { "id": "out-fraud", "label": "Fraud Decisioning", "kind": "external", "rel": "Kafka, 2 s", "dir": "out", "kind2": "async" }, { "id": "out-api", "label": "Channel Applications", "sub": "Profile lookup", "kind": "external", "rel": "REST 50 ms", "dir": "out" }, { "id": "out-bi", "label": "BI Platform", "kind": "external", "rel": "JDBC gold", "dir": "out" }, { "id": "out-ml", "label": "ML Platform", "kind": "external", "rel": "Delta share", "dir": "out", "kind2": "batch" }, { "id": "out-reg", "label": "Regulatory Reporting", "kind": "external", "rel": "signed daily", "dir": "out", "kind2": "batch" } ] }, "note": "Every interface has a named owner, a versioned contract and a published SLA.", "meta": { "v": "1.0", "owner": "Integration Architecture", "date": "2026-08" } }, { "id": "13-security-trust-zones", "title": "Security Architecture and Trust Zones", "layout": "zones", "canvas": { "width": 1740 }, "zones": [ { "title": "Untrusted", "kind": "trust", "nodes": [ { "id": "sz-int", "label": "Internet Clients", "sub": "Mobile, partners", "kind": "external" }, { "id": "sz-threat", "label": "Credential Abuse", "sub": "Replay, injection", "kind": "risk" } ] }, { "title": "Perimeter", "kind": "trust", "nodes": [ { "id": "sz-waf", "label": "WAF & API Gateway", "sub": "Rate limit, schema", "kind": "security" }, { "id": "sz-idp", "label": "Identity Provider", "sub": "Entra ID, MFA", "kind": "security" }, { "id": "sz-pl", "label": "Private Link", "sub": "No public endpoints", "kind": "security" } ] }, { "title": "Application", "kind": "trust", "nodes": [ { "id": "sz-kafka", "label": "Kafka Cluster", "sub": "mTLS, topic ACLs", "kind": "queue" }, { "id": "sz-comp", "label": "Processing Clusters", "sub": "No inbound internet", "kind": "app" }, { "id": "sz-api", "label": "Profile API", "sub": "OAuth2 scopes", "kind": "app" } ] }, { "title": "Data", "kind": "trust", "nodes": [ { "id": "sz-lake", "label": "Lakehouse Storage", "sub": "CMK encrypted", "kind": "store" }, { "id": "sz-tok", "label": "Tokenisation Vault", "sub": "PAN, national ID", "kind": "security" }, { "id": "sz-rbac", "label": "Catalog RBAC", "sub": "Row & column masks", "kind": "security" } ] }, { "title": "Control plane", "kind": "trust", "nodes": [ { "id": "sz-kms", "label": "Key Management", "sub": "HSM backed, rotated", "kind": "security" }, { "id": "sz-siem", "label": "SIEM & Audit", "sub": "Immutable, 7 years", "kind": "security" }, { "id": "sz-pam", "label": "Privileged Access", "sub": "Just in time", "kind": "security" } ] } ], "edges": [ { "from": "sz-int", "to": "sz-waf", "label": "TLS 1.3, mTLS" }, { "from": "sz-threat", "to": "sz-waf", "label": "blocked at edge", "kind": "error" }, { "from": "sz-waf", "to": "sz-api", "label": "token validated" }, { "from": "sz-idp", "to": "sz-rbac", "label": "group to grant" }, { "from": "sz-api", "to": "sz-lake", "label": "scoped read" }, { "from": "sz-comp", "to": "sz-tok", "label": "detokenise on policy" }, { "from": "sz-kms", "to": "sz-lake", "label": "envelope keys" }, { "from": "sz-rbac", "to": "sz-siem", "label": "access events", "kind": "async" } ], "note": "No workload crosses two zones without an authenticated, logged and policy-checked hop.", "meta": { "v": "1.0", "owner": "Security Architecture", "date": "2026-08" } }, { "id": "14-governance-controls-matrix", "title": "Governance Controls by Data Zone", "layout": "grid", "canvas": { "width": 1740 }, "laneHeaderWidth": 170, "columns": [ "Landing", "Bronze", "Silver", "Gold", "Consumption" ], "rows": [ { "title": "Classification", "cells": [ [ { "id": "g11", "label": "Auto scan on arrival", "kind": "security" } ], [ { "id": "g12", "label": "Tag PII columns", "kind": "security" } ], [ { "id": "g13", "label": "Sensitivity inherited", "kind": "security" } ], [ { "id": "g14", "label": "Business terms bound", "kind": "security" } ], [ { "id": "g15", "label": "Label on export", "kind": "security" } ] ] }, { "title": "PII protection", "cells": [ [ { "id": "g21", "label": "Encrypt at rest", "kind": "security" } ], [ { "id": "g22", "label": "Tokenise PAN & ID", "kind": "security" } ], [ { "id": "g23", "label": "Hash join keys", "kind": "security" } ], [ { "id": "g24", "label": "Dynamic masking", "kind": "security" } ], [ { "id": "g25", "label": "Purpose check", "kind": "decision" } ] ] }, { "title": "Access control", "cells": [ [ { "id": "g31", "label": "Platform team only", "kind": "app" } ], [ { "id": "g32", "label": "Engineers, no read PII", "kind": "app" } ], [ { "id": "g33", "label": "Domain owned grants", "kind": "app" } ], [ { "id": "g34", "label": "Role based, attested", "kind": "app" } ], [ { "id": "g35", "label": "Row filters by region", "kind": "app" } ] ] }, { "title": "Lineage & metadata", "cells": [ [ { "id": "g41", "label": "Source manifest", "kind": "integration" } ], [ { "id": "g42", "label": "Offset & file ancestry", "kind": "integration" } ], [ { "id": "g43", "label": "Job level lineage", "kind": "integration" } ], [ { "id": "g44", "label": "Column lineage", "kind": "integration" } ], [ { "id": "g45", "label": "Report to source trace", "kind": "integration" } ] ] }, { "title": "Retention & records", "cells": [ [ { "id": "g51", "label": "7 day TTL", "kind": "store" } ], [ { "id": "g52", "label": "10 years, WORM", "kind": "store" } ], [ { "id": "g53", "label": "7 years", "kind": "store" } ], [ { "id": "g54", "label": "Rebuildable, 3 years", "kind": "store" } ], [ { "id": "g55", "label": "Extracts expire 90 d", "kind": "store" } ] ] }, { "title": "Quality assurance", "cells": [ [ { "id": "g61", "label": "Control totals", "kind": "decision" } ], [ { "id": "g62", "label": "Schema conformance", "kind": "decision" } ], [ { "id": "g63", "label": "Dedup & integrity", "kind": "decision" } ], [ { "id": "g64", "label": "Reconciliation", "kind": "decision" } ], [ { "id": "g65", "label": "Freshness SLA", "kind": "decision" } ] ] } ], "note": "Every cell is an implemented control with a named owner; blank cells would be findings, not omissions.", "meta": { "v": "1.0", "owner": "Data Governance", "date": "2026-08" } }, { "id": "15-deployment-architecture", "title": "Deployment Topology and Failure Domains", "layout": "nested", "canvas": { "width": 1740 }, "boxes": [ { "title": "Primary Region — eu-west-1", "kind": "cloud", "dir": "row", "children": [ { "title": "Availability Zone A", "kind": "boundary", "nodes": [ { "id": "d-ka", "label": "Kafka Brokers", "sub": "12 nodes", "kind": "queue" }, { "id": "d-fa", "label": "Flink Task Managers", "sub": "Autoscaled", "kind": "app" }, { "id": "d-sa", "label": "Spark Workers", "sub": "Spot + on demand", "kind": "app" } ] }, { "title": "Availability Zone B", "kind": "boundary", "nodes": [ { "id": "d-kb", "label": "Kafka Brokers", "sub": "12 nodes", "kind": "queue" }, { "id": "d-fb", "label": "Flink Task Managers", "kind": "app" }, { "id": "d-sb", "label": "Spark Workers", "kind": "app" } ] }, { "title": "Availability Zone C", "kind": "boundary", "nodes": [ { "id": "d-kc", "label": "Kafka Brokers", "sub": "12 nodes", "kind": "queue" }, { "id": "d-af", "label": "Airflow Scheduler", "sub": "Active / standby", "kind": "app" }, { "id": "d-api", "label": "Profile API Pods", "kind": "app" } ] } ] }, { "title": "Regional Managed Services", "kind": "cloud", "dir": "row", "children": [ { "title": "Storage & catalog", "kind": "boundary", "nodes": [ { "id": "d-s3", "label": "Object Storage", "sub": "Cross region replication", "kind": "store" }, { "id": "d-uc", "label": "Catalog Metastore", "sub": "Multi AZ", "kind": "security" } ] }, { "title": "State & serving", "kind": "boundary", "nodes": [ { "id": "d-kv", "label": "Profile Key Value Store", "sub": "3 AZ replicas", "kind": "store" }, { "id": "d-wh", "label": "SQL Warehouse", "sub": "Serverless pools", "kind": "app" } ] } ] }, { "title": "DR Region — eu-central-1 (warm standby)", "kind": "cloud", "dir": "row", "children": [ { "title": "Standby compute", "kind": "boundary", "nodes": [ { "id": "d-kdr", "label": "Kafka Mirror Cluster", "sub": "Cluster Linking", "kind": "queue" }, { "id": "d-cdr", "label": "Pre-provisioned Compute", "sub": "Scaled to zero", "kind": "app" } ] }, { "title": "Standby data", "kind": "boundary", "nodes": [ { "id": "d-s3dr", "label": "Replicated Buckets", "sub": "RPO 15 min", "kind": "store" }, { "id": "d-ucdr", "label": "Catalog Replica", "kind": "security" } ] } ] } ], "outside": [ { "id": "d-dns", "label": "Global Traffic Manager", "sub": "Health based failover", "kind": "integration" } ], "edges": [ { "from": "d-ka", "to": "d-kdr", "label": "async mirror", "kind": "async" }, { "from": "d-s3", "to": "d-s3dr", "label": "CRR, 15 min", "kind": "async" }, { "from": "d-uc", "to": "d-ucdr", "label": "metadata sync", "kind": "async" }, { "from": "d-dns", "to": "d-api", "label": "primary" }, { "from": "d-dns", "to": "d-cdr", "label": "on failover", "kind": "error" } ], "note": "Loss of one AZ costs capacity, not availability. RTO 4 h, RPO 15 min for a region loss.", "meta": { "v": "1.0", "owner": "Platform Engineering", "date": "2026-08" } }, { "id": "16-cicd-environments", "title": "CI/CD Pipeline and Environment Promotion", "layout": "flow", "canvas": { "width": 1720 }, "chain": true, "align": "top", "stages": [ { "title": "Source", "nodes": [ { "id": "v-repo", "label": "Pipeline Repository", "sub": "Spark, Flink, DAGs", "kind": "app" }, { "id": "v-contract", "label": "Schema & DQ Rules", "sub": "Versioned with code", "kind": "app" }, { "id": "v-iac", "label": "Infrastructure as Code", "sub": "Terraform modules", "kind": "app" } ] }, { "title": "Build & test", "nodes": [ { "id": "v-unit", "label": "Unit & Contract Tests", "sub": "Sample fixtures", "kind": "app" }, { "id": "v-compat", "label": "Schema Compatibility", "sub": "Registry dry run", "kind": "decision" }, { "id": "v-scan", "label": "Security Scan", "sub": "SAST, deps, secrets", "kind": "decision" } ] }, { "title": "Non-production", "nodes": [ { "id": "v-dev", "label": "Development", "sub": "Synthetic data only", "kind": "app" }, { "id": "v-int", "label": "Integration", "sub": "Masked subset", "kind": "app" }, { "id": "v-perf", "label": "Performance", "sub": "Peak volume replay", "kind": "app" } ] }, { "title": "Gates", "nodes": [ { "id": "v-dq", "label": "Quality Regression", "sub": "Golden dataset diff", "kind": "decision" }, { "id": "v-cab", "label": "Change Approval", "sub": "Risk rated", "kind": "decision" } ] }, { "title": "Production release", "nodes": [ { "id": "v-blue", "label": "Blue / Green Jobs", "sub": "Savepoint restore", "kind": "app" }, { "id": "v-dag", "label": "DAG Sync", "sub": "Paused then enabled", "kind": "app" }, { "id": "v-flag", "label": "Feature Flags", "sub": "Per source rollout", "kind": "app" } ] }, { "title": "Verify", "nodes": [ { "id": "v-smoke", "label": "Smoke & Canary", "sub": "Shadow consumer", "kind": "app" }, { "id": "v-slo", "label": "SLO Watch", "sub": "30 min burn window", "kind": "app" }, { "id": "v-rb", "label": "Rollback", "sub": "Previous savepoint", "kind": "risk" } ] } ], "edges": [ { "from": "v-compat", "to": "v-repo", "label": "breaking, reject", "kind": "error" }, { "from": "v-dq", "to": "v-int", "label": "regression, return", "kind": "error" }, { "from": "v-slo", "to": "v-rb", "label": "breach", "kind": "error" }, { "from": "v-cab", "to": "v-blue", "label": "approved" } ], "note": "No production dataset is copied downward; lower environments use synthetic or masked data.", "meta": { "v": "1.0", "owner": "Platform Engineering", "date": "2026-08" } }, { "id": "17-observability-slo", "title": "Observability and SLA Monitoring Matrix", "layout": "grid", "canvas": { "width": 1740 }, "laneHeaderWidth": 165, "columns": [ "Emit", "Collect", "Store", "Detect", "Act" ], "rows": [ { "title": "Metrics", "cells": [ [ { "id": "o11", "label": "Broker, job, API", "kind": "app" } ], [ { "id": "o12", "label": "OTel collector", "kind": "integration" } ], [ { "id": "o13", "label": "Time series store", "kind": "store" } ], [ { "id": "o14", "label": "SLO burn alert", "kind": "decision" } ], [ { "id": "o15", "label": "Scale or page", "kind": "app" } ] ] }, { "title": "Logs", "cells": [ [ { "id": "o21", "label": "Structured JSON", "kind": "app" } ], [ { "id": "o22", "label": "Node agents", "kind": "integration" } ], [ { "id": "o23", "label": "Log lake, 90 d", "kind": "store" } ], [ { "id": "o24", "label": "Error pattern rule", "kind": "decision" } ], [ { "id": "o25", "label": "Ticket + runbook", "kind": "app" } ] ] }, { "title": "Traces", "cells": [ [ { "id": "o31", "label": "Spans on API & jobs", "kind": "app" } ], [ { "id": "o32", "label": "Tail sampling", "kind": "integration" } ], [ { "id": "o33", "label": "Trace store, 15 d", "kind": "store" } ], [ { "id": "o34", "label": "Latency outlier", "kind": "decision" } ], [ { "id": "o35", "label": "Hotspot fix", "kind": "app" } ] ] }, { "title": "Pipeline SLA", "cells": [ [ { "id": "o41", "label": "Lag, duration, offset", "kind": "app" } ], [ { "id": "o42", "label": "Airflow + Flink", "kind": "integration" } ], [ { "id": "o43", "label": "SLA history table", "kind": "store" } ], [ { "id": "o44", "label": "Miss prediction", "kind": "decision" } ], [ { "id": "o45", "label": "Escalate on call", "kind": "app" } ] ] }, { "title": "Data quality", "cells": [ [ { "id": "o51", "label": "Expectation results", "kind": "app" } ], [ { "id": "o52", "label": "Quality event topic", "kind": "queue" } ], [ { "id": "o53", "label": "Quality history", "kind": "store" } ], [ { "id": "o54", "label": "Threshold + drift", "kind": "decision" } ], [ { "id": "o55", "label": "Quarantine + steward", "kind": "app" } ] ] }, { "title": "Cost", "cells": [ [ { "id": "o61", "label": "Job & cluster tags", "kind": "app" } ], [ { "id": "o62", "label": "Billing export", "kind": "integration" } ], [ { "id": "o63", "label": "FinOps dataset", "kind": "store" } ], [ { "id": "o64", "label": "Budget variance", "kind": "decision" } ], [ { "id": "o65", "label": "Rightsize, reserve", "kind": "app" } ] ] } ], "note": "Data quality and cost are first-class signals, monitored on the same rails as infrastructure.", "meta": { "v": "1.0", "owner": "Platform Engineering", "date": "2026-08" } }, { "id": "18-replay-reprocessing", "title": "Replay and Historical Reprocessing", "layout": "flow", "canvas": { "width": 1700 }, "chain": true, "align": "top", "stages": [ { "title": "Trigger", "nodes": [ { "id": "y-logic", "label": "Business Logic Change", "sub": "New risk definition", "kind": "app" }, { "id": "y-defect", "label": "Pipeline Defect", "sub": "Wrong join or filter", "kind": "risk" }, { "id": "y-reg", "label": "Regulatory Restatement", "kind": "app" } ] }, { "title": "Scope", "nodes": [ { "id": "y-lin", "label": "Lineage Impact Query", "sub": "Which tables, which runs", "kind": "app" }, { "id": "y-win", "label": "Select Time Window", "sub": "Offsets or partitions", "kind": "decision" } ] }, { "title": "Source of truth", "nodes": [ { "id": "y-bronze", "label": "Bronze History", "sub": "Immutable, 10 years", "kind": "store" }, { "id": "y-tier", "label": "Kafka Tiered Storage", "sub": "Offset replay", "kind": "queue" } ] }, { "title": "Isolated rerun", "nodes": [ { "id": "y-pool", "label": "Dedicated Compute Pool", "sub": "No effect on live SLA", "kind": "app" }, { "id": "y-shadow", "label": "Shadow Tables", "sub": "Versioned suffix", "kind": "store" }, { "id": "y-job", "label": "New Logic Version", "sub": "Same code, new tag", "kind": "app" } ] }, { "title": "Compare", "nodes": [ { "id": "y-diff", "label": "Row & Measure Diff", "sub": "Old vs new", "kind": "decision" }, { "id": "y-dq", "label": "Quality Suite", "sub": "Full rule set", "kind": "decision" }, { "id": "y-sign", "label": "Steward Sign-off", "kind": "actor" } ] }, { "title": "Cut over", "nodes": [ { "id": "y-swap", "label": "Atomic Swap", "sub": "Delta table version", "kind": "app" }, { "id": "y-notify", "label": "Notify Consumers", "sub": "Restatement notice", "kind": "app" }, { "id": "y-arch", "label": "Archive Prior Version", "sub": "Audit evidence", "kind": "store" } ] } ], "edges": [ { "from": "y-diff", "to": "y-job", "label": "unexplained delta", "kind": "error" }, { "from": "y-tier", "to": "y-pool", "label": "replay from offset", "kind": "async" }, { "from": "y-bronze", "to": "y-pool", "label": "read partitions", "kind": "batch" }, { "from": "y-sign", "to": "y-swap", "label": "approved" } ], "note": "Live pipelines keep running throughout; the swap is the only moment consumers observe a change.", "meta": { "v": "1.0", "owner": "Data Engineering", "date": "2026-08" } }, { "id": "19-transaction-event-sequence", "title": "Card Transaction — Event Sequence", "layout": "sequence", "canvas": { "width": 1620 }, "lifelines": [ { "id": "sw", "label": "Card Switch", "kind": "external" }, { "id": "gw", "label": "Event Gateway", "kind": "integration" }, { "id": "kf", "label": "Kafka", "kind": "queue" }, { "id": "fl", "label": "Flink Enrichment", "kind": "app" }, { "id": "ps", "label": "Profile Store", "kind": "store" }, { "id": "lk", "label": "Bronze Lakehouse", "kind": "store" }, { "id": "fr", "label": "Fraud Service", "kind": "external" } ], "messages": [ { "from": "sw", "to": "gw", "label": "authorisation event", "kind": "call" }, { "from": "gw", "to": "gw", "label": "mTLS + schema validate", "kind": "self" }, { "from": "gw", "to": "kf", "label": "produce txn.auth.v2", "kind": "async" }, { "from": "gw", "to": "kf", "label": "invalid to DLQ", "kind": "error" }, { "from": "kf", "to": "fl", "label": "consume by customer key", "kind": "async" }, { "from": "fl", "to": "fl", "label": "dedup on event id", "kind": "self" }, { "from": "fl", "to": "ps", "label": "lookup 360 profile", "kind": "call" }, { "from": "ps", "to": "fl", "label": "profile + features", "kind": "return" }, { "from": "fl", "to": "fl", "label": "derive velocity signals", "kind": "self" }, { "from": "fl", "to": "kf", "label": "publish enriched event", "kind": "async" }, { "from": "fl", "to": "lk", "label": "sink raw + enriched", "kind": "async" }, { "from": "kf", "to": "fr", "label": "enriched.txn.v1", "kind": "async" }, { "from": "fr", "to": "sw", "label": "approve or challenge", "kind": "return" }, { "from": "fr", "to": "kf", "label": "decision outcome", "kind": "async" } ], "note": "Happy path plus schema rejection. Retry, rebalance and backpressure paths are omitted.", "meta": { "v": "1.0", "owner": "Risk Data Engineering", "date": "2026-08" } }, { "id": "20-customer-360-data-model", "title": "Customer 360 — Core Data Model", "layout": "er", "canvas": { "width": 1620, "cols": 3 }, "rowGap": 260, "entities": [ { "id": "xref", "name": "customer_xref", "kind": "integration", "row": 0, "col": 0, "attrs": [ "xref_id PK", "customer_id FK", "source_system", "source_key", "match_confidence", "valid_from / valid_to" ] }, { "id": "hh", "name": "dim_household", "kind": "app", "row": 0, "col": 1, "attrs": [ "household_id PK", "household_type", "primary_customer_id", "region_code" ] }, { "id": "prof", "name": "customer_360_profile", "kind": "store", "row": 0, "col": 2, "attrs": [ "customer_id PK, FK", "as_of_ts", "tenure_months", "relationship_value", "channel_affinity", "next_best_action" ] }, { "id": "acct", "name": "dim_account", "kind": "app", "row": 1, "col": 0, "attrs": [ "account_id PK", "customer_id FK", "product_code", "open_date / close_date", "status", "valid_from / valid_to" ] }, { "id": "cust", "name": "dim_customer", "kind": "app", "row": 1, "col": 1, "attrs": [ "customer_id PK", "household_id FK", "party_type", "name_token PII", "national_id_token PII", "kyc_status", "valid_from / valid_to" ] }, { "id": "risk", "name": "dim_risk_rating", "kind": "app", "row": 1, "col": 2, "attrs": [ "rating_id PK", "customer_id FK", "rating_scheme", "rating_value", "effective_ts" ] }, { "id": "txn", "name": "fact_transaction", "kind": "store", "row": 2, "col": 0, "attrs": [ "txn_id PK", "account_id FK", "event_ts / ingest_ts", "amount, currency", "merchant_category", "channel", "dedup_key" ] }, { "id": "intr", "name": "fact_interaction", "kind": "store", "row": 2, "col": 1, "attrs": [ "interaction_id PK", "customer_id FK", "event_ts", "channel, device_id", "session_id", "event_type" ] }, { "id": "rsk", "name": "fact_risk_event", "kind": "store", "row": 2, "col": 2, "attrs": [ "risk_event_id PK", "rating_id FK", "customer_id FK", "event_ts", "signal_type", "score, decision" ] } ], "relations": [ { "from": "xref", "to": "cust", "label": "N : 1", "from_side": "s", "to_side": "n1" }, { "from": "hh", "to": "cust", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "prof", "to": "cust", "label": "1 : 1", "from_side": "s", "to_side": "n3" }, { "from": "cust", "to": "acct", "label": "1 : N", "from_side": "w", "to_side": "e" }, { "from": "cust", "to": "risk", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "acct", "to": "txn", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "cust", "to": "intr", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "risk", "to": "rsk", "label": "1 : N", "from_side": "s", "to_side": "n" } ], "note": "PII columns hold surrogate tokens; clear values resolve only through the vault under policy.", "meta": { "v": "1.0", "owner": "Data Architecture", "date": "2026-08" } } ]