[ { "id": "01-system-context", "title": "Real-Time Analytics Platform — System Context", "layout": "context", "colWidth": 250, "system": { "label": "Real-Time Analytics Platform", "sub": "Azure · event to insight < 5 s" }, "groups": [ { "side": "left", "title": "Event producers", "nodes": [ { "id": "sdk", "label": "Web & Mobile Apps", "sub": "client SDK", "kind": "external", "rel": "HTTPS event batches", "dir": "in", "kind2": "async" }, { "id": "partner", "label": "Partner Systems", "sub": "REST · OAuth2", "kind": "external", "rel": "server-side events", "dir": "in", "kind2": "async", "icon": "fa5_cloud" }, { "id": "iot", "label": "Devices & Edge", "sub": "MQTT / AMQP", "kind": "external", "rel": "telemetry", "dir": "in", "kind2": "async" }, { "id": "oltp", "label": "Operational Databases", "sub": "Azure SQL · PostgreSQL", "kind": "external", "rel": "CDC log stream", "dir": "in", "kind2": "async" }, { "id": "kafka", "label": "Existing Kafka Estate", "kind": "external", "rel": "mirrored topics", "dir": "in", "kind2": "async" }, { "id": "ref", "label": "Reference Data Lake", "sub": "ADLS Gen2", "kind": "store", "rel": "reference dimensions", "dir": "in", "kind2": "batch" } ] }, { "side": "right", "title": "Analytics consumers", "nodes": [ { "id": "analyst", "label": "Business Analysts", "kind": "actor", "rel": "live dashboards" }, { "id": "ops", "label": "Operations & SRE", "kind": "actor", "rel": "operational views" }, { "id": "apps", "label": "Downstream Applications", "kind": "external", "rel": "gold event topics", "kind2": "async" }, { "id": "auto", "label": "Alerting & Automation", "sub": "Logic Apps", "kind": "external", "rel": "threshold triggers", "kind2": "async" } ] }, { "side": "top", "title": "Governance and platform", "nodes": [ { "id": "entra", "label": "Microsoft Entra ID", "kind": "security", "rel": "OAuth2 · RBAC", "dir": "in", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "purview", "label": "Microsoft Purview", "kind": "security", "rel": "catalogue · lineage", "kind2": "bidirectional", "icon": "azure_AzurePurviewAccount (Analytics)" }, { "id": "monitor", "label": "Azure Monitor", "kind": "platform", "rel": "telemetry · alerts" } ] }, { "side": "bottom", "title": "Enterprise data estate", "nodes": [ { "id": "edw", "label": "Enterprise Warehouse", "sub": "Microsoft Fabric", "kind": "external", "rel": "historical export", "kind2": "batch", "icon": "fa5_warehouse" }, { "id": "ml", "label": "Data Science & ML", "sub": "Azure Machine Learning", "kind": "external", "rel": "feature extracts", "kind2": "batch" } ] } ], "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "02-high-level-architecture", "title": "High-Level Platform Architecture", "layout": "flow", "chain": true, "align": "top", "stages": [ { "title": "Producers", "nodes": [ { "id": "h-iot", "label": "Devices & Edge", "sub": "MQTT / AMQP", "kind": "external" }, { "id": "h-sdk", "label": "App & Web SDK", "sub": "HTTPS batches", "kind": "external" }, { "id": "h-cdc", "label": "OLTP & Kafka Sources", "sub": "CDC · mirrored topics", "kind": "external" } ] }, { "title": "Ingestion", "nodes": [ { "id": "h-apim", "label": "API Management", "sub": "Front Door · WAF · quota", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "h-coll", "label": "Event Collector", "sub": "Container Apps", "kind": "app", "icon": "azure_container-apps-environments" }, { "id": "h-conn", "label": "Ingest Connectors", "sub": "IoT Hub · Debezium", "kind": "integration", "icon": "azure_iot-hub" } ] }, { "title": "Durable event log", "nodes": [ { "id": "h-sr", "label": "Schema Registry", "sub": "Avro · BACKWARD", "kind": "platform", "icon": "fa5_file_code" }, { "id": "h-eh", "label": "Azure Event Hubs", "sub": "256 partitions · 7 d", "kind": "queue" }, { "id": "h-cap", "label": "Event Hubs Capture", "sub": "ADLS Gen2 bronze", "kind": "store" } ] }, { "title": "Stream processing", "nodes": [ { "id": "h-dlq", "label": "Dead-Letter Hub", "sub": "14 d retention", "kind": "queue" }, { "id": "h-dbx", "label": "Azure Databricks", "sub": "Structured Streaming", "kind": "app" }, { "id": "h-win", "label": "Window & Enrich", "sub": "watermark 2 min", "kind": "app", "icon": "fa5_layer_group" } ] }, { "title": "Hot analytics store", "nodes": [ { "id": "h-delta", "label": "Delta Lake", "sub": "silver · gold", "kind": "store" }, { "id": "h-adx", "label": "Azure Data Explorer", "sub": "hot cache 31 d", "kind": "store" }, { "id": "h-mv", "label": "Materialized Views", "sub": "1 min · 5 min · 1 h", "kind": "store" } ] }, { "title": "Query and serving", "nodes": [ { "id": "h-redis", "label": "Azure Cache for Redis", "sub": "30 s TTL", "kind": "store" }, { "id": "h-qapi", "label": "Query API", "sub": "Container Apps", "kind": "integration", "icon": "azure_container-apps-environments" }, { "id": "h-dash", "label": "Dashboards", "sub": "Power BI · Grafana", "kind": "external", "icon": "powerbi" } ] } ], "edges": [ { "from": "h-eh", "to": "h-cap", "label": "Capture 15 min", "kind": "batch" }, { "from": "h-dbx", "to": "h-dlq", "label": "poison events", "kind": "error" }, { "from": "h-cap", "to": "h-dbx", "label": "replay", "kind": "batch" }, { "from": "h-mv", "to": "h-qapi", "label": "pre-aggregated" } ], "note": "Front Door, IoT Hub and the CDC connector are drawn individually in the container and integration views.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "03-layered-architecture", "title": "Layered Architecture", "layout": "bands", "layerHeaderWidth": 160, "bands": [ { "name": "Consumption", "nodes": [ { "id": "l-pbi", "label": "Power BI", "kind": "external" }, { "id": "l-graf", "label": "Managed Grafana", "kind": "external" }, { "id": "l-app", "label": "Analytics Applications", "kind": "external" }, { "id": "l-alert", "label": "Alerting & Automation", "kind": "external" } ] }, { "name": "Serving", "nodes": [ { "id": "l-qapi", "label": "Query API", "sub": "REST over KQL", "kind": "integration", "icon": "azure_container-apps-environments" }, { "id": "l-sem", "label": "KQL Functions", "sub": "semantic layer", "kind": "app" }, { "id": "l-cache", "label": "Result Cache", "sub": "Azure Cache for Redis", "kind": "store" }, { "id": "l-gold", "label": "Gold Event Topics", "kind": "queue" } ] }, { "name": "Analytics store", "nodes": [ { "id": "l-adx", "label": "Azure Data Explorer", "kind": "store" }, { "id": "l-mv", "label": "Materialized Views", "kind": "store" }, { "id": "l-ext", "label": "External Tables", "sub": "ADLS Delta", "kind": "store" } ] }, { "name": "Processing", "nodes": [ { "id": "l-str", "label": "Streaming Jobs", "sub": "Azure Databricks", "kind": "app", "icon": "azure_databricks" }, { "id": "l-enr", "label": "Enrichment & Joins", "kind": "app" }, { "id": "l-agg", "label": "Windowed Aggregation", "kind": "app" }, { "id": "l-rep", "label": "Replay & Backfill", "kind": "app" } ] }, { "name": "Event log", "nodes": [ { "id": "l-eh", "label": "Azure Event Hubs", "kind": "queue" }, { "id": "l-sr", "label": "Schema Registry", "kind": "platform", "icon": "fa5_file_code" }, { "id": "l-bronze", "label": "Bronze Capture", "sub": "ADLS Gen2", "kind": "store" }, { "id": "l-dlq", "label": "Dead-Letter Hub", "kind": "queue" } ] }, { "name": "Ingestion", "nodes": [ { "id": "l-apim", "label": "API Management", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "l-coll", "label": "Event Collector", "kind": "app", "icon": "azure_container-apps-environments" }, { "id": "l-iot", "label": "Azure IoT Hub", "kind": "integration" }, { "id": "l-cdc", "label": "CDC Connectors", "kind": "app" } ] }, { "name": "Platform services", "nodes": [ { "id": "l-entra", "label": "Microsoft Entra ID", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "l-kv", "label": "Azure Key Vault", "kind": "security" }, { "id": "l-mon", "label": "Azure Monitor", "kind": "platform" }, { "id": "l-pv", "label": "Microsoft Purview", "kind": "security", "icon": "azure_AzurePurviewAccount (Analytics)" }, { "id": "l-iac", "label": "Azure DevOps", "sub": "Terraform", "kind": "platform", "icon": "azure_devops" } ] } ], "edges": [ { "from": "l-pbi", "to": "l-qapi", "label": "HTTPS" }, { "from": "l-qapi", "to": "l-adx", "label": "KQL" }, { "from": "l-adx", "to": "l-str", "label": "ingest sink", "kind": "async" }, { "from": "l-str", "to": "l-eh", "label": "consume", "kind": "async" }, { "from": "l-eh", "to": "l-coll", "label": "publish", "kind": "async" }, { "from": "l-gold", "to": "l-alert", "label": "push", "kind": "async" } ], "note": "Dependencies point downward only. Gold topics are the one sanctioned shortcut past the serving layer.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "04-container-architecture", "title": "Container Architecture — Deployable Units", "layout": "nested", "boxes": [ { "title": "Azure Subscription · rta-prod", "kind": "cloud", "dir": "col", "children": [ { "title": "Ingestion spoke · snet-ingest", "kind": "boundary", "nodes": [ { "id": "c-afd", "label": "Azure Front Door", "sub": "WAF · TLS 1.3", "kind": "security" }, { "id": "c-apim", "label": "API Management", "sub": "Premium · per-tenant quota", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "c-coll", "label": "Event Collector", "sub": "Container Apps · FastAPI", "kind": "app", "icon": "azure_container-apps-environments" }, { "id": "c-iot", "label": "Azure IoT Hub", "sub": "per-device certificates", "kind": "integration" }, { "id": "c-cdc", "label": "CDC Connector", "sub": "Debezium on ACA", "kind": "app" } ] }, { "title": "Event backbone · snet-events", "kind": "boundary", "nodes": [ { "id": "c-eh", "label": "Azure Event Hubs", "sub": "Dedicated · 6 hubs", "kind": "queue" }, { "id": "c-sr", "label": "Schema Registry", "sub": "Avro · BACKWARD", "kind": "platform", "icon": "fa5_file_code" }, { "id": "c-cap", "label": "Event Hubs Capture", "sub": "Parquet · 15 min", "kind": "store" }, { "id": "c-dlq", "label": "Dead-Letter Hub", "sub": "14 d retention", "kind": "queue" } ] }, { "title": "Processing spoke · snet-compute", "kind": "boundary", "nodes": [ { "id": "c-dbx", "label": "Azure Databricks", "sub": "VNet-injected · no public IP", "kind": "app" }, { "id": "c-job", "label": "Streaming Jobs", "sub": "1 s micro-batch", "kind": "app", "icon": "azure_databricks" }, { "id": "c-ckpt", "label": "Checkpoint Store", "sub": "ADLS Gen2 ZRS", "kind": "store" }, { "id": "c-dim", "label": "Reference Delta Tables", "sub": "broadcast joins", "kind": "store" } ] }, { "title": "Serving spoke · snet-serve", "kind": "boundary", "nodes": [ { "id": "c-adx", "label": "Azure Data Explorer", "sub": "leader · 12 nodes", "kind": "store" }, { "id": "c-fol", "label": "ADX Follower", "sub": "read-only · BI", "kind": "store" }, { "id": "c-qapi", "label": "Query API", "sub": "Container Apps · autoscale", "kind": "integration", "icon": "azure_container-apps-environments" }, { "id": "c-redis", "label": "Azure Cache for Redis", "sub": "Premium 26 GB", "kind": "store" } ] } ] } ], "outside": [ { "id": "c-pbi", "label": "Power BI", "kind": "external" }, { "id": "c-graf", "label": "Managed Grafana", "kind": "external" }, { "id": "c-entra", "label": "Microsoft Entra ID", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "c-kv", "label": "Azure Key Vault", "kind": "security" }, { "id": "c-mon", "label": "Azure Monitor", "kind": "platform" }, { "id": "c-pv", "label": "Microsoft Purview", "kind": "security", "icon": "azure_AzurePurviewAccount (Analytics)" } ], "edges": [ { "from": "c-apim", "to": "c-coll", "label": "HTTPS" }, { "from": "c-coll", "to": "c-eh", "label": "AMQP publish", "kind": "async" }, { "from": "c-cdc", "to": "c-eh", "label": "Kafka protocol", "kind": "async" }, { "from": "c-eh", "to": "c-job", "label": "consumer group", "kind": "async" }, { "from": "c-job", "to": "c-adx", "label": "streaming ingest", "kind": "async" }, { "from": "c-job", "to": "c-dlq", "label": "poison event", "kind": "error" }, { "from": "c-adx", "to": "c-fol", "label": "follower attach" }, { "from": "c-qapi", "to": "c-fol", "label": "KQL" }, { "from": "c-pbi", "to": "c-fol", "label": "DirectQuery" } ], "note": "Every service reaches Event Hubs, ADLS and Key Vault over private endpoints; public network access is disabled.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "05-integration-catalogue", "title": "Integration and Interface Catalogue", "layout": "hub", "left": { "title": "Inbound producers", "nodes": [ { "id": "i-sdk", "label": "Web & Mobile SDK", "kind": "external", "rel": "HTTPS batch", "dir": "in", "kind2": "async" }, { "id": "i-part", "label": "Partner Services", "kind": "external", "rel": "REST · OAuth2", "dir": "in", "icon": "fa5_cloud" }, { "id": "i-iot", "label": "Devices & Edge", "kind": "external", "rel": "MQTT · AMQP", "dir": "in", "kind2": "async" }, { "id": "i-sql", "label": "Azure SQL Database", "kind": "external", "rel": "CDC · Debezium", "dir": "in", "kind2": "async" }, { "id": "i-kafka", "label": "Kafka Estate", "kind": "external", "rel": "MirrorMaker 2", "dir": "in", "kind2": "async" }, { "id": "i-sftp", "label": "Batch File Drops", "kind": "external", "rel": "SFTP to ADLS", "dir": "in", "kind2": "batch" } ] }, "centre": { "title": "Platform", "nodes": [ { "id": "i-apim", "label": "API Management", "sub": "every inbound interface", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "i-eh", "label": "Azure Event Hubs", "sub": "durable event log", "kind": "queue" }, { "id": "i-adx", "label": "Azure Data Explorer", "sub": "hot analytics store", "kind": "store" } ] }, "right": { "title": "Outbound consumers", "nodes": [ { "id": "o-pbi", "label": "Power BI", "kind": "external", "rel": "DirectQuery", "dir": "out" }, { "id": "o-graf", "label": "Managed Grafana", "kind": "external", "rel": "ADX data source", "dir": "out" }, { "id": "o-app", "label": "Downstream Apps", "kind": "external", "rel": "gold topics", "kind2": "async", "dir": "out" }, { "id": "o-auto", "label": "Logic Apps", "kind": "external", "rel": "threshold events", "kind2": "async", "dir": "out" }, { "id": "o-edw", "label": "Microsoft Fabric", "kind": "external", "rel": "continuous export", "kind2": "batch", "icon": "fa5_warehouse", "dir": "out" }, { "id": "o-ml", "label": "Azure Machine Learning", "kind": "external", "rel": "Delta share", "kind2": "batch", "dir": "out" } ] }, "note": "Every inbound interface authenticates with Entra ID; no shared keys are issued to producers.", "meta": { "v": "1.0", "owner": "Integration Architecture", "date": "2026-08" } }, { "id": "06-tenancy-workload-isolation", "title": "Tenancy and Workload Isolation", "layout": "nested", "boxes": [ { "title": "Azure Tenant · shared control plane", "kind": "cloud", "dir": "col", "children": [ { "title": "Identity and policy", "kind": "trust", "nodes": [ { "id": "t-entra", "label": "Microsoft Entra ID", "sub": "app registration per tenant", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "t-pol", "label": "Azure Policy", "sub": "deny public endpoints", "kind": "security" }, { "id": "t-kv", "label": "Azure Key Vault", "sub": "CMK per tenant tier", "kind": "security" } ] }, { "title": "Ingestion isolation", "kind": "boundary", "nodes": [ { "id": "t-shared", "label": "Shared Namespace", "sub": "partition key = tenant", "kind": "queue" }, { "id": "t-ded", "label": "Dedicated Namespace", "sub": "premium tenants", "kind": "queue" }, { "id": "t-quota", "label": "APIM Quota Policy", "sub": "RPS + burst per tenant", "kind": "integration" }, { "id": "t-noisy", "label": "Noisy Neighbour", "sub": "shared-tier saturation", "kind": "risk" } ] }, { "title": "Storage isolation", "kind": "boundary", "nodes": [ { "id": "t-db", "label": "ADX Database per Tier", "sub": "shared · premium", "kind": "store" }, { "id": "t-rls", "label": "Row-Level Security", "sub": "tenant_id predicate", "kind": "security" }, { "id": "t-adls", "label": "ADLS Container", "sub": "one per tenant", "kind": "store" } ] }, { "title": "Query isolation", "kind": "boundary", "nodes": [ { "id": "t-lead", "label": "Leader Cluster", "sub": "ingest and write", "kind": "store" }, { "id": "t-f1", "label": "Follower · Dashboards", "sub": "predictable latency", "kind": "store" }, { "id": "t-f2", "label": "Follower · Ad-hoc", "sub": "exports and notebooks", "kind": "store" }, { "id": "t-wg", "label": "Workload Groups", "sub": "concurrency caps", "kind": "platform" } ] } ] } ], "edges": [ { "from": "t-quota", "to": "t-shared", "label": "quota", "kind": "async" }, { "from": "t-shared", "to": "t-db", "label": "by tenant_id", "kind": "async" }, { "from": "t-ded", "to": "t-db", "label": "premium path", "kind": "async" }, { "from": "t-lead", "to": "t-f1", "label": "read replica" }, { "from": "t-lead", "to": "t-f2", "label": "read replica" }, { "from": "t-noisy", "to": "t-ded", "label": "promotion path", "kind": "error" } ], "note": "Shared tier is cost-optimised and rate-limited; premium tier buys its own namespace and follower.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "07-event-data-flow", "title": "Event Data Flow — Format, Cadence and Retention", "layout": "flow", "chain": true, "align": "top", "stages": [ { "title": "Source", "nodes": [ { "id": "d-cdc", "label": "Change Rows", "sub": "Avro · row image", "kind": "external" }, { "id": "d-app", "label": "Application Events", "sub": "JSON · 1.1 KB avg", "kind": "external" }, { "id": "d-tel", "label": "Device Telemetry", "sub": "protobuf · 240 B", "kind": "external" } ] }, { "title": "Capture and validate", "nodes": [ { "id": "d-idem", "label": "Idempotency Key", "sub": "event_id + payload hash", "kind": "app" }, { "id": "d-coll", "label": "Event Collector", "sub": "schema validation", "kind": "app", "icon": "azure_container-apps-environments" }, { "id": "d-rej", "label": "Rejected Events", "sub": "invalid schema", "kind": "risk" } ] }, { "title": "Durable log", "nodes": [ { "id": "d-bronze", "label": "Bronze Capture", "sub": "Parquet · 7 y · immutable", "kind": "store" }, { "id": "d-eh", "label": "Azure Event Hubs", "sub": "7 d · 256 partitions", "kind": "queue" }, { "id": "d-dlq", "label": "Dead-Letter Hub", "sub": "14 d", "kind": "queue" } ] }, { "title": "Process and enrich", "nodes": [ { "id": "d-join", "label": "Dimension Join", "sub": "broadcast Delta", "kind": "app" }, { "id": "d-win", "label": "Windowed Aggregation", "sub": "tumbling · sliding · session", "kind": "app" }, { "id": "d-wm", "label": "Watermark Gate", "sub": "lateness 2 min", "kind": "decision", "icon": "fa5_clock" } ] }, { "title": "Hot store", "nodes": [ { "id": "d-silver", "label": "Silver & Gold Delta", "sub": "ADLS · rebuildable", "kind": "store" }, { "id": "d-raw", "label": "ADX Raw Table", "sub": "31 d hot · 400 d total", "kind": "store" }, { "id": "d-mv", "label": "Materialized Views", "sub": "1 min · 5 min · 1 h", "kind": "store" } ] }, { "title": "Serve", "nodes": [ { "id": "d-gold", "label": "Gold Event Topics", "sub": "enriched · async", "kind": "queue" }, { "id": "d-qapi", "label": "Query API", "sub": "SQL and KQL", "kind": "integration", "icon": "azure_container-apps-environments" }, { "id": "d-pbi", "label": "Dashboards", "sub": "Power BI · Grafana", "kind": "external" } ] } ], "edges": [ { "from": "d-coll", "to": "d-rej", "label": "schema fail", "kind": "error" }, { "from": "d-eh", "to": "d-bronze", "label": "Capture 15 min", "kind": "batch" }, { "from": "d-win", "to": "d-dlq", "label": "unprocessable", "kind": "error" }, { "from": "d-bronze", "to": "d-join", "label": "replay", "kind": "batch" }, { "from": "d-mv", "to": "d-qapi", "label": "sub-second reads" } ], "note": "Bronze is the only zone that cannot be rebuilt; everything downstream is derivable from it.", "meta": { "v": "1.0", "owner": "Data Architecture", "date": "2026-08" } }, { "id": "08-storage-tiers-retention", "title": "Storage Zones, Tiering and Retention", "layout": "nested", "boxes": [ { "title": "Platform storage estate", "kind": "cloud", "dir": "col", "children": [ { "title": "Hot · Azure Data Explorer", "kind": "boundary", "nodes": [ { "id": "s-raw", "label": "Raw Event Table", "sub": "hot cache 31 d", "kind": "store" }, { "id": "s-mv", "label": "Materialized Views", "sub": "pre-aggregated metrics", "kind": "store" }, { "id": "s-dim", "label": "Dimension Tables", "sub": "reference lookups", "kind": "store" }, { "id": "s-late", "label": "Late Arrival Table", "sub": "beyond watermark", "kind": "store" } ] }, { "title": "Warm · ADLS Gen2 Delta Lake", "kind": "boundary", "nodes": [ { "id": "s-bronze", "label": "Bronze · Raw Capture", "sub": "immutable · system of record", "kind": "store" }, { "id": "s-silver", "label": "Silver · Enriched", "sub": "rebuildable from bronze", "kind": "store" }, { "id": "s-gold", "label": "Gold · Curated Marts", "sub": "rebuildable from silver", "kind": "store" } ] }, { "title": "Cold · archive and export", "kind": "boundary", "nodes": [ { "id": "s-cold", "label": "ADX Cold Tier", "sub": "32 d to 400 d", "kind": "store" }, { "id": "s-arch", "label": "Blob Archive", "sub": "7 y · legal hold", "kind": "store" }, { "id": "s-ext", "label": "External Tables", "sub": "hot + cold union query", "kind": "store" } ] }, { "title": "Operational state", "kind": "boundary", "nodes": [ { "id": "s-ckpt", "label": "Stream Checkpoints", "sub": "ADLS · per job", "kind": "store" }, { "id": "s-sr", "label": "Schema Registry", "sub": "versioned contracts", "kind": "platform", "icon": "fa5_file_code" }, { "id": "s-off", "label": "Offset & Watermark State", "sub": "RocksDB + ADLS", "kind": "store" }, { "id": "s-loss", "label": "Single Unrebuildable Zone", "sub": "bronze loss is terminal", "kind": "risk" } ] } ] } ], "outside": [ { "id": "s-pv", "label": "Microsoft Purview", "sub": "classification · lineage", "kind": "security", "icon": "azure_AzurePurviewAccount (Analytics)" }, { "id": "s-life", "label": "Lifecycle Policy", "sub": "hot to cool to archive", "kind": "platform" }, { "id": "s-kv", "label": "Azure Key Vault", "sub": "customer-managed keys", "kind": "security" } ], "edges": [ { "from": "s-raw", "to": "s-cold", "label": "cache eviction", "kind": "batch" }, { "from": "s-bronze", "to": "s-silver", "label": "rebuild", "kind": "batch" }, { "from": "s-silver", "to": "s-gold", "label": "curate", "kind": "batch" }, { "from": "s-bronze", "to": "s-arch", "label": "after 90 d", "kind": "batch" }, { "from": "s-cold", "to": "s-ext", "label": "union query" }, { "from": "s-life", "to": "s-arch", "label": "tier policy", "kind": "batch" } ], "note": "Retention is policy-driven per table; no data is deleted by a job, only by an expiry policy.", "meta": { "v": "1.0", "owner": "Data Architecture", "date": "2026-08" } }, { "id": "09-analytics-data-model", "title": "Analytics Data Model — Hot Store", "layout": "er", "canvas": { "width": 1400, "cols": 3 }, "rowGap": 200, "entities": [ { "id": "sc", "name": "schema_contract", "kind": "platform", "row": 0, "col": 0, "attrs": [ "schema_id PK", "event_type", "version", "compat_mode", "registered_at", "deprecated_at" ] }, { "id": "raw", "name": "events_raw", "kind": "store", "row": 0, "col": 1, "attrs": [ "event_id PK", "tenant_id", "event_type", "schema_id FK", "event_time", "ingest_time", "payload dynamic" ] }, { "id": "dlq", "name": "dlq_event", "kind": "store", "row": 0, "col": 2, "attrs": [ "dlq_id PK", "event_id FK", "tenant_id", "failure_reason", "raw_payload", "failed_at", "retry_count" ] }, { "id": "dimu", "name": "dim_user", "kind": "store", "row": 1, "col": 0, "attrs": [ "user_key PK", "tenant_id", "user_id_hash", "segment", "first_seen_at", "valid_from" ] }, { "id": "enr", "name": "events_enriched", "kind": "store", "row": 1, "col": 1, "attrs": [ "event_id PK/FK", "tenant_id", "user_key FK", "session_id FK", "product_key FK", "geo", "device", "event_time" ] }, { "id": "sess", "name": "session", "kind": "store", "row": 1, "col": 2, "attrs": [ "session_id PK", "tenant_id", "user_key FK", "started_at", "ended_at", "event_count", "is_open" ] }, { "id": "dimp", "name": "dim_product", "kind": "store", "row": 2, "col": 0, "attrs": [ "product_key PK", "sku", "category", "price_band", "valid_from" ] }, { "id": "mv1", "name": "mv_metric_1m", "kind": "store", "row": 2, "col": 1, "attrs": [ "bucket_start PK", "tenant_id PK", "metric_name PK", "dimensions", "value", "event_count" ] }, { "id": "mv2", "name": "mv_topn_5m", "kind": "store", "row": 2, "col": 2, "attrs": [ "bucket_start PK", "tenant_id PK", "dimension", "member", "rank", "value" ] } ], "relations": [ { "from": "sc", "to": "raw", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "raw", "to": "dlq", "label": "0 : N", "from_side": "e", "to_side": "w", "kind": "optional" }, { "from": "raw", "to": "enr", "label": "1 : 1", "from_side": "s", "to_side": "n" }, { "from": "dimu", "to": "enr", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "enr", "to": "sess", "label": "N : 1", "from_side": "e", "to_side": "w" }, { "from": "enr", "to": "mv1", "label": "N : 1", "from_side": "s", "to_side": "n" }, { "from": "dimp", "to": "mv1", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "mv1", "to": "mv2", "label": "1 : N", "from_side": "e", "to_side": "w" } ], "note": "Dimensions are slowly changing type 2; joins are on the surrogate key valid at event_time.", "meta": { "v": "1.0", "owner": "Data Architecture", "date": "2026-08" } }, { "id": "10-schema-contract-governance", "title": "Schema Contract Governance", "layout": "swimlane", "stages": [ "Propose", "Validate", "Register", "Publish", "Retire" ], "laneHeaderWidth": 150, "lanes": [ { "title": "Producer team", "cells": [ [ { "id": "p1", "label": "Schema pull request", "kind": "app" } ], [ { "id": "p2", "label": "Local compat check", "kind": "app" } ], [], [ { "id": "p4", "label": "SDK regenerated", "kind": "app" } ], [ { "id": "p5", "label": "Cut over to v(n)", "kind": "app" } ] ] }, { "title": "Platform CI", "cells": [ [ { "id": "q1", "label": "Lint and policy", "sub": "Azure DevOps", "kind": "app" } ], [ { "id": "q2", "label": "Backward-compat gate", "kind": "decision" } ], [ { "id": "q3", "label": "Push to registry", "kind": "app" } ], [ { "id": "q4", "label": "Contract tests", "kind": "app" } ], [ { "id": "q5", "label": "Sunset job", "sub": "90 d window", "kind": "app" } ] ] }, { "title": "Schema Registry", "cells": [ [], [ { "id": "r2", "label": "Diff against v(n-1)", "kind": "platform" } ], [ { "id": "r3", "label": "Avro schema v(n)", "kind": "platform" } ], [ { "id": "r4", "label": "Schema ID in header", "kind": "platform" } ], [ { "id": "r5", "label": "Marked deprecated", "kind": "platform" } ] ] }, { "title": "Consumers", "cells": [ [ { "id": "s1", "label": "Impact report", "kind": "app" } ], [ { "id": "s2", "label": "Consumer contract test", "kind": "app" } ], [], [ { "id": "s4", "label": "Resolve by schema ID", "kind": "app" } ], [ { "id": "s5", "label": "Migration confirmed", "kind": "decision" } ] ] } ], "edges": [ { "from": "q2", "to": "p1", "label": "breaking change rejected", "kind": "error" } ], "note": "Only backward-compatible changes reach the registry; a breaking change requires a new event type.", "meta": { "v": "1.0", "owner": "Data Governance", "date": "2026-08" } }, { "id": "11-stream-processing-pipeline", "title": "Stream Processing Pipeline", "layout": "flow", "chain": true, "align": "top", "stages": [ { "title": "Read", "nodes": [ { "id": "r-ck", "label": "Checkpoint & Offsets", "sub": "advanced after sink ack", "kind": "store" }, { "id": "r-eh", "label": "Event Hubs Source", "sub": "maxOffsetsPerTrigger", "kind": "queue" }, { "id": "r-bp", "label": "Backpressure Control", "sub": "trigger 1 s", "kind": "platform" } ] }, { "title": "Decode", "nodes": [ { "id": "r-bad", "label": "Undecodable Event", "kind": "risk" }, { "id": "r-avro", "label": "Avro Decode", "kind": "app" }, { "id": "r-sr", "label": "Schema Resolve", "sub": "by schema ID", "kind": "platform", "icon": "fa5_file_code" } ] }, { "title": "Enrich", "nodes": [ { "id": "r-state", "label": "Stateful Lookup", "sub": "RocksDB state store", "kind": "store" }, { "id": "r-bc", "label": "Broadcast Dimension Join", "sub": "Delta reference", "kind": "app" }, { "id": "r-miss", "label": "Dimension Miss", "sub": "late reference fallback", "kind": "decision" } ] }, { "title": "Window", "nodes": [ { "id": "r-wm", "label": "Watermark 2 min", "kind": "decision", "icon": "fa5_clock" }, { "id": "r-win", "label": "Window Assignment", "sub": "tumbling · sliding · session", "kind": "app", "icon": "fa5_layer_group" }, { "id": "r-agg", "label": "Stateful Aggregation", "sub": "per key and window", "kind": "app" } ] }, { "title": "Write", "nodes": [ { "id": "r-delta", "label": "Delta Silver Sink", "sub": "txnAppId idempotence", "kind": "store" }, { "id": "r-adx", "label": "ADX Streaming Sink", "sub": "ingest-by dedup tag", "kind": "store" }, { "id": "r-dlq", "label": "Dead-Letter Hub", "kind": "queue", "sub": "quarantine · replayable" } ] }, { "title": "Signal", "nodes": [ { "id": "r-gold", "label": "Gold Event Hub", "sub": "enriched output", "kind": "queue" }, { "id": "r-com", "label": "Checkpoint Commit", "sub": "after sink ack", "kind": "platform" }, { "id": "r-met", "label": "Pipeline Metrics", "sub": "lag · duration · rows", "kind": "platform" } ] } ], "edges": [ { "from": "r-avro", "to": "r-bad", "label": "decode failure", "kind": "error" }, { "from": "r-wm", "to": "r-dlq", "label": "beyond lateness", "kind": "error" } ], "note": "Checkpoints commit only after every sink acknowledges, giving effectively-once delivery end to end.", "meta": { "v": "1.0", "owner": "Streaming Engineering", "date": "2026-08" } }, { "id": "12-windowing-late-events", "title": "Windowing and Late-Arriving Events", "layout": "grid", "laneHeaderWidth": 175, "columns": [ "Window definition", "State kept", "Emit trigger", "Late within 2 min", "Beyond watermark" ], "rows": [ { "title": "Tumbling · 1 min", "cells": [ [ { "id": "w11", "label": "Fixed 60 s buckets", "kind": "app" } ], [ { "id": "w12", "label": "One aggregate per key", "kind": "store" } ], [ { "id": "w13", "label": "Watermark passes end", "kind": "decision" } ], [ { "id": "w14", "label": "Updated in place", "kind": "app" } ], [ { "id": "w15", "label": "Routed to late table", "kind": "queue" } ] ] }, { "title": "Sliding · 5 min / 1 min", "cells": [ [ { "id": "w21", "label": "Five overlapping windows", "kind": "app", "icon": "fa5_layer_group" } ], [ { "id": "w22", "label": "5x state per key", "kind": "store" } ], [ { "id": "w23", "label": "Every 1 min slide", "kind": "decision" } ], [ { "id": "w24", "label": "Overlaps recomputed", "kind": "app" } ], [ { "id": "w25", "label": "Dropped and counted", "kind": "risk" } ] ] }, { "title": "Session · 30 min gap", "cells": [ [ { "id": "w31", "label": "Gap-based boundary", "kind": "app" } ], [ { "id": "w32", "label": "Open session state", "kind": "store" } ], [ { "id": "w33", "label": "Inactivity timeout", "kind": "decision" } ], [ { "id": "w34", "label": "Session extended", "kind": "app" } ], [ { "id": "w35", "label": "New session opened", "kind": "app" } ] ] }, { "title": "Out-of-order handling", "cells": [ [ { "id": "w41", "label": "event_time not ingest_time", "kind": "app" } ], [ { "id": "w42", "label": "Per-partition watermark", "kind": "platform" } ], [ { "id": "w43", "label": "Min across partitions", "kind": "decision" } ], [ { "id": "w44", "label": "Reordered in window", "kind": "app" } ], [ { "id": "w45", "label": "Backfill job corrects", "kind": "app" } ] ] } ], "note": "Watermark lateness is 2 minutes; a slow partition holds the global watermark and is alerted on.", "meta": { "v": "1.0", "owner": "Streaming Engineering", "date": "2026-08" } }, { "id": "13-event-to-insight-sequence", "title": "Event to Insight — Critical Path", "layout": "sequence", "lifelines": [ { "id": "z-prod", "label": "Producer SDK", "kind": "external" }, { "id": "z-apim", "label": "API Management", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "z-coll", "label": "Event Collector", "kind": "app", "icon": "azure_container-apps-environments" }, { "id": "z-eh", "label": "Azure Event Hubs", "kind": "queue" }, { "id": "z-dbx", "label": "Databricks Stream", "kind": "app" }, { "id": "z-adx", "label": "Azure Data Explorer", "kind": "store" }, { "id": "z-dash", "label": "Dashboard", "kind": "actor" } ], "messages": [ { "from": "z-prod", "to": "z-apim", "label": "POST /v1/events · batch 500", "kind": "call" }, { "from": "z-apim", "to": "z-apim", "label": "validate token, quota", "kind": "self" }, { "from": "z-apim", "to": "z-coll", "label": "forward with tenant claim", "kind": "call" }, { "from": "z-coll", "to": "z-coll", "label": "schema check, event_id", "kind": "self" }, { "from": "z-coll", "to": "z-eh", "label": "publish · key = tenant", "kind": "async" }, { "from": "z-eh", "to": "z-coll", "label": "offset committed", "kind": "return" }, { "from": "z-coll", "to": "z-prod", "label": "202 Accepted · 200 ms", "kind": "return" }, { "from": "z-eh", "to": "z-dbx", "label": "micro-batch · 1 s trigger", "kind": "async" }, { "from": "z-dbx", "to": "z-dbx", "label": "enrich, window, watermark", "kind": "self" }, { "from": "z-dbx", "to": "z-adx", "label": "streaming ingest · dedup tag", "kind": "async" }, { "from": "z-dbx", "to": "z-eh", "label": "poison event to DLQ", "kind": "error" }, { "from": "z-adx", "to": "z-adx", "label": "materialized view update", "kind": "self" }, { "from": "z-dash", "to": "z-adx", "label": "KQL over view", "kind": "call" }, { "from": "z-adx", "to": "z-dash", "label": "result · 350 ms p95", "kind": "return" } ], "note": "Budget: 200 ms accept, 1.5 s trigger, 1.2 s process, 1.5 s ingest visibility. 4.4 s of a 5 s p95 target.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "14-query-serving-architecture", "title": "Query and Serving Architecture", "layout": "nested", "boxes": [ { "title": "Serving tier", "kind": "cloud", "dir": "col", "children": [ { "title": "Client edge", "kind": "boundary", "nodes": [ { "id": "y-afd", "label": "Azure Front Door", "sub": "TLS · geo routing", "kind": "security" }, { "id": "y-apim", "label": "API Management", "sub": "per-tenant RPS", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "y-api", "label": "Query API", "sub": "Container Apps · KEDA", "kind": "app", "icon": "azure_container-apps-environments" } ] }, { "title": "Acceleration", "kind": "boundary", "nodes": [ { "id": "y-redis", "label": "Azure Cache for Redis", "sub": "55% hit · 30 s TTL", "kind": "store" }, { "id": "y-mv", "label": "Materialized Views", "sub": "1 min · 5 min · 1 h", "kind": "store" }, { "id": "y-fn", "label": "KQL Functions", "sub": "governed semantics", "kind": "app" } ] }, { "title": "Azure Data Explorer", "kind": "boundary", "nodes": [ { "id": "y-lead", "label": "Leader Cluster", "sub": "ingest and write", "kind": "store" }, { "id": "y-f1", "label": "Follower · Dashboards", "sub": "read-only", "kind": "store" }, { "id": "y-f2", "label": "Follower · Ad-hoc", "sub": "exports · notebooks", "kind": "store" }, { "id": "y-wg", "label": "Workload Groups", "sub": "concurrency and memory caps", "kind": "platform" } ] }, { "title": "Consumption", "kind": "boundary", "nodes": [ { "id": "y-pbi", "label": "Power BI", "sub": "DirectQuery", "kind": "external" }, { "id": "y-graf", "label": "Managed Grafana", "kind": "external" }, { "id": "y-gold", "label": "Gold Event Topics", "kind": "queue" }, { "id": "y-nb", "label": "Notebooks & Ad-hoc", "kind": "external" } ] } ] } ], "outside": [ { "id": "y-entra", "label": "Microsoft Entra ID", "sub": "on-behalf-of tokens", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "y-pv", "label": "Microsoft Purview", "sub": "column classification", "kind": "security", "icon": "azure_AzurePurviewAccount (Analytics)" } ], "edges": [ { "from": "y-afd", "to": "y-apim", "label": "TLS 1.3" }, { "from": "y-apim", "to": "y-api", "label": "HTTPS" }, { "from": "y-api", "to": "y-redis", "label": "cache lookup · 15 ms" }, { "from": "y-api", "to": "y-mv", "label": "miss · 350 ms" }, { "from": "y-mv", "to": "y-f1", "label": "resolved on follower" }, { "from": "y-lead", "to": "y-f1", "label": "attach" }, { "from": "y-lead", "to": "y-f2", "label": "attach" }, { "from": "y-pbi", "to": "y-f1", "label": "DirectQuery KQL" }, { "from": "y-nb", "to": "y-f2", "label": "ad-hoc KQL" } ], "note": "Ad-hoc and export workloads never touch the cluster that serves dashboards.", "meta": { "v": "1.0", "owner": "Data & AI Architecture", "date": "2026-08" } }, { "id": "15-replay-and-backfill", "title": "Replay, Backfill and Correction", "layout": "flow", "chain": true, "align": "top", "stages": [ { "title": "Trigger", "nodes": [ { "id": "b-new", "label": "New Derived Metric", "kind": "app" }, { "id": "b-inc", "label": "Incident or Defect", "sub": "bad enrichment logic", "kind": "risk" }, { "id": "b-appr", "label": "Change Approval", "sub": "data owner sign-off", "kind": "decision" } ] }, { "title": "Source of truth", "nodes": [ { "id": "b-eh", "label": "Event Hubs Retention", "sub": "last 7 d", "kind": "queue" }, { "id": "b-bronze", "label": "Bronze Capture", "sub": "ADLS · 7 y", "kind": "store" }, { "id": "b-range", "label": "Offset Range Selector", "sub": "by event_time", "kind": "app" } ] }, { "title": "Replay job", "nodes": [ { "id": "b-ver", "label": "Pinned Code Version", "sub": "git SHA", "kind": "platform" }, { "id": "b-job", "label": "Isolated Replay Job", "sub": "separate Databricks pool", "kind": "app" }, { "id": "b-ck", "label": "Shadow Checkpoint", "sub": "no production offsets", "kind": "store" } ] }, { "title": "Shadow write", "nodes": [ { "id": "b-sdelta", "label": "Delta Shadow Branch", "kind": "store" }, { "id": "b-stbl", "label": "ADX Shadow Table", "sub": "_replay suffix", "kind": "store" }, { "id": "b-iso", "label": "No Consumer Reads", "sub": "shadow is invisible", "kind": "platform" } ] }, { "title": "Verify", "nodes": [ { "id": "b-rec", "label": "Row and Metric Reconciliation", "kind": "app" }, { "id": "b-var", "label": "Variance Gate", "sub": "below 0.1%", "kind": "decision" }, { "id": "b-dis", "label": "Discard and Re-run", "kind": "risk" } ] }, { "title": "Cut over", "nodes": [ { "id": "b-mv", "label": "Materialized View Rebuild", "kind": "app" }, { "id": "b-swap", "label": "Atomic Table Swap", "sub": "rename in ADX", "kind": "app" }, { "id": "b-not", "label": "Consumer Notice", "sub": "restated window", "kind": "platform" } ] } ], "edges": [ { "from": "b-var", "to": "b-dis", "label": "variance high", "kind": "error" }, { "from": "b-dis", "to": "b-job", "label": "re-run", "kind": "error", "route": "gutter" } ], "note": "Production consumers never read a shadow table; the swap is the only moment they see a change.", "meta": { "v": "1.0", "owner": "Platform Operations", "date": "2026-08" } }, { "id": "16-deployment-architecture", "title": "Deployment and Infrastructure", "layout": "nested", "boxes": [ { "title": "Azure · Sweden Central (primary)", "kind": "cloud", "dir": "col", "children": [ { "title": "Hub VNet · shared services", "kind": "boundary", "nodes": [ { "id": "n-fw", "label": "Azure Firewall", "sub": "egress control", "kind": "security", "icon": "azure_AzureFirewall (Networking)" }, { "id": "n-dns", "label": "Private DNS Zones", "sub": "privatelink.*", "kind": "platform", "icon": "azure_dns-zones" }, { "id": "n-bas", "label": "Azure Bastion", "sub": "no public RDP/SSH", "kind": "security" }, { "id": "n-er", "label": "ExpressRoute Gateway", "sub": "on-premises CDC", "kind": "integration", "icon": "azure_AzureExpressRoute (Networking)" } ] }, { "title": "Zone-redundant compute and messaging", "kind": "boundary", "dir": "row", "children": [ { "title": "Availability Zone 1", "kind": "boundary", "nodes": [ { "id": "n-eh1", "label": "Event Hubs CU", "kind": "queue" }, { "id": "n-dbx1", "label": "Databricks Workers", "kind": "app" }, { "id": "n-adx1", "label": "ADX Nodes x4", "kind": "store" } ] }, { "title": "Availability Zone 2", "kind": "boundary", "nodes": [ { "id": "n-eh2", "label": "Event Hubs CU", "kind": "queue" }, { "id": "n-dbx2", "label": "Databricks Workers", "kind": "app" }, { "id": "n-adx2", "label": "ADX Nodes x4", "kind": "store" } ] }, { "title": "Availability Zone 3", "kind": "boundary", "nodes": [ { "id": "n-eh3", "label": "Event Hubs CU", "kind": "queue" }, { "id": "n-dbx3", "label": "Databricks Workers", "kind": "app" }, { "id": "n-adx3", "label": "ADX Nodes x4", "kind": "store" } ] } ] }, { "title": "Zone-redundant state", "kind": "boundary", "nodes": [ { "id": "n-adls", "label": "ADLS Gen2", "sub": "ZRS · hierarchical", "kind": "store" }, { "id": "n-ck", "label": "Checkpoint Container", "sub": "per streaming job", "kind": "store" }, { "id": "n-kv", "label": "Azure Key Vault", "sub": "HSM · CMK", "kind": "security" }, { "id": "n-redis", "label": "Azure Cache for Redis", "sub": "zone-redundant premium", "kind": "store" } ] } ] }, { "title": "Azure · Germany West Central (DR)", "kind": "cloud", "dir": "col", "children": [ { "title": "Warm standby", "kind": "boundary", "nodes": [ { "id": "n-ehdr", "label": "Event Hubs Geo-DR Alias", "sub": "metadata replication", "kind": "queue" }, { "id": "n-adxdr", "label": "ADX Follower Cluster", "sub": "read-only standby", "kind": "store" }, { "id": "n-adlsdr", "label": "ADLS GRS Replica", "sub": "bronze archive", "kind": "store" }, { "id": "n-tf", "label": "Terraform Standby Stack", "sub": "apply on declare", "kind": "platform" } ] } ] } ], "outside": [ { "id": "n-mon", "label": "Azure Monitor", "sub": "cross-region workspace", "kind": "platform" }, { "id": "n-entra", "label": "Microsoft Entra ID", "sub": "global", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "n-do", "label": "Azure DevOps", "sub": "self-hosted agents", "kind": "platform", "icon": "azure_devops" } ], "edges": [ { "from": "n-eh1", "to": "n-ehdr", "label": "geo-DR pair", "kind": "async" }, { "from": "n-adls", "to": "n-adlsdr", "label": "GRS replication", "kind": "batch" }, { "from": "n-adx1", "to": "n-adxdr", "label": "attach", "kind": "async" } ], "note": "RTO 60 min, RPO 5 min. DR is warm: capacity is provisioned on declare, not held idle.", "meta": { "v": "1.0", "owner": "Cloud Platform", "date": "2026-08" } }, { "id": "17-cicd-and-environments", "title": "CI/CD and Environment Promotion", "layout": "flow", "chain": true, "align": "top", "stages": [ { "title": "Source", "nodes": [ { "id": "k-tf", "label": "Terraform Modules", "sub": "infrastructure as code", "kind": "platform" }, { "id": "k-git", "label": "Azure Repos", "sub": "trunk + short branches", "kind": "app" }, { "id": "k-sch", "label": "Schema Contracts", "sub": "versioned Avro", "kind": "platform", "icon": "fa5_file_code" } ] }, { "title": "Build and test", "nodes": [ { "id": "k-ut", "label": "Unit & PySpark Tests", "kind": "app" }, { "id": "k-comp", "label": "Schema Compat Gate", "kind": "decision" }, { "id": "k-sec", "label": "SAST & IaC Scan", "sub": "Defender for DevOps", "kind": "security" } ] }, { "title": "Dev", "nodes": [ { "id": "k-devdb", "label": "Ephemeral ADX Database", "sub": "torn down nightly", "kind": "store" }, { "id": "k-devjob", "label": "Dev Stream Job", "sub": "feature branch", "kind": "app" }, { "id": "k-gen", "label": "Synthetic Event Generator", "sub": "replayed production shape", "kind": "app" } ] }, { "title": "Test and UAT", "nodes": [ { "id": "k-shadow", "label": "Shadow Traffic", "sub": "5% mirrored", "kind": "app" }, { "id": "k-perf", "label": "Latency Gate", "sub": "5 s p95 end to end", "kind": "decision" }, { "id": "k-ct", "label": "Consumer Contract Tests", "kind": "app" } ] }, { "title": "Production", "nodes": [ { "id": "k-cab", "label": "Change Approval", "kind": "decision" }, { "id": "k-bg", "label": "Blue/Green Stream Job", "sub": "new consumer group", "kind": "app" }, { "id": "k-mvg", "label": "View Rebuild Guard", "sub": "no read downtime", "kind": "decision" } ] }, { "title": "Post-release", "nodes": [ { "id": "k-eb", "label": "Error Budget Update", "kind": "platform" }, { "id": "k-watch", "label": "48 h Watch", "sub": "lag and freshness", "kind": "platform" }, { "id": "k-rb", "label": "Auto Rollback", "sub": "on lag breach", "kind": "security" } ] } ], "edges": [ { "from": "k-comp", "to": "k-git", "label": "breaking", "kind": "error" }, { "from": "k-rb", "to": "k-bg", "label": "revert", "kind": "error" } ], "note": "Blue/green uses a second consumer group replaying from the same offsets, so no event is skipped at cut-over.", "meta": { "v": "1.0", "owner": "Platform Engineering", "date": "2026-08" } }, { "id": "18-observability-and-slo", "title": "Observability and Service Level Objectives", "layout": "grid", "laneHeaderWidth": 160, "columns": [ "Ingestion", "Event log", "Processing", "Hot store", "Serving" ], "rows": [ { "title": "Latency and lag", "cells": [ [ { "id": "o11", "label": "Accept p95 · 200 ms", "kind": "platform" } ], [ { "id": "o12", "label": "Consumer lag", "kind": "platform" } ], [ { "id": "o13", "label": "Batch duration", "kind": "platform" } ], [ { "id": "o14", "label": "Ingestion latency", "kind": "platform" } ], [ { "id": "o15", "label": "Query p95 · 2 s", "kind": "platform" } ] ] }, { "title": "Throughput", "cells": [ [ { "id": "o21", "label": "Events accepted/s", "kind": "platform" } ], [ { "id": "o22", "label": "Throughput units", "kind": "platform" } ], [ { "id": "o23", "label": "Rows per batch", "kind": "platform" } ], [ { "id": "o24", "label": "Rows ingested/min", "kind": "platform" } ], [ { "id": "o25", "label": "Concurrent queries", "kind": "platform" } ] ] }, { "title": "Errors", "cells": [ [ { "id": "o31", "label": "4xx / 5xx rate", "kind": "risk" } ], [ { "id": "o32", "label": "Quota throttles", "kind": "risk" } ], [ { "id": "o33", "label": "Failed batches", "kind": "risk" } ], [ { "id": "o34", "label": "Ingestion failures", "kind": "risk" } ], [ { "id": "o35", "label": "Query timeouts", "kind": "risk" } ] ] }, { "title": "Data quality", "cells": [ [ { "id": "o41", "label": "Schema rejects", "kind": "app" } ], [ { "id": "o42", "label": "Dead-letter depth", "kind": "queue" } ], [ { "id": "o43", "label": "Late and dropped events", "kind": "app" } ], [ { "id": "o44", "label": "Count reconciliation", "kind": "app" } ], [ { "id": "o45", "label": "Freshness watermark", "kind": "app" } ] ] }, { "title": "Alerting", "cells": [ [ { "id": "o51", "label": "Accept-rate burn alert", "kind": "security" } ], [ { "id": "o52", "label": "Lag over 60 s pages", "kind": "security" } ], [ { "id": "o53", "label": "Checkpoint stall pages", "kind": "security" } ], [ { "id": "o54", "label": "Freshness SLO burn", "kind": "security" } ], [ { "id": "o55", "label": "Query SLO burn", "kind": "security" } ] ] } ], "note": "Every event carries event_id and trace context, so a single record is traceable from SDK to dashboard.", "meta": { "v": "1.0", "owner": "SRE", "date": "2026-08" } }, { "id": "19-reliability-recovery-loop", "title": "Reliability and Recovery Loop", "layout": "cycle", "centre": { "label": "Analytics Platform" }, "nodes": [ { "id": "v1", "label": "Detect", "sub": "SLO burn · lag alert", "kind": "platform" }, { "id": "v2", "label": "Contain", "sub": "backpressure · scale out", "kind": "app", "icon": "fa5_compress_arrows_alt" }, { "id": "v3", "label": "Diagnose", "sub": "trace by event_id", "kind": "app" }, { "id": "v4", "label": "Recover", "sub": "checkpoint restart", "kind": "app", "icon": "fa5_life_ring" }, { "id": "v5", "label": "Replay", "sub": "bronze to shadow table", "kind": "app" }, { "id": "v6", "label": "Reconcile", "sub": "variance below 0.1%", "kind": "decision" } ], "ringLabels": [ "breach signal", "incident scoped", "root cause found", "service restored", "gap refilled", "verified clean" ], "rx": 430, "ry": 215, "note": "SLO 99.9% with no silent data loss. Retention in Event Hubs and bronze is what makes recovery possible.", "meta": { "v": "1.0", "owner": "SRE", "date": "2026-08" } }, { "id": "20-security-trust-zones", "title": "Security Architecture — Trust Zones", "layout": "zones", "zones": [ { "title": "Untrusted · internet", "kind": "trust", "nodes": [ { "id": "g-prod", "label": "Producer Applications", "kind": "external" }, { "id": "g-part", "label": "Partner Systems", "kind": "external", "icon": "fa5_cloud" }, { "id": "g-att", "label": "Forged Event Injection", "sub": "stolen client secret", "kind": "risk" } ] }, { "title": "Perimeter · edge services", "kind": "trust", "nodes": [ { "id": "g-afd", "label": "Azure Front Door", "sub": "WAF · DDoS · TLS 1.3", "kind": "security" }, { "id": "g-apim", "label": "API Management", "sub": "OAuth2 · quota · mTLS", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "g-iot", "label": "Azure IoT Hub", "sub": "per-device X.509", "kind": "integration" } ] }, { "title": "Private · ingestion and processing", "kind": "trust", "nodes": [ { "id": "g-coll", "label": "Event Collector", "sub": "managed identity", "kind": "app", "icon": "azure_container-apps-environments" }, { "id": "g-eh", "label": "Azure Event Hubs", "sub": "private endpoint only", "kind": "queue" }, { "id": "g-dbx", "label": "Azure Databricks", "sub": "VNet-injected · no public IP", "kind": "app" }, { "id": "g-entra", "label": "Microsoft Entra ID", "sub": "workload identity federation", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" } ] }, { "title": "Restricted · data and keys", "kind": "trust", "nodes": [ { "id": "g-adx", "label": "Azure Data Explorer", "sub": "CMK · row-level security", "kind": "store" }, { "id": "g-adls", "label": "ADLS Gen2", "sub": "CMK · immutable bronze", "kind": "store" }, { "id": "g-kv", "label": "Azure Key Vault", "sub": "HSM · 12-month rotation", "kind": "security" }, { "id": "g-aud", "label": "Audit Log", "sub": "write-once · 7 y", "kind": "security" } ] } ], "edges": [ { "from": "g-prod", "to": "g-afd", "label": "HTTPS 443" }, { "from": "g-part", "to": "g-apim", "label": "OAuth2 tokens" }, { "from": "g-att", "to": "g-apim", "label": "claim mismatch", "kind": "error" }, { "from": "g-afd", "to": "g-apim", "label": "WAF-filtered" }, { "from": "g-apim", "to": "g-coll", "label": "private link · mTLS" }, { "from": "g-iot", "to": "g-eh", "label": "routing endpoint", "kind": "async" }, { "from": "g-coll", "to": "g-eh", "label": "AMQP publish", "kind": "async" }, { "from": "g-eh", "to": "g-dbx", "label": "consumer group", "kind": "async" }, { "from": "g-dbx", "to": "g-adx", "label": "ingest", "kind": "async" }, { "from": "g-dbx", "to": "g-adls", "label": "checkpoints" }, { "from": "g-entra", "to": "g-kv", "label": "key access" }, { "from": "g-adx", "to": "g-aud", "label": "audit trail", "kind": "async" } ], "note": "No component holds a shared key. Every service-to-service hop uses a managed identity over a private endpoint.", "meta": { "v": "1.0", "owner": "Security Architecture", "date": "2026-08" } }, { "id": "21-identity-and-access-flow", "title": "Identity and Access — Producer and Consumer", "layout": "sequence", "lifelines": [ { "id": "a-prod", "label": "Producer Service", "kind": "external" }, { "id": "a-entra", "label": "Microsoft Entra ID", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "a-apim", "label": "API Management", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "a-coll", "label": "Event Collector", "kind": "app", "icon": "azure_container-apps-environments" }, { "id": "a-eh", "label": "Azure Event Hubs", "kind": "queue" }, { "id": "a-adx", "label": "Azure Data Explorer", "kind": "store" }, { "id": "a-an", "label": "Analyst", "kind": "actor" } ], "messages": [ { "from": "a-prod", "to": "a-entra", "label": "client credentials + certificate", "kind": "call" }, { "from": "a-entra", "to": "a-prod", "label": "token · scope events.write", "kind": "return" }, { "from": "a-prod", "to": "a-apim", "label": "POST /v1/events + bearer", "kind": "call" }, { "from": "a-apim", "to": "a-apim", "label": "validate JWT, pin tenant_id", "kind": "self" }, { "from": "a-apim", "to": "a-coll", "label": "forward with pinned claim", "kind": "call" }, { "from": "a-coll", "to": "a-eh", "label": "publish via managed identity", "kind": "async" }, { "from": "a-apim", "to": "a-prod", "label": "403 on tenant mismatch", "kind": "error" }, { "from": "a-an", "to": "a-entra", "label": "sign in + MFA", "kind": "call" }, { "from": "a-entra", "to": "a-an", "label": "id and access token", "kind": "return" }, { "from": "a-an", "to": "a-adx", "label": "KQL with on-behalf-of token", "kind": "call" }, { "from": "a-adx", "to": "a-adx", "label": "row-level security by tenant", "kind": "self" }, { "from": "a-adx", "to": "a-an", "label": "rows in scope only", "kind": "return" } ], "note": "Producer tokens live 60 minutes and are certificate-bound; analyst access is group-based and reviewed quarterly.", "meta": { "v": "1.0", "owner": "Security Architecture", "date": "2026-08" } } ]