[ { "id": "01-system-context", "title": "Distributed Workflow Orchestration Platform — System Context", "layout": "context", "canvas": { "width": 1740 }, "colWidth": 268, "system": { "label": "Distributed Workflow Orchestration Platform", "sub": "DAG execution · at-least-once · multi-tenant" }, "groups": [ { "side": "left", "title": "Clients and operators", "nodes": [ { "id": "apps", "label": "Business applications", "sub": "40 tenants", "kind": "external", "rel": "submit executions", "dir": "in", "icon": "fa5_building" }, { "id": "authors", "label": "Workflow authors", "sub": "domain engineers", "kind": "actor", "rel": "publish DAG versions", "dir": "in" }, { "id": "ops", "label": "Platform operations", "sub": "24x7 on-call", "kind": "actor", "rel": "inspect · retry · cancel", "kind2": "bidirectional", "icon": "role_devops" }, { "id": "portal", "label": "Operations console", "sub": "execution timeline", "kind": "external", "rel": "read status", "dir": "in", "icon": "fa5_desktop" } ] }, { "side": "top", "title": "Trigger sources", "nodes": [ { "id": "evtpub", "label": "Domain event publishers", "sub": "CustomerCreated, OrderPaid", "kind": "external", "rel": "event triggers", "dir": "in", "kind2": "async", "icon": "azure_event-grid-topics" }, { "id": "partner", "label": "Partner webhooks", "sub": "signed callbacks", "kind": "external", "rel": "inbound triggers", "dir": "in", "kind2": "async" } ] }, { "side": "right", "title": "Systems tasks act on", "nodes": [ { "id": "c360", "label": "Customer 360 platform", "kind": "external", "rel": "enrichment tasks", "kind2": "sync" }, { "id": "idv", "label": "Identity verification SaaS", "sub": "rate limited", "kind": "external", "rel": "identity-check tasks", "kind2": "sync" }, { "id": "http", "label": "Enterprise HTTP services", "sub": "180 endpoints", "kind": "external", "rel": "http tasks", "kind2": "sync", "icon": "fa5_plug" }, { "id": "llm", "label": "LLM gateway", "sub": "quota governed", "kind": "external", "rel": "rag tasks", "kind2": "sync" }, { "id": "notify", "label": "Notification providers", "sub": "email · SMS · push", "kind": "external", "rel": "notification tasks", "kind2": "async" } ] }, { "side": "bottom", "title": "Platform dependencies", "nodes": [ { "id": "entra", "label": "Microsoft Entra ID", "sub": "OAuth2 · managed identity", "kind": "external", "rel": "authenticate", "kind2": "sync", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "kv", "label": "Azure Key Vault", "sub": "handler credentials", "kind": "external", "rel": "resolve secrets", "kind2": "sync", "icon": "azure_key-vaults" }, { "id": "mon", "label": "Azure Monitor", "sub": "logs · metrics · traces", "kind": "external", "rel": "emit telemetry", "dir": "in", "kind2": "async", "icon": "azure_monitor" } ] } ], "note": "Task business logic is owned by domain teams and runs inside handlers the platform hosts but does not author.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "02-high-level-architecture", "title": "High-Level Architecture — Submission to Completion", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "middle", "nodeWidth": 178, "stages": [ { "title": "Define", "nodes": [ { "id": "wfsvc", "label": "Workflow Service", "sub": "validate · version · publish", "kind": "app", "icon": "azure_workflow" } ] }, { "title": "Admit", "nodes": [ { "id": "apim", "label": "API Management", "sub": "JWT · quota · throttle", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "execapi", "label": "Execution API", "sub": "idempotent submit", "kind": "app" } ] }, { "title": "Decide", "nodes": [ { "id": "orch", "label": "Orchestration Service", "sub": "stateless · competing", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "sched", "label": "Scheduler Service", "sub": "leader-elected", "kind": "app", "icon": "fa5_clock" } ] }, { "title": "Distribute", "nodes": [ { "id": "relay", "label": "Outbox Relay", "sub": "change feed", "kind": "app", "icon": "fa5_share_alt" }, { "id": "sb", "label": "Azure Service Bus", "sub": "task + completion queues", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" } ] }, { "title": "Execute", "nodes": [ { "id": "workers", "label": "Worker Fleet", "sub": "KEDA-scaled on AKS", "kind": "app", "icon": "fa5_cogs" }, { "id": "handlers", "label": "Task Handler Plugins", "sub": "http · c360 · rag · notify", "kind": "app" } ] }, { "title": "Record", "nodes": [ { "id": "cosmos", "label": "Cosmos DB", "sub": "source of truth", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "blob", "label": "Blob Storage", "sub": "claim-check payloads", "kind": "store", "icon": "azure_AzureBlobStorage (Storage)" } ] }, { "title": "Observe", "nodes": [ { "id": "monitor", "label": "Azure Monitor", "sub": "OpenTelemetry", "kind": "platform", "icon": "azure_monitor" }, { "id": "opapi", "label": "Operator API", "sub": "retry · replay · cancel", "kind": "app" } ] } ], "edges": [ { "from": "cosmos", "to": "relay", "label": "change feed", "kind": "async", "route": "gutter" }, { "from": "workers", "to": "sb", "label": "TaskCompleted", "kind": "async", "route": "gutter" }, { "from": "sb", "to": "orch", "label": "advance DAG", "kind": "async", "route": "gutter" }, { "from": "opapi", "to": "cosmos", "label": "intervene", "kind": "sync" } ], "note": "One journey, one direction, with a single loop: results return through the bus so the orchestrator advances the DAG without holding a thread.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" }, "stageGap": 30 }, { "id": "03-control-and-data-planes", "title": "Control, Transport and Execution Planes", "layout": "nested", "canvas": { "width": 1720 }, "boxes": [ { "title": "Control plane — state is authoritative here", "kind": "boundary", "dir": "col", "children": [ { "title": "Decision services", "kind": "plain", "nodes": [ { "id": "p_wf", "label": "Workflow Service", "kind": "app", "icon": "azure_workflow" }, { "id": "p_exec", "label": "Execution API", "kind": "app" }, { "id": "p_orch", "label": "Orchestration Service", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "p_sched", "label": "Scheduler Service", "kind": "app", "icon": "fa5_clock" }, { "id": "p_retry", "label": "Retry Service", "kind": "app", "icon": "fa5_redo" } ] }, { "title": "System of record", "kind": "plain", "nodes": [ { "id": "p_def", "label": "Definitions", "sub": "immutable versions", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "p_state", "label": "Execution state", "sub": "ETag guarded", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "p_events", "label": "Execution events", "sub": "append only", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "p_lease", "label": "Leases", "sub": "fencing tokens", "kind": "store", "icon": "fa5_lock" } ] } ] }, { "title": "Transport plane — carries work, never truth", "kind": "boundary", "dir": "row", "children": [ { "title": "Service Bus namespace", "kind": "plain", "nodes": [ { "id": "p_tq", "label": "Task queues", "sub": "by priority class", "kind": "queue" }, { "id": "p_cq", "label": "Completion queue", "kind": "queue" }, { "id": "p_sm", "label": "Scheduled messages", "sub": "retry backoff", "kind": "queue" }, { "id": "p_dlq", "label": "Dead-letter queues", "kind": "queue", "icon": "fa5_exclamation_triangle" } ] } ] }, { "title": "Execution plane — stateless and replaceable", "kind": "boundary", "dir": "row", "children": [ { "title": "Worker pools", "kind": "plain", "nodes": [ { "id": "p_wa", "label": "Pool A — fast HTTP", "sub": "p95 2s", "kind": "app", "icon": "fa5_cogs" }, { "id": "p_wb", "label": "Pool B — enrichment", "sub": "p95 40s", "kind": "app", "icon": "fa5_cogs" }, { "id": "p_wc", "label": "Pool C — inference", "sub": "GPU optional", "kind": "app", "icon": "fa5_cogs" } ] } ] } ], "outside": [ { "id": "p_targets", "label": "Downstream systems", "sub": "HTTP · SaaS · LLM", "kind": "external" } ], "edges": [ { "from": "p_orch", "to": "p_state", "label": "transactional batch", "kind": "sync" }, { "from": "p_state", "to": "p_tq", "label": "outbox relay", "kind": "async" }, { "from": "p_tq", "to": "p_wa", "label": "at-least-once", "kind": "async" }, { "from": "p_wa", "to": "p_targets", "label": "side effect", "kind": "sync" }, { "from": "p_wb", "to": "p_cq", "label": "TaskCompleted", "kind": "async" }, { "from": "p_cq", "to": "p_orch", "label": "advance DAG", "kind": "async" } ], "note": "A worker never writes execution state. It emits a result, and the control plane decides what that result means — which is exactly what makes a worker safe to kill mid-task.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "04-layered-architecture", "title": "Layered Architecture — What Depends on What", "layout": "bands", "canvas": { "width": 1700 }, "layerHeaderWidth": 158, "bands": [ { "name": "Experience", "nodes": [ { "id": "l_console", "label": "Operations console", "sub": "React · SSE timeline", "kind": "app", "icon": "fa5_desktop" }, { "id": "l_cli", "label": "Authoring CLI", "sub": "workflow lint · publish", "kind": "app", "icon": "fa5_terminal" }, { "id": "l_sdk", "label": "Client SDKs", "sub": ".NET · Python · Java", "kind": "app", "icon": "fa5_code" } ] }, { "name": "API", "nodes": [ { "id": "l_apim", "label": "API Management", "sub": "OAuth2 · quota", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "l_wfapi", "label": "Workflow API", "kind": "app" }, { "id": "l_exapi", "label": "Execution API", "kind": "app" }, { "id": "l_opapi", "label": "Operator API", "kind": "app" } ] }, { "name": "Orchestration", "nodes": [ { "id": "l_orch", "label": "Orchestration Service", "sub": "DAG advance", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "l_sched", "label": "Scheduler Service", "sub": "cron · leader", "kind": "app", "icon": "fa5_clock" }, { "id": "l_trig", "label": "Trigger Service", "sub": "event subscriptions", "kind": "app", "icon": "fa5_bolt" }, { "id": "l_retry", "label": "Retry Service", "sub": "backoff · circuit", "kind": "app", "icon": "fa5_redo" }, { "id": "l_relay", "label": "Outbox Relay", "sub": "change feed", "kind": "app", "icon": "fa5_share_alt" } ] }, { "name": "Execution", "nodes": [ { "id": "l_runtime", "label": "Worker Runtime", "sub": "lease · idempotency", "kind": "app", "icon": "fa5_cogs" }, { "id": "l_hsdk", "label": "Task Handler SDK", "sub": "execute(context)", "kind": "app" }, { "id": "l_plugins", "label": "Handler plugins", "sub": "7 task types", "kind": "app", "icon": "fa5_puzzle_piece" } ] }, { "name": "Messaging", "nodes": [ { "id": "l_sb", "label": "Azure Service Bus", "sub": "queues · sessions · DLQ", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" }, { "id": "l_eg", "label": "Azure Event Grid", "sub": "trigger ingress", "kind": "queue", "icon": "azure_event-grid-topics" } ] }, { "name": "State", "nodes": [ { "id": "l_cosmos", "label": "Azure Cosmos DB", "sub": "definitions · state · events", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "l_redis", "label": "Azure Managed Redis", "sub": "concurrency leases", "kind": "store", "icon": "azure_redis" }, { "id": "l_blob", "label": "Azure Blob Storage", "sub": "payloads · artifacts", "kind": "store", "icon": "azure_AzureBlobStorage (Storage)" }, { "id": "l_sql", "label": "Azure SQL", "sub": "reporting replica", "kind": "store", "icon": "azure_sql-database" } ] }, { "name": "Platform", "nodes": [ { "id": "l_aks", "label": "AKS", "sub": "zone redundant", "kind": "platform", "icon": "kubernetes" }, { "id": "l_entra", "label": "Microsoft Entra ID", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "l_kv", "label": "Azure Key Vault", "kind": "security", "icon": "azure_key-vaults" }, { "id": "l_mon", "label": "Azure Monitor", "sub": "OTel · App Insights", "kind": "platform", "icon": "azure_monitor" }, { "id": "l_acr", "label": "Container Registry", "kind": "platform", "icon": "azure_AzureContainerRegistry (Containers)" } ] } ], "edges": [ { "from": "l_console", "to": "l_apim", "label": "HTTPS", "kind": "sync" }, { "from": "l_exapi", "to": "l_orch", "label": "enqueue submit", "kind": "async" }, { "from": "l_orch", "to": "l_cosmos", "label": "state + outbox", "kind": "sync" }, { "from": "l_relay", "to": "l_sb", "label": "dispatch", "kind": "async" }, { "from": "l_sb", "to": "l_runtime", "label": "lease message", "kind": "async" }, { "from": "l_runtime", "to": "l_wfapi", "label": "definition fetch", "kind": "error", "route": "gutter" }, { "from": "l_retry", "to": "l_redis", "label": "circuit state", "kind": "sync" } ], "note": "One dependency points the wrong way and is drawn in red: workers read task definitions from the Workflow API. It is cached, read-only and version-pinned, and it is tracked as debt rather than hidden.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "05-container-architecture", "title": "Container & Component Architecture", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "AKS cluster — wfo-prod · Sweden Central · 3 zones", "kind": "cloud", "dir": "col", "children": [ { "title": "namespace: control", "kind": "boundary", "nodes": [ { "id": "c_exapi", "label": "execution-api", "sub": ".NET 9 · 4 pods", "kind": "app" }, { "id": "c_wfapi", "label": "workflow-api", "sub": ".NET 9 · 3 pods", "kind": "app" }, { "id": "c_opapi", "label": "operator-api", "sub": ".NET 9 · 2 pods", "kind": "app" }, { "id": "c_orch", "label": "orchestration-svc", "sub": "6-24 pods", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "c_sched", "label": "scheduler-svc", "sub": "3 pods · 1 leader", "kind": "app", "icon": "fa5_clock" }, { "id": "c_trig", "label": "trigger-svc", "sub": "3 pods", "kind": "app", "icon": "fa5_bolt" }, { "id": "c_retry", "label": "retry-svc", "sub": "3 pods", "kind": "app", "icon": "fa5_redo" }, { "id": "c_relay", "label": "outbox-relay", "sub": "StatefulSet · 4 leases", "kind": "app", "icon": "fa5_share_alt" } ] }, { "title": "namespace: workers", "kind": "boundary", "nodes": [ { "id": "c_wh", "label": "worker-http", "sub": "2-60 pods", "kind": "app", "icon": "fa5_cogs" }, { "id": "c_wc", "label": "worker-customer360", "sub": "2-30 pods", "kind": "app", "icon": "fa5_cogs" }, { "id": "c_wr", "label": "worker-rag", "sub": "0-20 pods", "kind": "app", "icon": "fa5_cogs" }, { "id": "c_wn", "label": "worker-notify", "sub": "2-15 pods", "kind": "app", "icon": "fa5_cogs" } ] }, { "title": "namespace: platform", "kind": "boundary", "nodes": [ { "id": "c_keda", "label": "KEDA operator", "sub": "queue-depth scalers", "kind": "platform", "icon": "kubernetes" }, { "id": "c_otel", "label": "OTel collector", "sub": "DaemonSet", "kind": "platform", "icon": "opentelemetry" }, { "id": "c_csi", "label": "Key Vault CSI driver", "kind": "security", "icon": "azure_key-vaults" }, { "id": "c_ing", "label": "Ingress controller", "sub": "internal only", "kind": "integration" } ] } ] }, { "title": "Azure managed services", "kind": "cloud", "dir": "row", "children": [ { "title": "Messaging", "kind": "plain", "nodes": [ { "id": "c_sb", "label": "Service Bus Premium", "sub": "4 messaging units", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" }, { "id": "c_eg", "label": "Event Grid", "sub": "trigger topics", "kind": "queue", "icon": "azure_event-grid-topics" } ] }, { "title": "State", "kind": "plain", "nodes": [ { "id": "c_cos", "label": "Cosmos DB", "sub": "autoscale 40K RU/s", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "c_red", "label": "Managed Redis", "sub": "concurrency leases", "kind": "store", "icon": "azure_redis" }, { "id": "c_blob", "label": "Blob Storage", "sub": "payloads · artifacts", "kind": "store", "icon": "azure_AzureBlobStorage (Storage)" }, { "id": "c_sql", "label": "Azure SQL", "sub": "reporting", "kind": "store", "icon": "azure_sql-database" } ] }, { "title": "Platform", "kind": "plain", "nodes": [ { "id": "c_apim", "label": "API Management", "sub": "Premium v2", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "c_kv", "label": "Key Vault", "kind": "security", "icon": "azure_key-vaults" }, { "id": "c_mon", "label": "Azure Monitor", "sub": "App Insights", "kind": "platform", "icon": "azure_monitor" }, { "id": "c_acr", "label": "Container Registry", "kind": "platform", "icon": "azure_AzureContainerRegistry (Containers)" } ] } ] } ], "outside": [ { "id": "c_entra", "label": "Microsoft Entra ID", "kind": "external", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "c_down", "label": "Downstream systems", "sub": "HTTP · SaaS · LLM", "kind": "external" } ], "edges": [ { "from": "c_apim", "to": "c_ing", "label": "mTLS", "kind": "sync" }, { "from": "c_orch", "to": "c_cos", "label": "batch write", "kind": "sync" }, { "from": "c_relay", "to": "c_sb", "label": "publish", "kind": "async" }, { "from": "c_sb", "to": "c_wh", "label": "peek-lock", "kind": "async" }, { "from": "c_keda", "to": "c_sb", "label": "queue depth", "kind": "sync" }, { "from": "c_wh", "to": "c_down", "label": "task call", "kind": "sync" }, { "from": "c_wr", "to": "c_blob", "label": "claim-check", "kind": "sync" } ], "note": "Every pod authenticates with a workload identity federated to Entra ID. No connection string is stored in the cluster; view 28 shows the token path.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "06-integration-architecture", "title": "Integration & Interface Catalogue", "layout": "hub", "canvas": { "width": 1700 }, "left": { "title": "Inbound — consumers and triggers", "nodes": [ { "id": "i_app", "label": "Business applications", "sub": "REST · OAuth2 client creds", "kind": "external", "rel": "POST /executions", "icon": "fa5_building" }, { "id": "i_cli", "label": "Authoring CLI / CI", "sub": "REST · OIDC", "kind": "external", "rel": "publish version" }, { "id": "i_con", "label": "Operations console", "sub": "REST + SSE", "kind": "external", "rel": "status stream", "icon": "fa5_desktop" }, { "id": "i_eg", "label": "Domain event publishers", "sub": "Event Grid CloudEvents", "kind": "external", "rel": "event trigger", "kind2": "async", "icon": "azure_event-grid-topics" }, { "id": "i_hook", "label": "Partner webhooks", "sub": "HMAC signed", "kind": "external", "rel": "inbound callback", "kind2": "async" } ] }, "centre": { "title": "Platform edge", "nodes": [ { "id": "i_apim", "label": "API Management", "sub": "Premium v2 · private", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "i_core", "label": "Orchestration Platform", "sub": "control + execution planes", "kind": "app" }, { "id": "i_egr", "label": "Egress policy", "sub": "Azure Firewall FQDN rules", "kind": "security" } ] }, "right": { "title": "Outbound — systems the platform calls", "nodes": [ { "id": "o_c360", "label": "Customer 360 platform", "sub": "REST · 400 rps cap", "kind": "external", "rel": "enrich", "dir": "out" }, { "id": "o_idv", "label": "Identity verification SaaS", "sub": "REST · 50 rps cap", "kind": "external", "rel": "verify", "dir": "out" }, { "id": "o_http", "label": "Enterprise HTTP services", "sub": "180 registered endpoints", "kind": "external", "rel": "generic http task", "dir": "out", "icon": "fa5_plug" }, { "id": "o_llm", "label": "LLM gateway", "sub": "token budget per tenant", "kind": "external", "rel": "rag task", "dir": "out" }, { "id": "o_not", "label": "Notification providers", "sub": "queued", "kind": "external", "rel": "notify", "dir": "out", "kind2": "async" }, { "id": "o_sql", "label": "Enterprise data warehouse", "sub": "nightly extract", "kind": "external", "rel": "execution facts", "dir": "out", "kind2": "batch" }, { "id": "o_mon", "label": "Azure Monitor / Sentinel", "sub": "OTLP + diagnostic settings", "kind": "external", "rel": "telemetry", "dir": "out", "kind2": "async", "icon": "azure_monitor" } ] }, "note": "Every outbound call carries the task's own managed identity or a Key Vault-resolved credential, plus the correlationId and taskExecutionId headers that make a downstream retry recognisable.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "07-messaging-topology", "title": "Messaging Topology & Queue Classes", "layout": "nested", "canvas": { "width": 1740 }, "boxes": [ { "title": "Service Bus Premium — shared namespace (sb-wfo-prod)", "kind": "cloud", "dir": "col", "children": [ { "title": "Work distribution — one queue per priority class", "kind": "boundary", "nodes": [ { "id": "q_hi", "label": "tasks-high", "sub": "lock 5 min · prefetch 8", "kind": "queue" }, { "id": "q_std", "label": "tasks-standard", "sub": "lock 5 min · prefetch 32", "kind": "queue" }, { "id": "q_bulk", "label": "tasks-bulk", "sub": "lock 5 min · prefetch 64", "kind": "queue" }, { "id": "q_long", "label": "tasks-longrunning", "sub": "sessions · renew to 2 h", "kind": "queue" } ] }, { "title": "Control and results", "kind": "boundary", "nodes": [ { "id": "q_done", "label": "task-completions", "sub": "sessions on executionId", "kind": "queue" }, { "id": "q_ctl", "label": "workflow-control", "sub": "cancel · pause · resume", "kind": "queue" }, { "id": "q_sched", "label": "scheduled-retries", "sub": "native scheduled messages", "kind": "queue", "icon": "fa5_clock" } ] }, { "title": "Failure handling", "kind": "boundary", "nodes": [ { "id": "q_dlq", "label": "Native DLQ (per queue)", "sub": "maxDelivery 10 · TTL expiry", "kind": "queue", "icon": "fa5_exclamation_triangle" }, { "id": "q_drain", "label": "dlq-drain", "sub": "moves to Cosmos store", "kind": "app", "icon": "fa5_hand_paper" }, { "id": "q_quar", "label": "Poison quarantine", "sub": "unparseable messages", "kind": "risk" } ] } ] }, { "title": "Dedicated namespace — tier-1 tenant escape hatch", "kind": "cloud", "dir": "row", "children": [ { "title": "sb-wfo-tenant-a", "kind": "boundary", "nodes": [ { "id": "q_ta", "label": "tasks-tenant-a", "sub": "noisy-neighbour isolation", "kind": "queue" }, { "id": "q_tac", "label": "completions-tenant-a", "kind": "queue" } ] } ] }, { "title": "Trigger ingress", "kind": "cloud", "dir": "row", "children": [ { "title": "Azure Event Grid", "kind": "boundary", "nodes": [ { "id": "q_egt", "label": "Custom topic", "sub": "CloudEvents 1.0", "kind": "queue", "icon": "azure_event-grid-topics" }, { "id": "q_egf", "label": "Subscription filters", "sub": "eventType + tenantId", "kind": "integration" }, { "id": "q_egd", "label": "Event Grid DLQ", "sub": "Blob container", "kind": "store", "icon": "azure_AzureBlobStorage (Storage)" } ] } ] } ], "outside": [ { "id": "m_relay", "label": "Outbox Relay", "kind": "app", "icon": "fa5_share_alt" }, { "id": "m_work", "label": "Worker pools", "kind": "app", "icon": "fa5_cogs" }, { "id": "m_orch", "label": "Orchestration Service", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "m_dlqs", "label": "Dead-letter store", "sub": "Cosmos container", "kind": "store" } ], "edges": [ { "from": "m_relay", "to": "q_std", "label": "dispatch", "kind": "async" }, { "from": "q_std", "to": "m_work", "label": "peek-lock", "kind": "async" }, { "from": "m_work", "to": "q_done", "label": "result", "kind": "async" }, { "from": "q_done", "to": "m_orch", "label": "advance", "kind": "async" }, { "from": "q_dlq", "to": "q_drain", "label": "drain loop", "kind": "error" }, { "from": "q_drain", "to": "m_dlqs", "label": "enriched record", "kind": "sync" }, { "from": "q_egf", "to": "m_orch", "label": "start execution", "kind": "async" } ], "note": "Service Bus has no message priority, so priority is expressed as separate queues with separate worker deployments and separate KEDA bounds. Sessions are used only where ordering is genuinely required.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "08-definition-lifecycle", "title": "Workflow Definition — Validation, Versioning and Binding", "layout": "flow", "canvas": { "width": 1740 }, "chain": true, "align": "middle", "nodeWidth": 196, "stages": [ { "title": "Author", "nodes": [ { "id": "d_dsl", "label": "Workflow DSL", "sub": "JSON or YAML DAG", "kind": "app" }, { "id": "d_cli", "label": "Authoring CLI", "sub": "lint · dry-run", "kind": "app", "icon": "fa5_terminal" } ] }, { "title": "Submit", "nodes": [ { "id": "d_api", "label": "Workflow API", "sub": "POST /workflows/{n}/versions", "kind": "app" } ] }, { "title": "Validate", "nodes": [ { "id": "d_schema", "label": "Schema valid?", "kind": "decision" }, { "id": "d_dag", "label": "Acyclic?", "sub": "Kahn topological sort", "kind": "decision" }, { "id": "d_types", "label": "Task types registered?", "kind": "decision" }, { "id": "d_policy", "label": "Tenant policy met?", "sub": "size · handlers · quota", "kind": "decision" }, { "id": "d_rej", "label": "Rejected before deploy", "sub": "422 · violation list", "kind": "risk", "icon": "fa5_ban" } ] }, { "title": "Publish", "nodes": [ { "id": "d_ver", "label": "Immutable version", "sub": "content hash · never edited", "kind": "store" }, { "id": "d_idx", "label": "Version index", "sub": "latest · pinned aliases", "kind": "store" } ] }, { "title": "Bind", "nodes": [ { "id": "d_bind", "label": "Execution binds version", "sub": "resolved once at submit", "kind": "app" }, { "id": "d_run", "label": "In-flight executions", "sub": "keep their own version", "kind": "app" } ] }, { "title": "Retire", "nodes": [ { "id": "d_dep", "label": "Deprecate version", "sub": "no new executions", "kind": "app" }, { "id": "d_del", "label": "Delete blocked", "sub": "while executions reference it", "kind": "risk", "icon": "fa5_ban" } ] } ], "edges": [ { "from": "d_schema", "to": "d_rej", "label": "422 + violations", "kind": "error" }, { "from": "d_dag", "to": "d_rej", "label": "cycle path", "kind": "error" }, { "from": "d_types", "to": "d_rej", "label": "unknown type", "kind": "error" }, { "from": "d_policy", "to": "d_rej", "label": "quota exceeded", "kind": "error" } ], "note": "Published versions are immutable and content-addressed. Schema evolution is additive: a new optional field publishes as a new version, and nothing already running observes the change.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "09-data-architecture", "title": "Data Architecture — Storage Zones by Ownership and Rebuildability", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Authoritative — losing this loses the platform", "kind": "boundary", "dir": "col", "children": [ { "title": "Cosmos DB · account cosmos-wfo-prod · session consistency", "kind": "plain", "nodes": [ { "id": "z_def", "label": "workflow-definitions", "sub": "PK /tenantId · no TTL", "kind": "store" }, { "id": "z_exec", "label": "executions", "sub": "PK /workflowExecutionId", "kind": "store" }, { "id": "z_evt", "label": "execution-events", "sub": "PK /workflowExecutionId · 90 d", "kind": "store" }, { "id": "z_dl", "label": "dead-letters", "sub": "PK /tenantId · 180 d", "kind": "store" }, { "id": "z_lease", "label": "leases", "sub": "PK /leaseName · fencing token", "kind": "store", "icon": "fa5_lock" }, { "id": "z_idem", "label": "idempotency-keys", "sub": "PK /taskExecutionId · 7 d", "kind": "store" }, { "id": "z_sched", "label": "schedules", "sub": "PK /tenantId", "kind": "store", "icon": "fa5_clock" } ] } ] }, { "title": "Payload and artifact — referenced, never inlined", "kind": "boundary", "dir": "row", "children": [ { "title": "Blob Storage · ZRS · immutable containers", "kind": "plain", "nodes": [ { "id": "z_in", "label": "task-payloads", "sub": "claim-check > 64 KB · 30 d", "kind": "store" }, { "id": "z_out", "label": "task-outputs", "sub": "30 d · lifecycle to cool", "kind": "store" }, { "id": "z_dlp", "label": "deadletter-payloads", "sub": "180 d · legal hold capable", "kind": "store" }, { "id": "z_arc", "label": "definition-archive", "sub": "WORM · 7 y", "kind": "store" } ] } ] }, { "title": "Derived — rebuildable from the event log", "kind": "boundary", "dir": "row", "children": [ { "title": "Reporting and cache", "kind": "plain", "nodes": [ { "id": "z_sql", "label": "Azure SQL reporting", "sub": "execution facts · 400 d", "kind": "store", "icon": "azure_sql-database" }, { "id": "z_red", "label": "Redis counters", "sub": "concurrency · circuit state", "kind": "store", "icon": "azure_redis" }, { "id": "z_proj", "label": "Status projections", "sub": "hot read model", "kind": "store" } ] } ] }, { "title": "Telemetry — operational, not a system of record", "kind": "boundary", "dir": "row", "children": [ { "title": "Azure Monitor", "kind": "plain", "nodes": [ { "id": "z_law", "label": "Log Analytics", "sub": "30 d hot · 2 y archive", "kind": "platform", "icon": "azure_log-analytics-workspaces" }, { "id": "z_ai", "label": "Application Insights", "sub": "traces · 5% sampled", "kind": "platform", "icon": "azure_application-insights" }, { "id": "z_aud", "label": "Audit log export", "sub": "immutable Blob · 7 y", "kind": "security" } ] } ] } ], "edges": [ { "from": "z_evt", "to": "z_sql", "label": "change feed", "kind": "async" }, { "from": "z_evt", "to": "z_proj", "label": "projection", "kind": "async" }, { "from": "z_exec", "to": "z_in", "label": "payload ref", "kind": "sync" }, { "from": "z_dl", "to": "z_dlp", "label": "payload snapshot", "kind": "sync" }, { "from": "z_law", "to": "z_aud", "label": "nightly export", "kind": "batch" } ], "note": "The rule that sizes the backup plan: only the authoritative zone is restored from backup. Everything in the derived zone is rebuilt by replaying execution-events, so its RPO is the log's, not its own.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "10-execution-data-model", "title": "Execution & Metadata Data Model", "layout": "er", "canvas": { "width": 1760, "cols": 4 }, "rowGap": 236, "entities": [ { "id": "tenant", "name": "tenant", "kind": "store", "row": 0, "col": 0, "attrs": [ "tenant_id PK", "name", "tier (bronze|silver|gold)", "max_concurrent_tasks", "max_concurrent_executions", "priority_class_default" ] }, { "id": "wfdef", "name": "workflow_definition", "kind": "store", "row": 0, "col": 1, "attrs": [ "workflow_id PK", "tenant_id FK -> tenant", "name", "latest_version", "owner_team" ] }, { "id": "wfver", "name": "workflow_version", "kind": "store", "row": 0, "col": 2, "attrs": [ "workflow_version_id PK", "workflow_id FK -> workflow_definition", "version (immutable)", "content_hash", "dag_json", "retry_policy_json", "published_at", "status (active|deprecated)" ] }, { "id": "sched", "name": "schedule", "kind": "store", "row": 0, "col": 3, "attrs": [ "schedule_id PK", "tenant_id FK -> tenant", "workflow_id FK -> workflow_definition", "cron_expression", "timezone", "next_fire_at", "misfire_policy" ] }, { "id": "trig", "name": "trigger_subscription", "kind": "store", "row": 1, "col": 0, "attrs": [ "subscription_id PK", "workflow_id FK -> workflow_definition", "event_type", "filter_expression", "input_mapping" ] }, { "id": "wfexec", "name": "workflow_execution", "kind": "store", "row": 1, "col": 1, "attrs": [ "workflow_execution_id PK", "tenant_id FK -> tenant", "workflow_version_id FK -> workflow_version", "status", "correlation_id", "idempotency_key", "input_ref", "output_ref", "created_at / started_at / completed_at", "_etag (optimistic concurrency)" ] }, { "id": "tkexec", "name": "task_execution", "kind": "store", "row": 1, "col": 2, "attrs": [ "task_execution_id PK", "workflow_execution_id FK -> workflow_execution", "task_id (DAG node)", "task_type", "status", "attempt_count", "depends_on[]", "priority_class", "next_visible_at", "_etag" ] }, { "id": "tkatt", "name": "task_attempt", "kind": "store", "row": 1, "col": 3, "attrs": [ "task_attempt_id PK", "task_execution_id FK -> task_execution", "attempt_no", "worker_id", "started_at / ended_at", "outcome", "error_code", "error_message", "trace_id / span_id" ] }, { "id": "evt", "name": "execution_event", "kind": "store", "row": 2, "col": 0, "attrs": [ "event_id PK", "workflow_execution_id FK -> workflow_execution", "sequence_no (gapless)", "event_type", "payload_json", "occurred_at", "actor" ] }, { "id": "outbox", "name": "outbox_message", "kind": "store", "row": 2, "col": 1, "attrs": [ "outbox_id PK", "workflow_execution_id FK -> workflow_execution", "destination_queue", "body_ref", "created_at", "published_at (null = pending)" ] }, { "id": "dl", "name": "dead_letter_record", "kind": "store", "row": 2, "col": 2, "attrs": [ "dead_letter_id PK", "task_execution_id FK -> task_execution", "tenant_id FK -> tenant", "attempt_count", "error_code / error_message", "stack_trace", "original_payload_ref", "disposition (open|retried|discarded)" ] }, { "id": "idem", "name": "idempotency_record", "kind": "store", "row": 2, "col": 3, "attrs": [ "task_execution_id PK", "handler_name", "side_effect_ref", "result_ref", "completed_at", "ttl_seconds" ] }, { "id": "lease", "name": "concurrency_lease", "kind": "store", "row": 3, "col": 1, "attrs": [ "lease_key PK (scope:tenant:type)", "holder_instance_id", "fencing_token (monotonic)", "acquired_at", "expires_at" ] } ], "relations": [ { "from": "tenant", "to": "wfdef", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "wfdef", "to": "wfver", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "tenant", "to": "trig", "label": "1 : N", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "wfdef", "to": "sched", "label": "1 : N", "from_side": "n", "to_side": "n", "kind": "optional" }, { "from": "wfver", "to": "wfexec", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "wfexec", "to": "tkexec", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "tkexec", "to": "tkatt", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "wfexec", "to": "evt", "label": "1 : N", "from_side": "s1", "to_side": "n" }, { "from": "wfexec", "to": "outbox", "label": "1 : N", "from_side": "s3", "to_side": "n" }, { "from": "tkexec", "to": "dl", "label": "1 : 0..1", "from_side": "s", "to_side": "n", "kind": "optional" }, { "from": "tkexec", "to": "idem", "label": "1 : 0..1", "from_side": "s3", "to_side": "n", "kind": "optional" } ], "note": "workflow_execution and task_execution share the /workflowExecutionId logical partition, which is what lets a state change and its outbox message be written in one transactional batch.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "11-execution-data-flow", "title": "Execution Data Flow — Origin to Consumption", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "align": "middle", "nodeWidth": 190, "stages": [ { "title": "Origin", "nodes": [ { "id": "f_cli", "label": "Client submit", "sub": "REST · sync", "kind": "external" }, { "id": "f_cron", "label": "Schedule fire", "sub": "cron · 1 min tick", "kind": "app", "icon": "fa5_clock" }, { "id": "f_evt", "label": "Domain event", "sub": "CloudEvents · async", "kind": "queue", "icon": "azure_event-grid-topics" } ] }, { "title": "Admit", "nodes": [ { "id": "f_api", "label": "Execution API", "sub": "validate · authorise", "kind": "app" }, { "id": "f_ck", "label": "Claim-check split", "sub": "> 64 KB to Blob", "kind": "app", "icon": "fa5_cut" } ] }, { "title": "Persist", "nodes": [ { "id": "f_batch", "label": "Transactional batch", "sub": "state + event + outbox", "kind": "app" }, { "id": "f_cos", "label": "executions container", "sub": "source of truth", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" } ] }, { "title": "Distribute", "nodes": [ { "id": "f_feed", "label": "Change feed", "sub": "ordered per partition", "kind": "app", "icon": "fa5_stream" }, { "id": "f_sb", "label": "Service Bus", "sub": "task queues", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" } ] }, { "title": "Produce", "nodes": [ { "id": "f_wrk", "label": "Worker", "sub": "handler executes", "kind": "app", "icon": "fa5_cogs" }, { "id": "f_res", "label": "Result + output ref", "sub": "completion message", "kind": "queue" } ] }, { "title": "Project", "nodes": [ { "id": "f_proj", "label": "Projection workers", "sub": "status · timeline · metrics", "kind": "app" }, { "id": "f_sqlp", "label": "Reporting load", "sub": "micro-batch · 5 min", "kind": "app", "icon": "azure_sql-database" } ] }, { "title": "Consume", "nodes": [ { "id": "f_con", "label": "Operations console", "sub": "SSE · sub-second", "kind": "external", "icon": "fa5_desktop" }, { "id": "f_dw", "label": "Data warehouse", "sub": "nightly extract", "kind": "external" }, { "id": "f_mon", "label": "Azure Monitor", "sub": "OTLP continuous", "kind": "platform", "icon": "azure_monitor" } ] } ], "edges": [ { "from": "f_ck", "to": "f_cos", "label": "payload ref only", "kind": "sync" }, { "from": "f_res", "to": "f_cos", "label": "state transition", "kind": "async", "route": "gutter" }, { "from": "f_cos", "to": "f_proj", "label": "change feed", "kind": "async", "route": "gutter" }, { "from": "f_sqlp", "to": "f_dw", "label": "nightly", "kind": "batch" } ], "note": "Cadence is on every arrow because batch and streaming otherwise look identical. Payloads travel by reference from the moment they exceed 64 KB, so message size stays bounded regardless of workload.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "12-event-log-and-projections", "title": "Execution Event Log & State Projection", "layout": "swimlane", "canvas": { "width": 1720 }, "laneHeaderWidth": 154, "stages": [ "Decide", "Append", "Project", "Serve" ], "lanes": [ { "title": "Command path", "cells": [ [ { "id": "e_orch", "label": "Orchestration Service", "sub": "validates transition", "kind": "app", "icon": "fa5_project_diagram" } ], [ { "id": "e_batch", "label": "Transactional batch", "sub": "state + event + outbox", "kind": "app" } ], [], [] ] }, { "title": "Event log", "cells": [ [], [ { "id": "e_log", "label": "execution-events", "sub": "gapless seq per execution", "kind": "store" } ], [ { "id": "e_feed", "label": "Change feed processor", "sub": "ordered · at-least-once", "kind": "app", "icon": "fa5_stream" } ], [] ] }, { "title": "Projectors", "cells": [ [], [], [ { "id": "e_ps", "label": "Status projector", "kind": "app" }, { "id": "e_pt", "label": "Timeline projector", "kind": "app" }, { "id": "e_pm", "label": "Metrics projector", "kind": "app" } ], [] ] }, { "title": "Read models", "cells": [ [], [], [], [ { "id": "e_rs", "label": "Execution status doc", "sub": "p95 < 40 ms read", "kind": "store" }, { "id": "e_rt", "label": "SSE status stream", "kind": "integration" }, { "id": "e_rf", "label": "SQL fact tables", "kind": "store", "icon": "azure_sql-database" } ] ] }, { "title": "Recovery", "cells": [ [], [], [ { "id": "e_replay", "label": "Replay from sequence", "sub": "operator initiated", "kind": "app", "icon": "fa5_history" } ], [ { "id": "e_rebuilt", "label": "Rebuilt projection", "sub": "swap on catch-up", "kind": "store" } ] ] } ], "edges": [ { "from": "e_orch", "to": "e_batch", "label": "one write", "kind": "sync" }, { "from": "e_batch", "to": "e_log", "label": "append", "kind": "sync" }, { "from": "e_log", "to": "e_feed", "label": "change feed", "kind": "async" }, { "from": "e_feed", "to": "e_ps", "kind": "async" }, { "from": "e_ps", "to": "e_rs", "kind": "sync" }, { "from": "e_pt", "to": "e_rt", "kind": "async" }, { "from": "e_pm", "to": "e_rf", "kind": "batch" }, { "from": "e_replay", "to": "e_rebuilt", "label": "rebuild", "kind": "batch" } ], "note": "This is event sourcing used for recovery and audit, not for reads. Current state is materialised in the same write as the event, so no read path depends on replaying the log — but every projection can be thrown away and rebuilt.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "13-submission-and-dispatch", "title": "Critical Flow — Submission, Persistence and First Dispatch", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "s_cli", "label": "Client application", "kind": "external" }, { "id": "s_apim", "label": "API Management", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "s_api", "label": "Execution API", "kind": "app" }, { "id": "s_cos", "label": "Cosmos DB", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "s_rel", "label": "Outbox Relay", "kind": "app" }, { "id": "s_sb", "label": "Service Bus", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" }, { "id": "s_wrk", "label": "Worker", "kind": "app" }, { "id": "s_orch", "label": "Orchestration Service", "kind": "app", "icon": "fa5_project_diagram" } ], "messages": [ { "from": "s_cli", "to": "s_apim", "label": "POST /workflows/customer-onboarding/executions", "kind": "call" }, { "from": "s_apim", "to": "s_apim", "label": "validate JWT · tenant quota · rate limit", "kind": "self" }, { "from": "s_apim", "to": "s_api", "label": "forward with tenantId claim", "kind": "call" }, { "from": "s_api", "to": "s_cos", "label": "read pinned workflow version", "kind": "call" }, { "from": "s_api", "to": "s_api", "label": "check Idempotency-Key header", "kind": "self" }, { "from": "s_api", "to": "s_cos", "label": "batch: execution PENDING + root tasks + event + outbox", "kind": "call" }, { "from": "s_cos", "to": "s_api", "label": "committed · _etag", "kind": "return" }, { "from": "s_api", "to": "s_cli", "label": "202 Accepted · executionId", "kind": "return" }, { "from": "s_cos", "to": "s_rel", "label": "change feed · pending outbox rows", "kind": "async" }, { "from": "s_rel", "to": "s_sb", "label": "send task message · MessageId = taskExecutionId", "kind": "async" }, { "from": "s_rel", "to": "s_cos", "label": "mark published_at", "kind": "call" }, { "from": "s_sb", "to": "s_wrk", "label": "peek-lock deliver", "kind": "async" }, { "from": "s_wrk", "to": "s_wrk", "label": "idempotency claim · execute handler", "kind": "self" }, { "from": "s_wrk", "to": "s_sb", "label": "TaskCompleted to completions queue", "kind": "async" }, { "from": "s_sb", "to": "s_orch", "label": "session-ordered completion", "kind": "async" }, { "from": "s_orch", "to": "s_cos", "label": "transition · dispatch ready successors", "kind": "call" } ], "note": "The client is answered at step 8, before any task runs. Everything after that point is recoverable from Cosmos alone, which is what makes the API able to promise 202 in under 300 ms.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "14-dag-dependency-resolution", "title": "DAG Dependency Resolution & Concurrent Fan-Out", "layout": "flow", "canvas": { "width": 1720 }, "chain": true, "align": "middle", "nodeWidth": 208, "stages": [ { "title": "Level 0 — root", "nodes": [ { "id": "g_a", "label": "create-customer", "sub": "http · no dependencies", "kind": "app" } ] }, { "title": "Level 1 — fan-out", "nodes": [ { "id": "g_b", "label": "verify-identity", "sub": "identity-check", "kind": "app" }, { "id": "g_c", "label": "score-risk", "sub": "customer360", "kind": "app" }, { "id": "g_d", "label": "fetch-credit-file", "sub": "http · 6 s p95", "kind": "app" } ] }, { "title": "Join gate", "nodes": [ { "id": "g_gate", "label": "All predecessors COMPLETED?", "sub": "counter reaches zero", "kind": "decision" }, { "id": "g_slot", "label": "Tenant slot available?", "kind": "decision" }, { "id": "g_def", "label": "Deferred", "sub": "re-evaluated on release", "kind": "risk", "icon": "fa5_hourglass_half" } ] }, { "title": "Level 2 — join", "nodes": [ { "id": "g_e", "label": "enrich-customer", "sub": "customer360", "kind": "app" } ] }, { "title": "Level 3", "nodes": [ { "id": "g_f", "label": "send-notification", "sub": "notification", "kind": "app" } ] }, { "title": "Terminal", "nodes": [ { "id": "g_end", "label": "Execution COMPLETED", "sub": "output assembled", "kind": "app", "icon": "fa5_check_circle" }, { "id": "g_fail", "label": "Any task DEAD_LETTERED", "sub": "execution FAILED", "kind": "risk" } ] } ], "edges": [ { "from": "g_b", "to": "g_gate", "kind": "sync" }, { "from": "g_c", "to": "g_gate", "kind": "sync" }, { "from": "g_d", "to": "g_gate", "kind": "sync" }, { "from": "g_gate", "to": "g_slot", "label": "ready", "kind": "sync" }, { "from": "g_slot", "to": "g_def", "label": "at limit", "kind": "error" }, { "from": "g_f", "to": "g_fail", "label": "on failure", "kind": "error" } ], "note": "Readiness is a pending-predecessor counter on each task document, decremented inside the same ETag-guarded write that records the predecessor's completion. Two orchestrators racing the same completion cannot both dispatch the successor.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "15-trigger-modes", "title": "Trigger Modes — Immediate, Scheduled, Event-Driven and Replay", "layout": "swimlane", "canvas": { "width": 1720 }, "laneHeaderWidth": 150, "stages": [ "Trigger", "Admit", "Create execution", "Dispatch" ], "lanes": [ { "title": "Immediate", "cells": [ [ { "id": "t_cli", "label": "Client POST", "sub": "synchronous request", "kind": "external" } ], [ { "id": "t_api", "label": "Execution API", "sub": "Idempotency-Key honoured", "kind": "app" } ], [ { "id": "t_e1", "label": "execution PENDING", "sub": "202 returned", "kind": "store" } ], [ { "id": "t_d1", "label": "Outbox → task queue", "kind": "queue" } ] ] }, { "title": "Scheduled", "cells": [ [ { "id": "t_cron", "label": "Cron definition", "sub": "IANA timezone · DST aware", "kind": "app", "icon": "fa5_clock" } ], [ { "id": "t_sch", "label": "Scheduler leader", "sub": "60 s tick · fencing token", "kind": "app" } ], [ { "id": "t_e2", "label": "Misfire policy applied", "sub": "skip · run-once · catch-up", "kind": "decision" } ], [ { "id": "t_d2", "label": "Same dispatch path", "kind": "queue" } ] ] }, { "title": "Event-driven", "cells": [ [ { "id": "t_ev", "label": "CloudEvent", "sub": "Event Grid custom topic", "kind": "queue", "icon": "azure_event-grid-topics" } ], [ { "id": "t_trg", "label": "Trigger Service", "sub": "filter + input mapping", "kind": "app", "icon": "fa5_bolt" } ], [ { "id": "t_e3", "label": "Dedupe on event id", "sub": "24 h window", "kind": "decision" } ], [ { "id": "t_d3", "label": "Same dispatch path", "kind": "queue" } ] ] }, { "title": "Operator replay", "cells": [ [ { "id": "t_op", "label": "Replay request", "sub": "from DLQ or history", "kind": "actor", "icon": "role_devops" } ], [ { "id": "t_opa", "label": "Operator API", "sub": "RBAC + reason required", "kind": "app" } ], [ { "id": "t_e4", "label": "New execution", "sub": "parent_execution_id linked", "kind": "store" } ], [ { "id": "t_d4", "label": "Same dispatch path", "kind": "queue" } ] ] } ], "edges": [ { "from": "t_cli", "to": "t_api", "kind": "sync" }, { "from": "t_api", "to": "t_e1", "kind": "sync" }, { "from": "t_e1", "to": "t_d1", "kind": "async" }, { "from": "t_cron", "to": "t_sch", "kind": "sync" }, { "from": "t_sch", "to": "t_e2", "kind": "sync" }, { "from": "t_e2", "to": "t_d2", "kind": "async" }, { "from": "t_ev", "to": "t_trg", "kind": "async" }, { "from": "t_trg", "to": "t_e3", "kind": "sync" }, { "from": "t_e3", "to": "t_d3", "kind": "async" }, { "from": "t_op", "to": "t_opa", "kind": "sync" }, { "from": "t_opa", "to": "t_e4", "kind": "sync" }, { "from": "t_e4", "to": "t_d4", "kind": "async" } ], "note": "Four entry points, one execution path. Every mode converges on the same execution document and the same outbox, so nothing downstream of admission needs to know how a workflow was started.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "16-task-state-machine", "title": "Task State Machine — Transitions, Guards and Terminal States", "layout": "flow", "canvas": { "width": 1740 }, "chain": false, "align": "middle", "nodeWidth": 202, "stages": [ { "title": "Created", "nodes": [ { "id": "m_pend", "label": "PENDING", "sub": "dependencies unmet", "kind": "app", "icon": "fa5_hourglass_start" } ] }, { "title": "Dispatched", "nodes": [ { "id": "m_sched", "label": "SCHEDULED", "sub": "outbox published", "kind": "app", "icon": "fa5_clock" }, { "id": "m_wait", "label": "WAITING_SLOT", "sub": "concurrency deferred", "kind": "app", "icon": "fa5_hourglass_half" } ] }, { "title": "Executing", "nodes": [ { "id": "m_run", "label": "RUNNING", "sub": "lock held · renewed", "kind": "app", "icon": "fa5_play_circle" }, { "id": "m_out", "label": "Handler outcome?", "kind": "decision" }, { "id": "m_lost", "label": "Lease lost", "sub": "worker died", "kind": "risk" } ] }, { "title": "Outcome", "nodes": [ { "id": "m_done", "label": "COMPLETED", "sub": "terminal", "kind": "app", "icon": "fa5_check_circle" }, { "id": "m_canc", "label": "CANCELLED", "sub": "terminal", "kind": "app", "icon": "fa5_ban" }, { "id": "m_to", "label": "TIMED_OUT", "sub": "task or workflow deadline", "kind": "risk" } ] }, { "title": "Failure triage", "nodes": [ { "id": "m_class", "label": "Retryable class?", "sub": "transient vs permanent", "kind": "decision" }, { "id": "m_budget", "label": "Attempts remaining?", "sub": "maxAttempts 5", "kind": "decision" } ] }, { "title": "Terminal", "nodes": [ { "id": "m_retry", "label": "RETRY_SCHEDULED", "sub": "backoff + jitter", "kind": "app", "icon": "fa5_redo" }, { "id": "m_dlq", "label": "DEAD_LETTERED", "sub": "terminal · operator owned", "kind": "risk" } ] } ], "edges": [ { "from": "m_pend", "to": "m_sched", "label": "deps met", "kind": "sync" }, { "from": "m_pend", "to": "m_wait", "label": "at limit", "kind": "error" }, { "from": "m_wait", "to": "m_sched", "label": "slot freed", "kind": "sync" }, { "from": "m_sched", "to": "m_run", "label": "lock acquired", "kind": "sync" }, { "from": "m_run", "to": "m_lost", "label": "lock expiry", "kind": "error" }, { "from": "m_lost", "to": "m_sched", "label": "redelivered", "kind": "async", "route": "gutter" }, { "from": "m_run", "to": "m_out", "kind": "sync" }, { "from": "m_out", "to": "m_done", "label": "success", "kind": "sync" }, { "from": "m_out", "to": "m_canc", "label": "cancel signal", "kind": "sync" }, { "from": "m_out", "to": "m_class", "label": "failure", "kind": "error" }, { "from": "m_out", "to": "m_to", "label": "deadline", "kind": "error" }, { "from": "m_class", "to": "m_budget", "label": "transient", "kind": "sync" }, { "from": "m_class", "to": "m_dlq", "label": "permanent", "kind": "error", "route": "gutter" }, { "from": "m_budget", "to": "m_retry", "label": "yes", "kind": "sync" }, { "from": "m_budget", "to": "m_dlq", "label": "exhausted", "kind": "error" }, { "from": "m_to", "to": "m_budget", "label": "if retryable", "kind": "error" }, { "from": "m_retry", "to": "m_sched", "label": "after delay", "kind": "async", "route": "gutter" } ], "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "17-delivery-and-idempotency", "title": "At-Least-Once Delivery — Duplicate Execution Without Duplicate Effect", "layout": "sequence", "canvas": { "width": 1720 }, "lifelines": [ { "id": "x_sb", "label": "Service Bus", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" }, { "id": "x_w1", "label": "Worker 1", "kind": "app" }, { "id": "x_w2", "label": "Worker 2", "kind": "app" }, { "id": "x_idem", "label": "Idempotency store", "kind": "store" }, { "id": "x_down", "label": "Downstream system", "kind": "external" }, { "id": "x_orch", "label": "Orchestration Service", "kind": "app", "icon": "fa5_project_diagram" }, { "id": "x_cos", "label": "Cosmos DB", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" } ], "messages": [ { "from": "x_sb", "to": "x_w1", "label": "deliver taskExecutionId 82f3 · attempt 1", "kind": "async" }, { "from": "x_w1", "to": "x_idem", "label": "claim 82f3 · IN_PROGRESS · lease 5 min", "kind": "call" }, { "from": "x_w1", "to": "x_down", "label": "POST /customers · Idempotency-Key: 82f3", "kind": "call" }, { "from": "x_down", "to": "x_w1", "label": "201 Created", "kind": "return" }, { "from": "x_w1", "to": "x_w1", "label": "pod evicted before recording result", "kind": "error" }, { "from": "x_sb", "to": "x_w2", "label": "lock expired · redeliver 82f3", "kind": "async" }, { "from": "x_w2", "to": "x_idem", "label": "read 82f3 → IN_PROGRESS, lease stale", "kind": "call" }, { "from": "x_w2", "to": "x_down", "label": "POST /customers · Idempotency-Key: 82f3", "kind": "call" }, { "from": "x_down", "to": "x_w2", "label": "200 OK · same resource, no second customer", "kind": "return" }, { "from": "x_w2", "to": "x_idem", "label": "mark COMPLETED · result ref", "kind": "call" }, { "from": "x_w2", "to": "x_sb", "label": "TaskCompleted · complete message", "kind": "async" }, { "from": "x_sb", "to": "x_orch", "label": "completion for 82f3", "kind": "async" }, { "from": "x_orch", "to": "x_cos", "label": "transition RUNNING → COMPLETED (ETag)", "kind": "call" }, { "from": "x_cos", "to": "x_orch", "label": "already COMPLETED → no-op, message settled", "kind": "return" } ], "note": "The platform guarantees the key, not the absence of duplicates. Where a downstream cannot honour an idempotency key, the handler declares itself non-retryable and the task dead-letters on first failure instead of risking a second side effect.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "18-worker-runtime-pipeline", "title": "Worker Runtime — From Message to Settlement", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "align": "middle", "nodeWidth": 190, "stages": [ { "title": "Receive", "nodes": [ { "id": "w_recv", "label": "Peek-lock receive", "sub": "prefetch 32", "kind": "queue" }, { "id": "w_env", "label": "Envelope validation", "sub": "schema + version", "kind": "app" } ] }, { "title": "Guard", "nodes": [ { "id": "w_canc", "label": "Cancelled or superseded?", "kind": "decision" }, { "id": "w_idem", "label": "Idempotency claim", "sub": "taskExecutionId", "kind": "store" }, { "id": "w_slot", "label": "Local semaphore", "sub": "per type · per tenant", "kind": "app", "icon": "fa5_lock" } ] }, { "title": "Prepare", "nodes": [ { "id": "w_res", "label": "Resolve handler", "sub": "plugin registry", "kind": "app", "icon": "fa5_puzzle_piece" }, { "id": "w_hyd", "label": "Hydrate payload", "sub": "claim-check from Blob", "kind": "store" }, { "id": "w_cred", "label": "Acquire credential", "sub": "workload identity", "kind": "security" } ] }, { "title": "Execute", "nodes": [ { "id": "w_exec", "label": "execute(context)", "sub": "handler contract", "kind": "app", "icon": "fa5_cogs" }, { "id": "w_renew", "label": "Lock auto-renewal", "sub": "background · to 2 h", "kind": "app", "icon": "fa5_sync" }, { "id": "w_wd", "label": "Timeout watchdog", "sub": "per-task deadline", "kind": "app", "icon": "fa5_stopwatch" } ] }, { "title": "Record", "nodes": [ { "id": "w_outp", "label": "Persist output", "sub": "Blob if > 64 KB", "kind": "store" }, { "id": "w_mark", "label": "Idempotency COMPLETED", "kind": "store" }, { "id": "w_emit", "label": "Emit TaskCompleted", "sub": "completions queue", "kind": "queue" } ] }, { "title": "Settle", "nodes": [ { "id": "w_ok", "label": "Complete message", "sub": "happy path", "kind": "app", "icon": "fa5_check_circle" }, { "id": "w_span", "label": "Close OTel span", "sub": "trace + task attributes", "kind": "platform" }, { "id": "w_ab", "label": "Abandon", "sub": "transient · immediate redeliver", "kind": "risk" }, { "id": "w_dl", "label": "Dead-letter", "sub": "poison envelope only", "kind": "risk" } ] } ], "edges": [ { "from": "w_exec", "to": "w_ab", "label": "transient error", "kind": "error" }, { "from": "w_env", "to": "w_dl", "label": "unparseable", "kind": "error", "route": "gutter" } ], "note": "The runtime does everything except decide what a failure means. It reports the outcome and settles the message; the retry policy and state transition are the control plane's decision, made from the workflow definition.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "19-retry-and-circuit-breaking", "title": "Retry, Backoff and Circuit Breaking", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "align": "middle", "nodeWidth": 194, "stages": [ { "title": "Failure", "nodes": [ { "id": "r_fail", "label": "Task attempt failed", "sub": "exception · 5xx · timeout", "kind": "risk" }, { "id": "r_att", "label": "task_attempt written", "sub": "error code + trace id", "kind": "store" } ] }, { "title": "Classify", "nodes": [ { "id": "r_cls", "label": "Error taxonomy", "sub": "transient · throttled · permanent", "kind": "app" }, { "id": "r_dec", "label": "Retryable?", "kind": "decision" } ] }, { "title": "Budget", "nodes": [ { "id": "r_pol", "label": "Retry policy", "sub": "max 5 · exp · 2 s → 5 m", "kind": "app" }, { "id": "r_bud", "label": "Attempts remaining?", "kind": "decision" } ] }, { "title": "Backoff", "nodes": [ { "id": "r_calc", "label": "delay = min(2s·2ⁿ, 5m)", "sub": "full jitter applied", "kind": "app", "icon": "fa5_stopwatch" }, { "id": "r_msg", "label": "Scheduled message", "sub": "Service Bus enqueue time", "kind": "queue", "icon": "fa5_clock" } ] }, { "title": "Circuit", "nodes": [ { "id": "r_cb", "label": "Handler circuit state", "sub": "Redis · per task type", "kind": "store" }, { "id": "r_open", "label": "Circuit open?", "kind": "decision" }, { "id": "r_probe", "label": "Half-open probe", "sub": "1 in flight", "kind": "app" } ] }, { "title": "Outcome", "nodes": [ { "id": "r_again", "label": "Re-dispatched", "sub": "attempt n+1", "kind": "app", "icon": "fa5_redo" }, { "id": "r_hold", "label": "Held, not failed", "sub": "circuit open · no budget burn", "kind": "app", "icon": "fa5_hourglass_half" }, { "id": "r_dlq", "label": "Dead-lettered", "sub": "operator owned", "kind": "risk" } ] } ], "edges": [ { "from": "r_dec", "to": "r_dlq", "label": "permanent", "kind": "error", "route": "gutter" }, { "from": "r_bud", "to": "r_dlq", "label": "exhausted", "kind": "error", "route": "gutter" }, { "from": "r_open", "to": "r_hold", "label": "open", "kind": "error" }, { "from": "r_open", "to": "r_again", "label": "closed", "kind": "sync" } ], "note": "Retries are re-enqueued as new scheduled messages rather than abandoned back onto the queue. Backoff can then exceed the lock duration, jitter is real, and the attempt count lives in state where the workflow definition can govern it.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "20-dead-letter-recovery", "title": "Dead-Letter Capture and Operator Recovery", "layout": "swimlane", "canvas": { "width": 1720 }, "laneHeaderWidth": 158, "stages": [ "Detect", "Capture", "Triage", "Disposition" ], "lanes": [ { "title": "Automatic", "cells": [ [ { "id": "k_exh", "label": "Retries exhausted", "sub": "application dead-letter", "kind": "risk" }, { "id": "k_nat", "label": "Native DLQ", "sub": "maxDelivery · TTL · poison", "kind": "queue" } ], [ { "id": "k_drain", "label": "dlq-drain job", "sub": "60 s poll · both sources", "kind": "app", "icon": "fa5_hand_paper" } ], [ { "id": "k_auto", "label": "Auto-classification", "sub": "error code → cause group", "kind": "app" } ], [] ] }, { "title": "Dead-letter store", "cells": [ [], [ { "id": "k_rec", "label": "dead_letter_record", "sub": "full error + attempt history", "kind": "store" }, { "id": "k_pay", "label": "Payload snapshot", "sub": "Blob · 180 d", "kind": "store" } ], [ { "id": "k_srch", "label": "Searchable index", "sub": "tenant · type · errorCode", "kind": "store" } ], [] ] }, { "title": "Operator", "cells": [ [ { "id": "k_alert", "label": "DLQ depth alert", "sub": "> 50 in 15 min", "kind": "platform" } ], [], [ { "id": "k_insp", "label": "Inspect timeline", "sub": "events + traces + payload", "kind": "actor", "icon": "role_devops" } ], [ { "id": "k_retry", "label": "Retry task", "sub": "same execution", "kind": "app", "icon": "fa5_redo" }, { "id": "k_rep", "label": "Replay execution", "sub": "new id · parent linked", "kind": "app", "icon": "fa5_history" }, { "id": "k_disc", "label": "Discard", "sub": "reason mandatory", "kind": "app", "icon": "fa5_ban" }, { "id": "k_term", "label": "Terminate execution", "sub": "compensations fire", "kind": "risk" } ] ] }, { "title": "Effect on state", "cells": [ [], [], [], [ { "id": "k_new", "label": "New task_attempt", "sub": "attempt count preserved", "kind": "store" }, { "id": "k_fail", "label": "Execution FAILED", "sub": "audited with actor", "kind": "store" } ] ] } ], "edges": [ { "from": "k_exh", "to": "k_drain", "kind": "error" }, { "from": "k_nat", "to": "k_drain", "kind": "error" }, { "from": "k_drain", "to": "k_rec", "kind": "sync" }, { "from": "k_rec", "to": "k_srch", "kind": "sync" }, { "from": "k_auto", "to": "k_srch", "kind": "sync" }, { "from": "k_alert", "to": "k_insp", "kind": "async" }, { "from": "k_srch", "to": "k_insp", "kind": "sync" }, { "from": "k_insp", "to": "k_retry", "kind": "sync" }, { "from": "k_retry", "to": "k_new", "kind": "sync" }, { "from": "k_term", "to": "k_fail", "kind": "error" } ], "note": "Two failure sources, one queue for humans. Native Service Bus dead-letters are infrastructure problems and application dead-letters are business problems, but an operator should never have to know which console to open.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "21-backpressure-and-limits", "title": "Backpressure — Control Scope Against Enforcement Point", "layout": "grid", "canvas": { "width": 1720 }, "laneHeaderWidth": 166, "columns": [ "API admission", "Dispatch decision", "Worker runtime", "Infrastructure" ], "rows": [ { "title": "Tenant", "cells": [ [ { "id": "b_t1", "label": "Submission quota", "sub": "APIM · 600/min gold", "kind": "integration" } ], [ { "id": "b_t2", "label": "Concurrency lease", "sub": "Redis · 500 / 100 / 20", "kind": "store" } ], [ { "id": "b_t3", "label": "Per-tenant prefetch cap", "kind": "app" } ], [ { "id": "b_t4", "label": "Dedicated namespace", "sub": "tier-1 escape hatch", "kind": "queue" } ] ] }, { "title": "Task type", "cells": [ [], [ { "id": "b_y2", "label": "Per-type lease", "sub": "identity-check ≤ 50", "kind": "store" } ], [ { "id": "b_y3", "label": "Handler semaphore", "sub": "in-process", "kind": "app", "icon": "fa5_lock" } ], [ { "id": "b_y4", "label": "Deployment per type", "sub": "independent scaling", "kind": "platform", "icon": "kubernetes" } ] ] }, { "title": "Downstream", "cells": [ [], [ { "id": "b_d2", "label": "Circuit state check", "sub": "skip dispatch when open", "kind": "decision" } ], [ { "id": "b_d3", "label": "Adaptive rate limiter", "sub": "honours Retry-After", "kind": "app" } ], [ { "id": "b_d4", "label": "LLM token budget", "sub": "per tenant per hour", "kind": "security" } ] ] }, { "title": "Queue pressure", "cells": [ [ { "id": "b_q1", "label": "429 above depth", "sub": "> 200K messages", "kind": "risk" } ], [ { "id": "b_q2", "label": "Priority class routing", "sub": "high · standard · bulk", "kind": "queue" } ], [ { "id": "b_q3", "label": "Max concurrent calls", "sub": "8 · 32 · 64", "kind": "app" } ], [ { "id": "b_q4", "label": "KEDA bounds", "sub": "min/max replicas", "kind": "platform", "icon": "kubernetes" } ] ] }, { "title": "Cluster", "cells": [ [], [], [ { "id": "b_c3", "label": "Pod resource limits", "sub": "CPU · memory · GOMAXPROCS", "kind": "platform" } ], [ { "id": "b_c4", "label": "Node pool maxima", "sub": "cluster autoscaler ceiling", "kind": "platform" } ] ] } ], "note": "Admission is the only place a request is refused. Everywhere after it, pressure is expressed as delay rather than rejection, because a task already accepted has been promised at-least-once execution.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "22-deployment-architecture", "title": "Deployment & Infrastructure — Sweden Central, Three Zones", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Azure subscription · sub-wfo-prod · Sweden Central", "kind": "cloud", "dir": "col", "children": [ { "title": "Hub VNet 10.10.0.0/16 — shared connectivity", "kind": "boundary", "nodes": [ { "id": "n_fw", "label": "Azure Firewall Premium", "sub": "FQDN egress allow-list", "kind": "security", "icon": "azure_firewalls" }, { "id": "n_bas", "label": "Azure Bastion", "sub": "no public SSH", "kind": "security", "icon": "azure_bastions" }, { "id": "n_er", "label": "ExpressRoute gateway", "sub": "corporate reachability", "kind": "integration", "icon": "azure_AzureExpressRoute (Networking)" }, { "id": "n_dns", "label": "Private DNS zones", "sub": "privatelink.*", "kind": "platform", "icon": "azure_dns-zones" } ] }, { "title": "Spoke VNet 10.20.0.0/16 — AKS wfo-prod", "kind": "boundary", "nodes": [ { "id": "n_sys", "label": "System node pool", "sub": "3 nodes · 1 per zone", "kind": "platform", "icon": "kubernetes" }, { "id": "n_ctl", "label": "Control node pool", "sub": "D8s v5 · 3-9 nodes", "kind": "platform", "icon": "kubernetes" }, { "id": "n_wrk", "label": "Worker node pool", "sub": "spot + on-demand · 3-40", "kind": "platform", "icon": "kubernetes" }, { "id": "n_apim", "label": "APIM subnet", "sub": "Premium v2 · zone redundant", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "n_pe", "label": "Private endpoint subnet", "sub": "no public data plane", "kind": "security" } ] }, { "title": "Zone-redundant managed services", "kind": "boundary", "nodes": [ { "id": "n_sb", "label": "Service Bus Premium", "sub": "4 MU · zone redundant", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" }, { "id": "n_cos", "label": "Cosmos DB", "sub": "single write region · ZR", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "n_red", "label": "Managed Redis", "sub": "Enterprise · ZR", "kind": "store", "icon": "azure_redis" }, { "id": "n_blob", "label": "Blob Storage", "sub": "ZRS + GRS for archive", "kind": "store", "icon": "azure_AzureBlobStorage (Storage)" }, { "id": "n_sql", "label": "Azure SQL", "sub": "Business Critical · ZR", "kind": "store", "icon": "azure_sql-database" }, { "id": "n_kv", "label": "Key Vault", "sub": "Premium · HSM keys", "kind": "security", "icon": "azure_key-vaults" } ] } ] }, { "title": "Secondary region · Norway East — warm standby, RTO 4 h / RPO 5 min", "kind": "cloud", "dir": "row", "children": [ { "title": "DR footprint", "kind": "boundary", "nodes": [ { "id": "n_cos2", "label": "Cosmos read region", "sub": "async replica · promotable", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "n_blob2", "label": "Blob GRS secondary", "kind": "store", "icon": "azure_AzureBlobStorage (Storage)" }, { "id": "n_acr2", "label": "ACR geo-replica", "kind": "platform", "icon": "azure_AzureContainerRegistry (Containers)" }, { "id": "n_aks2", "label": "AKS cluster (IaC)", "sub": "scaled to zero", "kind": "platform", "icon": "kubernetes" }, { "id": "n_sb2", "label": "Service Bus namespace", "sub": "empty · no geo-DR pairing", "kind": "risk" } ] } ] } ], "outside": [ { "id": "n_entra", "label": "Microsoft Entra ID", "sub": "global", "kind": "external", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "n_mon", "label": "Azure Monitor", "sub": "region-paired workspace", "kind": "platform", "icon": "azure_monitor" }, { "id": "n_git", "label": "GitHub Actions", "sub": "OIDC federated", "kind": "external" } ], "edges": [ { "from": "n_wrk", "to": "n_fw", "label": "all egress", "kind": "sync" }, { "from": "n_ctl", "to": "n_pe", "label": "private endpoints", "kind": "sync" }, { "from": "n_cos", "to": "n_cos2", "label": "async replication", "kind": "async" }, { "from": "n_git", "to": "n_acr2", "label": "image push", "kind": "batch" }, { "from": "n_ctl", "to": "n_entra", "label": "workload identity", "kind": "sync" } ], "note": "Service Bus is drawn red in the DR footprint on purpose: in-flight messages do not fail over. Regional recovery replays from Cosmos, which is why the queue was never allowed to be the state store.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "23-cicd-and-environments", "title": "CI/CD — How a Change Reaches Production, and What Stops a Bad One", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "align": "middle", "nodeWidth": 182, "stages": [ { "title": "Source", "nodes": [ { "id": "v_repo", "label": "GitHub monorepo", "sub": "services · workers · IaC", "kind": "external", "icon": "githubactions" }, { "id": "v_pr", "label": "Pull request", "sub": "2 reviewers · CODEOWNERS", "kind": "decision" } ] }, { "title": "Build", "nodes": [ { "id": "v_ci", "label": "GitHub Actions", "sub": "OIDC to Azure · no PAT", "kind": "app" }, { "id": "v_unit", "label": "Unit + mutation tests", "sub": "80% line gate", "kind": "decision" }, { "id": "v_img", "label": "Container build", "sub": "distroless · SBOM", "kind": "app" } ] }, { "title": "Verify", "nodes": [ { "id": "v_int", "label": "Integration tests", "sub": "Cosmos + SB emulators", "kind": "decision" }, { "id": "v_con", "label": "Contract tests", "sub": "task + workflow events", "kind": "decision" }, { "id": "v_sec", "label": "Security scan", "sub": "Defender · Trivy · secrets", "kind": "security" }, { "id": "v_iac", "label": "Terraform plan", "sub": "policy-as-code gate", "kind": "decision" } ] }, { "title": "Package", "nodes": [ { "id": "v_acr", "label": "Push to ACR", "sub": "immutable digest tag", "kind": "platform", "icon": "azure_AzureContainerRegistry (Containers)" }, { "id": "v_sign", "label": "Sign + attest", "sub": "Notation · provenance", "kind": "security" }, { "id": "v_helm", "label": "Helm chart to OCI", "sub": "versioned with image", "kind": "platform", "icon": "helm" } ] }, { "title": "Promote", "nodes": [ { "id": "v_dev", "label": "dev", "sub": "auto on merge", "kind": "app", "icon": "fa5_layer_group" }, { "id": "v_stg", "label": "staging", "sub": "load + chaos suite", "kind": "app", "icon": "fa5_layer_group" }, { "id": "v_apr", "label": "Change approval", "sub": "manual · change record", "kind": "decision" } ] }, { "title": "Production", "nodes": [ { "id": "v_prd", "label": "Rolling control plane", "sub": "PDB · maxUnavailable 1", "kind": "app" }, { "id": "v_wrk", "label": "Worker drain-and-replace", "sub": "finish lease, then exit", "kind": "app", "icon": "fa5_cogs" }, { "id": "v_rb", "label": "Auto rollback", "sub": "SLO burn > 2% in 15 min", "kind": "risk", "icon": "fa5_undo" } ] } ], "edges": [ { "from": "v_pr", "to": "v_ci", "label": "on merge", "kind": "sync" }, { "from": "v_stg", "to": "v_apr", "kind": "sync" }, { "from": "v_rb", "to": "v_acr", "label": "previous digest", "kind": "error", "route": "gutter" } ], "note": "Zero-downtime rests on one worker behaviour: on SIGTERM a worker stops receiving, finishes or abandons its in-flight lease, then exits. terminationGracePeriodSeconds is set above the longest task's lock renewal window.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "24-autoscaling", "title": "Queue-Driven Autoscaling with KEDA", "layout": "flow", "canvas": { "width": 1740 }, "chain": true, "align": "middle", "nodeWidth": 200, "stages": [ { "title": "Signal", "nodes": [ { "id": "a_depth", "label": "Active message count", "sub": "per task queue", "kind": "queue" }, { "id": "a_age", "label": "Oldest message age", "sub": "the SLO that matters", "kind": "queue", "icon": "fa5_stopwatch" } ] }, { "title": "Decide", "nodes": [ { "id": "a_keda", "label": "KEDA ScaledObject", "sub": "azure-servicebus trigger", "kind": "platform", "icon": "kubernetes" }, { "id": "a_tgt", "label": "queueLength 20 / replica", "sub": "cooldown 300 s", "kind": "app" }, { "id": "a_hpa", "label": "HPA behaviour", "sub": "up fast · down slow", "kind": "platform", "icon": "k8s_hpa" } ] }, { "title": "Scale out", "nodes": [ { "id": "a_pods", "label": "Worker replicas 2 → 60", "sub": "per deployment bounds", "kind": "app", "icon": "fa5_cogs" }, { "id": "a_ca", "label": "Cluster autoscaler", "sub": "spot first, on-demand fallback", "kind": "platform" } ] }, { "title": "Absorb", "nodes": [ { "id": "a_tp", "label": "900 tasks/s sustained", "sub": "peak drain target", "kind": "app" }, { "id": "a_lease", "label": "Tenant leases still cap", "sub": "fairness survives scale-out", "kind": "store" }, { "id": "a_wall", "label": "Downstream ceiling", "sub": "400 rps · scaling cannot pass it", "kind": "risk" } ] }, { "title": "Scale in", "nodes": [ { "id": "a_cool", "label": "Cooldown then scale in", "sub": "5 min stable", "kind": "app" }, { "id": "a_zero", "label": "worker-rag → 0", "sub": "scale-to-zero, cold start 25 s", "kind": "app" }, { "id": "a_pdb", "label": "PodDisruptionBudget", "sub": "min 2 during drain", "kind": "platform", "icon": "kubernetes" } ] } ], "edges": [ { "from": "a_wall", "to": "a_keda", "label": "circuit opens · stop scaling", "kind": "error", "route": "gutter" }, { "from": "a_age", "to": "a_keda", "label": "primary trigger", "kind": "sync" } ], "note": "Depth alone is the wrong trigger: 200,000 cheap messages and 200 expensive ones look identical. Oldest-message-age is scaled against the task scheduling SLO of p95 under one second, and depth is the secondary signal.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "25-observability", "title": "Observability — Signal Type Against Pipeline Stage", "layout": "grid", "canvas": { "width": 1740 }, "laneHeaderWidth": 168, "columns": [ "Instrument", "Collect", "Store", "Consume" ], "rows": [ { "title": "Traces", "cells": [ [ { "id": "y_t1", "label": "W3C traceparent", "sub": "carried in SB app properties", "kind": "app", "icon": "opentelemetry" } ], [ { "id": "y_t2", "label": "OTel collector", "sub": "DaemonSet · OTLP", "kind": "platform" } ], [ { "id": "y_t3", "label": "Application Insights", "sub": "5% head · 100% error tail", "kind": "store", "icon": "azure_application-insights" } ], [ { "id": "y_t4", "label": "Execution waterfall", "sub": "why did 123 take 47 s", "kind": "app" } ] ] }, { "title": "Metrics", "cells": [ [ { "id": "y_m1", "label": "RED + queue depth", "sub": "task duration by type", "kind": "app" } ], [ { "id": "y_m2", "label": "Managed Prometheus", "sub": "scrape + remote write", "kind": "platform", "icon": "prometheus" } ], [ { "id": "y_m3", "label": "Azure Monitor metrics", "sub": "93 d retention", "kind": "store" } ], [ { "id": "y_m4", "label": "SLO dashboards", "sub": "Grafana · burn rate", "kind": "app", "icon": "grafana" } ] ] }, { "title": "Logs", "cells": [ [ { "id": "y_l1", "label": "Structured JSON", "sub": "tenant · execution · task ids", "kind": "app" } ], [ { "id": "y_l2", "label": "Container Insights", "sub": "stdout · no PII", "kind": "platform" } ], [ { "id": "y_l3", "label": "Log Analytics", "sub": "30 d hot · 2 y archive", "kind": "store", "icon": "azure_log-analytics-workspaces" } ], [ { "id": "y_l4", "label": "KQL investigation", "sub": "saved incident queries", "kind": "app" } ] ] }, { "title": "Business events", "cells": [ [ { "id": "y_b1", "label": "ExecutionStarted/Failed", "sub": "domain events, not logs", "kind": "app" } ], [ { "id": "y_b2", "label": "Change feed", "sub": "ordered per execution", "kind": "app", "icon": "fa5_stream" } ], [ { "id": "y_b3", "label": "SQL fact tables", "sub": "400 d", "kind": "store", "icon": "azure_sql-database" } ], [ { "id": "y_b4", "label": "Tenant SLA reporting", "sub": "monthly attestation", "kind": "app" } ] ] }, { "title": "Alerting", "cells": [ [ { "id": "y_a1", "label": "SLO definitions", "sub": "as code, versioned", "kind": "app" } ], [ { "id": "y_a2", "label": "Alert rules", "sub": "burn rate · DLQ · lag", "kind": "platform" } ], [ { "id": "y_a3", "label": "Action groups", "sub": "severity routed", "kind": "platform" } ], [ { "id": "y_a4", "label": "On-call + runbook", "sub": "every alert links one", "kind": "actor", "icon": "role_devops" } ] ] } ], "note": "Six identifiers travel together on every log line, span and message: traceId, spanId, correlationId, tenantId, workflowExecutionId and taskExecutionId. Any one of them reconstructs the whole picture.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "26-operations-loop", "title": "Operations Lifecycle — The Loop That Has to Close", "layout": "cycle", "canvas": { "width": 1560 }, "centre": { "label": "Running platform" }, "rx": 452, "ry": 214, "nodes": [ { "id": "o_obs", "label": "Observe", "sub": "SLO · queue age · DLQ", "kind": "platform" }, { "id": "o_det", "label": "Detect", "sub": "burn-rate alert fires", "kind": "platform", "icon": "fa5_bell" }, { "id": "o_tri", "label": "Triage", "sub": "timeline + trace + payload", "kind": "actor", "icon": "role_devops" }, { "id": "o_act", "label": "Act", "sub": "retry · replay · cancel · scale", "kind": "app" }, { "id": "o_rec", "label": "Recover", "sub": "verify SLO restored", "kind": "app", "icon": "fa5_check_circle" }, { "id": "o_learn", "label": "Learn", "sub": "postmortem · chaos test", "kind": "app", "icon": "fa5_lightbulb" } ], "ringLabels": [ "burn rate", "page", "root cause", "intervention", "green", "new guardrail" ], "note": "The loop only closes at Learn: every incident produces either a chaos experiment that reproduces it or a policy that prevents it. Without that edge this is an escalation path, not a lifecycle.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "27-security-trust-zones", "title": "Security Architecture — Trust Zones and What Crosses Them", "layout": "zones", "canvas": { "width": 1740 }, "zones": [ { "title": "Internet — untrusted", "kind": "trust", "nodes": [ { "id": "sz_cli", "label": "Client applications", "sub": "confidential clients", "kind": "external" }, { "id": "sz_hook", "label": "Partner callers", "sub": "HMAC + IP allow-list", "kind": "external" }, { "id": "sz_att", "label": "Untrusted caller", "sub": "credential stuffing · replay", "kind": "risk", "icon": "fa5_user_secret" } ] }, { "title": "Perimeter — Azure edge, public ingress ends here", "kind": "trust", "nodes": [ { "id": "sz_afd", "label": "Front Door + WAF", "sub": "OWASP ruleset · TLS 1.3", "kind": "security", "icon": "azure_AzureFrontDoor (Networking)" }, { "id": "sz_ddos", "label": "DDoS Protection", "sub": "network tier", "kind": "security", "icon": "azure_ddos-protection-plans" }, { "id": "sz_apim", "label": "API Management", "sub": "validate-jwt · quota · schema", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" } ] }, { "title": "Application — private VNet, no public endpoint", "kind": "trust", "nodes": [ { "id": "sz_ctl", "label": "Control namespace", "sub": "APIs · orchestrator · scheduler", "kind": "app" }, { "id": "sz_wrk", "label": "Worker namespace", "sub": "network policy denies east-west", "kind": "app", "icon": "fa5_cogs" }, { "id": "sz_fw", "label": "Azure Firewall", "sub": "FQDN egress allow-list", "kind": "security", "icon": "azure_firewalls" } ] }, { "title": "Data — private endpoints only, RBAC data plane", "kind": "trust", "nodes": [ { "id": "sz_cos", "label": "Cosmos DB", "sub": "keys disabled · CMK", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "sz_sb", "label": "Service Bus", "sub": "SAS disabled · Entra RBAC", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" }, { "id": "sz_blob", "label": "Blob Storage", "sub": "no anonymous · CMK", "kind": "store", "icon": "azure_AzureBlobStorage (Storage)" }, { "id": "sz_red", "label": "Managed Redis", "sub": "no persistence of payloads", "kind": "store", "icon": "azure_redis" }, { "id": "sz_kv", "label": "Key Vault", "sub": "HSM · purge protection", "kind": "security", "icon": "azure_key-vaults" } ] }, { "title": "Identity and assurance — control plane", "kind": "trust", "nodes": [ { "id": "sz_entra", "label": "Microsoft Entra ID", "sub": "OAuth2 · Conditional Access", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "sz_mi", "label": "Workload identities", "sub": "federated · no secrets", "kind": "security" }, { "id": "sz_pim", "label": "PIM for operator roles", "sub": "JIT · approval · 4 h", "kind": "security" }, { "id": "sz_sen", "label": "Defender + Sentinel", "sub": "detections · UEBA", "kind": "security", "icon": "azure_sentinel" }, { "id": "sz_aud", "label": "Immutable audit log", "sub": "WORM Blob · 7 y", "kind": "security" } ] } ], "edges": [ { "from": "sz_cli", "to": "sz_afd", "label": "HTTPS · TLS 1.3", "kind": "sync" }, { "from": "sz_att", "to": "sz_afd", "label": "blocked", "kind": "error" }, { "from": "sz_afd", "to": "sz_apim", "label": "private origin", "kind": "sync" }, { "from": "sz_apim", "to": "sz_ctl", "label": "mTLS + tenant claim", "kind": "sync" }, { "from": "sz_ctl", "to": "sz_cos", "label": "Private Link · Entra RBAC", "kind": "sync" }, { "from": "sz_wrk", "to": "sz_fw", "label": "egress · FQDN filtered", "kind": "sync" }, { "from": "sz_mi", "to": "sz_cos", "label": "token, not key", "kind": "sync" }, { "from": "sz_apim", "to": "sz_entra", "label": "JWKS validation", "kind": "sync", "route": "gutter" }, { "from": "sz_ctl", "to": "sz_aud", "label": "audit events", "kind": "async" } ], "note": "Task payloads may contain personal data, so the classification travels with the workflow definition: a task marked restricted may only bind to handlers whose egress FQDN is on the approved list for that classification.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "28-identity-and-token-flow", "title": "Secretless Access — Token Path from Caller to Data Plane", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "id_cli", "label": "Client application", "kind": "external" }, { "id": "id_ent", "label": "Microsoft Entra ID", "kind": "security", "icon": "azure_AzureActiveDirectory (Identity)" }, { "id": "id_apim", "label": "API Management", "kind": "integration", "icon": "azure_AzureAPIManagement (Web)" }, { "id": "id_api", "label": "Execution API pod", "kind": "app" }, { "id": "id_kv", "label": "Key Vault", "kind": "security", "icon": "azure_key-vaults" }, { "id": "id_cos", "label": "Cosmos DB", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "id_sb", "label": "Service Bus", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" } ], "messages": [ { "from": "id_cli", "to": "id_ent", "label": "client_credentials · scope api://wfo/.default", "kind": "call" }, { "from": "id_ent", "to": "id_cli", "label": "access token · 60 min · appid + roles", "kind": "return" }, { "from": "id_cli", "to": "id_apim", "label": "POST /executions · Bearer", "kind": "call" }, { "from": "id_apim", "to": "id_ent", "label": "fetch + cache JWKS", "kind": "call" }, { "from": "id_apim", "to": "id_apim", "label": "validate-jwt · scope · tenant claim · quota", "kind": "self" }, { "from": "id_apim", "to": "id_api", "label": "forward over mTLS · x-tenant-id", "kind": "call" }, { "from": "id_api", "to": "id_ent", "label": "exchange projected SA token (workload identity federation)", "kind": "call" }, { "from": "id_ent", "to": "id_api", "label": "token for Cosmos and Service Bus scopes", "kind": "return" }, { "from": "id_api", "to": "id_cos", "label": "write execution · RBAC data plane, no account key", "kind": "call" }, { "from": "id_api", "to": "id_sb", "label": "send · Entra RBAC, SAS disabled", "kind": "call" }, { "from": "id_api", "to": "id_kv", "label": "resolve legacy downstream credential", "kind": "call" }, { "from": "id_kv", "to": "id_api", "label": "secret · cached 10 min in memory only", "kind": "return" }, { "from": "id_api", "to": "id_apim", "label": "202 Accepted", "kind": "return" } ], "note": "No connection string or account key exists anywhere in the cluster, so none can be stolen from a compromised pod. The only secrets in Key Vault are third-party credentials for downstreams that cannot speak Entra.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "29-multi-tenant-isolation", "title": "Multi-Tenancy — Four Layers of Isolation, and What Is Shared", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Layer 1 — Identity and admission", "kind": "boundary", "dir": "row", "children": [ { "title": "Every request carries a tenant", "kind": "plain", "nodes": [ { "id": "u_app", "label": "App registration per tenant", "sub": "appid → tenantId map", "kind": "security" }, { "id": "u_sub", "label": "APIM subscription", "sub": "own key · own quota", "kind": "integration" }, { "id": "u_claim", "label": "tenantId claim required", "sub": "request rejected without it", "kind": "security" }, { "id": "u_rbac", "label": "Tenant-scoped RBAC", "sub": "operator roles per tenant", "kind": "security" } ] } ] }, { "title": "Layer 2 — Logical data isolation on shared infrastructure", "kind": "boundary", "dir": "row", "children": [ { "title": "Tenant is part of every key", "kind": "plain", "nodes": [ { "id": "u_pk", "label": "tenantId in partition key", "sub": "definitions · dead-letters", "kind": "store" }, { "id": "u_filt", "label": "Mandatory query filter", "sub": "enforced in data layer", "kind": "app" }, { "id": "u_redk", "label": "Redis key prefix", "sub": "t:{tenantId}:*", "kind": "store" }, { "id": "u_blob", "label": "Container per tenant", "sub": "user-delegation SAS", "kind": "store" }, { "id": "u_msg", "label": "tenantId message property", "sub": "filterable · auditable", "kind": "queue" } ] } ] }, { "title": "Layer 3 — Runtime fairness", "kind": "boundary", "dir": "row", "children": [ { "title": "Noisy neighbours are bounded, not blocked", "kind": "plain", "nodes": [ { "id": "u_lease", "label": "Concurrency lease", "sub": "gold 500 · silver 100 · bronze 20", "kind": "store" }, { "id": "u_prio", "label": "Priority class", "sub": "queue selection per tier", "kind": "queue" }, { "id": "u_cb", "label": "Per-tenant circuit", "sub": "one tenant cannot trip all", "kind": "app" }, { "id": "u_obs", "label": "Per-tenant SLO view", "sub": "own dashboard + alerting", "kind": "platform" } ] } ] }, { "title": "Layer 4 — Physical isolation, tier-1 escape hatch", "kind": "boundary", "dir": "row", "children": [ { "title": "Bought, not default", "kind": "plain", "nodes": [ { "id": "u_ns", "label": "Dedicated SB namespace", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" }, { "id": "u_pool", "label": "Dedicated node pool", "sub": "taint + toleration", "kind": "platform", "icon": "kubernetes" }, { "id": "u_cont", "label": "Dedicated Cosmos container", "sub": "own RU allocation", "kind": "store", "icon": "azure_AzureCosmosDb (Databases)" }, { "id": "u_kv", "label": "Dedicated Key Vault", "sub": "own CMK", "kind": "security", "icon": "azure_key-vaults" } ] } ] }, { "title": "Deliberately shared — the honest part of the model", "kind": "boundary", "dir": "row", "children": [ { "title": "A fault here is a fault for everyone", "kind": "plain", "nodes": [ { "id": "u_r1", "label": "Control plane pods", "sub": "all tenants, one deployment", "kind": "risk" }, { "id": "u_r2", "label": "AKS cluster", "sub": "one control plane", "kind": "risk" }, { "id": "u_r3", "label": "Cosmos RU pool", "sub": "shared autoscale ceiling", "kind": "risk" }, { "id": "u_r4", "label": "Monitor workspace", "sub": "RBAC-filtered, not separated", "kind": "risk" } ] } ] } ], "note": "Isolation is bought in layers, and the last box says what a bronze tenant does not get. A tier-1 tenant that needs a blast radius of one must buy layer 4; nothing in layers 1 to 3 delivers that.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "30-leader-election-and-fencing", "title": "Leader Election — Split Brain and the Fencing Token That Stops It", "layout": "sequence", "canvas": { "width": 1700 }, "lifelines": [ { "id": "le_a", "label": "Scheduler A", "kind": "app", "icon": "fa5_clock" }, { "id": "le_b", "label": "Scheduler B", "kind": "app", "icon": "fa5_clock" }, { "id": "le_l", "label": "Lease document (Cosmos)", "kind": "store" }, { "id": "le_s", "label": "Schedule store", "kind": "store" }, { "id": "le_sb", "label": "Service Bus", "kind": "queue", "icon": "azure_AzureServiceBus (Integration)" }, { "id": "le_m", "label": "Azure Monitor", "kind": "platform", "icon": "azure_monitor" } ], "messages": [ { "from": "le_a", "to": "le_l", "label": "conditional write if expired · token 41", "kind": "call" }, { "from": "le_l", "to": "le_a", "label": "acquired · fencing token 41 · TTL 30 s", "kind": "return" }, { "from": "le_a", "to": "le_a", "label": "renew every 10 s", "kind": "self" }, { "from": "le_a", "to": "le_s", "label": "claim due schedules with token 41", "kind": "call" }, { "from": "le_a", "to": "le_sb", "label": "enqueue execution starts", "kind": "async" }, { "from": "le_a", "to": "le_a", "label": "35 s stop-the-world pause · renewal missed", "kind": "error" }, { "from": "le_b", "to": "le_l", "label": "lease expired · acquire · token 42", "kind": "call" }, { "from": "le_l", "to": "le_b", "label": "acquired · fencing token 42", "kind": "return" }, { "from": "le_b", "to": "le_s", "label": "claim due schedules with token 42", "kind": "call" }, { "from": "le_a", "to": "le_s", "label": "resumes · writes with stale token 41", "kind": "error" }, { "from": "le_s", "to": "le_a", "label": "rejected · 41 < 42", "kind": "return" }, { "from": "le_a", "to": "le_a", "label": "self-demote · stop firing", "kind": "self" }, { "from": "le_b", "to": "le_m", "label": "LeaderChanged event · alert", "kind": "async" } ], "note": "The lease alone does not prevent split brain, because A cannot know it lost the lease until it tries to use it. The fencing token does: every write carries it, and the store rejects anything below the highest token it has seen.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "31-failure-modes", "title": "Failure Modes, Mitigations and Accepted Residual Risk", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Execution plane — expected to fail constantly", "kind": "boundary", "dir": "row", "children": [ { "title": "Failure → mitigation", "kind": "plain", "nodes": [ { "id": "x_1", "label": "Worker crash mid-task", "sub": "lock expires · redelivered", "kind": "risk" }, { "id": "x_2", "label": "Spot node reclaimed", "sub": "30 s notice · PDB · drain", "kind": "risk" }, { "id": "x_3", "label": "Handler hangs", "sub": "watchdog cancels at deadline", "kind": "risk" }, { "id": "x_4", "label": "Poison message", "sub": "envelope check · DLQ, no retry", "kind": "risk" }, { "id": "x_5", "label": "Duplicate delivery", "sub": "idempotency key · view 17", "kind": "risk" } ] } ] }, { "title": "Control plane — must fail safe", "kind": "boundary", "dir": "row", "children": [ { "title": "Failure → mitigation", "kind": "plain", "nodes": [ { "id": "x_6", "label": "Orchestrator dies mid-transition", "sub": "ETag write atomic · replay safe", "kind": "risk" }, { "id": "x_7", "label": "Outbox relay lag", "sub": "checkpointed feed · resumes", "kind": "risk" }, { "id": "x_8", "label": "Double dispatch", "sub": "MessageId dedupe · 10 min", "kind": "risk" }, { "id": "x_9", "label": "Scheduler split brain", "sub": "fencing token · view 30", "kind": "risk" }, { "id": "x_10", "label": "Bad definition deployed", "sub": "immutable versions · pinned", "kind": "risk" } ] } ] }, { "title": "Dependency failures — the behaviour must be stated, not discovered", "kind": "boundary", "dir": "row", "children": [ { "title": "Failure → declared behaviour", "kind": "plain", "nodes": [ { "id": "x_11", "label": "Cosmos 429 throttling", "sub": "SDK retry · autoscale · shed writes", "kind": "risk" }, { "id": "x_12", "label": "Cosmos region down", "sub": "reads fail over · writes 503", "kind": "risk" }, { "id": "x_13", "label": "Service Bus down", "sub": "submissions accepted · outbox drains", "kind": "risk" }, { "id": "x_14", "label": "Redis down", "sub": "leases fail open at reduced limit", "kind": "risk" }, { "id": "x_15", "label": "Downstream saturated", "sub": "circuit opens · tasks held", "kind": "risk" }, { "id": "x_16", "label": "Network partition", "sub": "no dispatch · no state loss", "kind": "risk" } ] } ] }, { "title": "Accepted residual risk for V1 — signed off, not overlooked", "kind": "boundary", "dir": "row", "children": [ { "title": "Known and priced", "kind": "plain", "nodes": [ { "id": "x_17", "label": "Single write region", "sub": "RTO 4 h · RPO 5 min", "kind": "risk" }, { "id": "x_18", "label": "In-flight messages not replicated", "sub": "rebuilt from Cosmos on failover", "kind": "risk" }, { "id": "x_19", "label": "Exactly-once not offered", "sub": "at-least-once + idempotency", "kind": "risk" }, { "id": "x_20", "label": "500-task hot partition", "sub": "10K RU / 20 GB partition cap", "kind": "risk" }, { "id": "x_21", "label": "Shared control plane", "sub": "no per-tenant blast radius", "kind": "risk" } ] } ] } ], "outside": [ { "id": "x_chaos", "label": "Chaos experiments", "sub": "each row has a test in staging", "kind": "app", "icon": "azure_azure-chaos-studio" }, { "id": "x_run", "label": "Runbooks", "sub": "one per failure mode", "kind": "app", "icon": "fa5_book" } ], "edges": [ { "from": "x_chaos", "to": "x_1", "label": "pod kill", "kind": "error" }, { "from": "x_chaos", "to": "x_13", "label": "namespace block", "kind": "error" }, { "from": "x_chaos", "to": "x_16", "label": "NSG partition", "kind": "error" }, { "from": "x_run", "to": "x_12", "label": "failover drill", "kind": "sync" } ], "note": "Twenty-one failure modes, each with a stated behaviour and a chaos experiment that proves it in staging. The last box is the part a reviewer should argue with: it is what this design chose not to solve in V1.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } } ]