[ { "id": "01-system-context", "title": "LLM Traffic Manager — System Context", "layout": "context", "canvas": { "width": 1620 }, "colWidth": 265, "system": { "label": "LLM Traffic Manager", "sub": "Authorize · meter · route" }, "groups": [ { "side": "left", "title": "Traffic sources", "nodes": [ { "id": "apps", "label": "Product Applications", "sub": "chat, search, copilots", "kind": "external", "rel": "authorize + call", "dir": "in" }, { "id": "agents", "label": "Agent Workloads", "sub": "bursty, multi-step", "kind": "external", "rel": "high fan-out", "dir": "in" }, { "id": "batch", "label": "Offline & Batch Jobs", "sub": "evals, backfills", "kind": "external", "rel": "bulk tier", "dir": "in", "kind2": "batch" } ] }, { "side": "right", "title": "LLM providers", "nodes": [ { "id": "aoai", "label": "Azure OpenAI", "kind": "external", "rel": "completions", "icon": "azure_azure-openai" }, { "id": "claude", "label": "Anthropic Claude", "kind": "external", "rel": "completions", "icon": "anthropic" }, { "id": "gemini", "label": "Google Gemini", "sub": "Vertex AI", "kind": "external", "rel": "completions", "icon": "di2_googlecloud" }, { "id": "vllm", "label": "Mixtral on vLLM", "sub": "self-hosted GPU", "kind": "external", "rel": "completions", "icon": "vllm" } ] }, { "side": "top", "title": "Governance", "nodes": [ { "id": "admin", "label": "Platform Admin", "sub": "sets policy", "kind": "actor", "rel": "policy-as-code", "dir": "in" }, { "id": "finops", "label": "FinOps & Billing", "kind": "external", "rel": "usage + cost", "icon": "fa5_file_invoice_dollar" } ] }, { "side": "bottom", "title": "Enterprise services", "nodes": [ { "id": "idp", "label": "Keycloak IdP", "sub": "OIDC, tenant realms", "kind": "security", "rel": "tenant identity", "dir": "in", "icon": "keycloak" }, { "id": "obs", "label": "Observability Platform", "sub": "Prometheus · Grafana", "kind": "platform", "rel": "metrics, traces", "icon": "prometheus" }, { "id": "lake", "label": "Data Lakehouse", "sub": "Iceberg on MinIO", "kind": "external", "rel": "usage export", "kind2": "batch", "icon": "minio" } ] } ], "note": "Prompts and completions traverse the egress gateway but are never persisted by the limiter — only token counts, scopes and decisions.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice", "date": "2026-08" } }, { "id": "02-high-level-architecture", "title": "High-Level Architecture — the path of one request", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "nodeWidth": 205, "stages": [ { "title": "Client edge", "nodes": [ { "id": "sdk", "label": "Application / SDK", "sub": "Go · Python · TS", "kind": "external", "icon": "python" }, { "id": "envoy", "label": "Envoy Gateway", "sub": "ext_authz filter", "kind": "integration", "icon": "envoy" } ] }, { "title": "Decision", "nodes": [ { "id": "limiterd", "label": "limiterd", "sub": "Go · gRPC · stateless", "kind": "app", "icon": "go" }, { "id": "lease", "label": "Local Lease Buckets", "sub": "in-process · 0.3 ms", "kind": "app", "icon": "mat_memory" } ] }, { "title": "Coordination", "nodes": [ { "id": "valkey", "label": "Valkey Cluster", "sub": "16 shards · Lua", "kind": "store", "icon": "redis" }, { "id": "resv", "label": "Reservation Ledger", "sub": "TTL 120 s", "kind": "store", "icon": "fa5_ticket_alt" } ] }, { "title": "Egress", "nodes": [ { "id": "gw", "label": "LLM Gateway", "sub": "provider adapters", "kind": "integration", "icon": "fa5_exchange_alt" }, { "id": "brk", "label": "Circuit Breakers", "sub": "per provider + model", "kind": "app", "icon": "fa5_traffic_light" } ] }, { "title": "Providers", "nodes": [ { "id": "prov1", "label": "Azure OpenAI", "kind": "external", "icon": "azure_azure-openai" }, { "id": "prov2", "label": "Claude · Gemini", "kind": "external", "icon": "anthropic" }, { "id": "prov3", "label": "Mixtral on vLLM", "kind": "external", "icon": "vllm" } ] }, { "title": "Accounting", "nodes": [ { "id": "kafka", "label": "Kafka usage.v1", "sub": "24 partitions", "kind": "queue", "icon": "kafka" }, { "id": "flink", "label": "Flink Aggregator", "sub": "exactly-once", "kind": "app", "icon": "flink" }, { "id": "ch", "label": "ClickHouse Ledger", "sub": "usage + cost", "kind": "store", "icon": "clickhouse" } ] } ], "edges": [ { "from": "gw", "to": "limiterd", "label": "commit actuals", "kind": "async", "route": "gutter" }, { "from": "ch", "to": "valkey", "label": "budget refresh", "kind": "async", "route": "gutter" } ], "note": "The hot path is edge → limiterd → local lease. Valkey is touched only when a lease is exhausted; PostgreSQL and ClickHouse are never on it.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "03-layered-architecture", "title": "Layered Architecture", "layout": "bands", "canvas": { "width": 1700 }, "layerHeaderWidth": 168, "bands": [ { "name": "Client & Edge", "nodes": [ { "label": "Application SDK", "sub": "authorize + commit", "kind": "external", "icon": "python" }, { "label": "Envoy Gateway", "sub": "ext_authz, mTLS", "kind": "integration", "icon": "envoy" }, { "label": "Tenant Router", "sub": "Maglev hash", "kind": "integration", "icon": "fa5_random" }, { "label": "Admission Control", "sub": "tier shedding", "kind": "app", "icon": "fa5_traffic_light" } ] }, { "name": "Decision", "nodes": [ { "label": "Authorize API", "sub": "gRPC + HTTP", "kind": "app", "icon": "grpc" }, { "label": "Policy Evaluator", "sub": "scope cascade", "kind": "app", "icon": "fa5_sitemap" }, { "label": "Algorithm Engine", "sub": "bucket · sliding window", "kind": "app", "icon": "fa5_tachometer_alt" }, { "label": "Reason & Retry Builder", "sub": "429 semantics", "kind": "app", "icon": "fa5_stopwatch" } ] }, { "name": "Coordination", "nodes": [ { "label": "Lease Manager", "sub": "250 ms grants", "kind": "app", "icon": "fa5_hourglass_half" }, { "label": "Atomic Limit Scripts", "sub": "Lua · single slot", "kind": "app", "icon": "redis" }, { "label": "Reservation Ledger", "sub": "estimate vs actual", "kind": "store", "icon": "fa5_ticket_alt" }, { "label": "Concurrency Semaphores", "sub": "slot lease + reaper", "kind": "store", "icon": "fa5_lock" } ] }, { "name": "Egress & Providers", "nodes": [ { "label": "LLM Gateway", "sub": "OpenAI-compatible", "kind": "integration", "icon": "fa5_exchange_alt" }, { "label": "Provider Adapters", "sub": "four upstreams", "kind": "app", "icon": "fa5_plug" }, { "label": "Failover Router", "sub": "weighted + health", "kind": "app", "icon": "fa5_route" }, { "label": "Usage Extractor", "sub": "reads token counts", "kind": "app", "icon": "fa5_calculator" } ] }, { "name": "Control", "nodes": [ { "label": "Policy API", "sub": "Go · REST", "kind": "app", "icon": "go" }, { "label": "Policy Store", "sub": "PostgreSQL 16", "kind": "store", "icon": "postgresql" }, { "label": "Policy Bus", "sub": "Kafka · compacted", "kind": "queue", "icon": "kafka" }, { "label": "Admin Console", "sub": "React", "kind": "app", "icon": "react" } ] }, { "name": "Accounting & Insight", "nodes": [ { "label": "Usage Collector", "sub": "at-least-once", "kind": "app", "icon": "fa5_receipt" }, { "label": "Stream Aggregator", "sub": "Apache Flink", "kind": "app", "icon": "flink" }, { "label": "Usage Ledger", "sub": "ClickHouse", "kind": "store", "icon": "clickhouse" }, { "label": "Cost & Budget Marts", "sub": "dbt models", "kind": "store", "icon": "dbt" } ] }, { "name": "Platform", "nodes": [ { "label": "Kubernetes", "sub": "3 AZ per region", "kind": "platform", "icon": "kubernetes" }, { "label": "Identity & Secrets", "sub": "Keycloak · Vault", "kind": "security", "icon": "keycloak" }, { "label": "Observability", "sub": "OTel · Prom · Grafana", "kind": "platform", "icon": "opentelemetry" }, { "label": "GitOps Delivery", "sub": "Argo CD · Rollouts", "kind": "platform", "icon": "argocd" } ] } ], "note": "A layer calls only the layer below it. The one deliberate exception is Accounting, fed asynchronously from Decision and Egress and never called back by them.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "04-container-architecture", "title": "Container Architecture (C4 Level 2)", "layout": "nested", "canvas": { "width": 1150 }, "boxes": [ { "title": "Data plane — one per region", "kind": "boundary", "dir": "col", "children": [ { "title": "Edge", "kind": "plain", "nodes": [ { "id": "c_envoy", "label": "Envoy Gateway", "sub": "ext_authz · mTLS", "kind": "integration", "icon": "envoy" }, { "id": "c_adm", "label": "Admission Controller", "sub": "tier + queue", "kind": "app", "icon": "fa5_traffic_light" } ] }, { "title": "Decision services", "kind": "plain", "nodes": [ { "id": "c_lim", "label": "limiterd", "sub": "Go · 60 pods", "kind": "app", "icon": "go" }, { "id": "c_gw", "label": "llm-gateway", "sub": "Go · 30 pods", "kind": "integration", "icon": "fa5_exchange_alt" } ] }, { "title": "Coordination state", "kind": "plain", "nodes": [ { "id": "c_vk", "label": "Valkey Cluster", "sub": "16 shards · 3 AZ", "kind": "store", "icon": "redis" }, { "id": "c_reap", "label": "Reservation Reaper", "sub": "sweeps expiries", "kind": "app", "icon": "fa5_sync_alt" } ] } ] }, { "title": "Control plane — global", "kind": "boundary", "dir": "col", "children": [ { "title": "Policy", "kind": "plain", "nodes": [ { "id": "c_papi", "label": "policy-api", "sub": "Go · REST", "kind": "app", "icon": "go" }, { "id": "c_pg", "label": "PostgreSQL", "sub": "Patroni · 3 nodes", "kind": "store", "icon": "postgresql" }, { "id": "c_bus", "label": "policy.v1", "sub": "Kafka · compacted", "kind": "queue", "icon": "kafka" } ] }, { "title": "Administration", "kind": "plain", "nodes": [ { "id": "c_ui", "label": "Admin Console", "sub": "React SPA", "kind": "app", "icon": "react" }, { "id": "c_gitops", "label": "Policy-as-code", "sub": "Git + Argo CD", "kind": "platform", "icon": "argocd" } ] } ] }, { "title": "Usage plane — global", "kind": "boundary", "dir": "col", "children": [ { "title": "Ingest", "kind": "plain", "nodes": [ { "id": "c_uc", "label": "usage-collector", "sub": "Go · batched", "kind": "app", "icon": "fa5_receipt" }, { "id": "c_kafka", "label": "usage.v1", "sub": "Kafka · 24 parts", "kind": "queue", "icon": "kafka" } ] }, { "title": "Process & serve", "kind": "plain", "nodes": [ { "id": "c_flink", "label": "Flink Aggregator", "sub": "1 m / 1 h / 1 d", "kind": "app", "icon": "flink" }, { "id": "c_ch", "label": "ClickHouse", "sub": "usage + cost marts", "kind": "store", "icon": "clickhouse" }, { "id": "c_graf", "label": "Grafana", "sub": "tenant dashboards", "kind": "platform", "icon": "grafana" } ] } ] } ], "outside": [ { "id": "c_idp", "label": "Keycloak", "sub": "OIDC", "kind": "security", "icon": "keycloak" }, { "id": "c_vault", "label": "Vault / OpenBao", "sub": "provider keys", "kind": "security", "icon": "vault" }, { "id": "c_prov", "label": "LLM Providers", "sub": "four upstreams", "kind": "external", "icon": "fa5_cloud" } ], "edges": [ { "from": "c_envoy", "to": "c_lim", "label": "Check gRPC", "kind": "sync" }, { "from": "c_lim", "to": "c_vk", "label": "EVALSHA", "kind": "sync" }, { "from": "c_gw", "to": "c_prov", "label": "HTTPS", "kind": "sync" }, { "from": "c_gw", "to": "c_uc", "label": "usage event", "kind": "async" }, { "from": "c_papi", "to": "c_bus", "label": "outbox relay", "kind": "async" }, { "from": "c_bus", "to": "c_lim", "label": "policy push", "kind": "async", "route": "gutter" }, { "from": "c_gw", "to": "c_vault", "label": "key lease", "kind": "sync" } ], "note": "Only edges that carry a contract are drawn. Health probes, metric scrapes and trace exports are omitted — view 20 has them.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "05-integration-architecture", "title": "Integration Architecture — every interface, both directions", "layout": "hub", "canvas": { "width": 1620 }, "left": { "title": "Traffic sources", "nodes": [ { "id": "i_sdk", "label": "Native SDK", "sub": "Go · Python · TS", "kind": "external", "rel": "gRPC Check", "dir": "in", "icon": "python" }, { "id": "i_envoy", "label": "Envoy ext_authz", "sub": "no app change", "kind": "integration", "rel": "gRPC v3", "dir": "in", "icon": "envoy" }, { "id": "i_openai", "label": "OpenAI-compatible API", "sub": "drop-in base_url", "kind": "integration", "rel": "HTTPS", "dir": "in", "icon": "openai" }, { "id": "i_lc", "label": "LangChain / LlamaIndex", "sub": "callback middleware", "kind": "external", "rel": "HTTPS", "dir": "in", "icon": "python" }, { "id": "i_batch", "label": "Batch Scheduler", "sub": "bulk tier", "kind": "external", "rel": "queued admission", "dir": "in", "kind2": "batch", "icon": "fa5_stream" } ] }, "centre": { "title": "Platform", "nodes": [ { "id": "hub", "label": "LLM Traffic Manager", "sub": "authorize · meter · route", "kind": "app" } ] }, "right": { "title": "Dependencies and sinks", "nodes": [ { "id": "i_aoai", "label": "Azure OpenAI", "sub": "REST + Entra", "kind": "external", "rel": "completions", "dir": "out", "icon": "azure_azure-openai" }, { "id": "i_claude", "label": "Anthropic Claude", "sub": "Messages API", "kind": "external", "rel": "completions", "dir": "out", "icon": "anthropic" }, { "id": "i_gem", "label": "Google Gemini", "sub": "Vertex AI", "kind": "external", "rel": "completions", "dir": "out", "icon": "di2_googlecloud" }, { "id": "i_vllm", "label": "Mixtral on vLLM", "sub": "OpenAI-compatible", "kind": "external", "rel": "completions", "dir": "out", "icon": "vllm" }, { "id": "i_kc", "label": "Keycloak", "sub": "OIDC / JWKS", "kind": "security", "rel": "token check", "dir": "out", "icon": "keycloak" }, { "id": "i_vault", "label": "Vault / OpenBao", "sub": "dynamic secrets", "kind": "security", "rel": "key lease", "dir": "out", "icon": "vault" }, { "id": "i_ch", "label": "ClickHouse", "sub": "usage ledger", "kind": "store", "rel": "usage rows", "dir": "out", "kind2": "async", "icon": "clickhouse" }, { "id": "i_prom", "label": "Prometheus", "sub": "OTLP + scrape", "kind": "platform", "rel": "metrics", "dir": "out", "kind2": "async", "icon": "prometheus" }, { "id": "i_lake", "label": "Data Lakehouse", "sub": "Iceberg on MinIO", "kind": "external", "rel": "nightly export", "dir": "out", "kind2": "batch", "icon": "minio" } ] }, "note": "Four ways in, one contract. Every inbound path resolves to the same Check RPC, so a policy behaves identically whichever mode a team adopts.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "06-integration-modes", "title": "Integration Modes — who closes the reconciliation loop", "layout": "swimlane", "canvas": { "width": 1740 }, "laneHeaderWidth": 185, "stages": [ "Request", "Authorize", "Execute", "Observe actuals", "Reconcile" ], "lanes": [ { "title": "A · Inline proxy (default)", "cells": [ [ { "label": "Envoy ext_authz", "sub": "zero app change", "kind": "integration", "icon": "envoy" } ], [ { "label": "Check RPC", "sub": "p99 < 10 ms", "kind": "app", "icon": "grpc" } ], [ { "label": "Gateway calls provider", "kind": "integration", "icon": "fa5_exchange_alt" } ], [ { "label": "Gateway parses usage block", "kind": "app", "icon": "fa5_calculator" } ], [ { "label": "Automatic commit", "sub": "client cannot skip it", "kind": "app", "icon": "fa5_check_circle" } ] ] }, { "title": "B · SDK / API", "cells": [ [ { "label": "App calls /v1/authorize", "kind": "external", "icon": "python" } ], [ { "label": "HTTP decision + request_id", "kind": "app", "icon": "fa5_ticket_alt" } ], [ { "label": "App calls provider directly", "kind": "external" } ], [ { "label": "App reads provider usage", "kind": "external", "icon": "fa5_calculator" } ], [ { "label": "App calls /v1/commit", "sub": "reaper covers crashes", "kind": "app", "icon": "fa5_undo" } ] ] }, { "title": "C · Queued bulk", "cells": [ [ { "label": "Job enqueued", "sub": "priority BULK", "kind": "queue", "icon": "fa5_stream" } ], [ { "label": "Admission at dequeue", "sub": "spare capacity only", "kind": "decision" } ], [ { "label": "Worker pool executes", "kind": "app", "icon": "fa5_cubes" } ], [ { "label": "Worker reports actuals", "kind": "app", "icon": "fa5_receipt" } ], [ { "label": "Credit returned to pool", "kind": "app", "icon": "fa5_coins" } ] ] } ], "note": "Mode A is the default because the gateway sees the provider's own usage block, so reconciliation cannot be skipped by a misbehaving client.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "07-rate-limit-state-architecture", "title": "Where Rate-Limit State Lives — three tiers, one hot path", "layout": "nested", "canvas": { "width": 1420 }, "boxes": [ { "title": "L1 · In-process — per limiter pod (RAM)", "kind": "boundary", "nodes": [ { "id": "s_lease", "label": "Lease Buckets", "sub": "granted RPM / TPM slice", "kind": "app", "icon": "mat_memory" }, { "id": "s_pol", "label": "Policy Cache", "sub": "last-known-good", "kind": "app", "icon": "fa5_sliders_h" }, { "id": "s_neg", "label": "Negative Cache", "sub": "denied scope, 200 ms", "kind": "app", "icon": "fa5_ban" }, { "id": "s_l1c", "label": "≈ 0.3 ms p99", "sub": "92% of decisions", "kind": "decision" } ] }, { "title": "L2 · Shared coordination", "kind": "boundary", "nodes": [ { "id": "s_rpm", "label": "RPM / TPM Counters", "sub": "windowed, TTL-scoped", "kind": "store", "icon": "redis" }, { "id": "s_sem", "label": "Concurrency Semaphores", "sub": "sorted set + TTL", "kind": "store", "icon": "fa5_lock" }, { "id": "s_res", "label": "Reservation Ledger", "sub": "estimate held 120 s", "kind": "store", "icon": "fa5_ticket_alt" }, { "id": "s_bud", "label": "Budget Counters", "sub": "spend to date", "kind": "store", "icon": "fa5_coins" }, { "id": "s_l2c", "label": "≈ 1.5 ms p99", "sub": "8% of decisions", "kind": "decision" } ] }, { "title": "L3 · Durable — system of record, never on the hot path", "kind": "boundary", "nodes": [ { "id": "s_pg", "label": "PostgreSQL", "sub": "policy + hierarchy", "kind": "store", "icon": "postgresql" }, { "id": "s_ch", "label": "ClickHouse", "sub": "usage ledger", "kind": "store", "icon": "clickhouse" }, { "id": "s_obj", "label": "MinIO Archive", "sub": "Iceberg, 7 years", "kind": "store", "icon": "minio" }, { "id": "s_l3c", "label": "10–200 ms", "sub": "0% of decisions", "kind": "decision" } ] } ], "outside": [ { "id": "s_risk", "label": "Cold pod: no lease yet", "sub": "first call goes to L2", "kind": "risk", "icon": "fa5_exclamation_triangle" }, { "id": "s_bus", "label": "Kafka policy.v1 + usage.v1", "sub": "the only L3 → L1 path", "kind": "queue", "icon": "kafka" } ], "edges": [ { "from": "s_lease", "to": "s_rpm", "label": "refill every 250 ms", "kind": "sync" }, { "from": "s_res", "to": "s_ch", "label": "committed usage", "kind": "async" }, { "from": "s_pg", "to": "s_bus", "label": "outbox", "kind": "async" }, { "from": "s_bus", "to": "s_pol", "label": "policy push", "kind": "async", "route": "gutter" }, { "from": "s_ch", "to": "s_bud", "label": "budget rollup, 60 s", "kind": "async" } ], "note": "Answer to design question 1: state lives in all three tiers, but authority lives only in L2 for counters and L3 for policy. L1 holds a bounded, expiring claim on L2.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "08-policy-and-usage-data-model", "title": "Policy and Usage Data Model", "layout": "er", "canvas": { "width": 1720, "cols": 4 }, "rowGap": 235, "entities": [ { "id": "org", "name": "organisation", "kind": "store", "row": 0, "col": 0, "attrs": [ "org_id PK", "name", "tier ENUM(ENT,PRO,FREE)", "fail_mode ENUM(OPEN,LOCAL,CLOSED)", "budget_usd_month" ] }, { "id": "team", "name": "team", "kind": "store", "row": 0, "col": 1, "attrs": [ "team_id PK", "org_id FK -> organisation", "name", "cost_centre" ] }, { "id": "prin", "name": "principal", "kind": "store", "row": 0, "col": 2, "attrs": [ "principal_id PK", "team_id FK -> team", "subject (OIDC sub)", "api_key_hash", "priority" ] }, { "id": "prov", "name": "provider", "kind": "store", "row": 0, "col": 3, "attrs": [ "provider_id PK", "name", "region", "global_rpm", "global_tpm", "health_state" ] }, { "id": "model", "name": "model", "kind": "store", "row": 1, "col": 3, "attrs": [ "model_id PK", "provider_id FK -> provider", "name", "context_window", "price_in_per_1k", "price_out_per_1k" ] }, { "id": "pol", "name": "policy", "kind": "store", "row": 1, "col": 1, "attrs": [ "policy_id PK", "scope_type ENUM(ORG,TEAM,USER,MODEL,PROVIDER)", "scope_id", "model_id FK -> model NULL", "rpm", "input_tpm", "output_tpm", "max_concurrency", "algorithm ENUM(BUCKET,SLIDING)", "version", "effective_from" ] }, { "id": "res", "name": "reservation", "kind": "store", "row": 1, "col": 2, "attrs": [ "request_id PK", "principal_id FK -> principal", "model_id FK -> model", "reserved_input", "reserved_output", "slot_held", "expires_at", "state ENUM(HELD,COMMITTED,SWEPT)" ] }, { "id": "aud", "name": "decision_audit", "kind": "store", "row": 2, "col": 0, "attrs": [ "decision_id PK", "request_id", "verdict ENUM(ALLOW,REJECT)", "reason_code", "limiting_scope", "policy_version", "decided_at" ] }, { "id": "use", "name": "usage_event", "kind": "store", "row": 2, "col": 2, "attrs": [ "event_id PK", "request_id FK -> reservation", "input_tokens", "output_tokens", "latency_ms", "status", "cost_usd", "occurred_at" ] }, { "id": "ledg", "name": "budget_ledger", "kind": "store", "row": 2, "col": 3, "attrs": [ "org_id FK -> organisation", "period (month)", "spend_usd", "forecast_usd", "updated_at" ] } ], "relations": [ { "from": "org", "to": "team", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "team", "to": "prin", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "prov", "to": "model", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "org", "to": "pol", "label": "1 : N", "from_side": "s", "to_side": "w" }, { "from": "model", "to": "pol", "label": "0 : N", "from_side": "s", "to_side": "s", "kind": "optional" }, { "from": "prin", "to": "res", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "res", "to": "use", "label": "1 : 1", "from_side": "s", "to_side": "n" }, { "from": "use", "to": "ledg", "label": "N : 1", "from_side": "e", "to_side": "w" }, { "from": "res", "to": "aud", "label": "1 : N", "from_side": "w", "to_side": "e" } ], "note": "One policy row per scope, so a limit at any level is added or removed without a schema change. No prompt or completion text appears anywhere in this model.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "09-usage-accounting-data-flow", "title": "Usage Accounting — from decision to invoice", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "nodeWidth": 215, "stages": [ { "title": "Emit", "nodes": [ { "id": "u_dec", "label": "Decision Events", "sub": "verdict + reason", "kind": "app", "icon": "fa5_gavel" }, { "id": "u_cmp", "label": "Completion Events", "sub": "actual tokens", "kind": "app", "icon": "fa5_receipt" } ] }, { "title": "Transport", "nodes": [ { "id": "u_k", "label": "Kafka usage.v1", "sub": "24 parts · 7-day retain", "kind": "queue", "icon": "kafka" }, { "id": "u_sr", "label": "Schema Registry", "sub": "Apicurio · Avro", "kind": "app", "icon": "fa5_file_contract" } ] }, { "title": "Aggregate", "nodes": [ { "id": "u_fl", "label": "Flink Windows", "sub": "1 m / 1 h / 1 d", "kind": "app", "icon": "flink" }, { "id": "u_cost", "label": "Cost Calculator", "sub": "price × tokens", "kind": "app", "icon": "fa5_calculator" }, { "id": "u_dedup", "label": "Idempotent Upsert", "sub": "keyed on request_id", "kind": "app", "icon": "fa5_fingerprint" } ] }, { "title": "Store", "nodes": [ { "id": "u_ledger", "label": "Usage Ledger", "sub": "ClickHouse · 400 d", "kind": "store", "icon": "clickhouse" }, { "id": "u_mart", "label": "Cost Marts", "sub": "dbt · per org/team", "kind": "store", "icon": "dbt" }, { "id": "u_arch", "label": "Cold Archive", "sub": "Iceberg on MinIO", "kind": "store", "icon": "minio" } ] }, { "title": "Serve", "nodes": [ { "id": "u_graf", "label": "Tenant Dashboards", "sub": "Grafana", "kind": "platform", "icon": "grafana" }, { "id": "u_bill", "label": "Billing Export", "sub": "signed monthly CSV", "kind": "integration", "icon": "fa5_file_invoice_dollar" }, { "id": "u_budget", "label": "Budget Enforcer", "sub": "60 s refresh", "kind": "app", "icon": "fa5_balance_scale" } ] } ], "edges": [ { "from": "u_budget", "to": "u_dec", "label": "budget counters back to L2", "kind": "async", "route": "gutter" } ], "note": "At-least-once transport with an idempotent upsert keyed on request_id. Billing reads the ledger, never the stream, so a replay cannot double-charge a tenant.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "10-authorization-hot-path", "title": "Authorization Hot Path — one request, p99 under 10 ms", "layout": "sequence", "canvas": { "width": 1700 }, "lifelines": [ { "id": "app", "label": "Application", "kind": "external" }, { "id": "envoy", "label": "Envoy Gateway", "kind": "integration" }, { "id": "lim", "label": "limiterd", "kind": "app" }, { "id": "vk", "label": "Valkey Shard", "kind": "store" }, { "id": "gw", "label": "llm-gateway", "kind": "integration" }, { "id": "prov", "label": "LLM Provider", "kind": "external" } ], "messages": [ { "from": "app", "to": "envoy", "label": "POST /v1/chat/completions", "kind": "call" }, { "from": "envoy", "to": "lim", "label": "Check(tenant, model, 5k in, 2k out)", "kind": "call" }, { "from": "lim", "to": "lim", "label": "resolve scope chain from cached policy", "kind": "self" }, { "from": "lim", "to": "lim", "label": "try local lease — exhausted", "kind": "self" }, { "from": "lim", "to": "vk", "label": "EVALSHA check_and_commit(keys)", "kind": "call" }, { "from": "vk", "to": "lim", "label": "ALLOW + lease 40 req / 250 ms", "kind": "return" }, { "from": "lim", "to": "vk", "label": "reserve 7 000 tokens + slot, TTL 120 s", "kind": "call" }, { "from": "lim", "to": "envoy", "label": "ALLOW request_id=req-123", "kind": "return" }, { "from": "envoy", "to": "gw", "label": "forward with request_id", "kind": "call" }, { "from": "gw", "to": "prov", "label": "upstream completion", "kind": "call" }, { "from": "prov", "to": "gw", "label": "completion + usage block", "kind": "return" }, { "from": "gw", "to": "lim", "label": "Commit(actual 5 800)", "kind": "async" }, { "from": "lim", "to": "vk", "label": "release 1 200 tokens + free slot", "kind": "call" }, { "from": "gw", "to": "app", "label": "200 OK", "kind": "return" } ], "note": "Steps 5 to 7 are skipped on roughly 92% of requests, where the local lease still has capacity. That is what buys the latency budget.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "11-hierarchical-limit-evaluation", "title": "Hierarchical Limit Evaluation — every scope must pass", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "nodeWidth": 200, "stages": [ { "title": "Resolve", "nodes": [ { "id": "h_ctx", "label": "Request Context", "sub": "from JWT claims only", "kind": "app", "icon": "fa5_id_card" }, { "id": "h_chain", "label": "Scope Chain", "sub": "5 keys, ordered", "kind": "app", "icon": "fa5_sitemap" } ] }, { "title": "Organisation", "nodes": [ { "id": "h_orpm", "label": "Org RPM / TPM", "sub": "500 rpm · 2M tpm", "kind": "decision" }, { "id": "h_obud", "label": "Org Budget", "sub": "monthly USD cap", "kind": "decision" } ] }, { "title": "Team", "nodes": [ { "id": "h_trpm", "label": "Team RPM / TPM", "sub": "share of org", "kind": "decision" } ] }, { "title": "Principal", "nodes": [ { "id": "h_urpm", "label": "User RPM", "sub": "10 req/s", "kind": "decision" }, { "id": "h_ucon", "label": "User Concurrency", "sub": "max in flight", "kind": "decision" } ] }, { "title": "Model", "nodes": [ { "id": "h_mrpm", "label": "Model RPM / TPM", "sub": "per model policy", "kind": "decision" }, { "id": "h_mcon", "label": "Model Concurrency", "sub": "20 in flight", "kind": "decision" } ] }, { "title": "Provider", "nodes": [ { "id": "h_prpm", "label": "Provider Global RPM", "sub": "protects upstream", "kind": "decision" }, { "id": "h_phealth", "label": "Provider Health", "sub": "breaker state", "kind": "decision" } ] }, { "title": "Outcome", "nodes": [ { "id": "h_allow", "label": "ALLOW + reserve", "sub": "request_id issued", "kind": "app", "icon": "fa5_check_circle" }, { "id": "h_rej", "label": "REJECT 429", "sub": "reason + retry_after", "kind": "risk", "icon": "fa5_ban" } ] } ], "edges": [ { "from": "h_obud", "to": "h_rej", "label": "BUDGET_EXCEEDED", "kind": "error", "route": "gutter" }, { "from": "h_prpm", "to": "h_rej", "label": "PROVIDER_SATURATED", "kind": "error" } ], "note": "Scopes are evaluated in a fixed order so the reason code is deterministic. The first failing scope names the rejection; retry_after comes from that scope's window.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "12-atomic-multi-limit-commit", "title": "Atomic Multi-Limit Commit — all limits, or none", "layout": "swimlane", "canvas": { "width": 1760 }, "laneHeaderWidth": 190, "stages": [ "Resolve keys", "Dry-run all limits", "Commit or abort", "Reserve", "Respond" ], "lanes": [ { "title": "limiterd (Go)", "cells": [ [ { "label": "Build key set", "sub": "hash tag {org:acme}", "kind": "app", "icon": "fa5_key" } ], [ { "label": "Single EVALSHA", "sub": "one round trip", "kind": "app", "icon": "grpc" } ], [], [ { "label": "Record request_id", "kind": "app", "icon": "fa5_ticket_alt" } ], [ { "label": "ALLOW / REJECT", "sub": "+ limiting scope", "kind": "app", "icon": "fa5_gavel" } ] ] }, { "title": "Tenant shard (Valkey)", "cells": [ [ { "label": "Org, team, user, model keys", "sub": "same slot by design", "kind": "store", "icon": "redis" } ], [ { "label": "Read every counter", "sub": "no mutation yet", "kind": "app", "icon": "fa5_eye" } ], [ { "label": "INCR all or none", "sub": "Lua is single-threaded", "kind": "app", "icon": "fa5_lock" } ], [ { "label": "ZADD reservation", "sub": "score = expiry", "kind": "store", "icon": "fa5_hourglass_half" } ], [ { "label": "Verdict + retry_after", "kind": "app", "icon": "fa5_stopwatch" } ] ] }, { "title": "Provider shard (Valkey)", "cells": [ [ { "label": "Global provider key", "sub": "different slot", "kind": "store", "icon": "redis" } ], [ { "label": "Read provider counter", "kind": "app", "icon": "fa5_eye" } ], [ { "label": "Compensating release", "sub": "if tenant leg aborts", "kind": "app", "icon": "fa5_undo" } ], [], [ { "label": "Provider headroom", "kind": "app", "icon": "fa5_tachometer_alt" } ] ] } ], "note": "Answer to design question 2: tenant scopes are made atomic by co-locating their keys on one slot; the provider scope cannot be, so it is a second leg with a compensating release.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "13-token-reservation-and-reconciliation", "title": "Token Reservation and Reconciliation", "layout": "cycle", "canvas": { "width": 1500 }, "centre": { "label": "Token Ledger" }, "rx": 430, "ry": 215, "nodes": [ { "label": "Estimate", "sub": "5 000 in + 2 000 out", "kind": "app", "icon": "fa5_calculator", "id": "r_est" }, { "label": "Reserve", "sub": "7 000 held, TTL 120 s", "kind": "store", "icon": "fa5_ticket_alt", "id": "r_res" }, { "label": "Execute", "sub": "provider completion", "kind": "integration", "icon": "fa5_exchange_alt", "id": "r_exec" }, { "label": "Observe", "sub": "usage block: 5 800", "kind": "app", "icon": "fa5_eye", "id": "r_obs" }, { "label": "Reconcile", "sub": "release 1 200 unused", "kind": "app", "icon": "fa5_undo", "id": "r_rec" }, { "label": "Sweep", "sub": "reaper frees expired", "kind": "app", "icon": "fa5_sync_alt", "id": "r_sweep" } ], "ringLabels": [ "request_id issued", "slot + tokens held", "actual usage returned", "delta computed", "counters corrected", "leaked holds recovered" ], "note": "Answer to design question 3. The reaper is what makes over-reservation safe: a client that crashes after ALLOW costs the tenant at most 120 seconds of held quota.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "14-quota-lease-protocol", "title": "Quota Lease Protocol — how ten limiters share one quota", "layout": "sequence", "canvas": { "width": 1620 }, "lifelines": [ { "id": "la", "label": "limiterd A", "kind": "app" }, { "id": "lb", "label": "limiterd B", "kind": "app" }, { "id": "vk", "label": "Valkey Shard", "kind": "store" }, { "id": "pol", "label": "Policy Cache", "kind": "app" } ], "messages": [ { "from": "pol", "to": "la", "label": "org acme: 500 rpm, algorithm=BUCKET", "kind": "async" }, { "from": "la", "to": "vk", "label": "LEASE org:acme window=250 ms", "kind": "call" }, { "from": "vk", "to": "la", "label": "grant 40 of 125 in window", "kind": "return" }, { "from": "la", "to": "la", "label": "serve 40 decisions from RAM", "kind": "self" }, { "from": "lb", "to": "vk", "label": "LEASE org:acme window=250 ms", "kind": "call" }, { "from": "vk", "to": "lb", "label": "grant 25 — sized to observed demand", "kind": "return" }, { "from": "la", "to": "vk", "label": "RENEW + return 6 unused", "kind": "call" }, { "from": "vk", "to": "la", "label": "grant 55 — demand rising", "kind": "return" }, { "from": "lb", "to": "vk", "label": "LEASE — window already exhausted", "kind": "call" }, { "from": "vk", "to": "lb", "label": "DENY, retry_after 0.18 s", "kind": "error" }, { "from": "lb", "to": "lb", "label": "reject locally, no further round trips", "kind": "self" } ], "note": "Answer to design question 5 on consistency: overshoot is bounded by one unreturned lease per pod per window — measured at 0.9% against a 500 rpm quota with 60 pods.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "15-priority-and-congestion", "title": "Priority, Tiers and Congestion Behaviour", "layout": "grid", "canvas": { "width": 1760 }, "laneHeaderWidth": 185, "columns": [ "Normal · under 70%", "Elevated · 70–90%", "Saturated · over 90%", "Coordination degraded" ], "rows": [ { "title": "Enterprise · HIGH", "cells": [ [ { "label": "Serve immediately", "kind": "app", "icon": "fa5_check_circle" } ], [ { "label": "Serve", "sub": "20% headroom reserved", "kind": "app", "icon": "fa5_check" } ], [ { "label": "Serve from reserved headroom", "kind": "app", "icon": "fa5_shield_alt" } ], [ { "label": "FAIL OPEN", "sub": "local bucket × 1.0", "kind": "decision" } ] ] }, { "title": "Pro · MEDIUM", "cells": [ [ { "label": "Serve immediately", "kind": "app", "icon": "fa5_check_circle" } ], [ { "label": "Serve", "kind": "app", "icon": "fa5_check" } ], [ { "label": "Queue 200 ms, then 429", "kind": "queue", "icon": "fa5_hourglass_half" } ], [ { "label": "LOCAL fallback", "sub": "bucket × 0.8", "kind": "decision" } ] ] }, { "title": "Free · LOW", "cells": [ [ { "label": "Serve immediately", "kind": "app", "icon": "fa5_check_circle" } ], [ { "label": "Shed on burst", "kind": "app", "icon": "fa5_traffic_light" } ], [ { "label": "Shed first — 429", "kind": "risk", "icon": "fa5_ban" } ], [ { "label": "FAIL CLOSED", "kind": "risk", "icon": "fa5_lock" } ] ] }, { "title": "Batch · BULK", "cells": [ [ { "label": "Spare capacity only", "kind": "app", "icon": "fa5_stream" } ], [ { "label": "Deferred to queue", "kind": "queue" } ], [ { "label": "Paused", "kind": "risk", "icon": "fa5_pause" } ], [ { "label": "Paused", "kind": "risk", "icon": "fa5_pause" } ] ] } ], "note": "Headroom is reserved, not borrowed: the 20% HIGH reserve is deducted from the shared pool at policy publish time, so saturation cannot consume it.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "16-provider-limits-and-failover", "title": "Provider Limits, Routing and Failover", "layout": "flow", "canvas": { "width": 1760 }, "chain": true, "nodeWidth": 210, "stages": [ { "title": "Request", "nodes": [ { "id": "p_req", "label": "Requested Model", "sub": "alias or exact", "kind": "app", "icon": "fa5_tag" }, { "id": "p_pol", "label": "Routing Policy", "sub": "per tenant + model", "kind": "app", "icon": "fa5_map_signs" } ] }, { "title": "Selection", "nodes": [ { "id": "p_sel", "label": "Provider Selector", "sub": "weighted + health", "kind": "app", "icon": "fa5_route" }, { "id": "p_res", "label": "Residency Filter", "sub": "EU data stays EU", "kind": "decision" } ] }, { "title": "Provider guards", "nodes": [ { "id": "p_rpm", "label": "Provider RPM / TPM", "sub": "global counters", "kind": "decision" }, { "id": "p_brk", "label": "Circuit Breaker", "sub": "3 fails / 2 s", "kind": "decision" }, { "id": "p_bh", "label": "Upstream 429 Feedback", "sub": "adaptive throttle", "kind": "app", "icon": "fa5_wave_square" } ] }, { "title": "Primary route", "nodes": [ { "id": "p_aoai", "label": "Azure OpenAI", "sub": "PTU + pay-go", "kind": "external", "icon": "azure_azure-openai" }, { "id": "p_claude", "label": "Anthropic Claude", "sub": "Messages API", "kind": "external", "icon": "anthropic" } ] }, { "title": "Fallback route", "nodes": [ { "id": "p_gem", "label": "Google Gemini", "sub": "Vertex AI", "kind": "external", "icon": "di2_googlecloud" }, { "id": "p_vllm", "label": "Mixtral on vLLM", "sub": "self-hosted GPU", "kind": "external", "icon": "vllm" } ] }, { "title": "Outcome", "nodes": [ { "id": "p_ok", "label": "Response + usage", "sub": "provider recorded", "kind": "app", "icon": "fa5_check_circle" }, { "id": "p_fail", "label": "503 provider_unavailable", "sub": "retry_after from breaker", "kind": "risk", "icon": "fa5_ban" } ] } ], "edges": [ { "from": "p_aoai", "to": "p_gem", "label": "breaker open", "kind": "error" }, { "from": "p_claude", "to": "p_vllm", "label": "quota exhausted", "kind": "error" } ], "note": "Failover is opt-in per tenant policy. A tenant pinned to one provider for compliance reasons receives 503 rather than a silent reroute, and the policy records that choice.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "17-deployment-architecture", "title": "Deployment Architecture — active/active, two regions", "layout": "nested", "canvas": { "width": 1120 }, "boxes": [ { "title": "Region EU-West · active", "kind": "cloud", "dir": "col", "children": [ { "title": "Edge · 3 AZ", "kind": "boundary", "nodes": [ { "id": "d_lb", "label": "L4 Load Balancer", "sub": "MetalLB / cloud LB", "kind": "integration", "icon": "pattern_loadbalancer" }, { "id": "d_envoy", "label": "Envoy Gateway", "sub": "9 pods", "kind": "integration", "icon": "envoy" } ] }, { "title": "Kubernetes · data plane", "kind": "boundary", "nodes": [ { "id": "d_lim", "label": "limiterd", "sub": "60 pods · HPA on rps", "kind": "app", "icon": "kubernetes" }, { "id": "d_gw", "label": "llm-gateway", "sub": "30 pods", "kind": "app", "icon": "kubernetes" } ] }, { "title": "Stateful · 3 AZ", "kind": "boundary", "nodes": [ { "id": "d_vk", "label": "Valkey Cluster", "sub": "16 shards + replicas", "kind": "store", "icon": "redis" }, { "id": "d_pg", "label": "PostgreSQL", "sub": "Patroni · sync replica", "kind": "store", "icon": "postgresql" } ] }, { "title": "Usage plane", "kind": "boundary", "nodes": [ { "id": "d_kf", "label": "Kafka", "sub": "6 brokers · KRaft", "kind": "queue", "icon": "kafka" }, { "id": "d_fl", "label": "Flink", "sub": "6 task managers", "kind": "app", "icon": "flink" }, { "id": "d_ch", "label": "ClickHouse", "sub": "6 nodes · 2 replicas", "kind": "store", "icon": "clickhouse" } ] } ] }, { "title": "Region US-East · active", "kind": "cloud", "dir": "col", "children": [ { "title": "Edge", "kind": "boundary", "nodes": [ { "id": "d2_envoy", "label": "Envoy Gateway", "sub": "6 pods", "kind": "integration", "icon": "envoy" } ] }, { "title": "Kubernetes · data plane", "kind": "boundary", "nodes": [ { "id": "d2_lim", "label": "limiterd", "sub": "40 pods", "kind": "app", "icon": "kubernetes" }, { "id": "d2_gw", "label": "llm-gateway", "sub": "20 pods", "kind": "app", "icon": "kubernetes" } ] }, { "title": "Stateful", "kind": "boundary", "nodes": [ { "id": "d2_vk", "label": "Valkey Cluster", "sub": "regional, not replicated", "kind": "store", "icon": "redis" }, { "id": "d2_pg", "label": "PostgreSQL", "sub": "read replica", "kind": "store", "icon": "postgresql" } ] } ] }, { "title": "Global services", "kind": "boundary", "dir": "col", "children": [ { "title": "Routing & control", "kind": "boundary", "nodes": [ { "id": "d_dns", "label": "GeoDNS + Anycast", "sub": "latency routing", "kind": "integration", "icon": "net_router" }, { "id": "d_reconcile", "label": "Global Budget Reconciler", "sub": "60 s, eventually consistent", "kind": "app", "icon": "fa5_balance_scale" } ] }, { "title": "Platform", "kind": "boundary", "nodes": [ { "id": "d_argo", "label": "Argo CD", "sub": "GitOps, both regions", "kind": "platform", "icon": "argocd" }, { "id": "d_vault", "label": "Vault / OpenBao", "sub": "HA, 5 nodes", "kind": "security", "icon": "vault" } ] } ] } ], "outside": [ { "id": "d_prov", "label": "LLM Providers", "sub": "regional endpoints", "kind": "external", "icon": "fa5_cloud" }, { "id": "d_gpu", "label": "Self-hosted GPU Pool", "sub": "vLLM · A100 × 16", "kind": "external", "icon": "vllm" } ], "edges": [ { "from": "d_dns", "to": "d_envoy", "label": "primary", "kind": "sync" }, { "from": "d_dns", "to": "d2_envoy", "label": "secondary", "kind": "sync" }, { "from": "d_pg", "to": "d2_pg", "label": "streaming replication", "kind": "async" }, { "from": "d_reconcile", "to": "d2_vk", "label": "budget sync", "kind": "async", "route": "gutter" }, { "from": "d_gw", "to": "d_gpu", "label": "private link", "kind": "sync" } ], "note": "Rate-limit counters are deliberately regional. A globally strict quota would put a cross-region round trip on the hot path; instead each region gets a share and the reconciler corrects drift every 60 s.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "18-policy-control-plane", "title": "Policy Control Plane — from admin edit to live decision", "layout": "flow", "canvas": { "width": 1780 }, "chain": true, "nodeWidth": 205, "stages": [ { "title": "Author", "nodes": [ { "id": "y_admin", "label": "Platform Admin", "kind": "actor" }, { "id": "y_ui", "label": "Admin Console", "sub": "React", "kind": "app", "icon": "react" }, { "id": "y_git", "label": "Policy-as-code Repo", "sub": "YAML, reviewed", "kind": "app", "icon": "git" } ] }, { "title": "Validate", "nodes": [ { "id": "y_api", "label": "policy-api", "sub": "Go · REST", "kind": "app", "icon": "go" }, { "id": "y_guard", "label": "Guardrail Check", "sub": "child ≤ parent quota", "kind": "decision" }, { "id": "y_dry", "label": "Dry-run on Replay", "sub": "24 h of traffic", "kind": "app", "icon": "fa5_redo" } ] }, { "title": "Persist", "nodes": [ { "id": "y_pg", "label": "Policy Store", "sub": "PostgreSQL · versioned", "kind": "store", "icon": "postgresql" }, { "id": "y_out", "label": "Transactional Outbox", "sub": "same commit", "kind": "store", "icon": "fa5_inbox" } ] }, { "title": "Publish", "nodes": [ { "id": "y_relay", "label": "Outbox Relay", "sub": "at-least-once", "kind": "app", "icon": "fa5_share_alt" }, { "id": "y_topic", "label": "policy.v1", "sub": "Kafka · log-compacted", "kind": "queue", "icon": "kafka" } ] }, { "title": "Propagate", "nodes": [ { "id": "y_lim", "label": "limiterd Policy Cache", "sub": "p99 < 2 s", "kind": "app", "icon": "fa5_sliders_h" }, { "id": "y_gw", "label": "Gateway Route Cache", "sub": "same topic", "kind": "app", "icon": "fa5_map_signs" } ] }, { "title": "Verify", "nodes": [ { "id": "y_metric", "label": "policy_version Gauge", "sub": "per pod", "kind": "platform", "icon": "prometheus" }, { "id": "y_alert", "label": "Staleness Alert", "sub": "fires above 60 s", "kind": "risk", "icon": "fa5_bell" } ] } ], "edges": [ { "from": "y_metric", "to": "y_alert", "label": "min version lag", "kind": "sync" } ], "note": "Answer to design question 5 on propagation. A compacted topic means a pod that restarts replays only the current policy set, so cold start does not depend on PostgreSQL being reachable.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "19-cicd-and-progressive-delivery", "title": "Delivery Pipeline — code, scripts, policy and infrastructure", "layout": "swimlane", "canvas": { "width": 1760 }, "laneHeaderWidth": 175, "stages": [ "Commit", "Build", "Verify", "Canary", "Production" ], "lanes": [ { "title": "limiterd service", "cells": [ [ { "label": "PR + review", "kind": "app", "icon": "git" } ], [ { "label": "Go build, SBOM, cosign", "kind": "app", "icon": "go" } ], [ { "label": "Race + fuzz + 10k rps soak", "kind": "app", "icon": "fa5_vial" } ], [ { "label": "Argo Rollouts 5%", "sub": "SLO analysis gate", "kind": "platform", "icon": "argocd" } ], [ { "label": "Full rollout", "sub": "auto-rollback on p99", "kind": "platform", "icon": "kubernetes" } ] ] }, { "title": "Lua limit scripts", "cells": [ [ { "label": "Script in repo", "kind": "app", "icon": "fa5_file_code" } ], [ { "label": "SHA pinned to release", "kind": "app", "icon": "fa5_fingerprint" } ], [ { "label": "Golden-file limit tests", "kind": "app", "icon": "fa5_vial" } ], [ { "label": "Shadow dual-run compare", "kind": "app", "icon": "fa5_clone" } ], [ { "label": "SCRIPT LOAD on all shards", "kind": "store", "icon": "redis" } ] ] }, { "title": "Policies & config", "cells": [ [ { "label": "Policy-as-code PR", "kind": "app", "icon": "git" } ], [ { "label": "Schema + guardrail lint", "kind": "app", "icon": "fa5_file_contract" } ], [ { "label": "Dry-run on traffic replay", "kind": "app", "icon": "fa5_redo" } ], [ { "label": "One canary tenant", "kind": "app", "icon": "fa5_user_check" } ], [ { "label": "Published to policy.v1", "kind": "queue", "icon": "kafka" } ] ] }, { "title": "Infrastructure", "cells": [ [ { "label": "OpenTofu PR", "kind": "app", "icon": "terraform" } ], [ { "label": "Plan + cost diff", "kind": "app", "icon": "fa5_calculator" } ], [ { "label": "OPA policy checks", "kind": "security", "icon": "fa5_gavel" } ], [ { "label": "Apply to staging", "kind": "platform", "icon": "argocd" } ], [ { "label": "Apply via Argo CD", "kind": "platform", "icon": "argocd" } ] ] } ], "note": "Lua scripts are versioned and rolled out separately from the binary, because a change to the limit algorithm is a behaviour change even when no Go code moved.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "20-observability-and-slos", "title": "Observability — signal type against pipeline stage", "layout": "grid", "canvas": { "width": 1790 }, "laneHeaderWidth": 170, "columns": [ "Edge & admission", "Decision", "Coordination", "Egress & providers", "Usage & cost" ], "rows": [ { "title": "Metrics", "cells": [ [ { "label": "Throughput", "kind": "platform", "sub": "rps · shed · queue", "icon": "prometheus" } ], [ { "label": "Decision latency", "kind": "platform", "sub": "p50 / p99 · verdicts", "icon": "prometheus" } ], [ { "label": "Shard health", "kind": "platform", "sub": "Lua p99 · lease hit", "icon": "prometheus" } ], [ { "label": "Provider health", "kind": "platform", "sub": "latency · breaker · 429", "icon": "prometheus" } ], [ { "label": "Consumption", "kind": "platform", "sub": "tokens/min · burn rate", "icon": "prometheus" } ] ] }, { "title": "Logs", "cells": [ [ { "label": "Access log", "kind": "platform", "sub": "no request body", "icon": "loki" } ], [ { "label": "Decision audit", "kind": "platform", "sub": "1% + all rejects", "icon": "loki" } ], [ { "label": "Script errors", "kind": "platform", "sub": "failover events", "icon": "loki" } ], [ { "label": "Provider errors", "kind": "platform", "sub": "bodies redacted", "icon": "loki" } ], [ { "label": "Ledger gaps", "kind": "platform", "sub": "reconciliation", "icon": "loki" } ] ] }, { "title": "Traces", "cells": [ [ { "label": "Ingress span", "kind": "platform", "icon": "tempo" } ], [ { "label": "Check span", "kind": "platform", "sub": "scope resolution", "icon": "tempo" } ], [ { "label": "Valkey span", "kind": "platform", "sub": "one per lease refill", "icon": "tempo" } ], [ { "label": "Upstream span", "kind": "platform", "sub": "provider call", "icon": "tempo" } ], [ { "label": "Commit span", "kind": "platform", "sub": "linked by request_id", "icon": "tempo" } ] ] }, { "title": "SLOs & alerts", "cells": [ [ { "label": "Availability 99.99%", "kind": "decision", "sub": "52 min / year" } ], [ { "label": "p99 < 10 ms", "kind": "decision", "sub": "budget 4.3 min/mo" } ], [ { "label": "Shard p99 < 1.5 ms", "kind": "decision", "sub": "lease hit ≥ 90%" } ], [ { "label": "Failover < 5 s", "kind": "decision", "sub": "trip to reroute" } ], [ { "label": "Staleness < 60 s", "kind": "decision", "sub": "ledger lag < 5 min" } ] ] } ], "note": "Every signal is emitted through OpenTelemetry and correlated by request_id, which is the same identifier the reservation ledger and the usage row carry.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "21-security-trust-zones", "title": "Security Trust Zones", "layout": "zones", "canvas": { "width": 1720 }, "zones": [ { "title": "Internet", "kind": "trust", "nodes": [ { "id": "z_client", "label": "Client Applications", "sub": "tenant workloads", "kind": "external", "icon": "python" }, { "id": "z_admin", "label": "Platform Administrators", "kind": "actor" } ] }, { "title": "Perimeter", "kind": "trust", "nodes": [ { "id": "z_waf", "label": "WAF + DDoS Shield", "sub": "L7 rules", "kind": "security", "icon": "pattern_firewall" }, { "id": "z_envoy", "label": "Envoy Gateway", "sub": "TLS 1.3 termination", "kind": "integration", "icon": "envoy" }, { "id": "z_kc", "label": "Keycloak", "sub": "OIDC, realm per tenant", "kind": "security", "icon": "keycloak" } ] }, { "title": "Application zone", "kind": "trust", "nodes": [ { "id": "z_lim", "label": "limiterd", "sub": "SPIFFE identity", "kind": "app", "icon": "go" }, { "id": "z_gw", "label": "llm-gateway", "sub": "SPIFFE identity", "kind": "app", "icon": "fa5_exchange_alt" }, { "id": "z_papi", "label": "policy-api", "sub": "admin scope only", "kind": "app", "icon": "go" } ] }, { "title": "State & analytics zone", "kind": "trust", "nodes": [ { "id": "z_vk", "label": "Valkey Cluster", "sub": "ACL per service", "kind": "store", "icon": "redis" }, { "id": "z_pg", "label": "PostgreSQL", "sub": "row-level security", "kind": "store", "icon": "postgresql" }, { "id": "z_ch", "label": "ClickHouse", "sub": "tenant-scoped views", "kind": "store", "icon": "clickhouse" } ] }, { "title": "Egress zone", "kind": "trust", "nodes": [ { "id": "z_vault", "label": "Vault / OpenBao", "sub": "dynamic provider keys", "kind": "security", "icon": "vault" }, { "id": "z_eg", "label": "Egress Proxy", "sub": "FQDN allow-list", "kind": "security", "icon": "pattern_firewall" } ] }, { "title": "Providers (untrusted)", "kind": "trust", "nodes": [ { "id": "z_prov", "label": "Managed LLM Providers", "sub": "Azure · Anthropic · Google", "kind": "external", "icon": "fa5_cloud" }, { "id": "z_vllm", "label": "Self-hosted vLLM", "sub": "private subnet, no egress", "kind": "external", "icon": "vllm" } ] } ], "edges": [ { "from": "z_client", "to": "z_envoy", "label": "HTTPS + OIDC bearer", "kind": "sync" }, { "from": "z_envoy", "to": "z_kc", "label": "JWKS validation", "kind": "sync" }, { "from": "z_envoy", "to": "z_lim", "label": "mTLS, SPIFFE SVID", "kind": "sync" }, { "from": "z_lim", "to": "z_vk", "label": "TLS + ACL", "kind": "sync" }, { "from": "z_papi", "to": "z_pg", "label": "TLS, RLS by tenant", "kind": "sync" }, { "from": "z_gw", "to": "z_vault", "label": "SVID-authenticated key lease", "kind": "sync" }, { "from": "z_gw", "to": "z_eg", "label": "prompt in transit, never stored", "kind": "sync" }, { "from": "z_eg", "to": "z_prov", "label": "TLS 1.3, allow-list", "kind": "sync" }, { "from": "z_admin", "to": "z_kc", "label": "SSO + step-up MFA", "kind": "sync" } ], "note": "The limiter is deliberately outside the prompt data path. It receives token counts and scope claims, so a compromise of the decision plane exposes no customer content.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "22-identity-and-tenant-isolation", "title": "Identity and Tenant Isolation — proving who is asking", "layout": "sequence", "canvas": { "width": 1740 }, "lifelines": [ { "id": "app", "label": "Client App", "kind": "external", "icon": "fa5_desktop" }, { "id": "kc", "label": "Keycloak", "kind": "security" }, { "id": "envoy", "label": "Envoy Gateway", "kind": "integration" }, { "id": "lim", "label": "limiterd", "kind": "app" }, { "id": "gw", "label": "llm-gateway", "kind": "app" }, { "id": "vault", "label": "Vault", "kind": "security" }, { "id": "prov", "label": "Provider", "kind": "external", "icon": "fa5_cloud" } ], "messages": [ { "from": "app", "to": "kc", "label": "client_credentials, tenant realm", "kind": "call" }, { "from": "kc", "to": "app", "label": "JWT: org, team, sub, tier", "kind": "return" }, { "from": "app", "to": "envoy", "label": "request + Bearer token", "kind": "call" }, { "from": "envoy", "to": "envoy", "label": "verify signature, cached JWKS", "kind": "self" }, { "from": "envoy", "to": "lim", "label": "ext_authz Check over mTLS", "kind": "call" }, { "from": "lim", "to": "lim", "label": "scope keys from claims, never from body", "kind": "self" }, { "from": "lim", "to": "envoy", "label": "ALLOW + request_id", "kind": "return" }, { "from": "envoy", "to": "gw", "label": "forward, claims in headers", "kind": "call" }, { "from": "gw", "to": "vault", "label": "lease provider key for this tenant", "kind": "call" }, { "from": "vault", "to": "gw", "label": "credential, TTL 15 min", "kind": "return" }, { "from": "gw", "to": "prov", "label": "upstream call", "kind": "call" }, { "from": "prov", "to": "gw", "label": "completion + usage", "kind": "return" }, { "from": "gw", "to": "app", "label": "response, prompt not persisted", "kind": "return" }, { "from": "envoy", "to": "app", "label": "401 on tenant mismatch", "kind": "error" } ], "note": "A tenant cannot spend another tenant's quota because the scope key is derived from signed claims, not from any field the caller supplies in the request body.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "23-failure-modes-and-degradation", "title": "Failure Modes and the Degradation Ladder", "layout": "flow", "canvas": { "width": 1790 }, "chain": true, "nodeWidth": 200, "align": "top", "stages": [ { "title": "Fault", "nodes": [ { "id": "f_vk", "label": "Valkey shard lost", "kind": "risk", "icon": "redis" }, { "id": "f_bus", "label": "Policy bus lag", "kind": "risk", "icon": "kafka" }, { "id": "f_pg", "label": "PostgreSQL outage", "kind": "risk", "icon": "postgresql" }, { "id": "f_prov", "label": "Provider outage", "kind": "risk", "icon": "fa5_cloud" } ] }, { "title": "Detection", "nodes": [ { "id": "f_probe", "label": "Health probe", "sub": "3 fails in 2 s", "kind": "decision" }, { "id": "f_lag", "label": "Consumer lag alarm", "sub": "over 60 s", "kind": "decision" } ] }, { "title": "Degradation", "nodes": [ { "id": "f_local", "label": "Local bucket fallback", "sub": "quota ÷ pod count", "kind": "app", "icon": "mat_memory" }, { "id": "f_stale", "label": "Stale-but-serving policy", "sub": "last-known-good", "kind": "app", "icon": "fa5_sliders_h" }, { "id": "f_ro", "label": "Read-only control plane", "sub": "no policy edits", "kind": "app", "icon": "fa5_lock" }, { "id": "f_over", "label": "Provider failover", "sub": "if tenant permits", "kind": "app", "icon": "fa5_route" } ] }, { "title": "Tenant fail policy", "nodes": [ { "id": "f_open", "label": "HIGH → FAIL OPEN", "sub": "availability wins", "kind": "decision" }, { "id": "f_locald", "label": "STANDARD → LOCAL", "sub": "bounded overshoot", "kind": "decision" }, { "id": "f_closed", "label": "FREE → FAIL CLOSED", "sub": "cost wins", "kind": "decision" } ] }, { "title": "Blast radius", "nodes": [ { "id": "f_b1", "label": "Overshoot ≤ 15%", "sub": "one shard, one window", "kind": "app", "icon": "fa5_percent" }, { "id": "f_b2", "label": "Hot path unaffected", "sub": "PostgreSQL is not on it", "kind": "app", "icon": "fa5_check_circle" }, { "id": "f_b3", "label": "Free tier sees 429", "sub": "paying tenants do not", "kind": "risk", "icon": "fa5_ban" } ] }, { "title": "Recovery", "nodes": [ { "id": "f_r1", "label": "Shard rejoin + resync", "sub": "counters rebuilt from TTL", "kind": "app", "icon": "fa5_sync_alt" }, { "id": "f_r2", "label": "Reaper sweep", "sub": "frees leaked holds", "kind": "app", "icon": "fa5_undo" }, { "id": "f_r3", "label": "Ledger replay", "sub": "Kafka 7-day retention", "kind": "app", "icon": "kafka" } ] } ], "edges": [ { "from": "f_prov", "to": "f_over", "label": "breaker", "kind": "error", "route": "gutter" } ], "note": "Answer to design question 4: the failure mode is a per-tenant policy field, not a global constant, because the right answer differs by what the tenant is paying for.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } }, { "id": "24-scaling-to-one-million-decisions", "title": "Scaling to One Million Decisions per Second", "layout": "nested", "canvas": { "width": 1120 }, "boxes": [ { "title": "Tier 1 · Routing", "kind": "boundary", "dir": "col", "children": [ { "title": "Affinity", "kind": "plain", "nodes": [ { "id": "x_dns", "label": "GeoDNS + Anycast", "sub": "region by latency", "kind": "integration", "icon": "net_router" }, { "id": "x_hash", "label": "Maglev Hash", "sub": "on org_id header", "kind": "integration", "icon": "envoy" } ] }, { "title": "Why it matters", "kind": "plain", "nodes": [ { "id": "x_why1", "label": "Affinity raises lease hit rate", "sub": "78% → 92%", "kind": "decision" } ] } ] }, { "title": "Tier 2 · Decision pods", "kind": "boundary", "dir": "col", "children": [ { "title": "limiterd pools", "kind": "plain", "nodes": [ { "id": "x_p1", "label": "Pool A", "sub": "orgs 0–85", "kind": "app", "icon": "kubernetes" }, { "id": "x_p2", "label": "Pool B", "sub": "orgs 86–170", "kind": "app", "icon": "kubernetes" }, { "id": "x_p3", "label": "Pool C", "sub": "orgs 171–255", "kind": "app", "icon": "kubernetes" } ] }, { "title": "Unit economics", "kind": "plain", "nodes": [ { "id": "x_why2", "label": "1 pod ≈ 18k dec/s", "sub": "4 vCPU · 2 GiB", "kind": "decision" } ] } ] }, { "title": "Tier 3 · Coordination shards", "kind": "boundary", "dir": "col", "children": [ { "title": "Valkey Cluster", "kind": "plain", "nodes": [ { "id": "x_s1", "label": "Shards 0–7", "sub": "hash tag {org:id}", "kind": "store", "icon": "redis" }, { "id": "x_s2", "label": "Shards 8–15", "sub": "replica per AZ", "kind": "store", "icon": "redis" } ] }, { "title": "Unit economics", "kind": "plain", "nodes": [ { "id": "x_why3", "label": "1 shard ≈ 90k Lua/s", "sub": "8% of traffic reaches it", "kind": "decision" } ] } ] }, { "title": "Tier 4 · Hot tenant fan-out", "kind": "boundary", "dir": "col", "children": [ { "title": "Sub-sharding", "kind": "plain", "nodes": [ { "id": "x_hot", "label": "Split above 5k rps", "sub": "{org:acme#0..7}", "kind": "app", "icon": "fa5_expand_arrows_alt" }, { "id": "x_merge", "label": "Merge on cool-down", "sub": "5 min below threshold", "kind": "app", "icon": "fa5_compress_arrows_alt" } ] }, { "title": "Cost of the split", "kind": "plain", "nodes": [ { "id": "x_why4", "label": "Quota divided N ways", "sub": "overshoot rises to 3%", "kind": "risk" } ] } ] } ], "outside": [ { "id": "x_risk", "label": "Single org on one slot", "sub": "the real ceiling, not total rps", "kind": "risk", "icon": "fa5_exclamation_triangle" } ], "edges": [ { "from": "x_hash", "to": "x_p1", "label": "sticky by org", "kind": "sync" }, { "from": "x_p1", "to": "x_s1", "label": "lease refill only", "kind": "sync" }, { "from": "x_s1", "to": "x_hot", "label": "hot key detected", "kind": "async" }, { "from": "x_risk", "to": "x_hot", "label": "mitigated", "kind": "error", "route": "gutter" } ], "note": "Answer to design question 6. Total throughput scales linearly with pods and shards; the binding constraint is a single organisation's keys landing on one Redis slot, which tier 4 exists to relieve.", "meta": { "v": "1.0", "owner": "Data & AI Global Practice" } } ]