Deliverables

flowchart TB
  users(["Users"]) --> dns["Global DNS<br/><i>latency routing + health checks</i>"]

  subgraph r1["Region: primary"]
    direction TB
    lb1["Load Balancer<br/><i>public</i>"]
    subgraph az1["Zone A"]
      n1["Node Pool<br/><i>api ×3, worker ×2</i>"]
    end
    subgraph az2["Zone B"]
      n2["Node Pool<br/><i>api ×3, worker ×2</i>"]
    end
    subgraph az3["Zone C"]
      n3["Node Pool<br/><i>api ×2, worker ×1</i>"]
    end
    db1[("Primary DB<br/><i>synchronous replica in Zone B</i>")]
    lb1 --> n1
    lb1 --> n2
    lb1 --> n3
    n1 --> db1
    n2 --> db1
    n3 --> db1
  end

  subgraph r2["Region: secondary"]
    direction TB
    lb2["Load Balancer<br/><i>warm</i>"]
    n4["Node Pool<br/><i>api ×2, scaled down</i>"]
    db2[("Read Replica<br/><i>asynchronous — RPO 30s</i>")]
    lb2 --> n4 --> db2
  end

  dns --> lb1
  dns -.->|"failover only"| lb2
  db1 -.->|"async replication"| db2

flowchart TB
  subgraph pri["Primary site"]
    direction TB
    p1["Application tier"]
    p2[("Transactional DB")]
    p3[("Object storage")]
    p4["Secrets + config"]
  end
  subgraph dr["Recovery site — warm"]
    direction TB
    d1["Application tier<br/><i>scaled to 0 · IaC ready</i>"]
    d2[("DB replica<br/><i>async · lag &lt; 30s</i>")]
    d3[("Storage replica<br/><i>cross-region · 15 min</i>")]
    d4["Secrets replicated"]
  end

  p2 ==>|"async"| d2
  p3 ==>|"async"| d3
  p4 ==>|"sync"| d4

  subgraph run["Recovery sequence — tested 2026-05-18"]
    direction TB
    s1["1. Declare · 5 min"]
    s2["2. Promote replica · 8 min"]
    s3["3. Scale app tier · 6 min"]
    s4["4. Repoint DNS · 5 min<br/><i>TTL 60s</i>"]
    s5["5. Verify + reconcile · 12 min"]
    s1 --> s2 --> s3 --> s4 --> s5
  end
  dr --> run
  run --> res(["Restored — measured RTO 36 min"])

flowchart TB
  u(["Users"]) --> gslb["Global traffic manager<br/><i>health + latency routing</i>"]

  subgraph ra["Region A — write primary"]
    direction TB
    a1["Edge + API"]
    a2["Services"]
    a3[("Primary DB<br/><i>accepts writes</i>")]
    a1 --> a2 --> a3
  end

  subgraph rb["Region B — read + standby"]
    direction TB
    b1["Edge + API"]
    b2["Services"]
    b3[("Replica<br/><i>read only · lag &lt; 2s</i>")]
    b1 --> b2 --> b3
  end

  gslb -->|"reads: nearest"| a1
  gslb -->|"reads: nearest"| b1
  gslb -->|"writes: always A"| a1
  b2 -.->|"write forwarding<br/>+45ms"| a2
  a3 ==>|"async replication"| b3
  arb{{"Failover arbitration<br/><i>manual promote · documented<br/>RPO &le; 2s · RTO 15 min</i>"}}
  arb -.-> a3
  arb -.-> b3

flowchart TB
  s(["Alert: order submit p99 > 900 ms<br/>for 5 min"])
  s --> c1["1. Confirm scope<br/><i>one region or global?</i>"]
  c1 --> q1{"Global?"}
  q1 -->|"no"| a1["2a. Check that region's<br/>dependency health"]
  q1 -->|"yes"| a2["2b. Check shared dependencies:<br/>DB, cache, identity"]
  a1 --> q2{"Region-local cause?"}
  q2 -->|"yes"| m1["3. Shift traffic away<br/><i>runbook OPS-04</i>"]
  q2 -->|"no"| a2
  a2 --> q3{"DB CPU > 80%<br/>or replication lag > 10s?"}
  q3 -->|"yes"| m2["4. Enable read-path cache bypass=off<br/>+ raise connection limit<br/><i>flag: cache.aggressive</i>"]
  q3 -->|"no"| q4{"Recent deploy<br/>in last 60 min?"}
  q4 -->|"yes"| m3["5. Roll back<br/><i>runbook OPS-01</i>"]
  q4 -->|"no"| esc(["6. Escalate to<br/>Order team on-call<br/><i>stop investigating</i>"])
  m1 --> v["7. Verify p99 recovers<br/>within 10 min"]
  m2 --> v
  m3 --> v
  v --> q5{"Recovered?"}
  q5 -->|"yes"| done(["Close · record in incident log"])
  q5 -->|"no"| esc