[ { "id": "01-system-context", "title": "Video Encoding & Packaging Pipeline — System Context", "layout": "context", "colWidth": 265, "system": { "label": "Encoding & Packaging Pipeline", "sub": "master in, playable catalogue out" }, "groups": [ { "side": "left", "title": "Content supply", "nodes": [ { "id": "studio", "label": "Rights holder", "kind": "external", "rel": "delivers mezzanine", "dir": "in", "kind2": "batch" }, { "id": "inhouse", "label": "In-house production", "kind": "external", "rel": "delivers mezzanine", "dir": "in", "kind2": "batch" }, { "id": "ops", "label": "Content operations", "kind": "actor", "rel": "overrides, waives, publishes", "dir": "in" } ] }, { "side": "right", "title": "Delivery and viewing", "nodes": [ { "id": "cdn", "label": "Media CDN + origin", "kind": "external", "rel": "manifests, segments", "icon": "gcp_cloudcdn" }, { "id": "player", "label": "Player estate", "kind": "external", "sub": "TV, mobile, web", "rel": "playback telemetry", "dir": "in", "kind2": "async" }, { "id": "catalogue", "label": "Catalogue service", "kind": "external", "rel": "playability per title" } ] }, { "side": "top", "title": "Governing systems", "nodes": [ { "id": "rights", "label": "Rights & windows", "kind": "external", "rel": "publishable territories", "dir": "in" }, { "id": "drm", "label": "DRM licence service", "kind": "external", "rel": "content keys", "dir": "in", "kind2": "bidirectional" } ] }, { "side": "bottom", "title": "Out of scope", "nodes": [ { "id": "abr", "label": "Player ABR logic", "kind": "external", "sub": "not ours" }, { "id": "subs", "label": "Subtitle authoring", "kind": "external", "sub": "upstream" }, { "id": "live", "label": "Live contribution", "kind": "external", "sub": "separate pipeline" } ] } ], "note": "The boundary is master-in to manifest-published. The CDN, the player and the rights system are consumed, not owned.", "meta": { "v": "1.0", "owner": "Media Platform Architecture", "date": "2026-10" } }, { "id": "02-high-level-architecture", "title": "High-Level Architecture — Master to Playable", "layout": "flow", "chain": true, "align": "middle", "nodeWidth": 200, "stages": [ { "title": "Deliver", "nodes": [ { "id": "intake", "label": "Delivery intake", "sub": "resumable upload / pull", "kind": "integration" }, { "id": "masters", "label": "Master store", "sub": "dual-region, archive", "kind": "store", "icon": "gcp_cloudstorage" } ] }, { "title": "Conform", "nodes": [ { "id": "probe", "label": "Probe & conformance", "sub": "126 TB/day in" }, { "id": "remed", "label": "Remediation", "sub": "telecine, loudness" } ] }, { "title": "Decide", "nodes": [ { "id": "analyse", "label": "Complexity analysis", "sub": "shots, grain, motion" }, { "id": "ladder", "label": "Ladder solver", "sub": "per title", "icon": "gcp_cloudrun" }, { "id": "recipes", "label": "Recipe store", "sub": "immutable, addressed", "kind": "store", "icon": "gcp_cloudstorage" } ] }, { "title": "Encode", "nodes": [ { "id": "sched", "label": "Lane scheduler", "sub": "deadline priority", "kind": "platform", "icon": "gcp_gke" }, { "id": "fleet", "label": "Batch encode fleet", "sub": "Spot VMs, 18%/h reclaim", "icon": "gcp_compute" }, { "id": "stitch", "label": "Stitch & verify", "sub": "boundary check" } ] }, { "title": "Prove", "nodes": [ { "id": "gate", "label": "QC gate", "sub": "blocking", "kind": "decision" }, { "id": "scores", "label": "Score store", "sub": "per rung, per window", "kind": "store" } ] }, { "title": "Package", "nodes": [ { "id": "pack", "label": "Segmenter & encryptor", "sub": "common encryption" }, { "id": "segs", "label": "Rendition store", "sub": "content-addressed", "kind": "store", "icon": "gcp_cloudstorage" } ] }, { "title": "Publish", "nodes": [ { "id": "pub", "label": "Manifest publisher", "sub": "atomic swap", "kind": "integration", "icon": "gcp_cloudrun" }, { "id": "proj", "label": "Catalogue projection", "sub": "title to manifest", "kind": "store" } ] } ], "edges": [ { "from": "intake", "to": "masters", "label": "write once" }, { "from": "masters", "to": "fleet", "label": "chunk reads", "route": "gutter" }, { "from": "gate", "to": "ladder", "label": "new recipe version", "kind": "error", "route": "gutter" } ], "note": "The spine is the happy path. The red return is the only loop: a failed gate revises the recipe, never retries the same one.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "03-actors-and-journeys", "title": "Actors and Their Core Journeys", "layout": "actors", "cardWidth": 268, "canvas": { "width": 1740 }, "groups": [ { "title": "People who watch", "kind": "boundary", "actors": [ { "id": "viewer", "label": "Viewer on a big screen", "sub": "assumed 40 M MAU", "goal": "I press play on the thing everyone is talking about and it looks like television, immediately, with no spinner and no soft first ten seconds.", "journeys": [ { "id": "j-watch", "label": "Watch a new release", "sub": "premiere night" }, { "label": "Resume something old" } ] }, { "id": "tail", "label": "Viewer in the long tail", "sub": "72% of catalogue hours", "goal": "I picked a 1974 documentary nobody else is watching tonight, and I would rather not be punished for having unusual taste.", "journeys": [ { "id": "j-cold", "label": "Play a cold-stored title", "sub": "rendition not resident" } ] } ] }, { "title": "People who ship the content", "kind": "boundary", "actors": [ { "id": "cops", "label": "Content operations", "sub": "assumed 24 staff", "goal": "I need tomorrow's premiere playable on every device before the embargo lifts, and I need to know hours in advance if it will not be.", "journeys": [ { "id": "j-air", "label": "Get a title to air", "sub": "1,800 versions/day" }, { "label": "Waive a narrow QC failure" } ] }, { "id": "qc", "label": "Video quality engineer", "sub": "owns the gate", "goal": "I want the gate to fail the things viewers would notice, and I want to know which encoder build caused a regression without bisecting the catalogue.", "journeys": [ { "label": "Tune a ladder from telemetry" }, { "label": "Roll out a new encoder build" } ] }, { "id": "finops", "label": "Media FinOps", "sub": "cost per delivered hour", "goal": "I want to see what a codec or a rung costs before it is switched on for 400,000 titles, not in next month's invoice.", "journeys": [ { "label": "Price a codec addition" } ] } ] }, { "title": "Partners, systems and machines", "kind": "cloud", "actors": [ { "id": "studio2", "label": "Rights holder", "kind": "external", "sub": "assumed 310 suppliers", "goal": "I hand over one master in the format we agreed and get told within the hour whether you accepted it, not three days later.", "journeys": [ { "label": "Deliver and redeliver a master" } ] }, { "id": "drm2", "label": "DRM licence service", "kind": "external", "sub": "3 systems", "goal": "Issue a licence for a key the pipeline never stored itself.", "journeys": [ { "label": "Serve a playback licence" } ] }, { "id": "sre", "label": "Platform SRE", "kind": "platform", "sub": "on call", "goal": "When 60% of the fleet is taken away at 03:00 I want the queue to drain anyway and nobody to page me.", "journeys": [ { "label": "Survive a reclaim storm" } ] } ] } ], "legend": false, "note": "Goals are in the actor's voice. Journeys with an id get their own map; the rest are named so the set cannot pretend they do not exist.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "04-journey-watch-a-new-release", "title": "Journey — Viewer Watches a New Release on Premiere Night", "layout": "journey", "cellWidth": 206, "actor": { "label": "Viewer on a big screen", "sub": "assumed 40 M MAU", "goal": "Press play on tonight's premiere and have it look like television from the first frame", "trigger": "A midnight embargo lift, a trailer, a group chat", "success": "4K within two seconds, no visible step-up, no stall in the first minute" }, "phases": [ { "title": "Decide to watch", "sub": "outside our boundary" }, { "title": "Press play", "moment": true }, { "title": "First ten seconds", "moment": true }, { "title": "Settle in" }, { "title": "Tell someone" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Sees the row on the home page" } ], [ { "label": "Presses play" } ], [ { "label": "Judges the picture" } ], [ { "label": "Watches 48 minutes" } ], [ { "label": "Recommends it, or does not" } ] ] }, { "title": "What we must have ready", "kind": "system", "cells": [ [ { "label": "Manifest published", "kind": "integration" } ], [ { "label": "Launch subset resident", "kind": "store" }, { "label": "Licence issued", "kind": "security" } ], [ { "label": "Top rung reachable", "kind": "store" } ], [ { "label": "Full ladder filled in", "kind": "store" } ], [ { "label": "Telemetry collected", "kind": "platform" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Delighted", "Fine", "Frustrated" ], "points": [ 1, 1, 0, 2, 2 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Nothing published yet" } ], [ { "label": "Soft first 10 s: top rung missing" } ], [], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [], [ { "label": "Atomic manifest swap" } ], [ { "label": "Launch subset pre-encoded" } ], [ { "label": "Later manifest revision" } ], [ { "label": "Post-hoc gate" } ] ] } ], "chain": true, "note": "The trough is the third phase, and it is structural: the ladder a premiere needs resident at second zero is the launch subset, not the full set.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "05-journey-play-a-cold-title", "title": "Journey — Viewer Plays a Cold-Stored Long-Tail Title", "layout": "journey", "cellWidth": 210, "actor": { "label": "Viewer in the long tail", "sub": "72% of hours, under 0.1% of viewing", "goal": "Watch an obscure title at the quality my screen deserves", "trigger": "A search, a deep link, a critic's list", "success": "Plays within the same couple of seconds as anything else" }, "phases": [ { "title": "Find it", "sub": "search or deep link" }, { "title": "Press play", "moment": true }, { "title": "First segment", "moment": true }, { "title": "Keep watching" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Opens the title page" } ], [ { "label": "Presses play" } ], [ { "label": "Waits, or does not notice" } ], [ { "label": "Watches to the end" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "Manifest says what exists", "kind": "integration" } ], [ { "label": "Cache miss on the top rung", "kind": "risk" }, { "label": "Request coalesced", "kind": "queue" } ], [ { "label": "On-demand generation", "kind": "app" }, { "label": "Nearest rung served", "kind": "store" } ], [ { "label": "Demand tier promoted", "kind": "platform" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Delighted", "Fine", "Frustrated" ], "points": [ 1, 1, 0, 1 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [], [ { "label": "p99 miss: 3.5 s to first byte" } ], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [], [ { "label": "Coalescing, so one encode serves all" } ], [ { "label": "Fall back a rung rather than wait" } ], [ { "label": "Second viewer gets a hit" } ] ] } ], "chain": true, "note": "This journey only exists because renditions are a cache. Its trough is the price of the critical design decision, paid where it is cheapest.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "06-journey-get-a-title-to-air", "title": "Journey — Content Operations Gets a Title to Air", "layout": "journey", "cellWidth": 202, "actor": { "label": "Content operations", "sub": "assumed 24 staff, 1,800 versions/day", "goal": "Have tomorrow's premiere playable everywhere before the embargo lifts", "trigger": "A delivery landing, or a dated release calendar", "success": "Published, gate-passed, with hours of warning if it will not make it" }, "phases": [ { "title": "Delivery lands" }, { "title": "Conformance", "moment": true }, { "title": "Encode & prove" }, { "title": "Publish", "moment": true }, { "title": "Watch it live" } ], "lanes": [ { "title": "What they do", "kind": "step", "cells": [ [ { "label": "Books it into a lane" } ], [ { "label": "Reads the rejection reason" }, { "label": "Chases a redelivery" } ], [ { "label": "Watches chunk-seconds" } ], [ { "label": "Approves the swap" } ], [ { "label": "Rolls back if needed" } ] ] }, { "title": "What the platform does", "kind": "system", "cells": [ [ { "label": "Checksum & probe", "kind": "app" } ], [ { "label": "Conformance verdict", "kind": "decision" } ], [ { "label": "Lane scheduling", "kind": "platform" }, { "label": "Blocking QC gate", "kind": "decision" } ], [ { "label": "Atomic manifest swap", "kind": "integration" } ], [ { "label": "Rollback in 120 s", "kind": "app" } ] ] }, { "title": "How it feels", "kind": "emotion", "levels": [ "Confident", "Fine", "Exposed" ], "points": [ 1, 0, 1, 1, 1 ] }, { "title": "Where it hurts", "kind": "pain", "cells": [ [], [ { "label": "Rejection 3 days before air" } ], [ { "label": "No warning a deadline will slip" } ], [], [] ] }, { "title": "What answers it", "kind": "gain", "cells": [ [ { "label": "Verify before scheduling" } ], [ { "label": "Verdict within the hour" } ], [ { "label": "Chunk-seconds, not stage names" } ], [ { "label": "Launch subset first" } ], [ { "label": "Rollback needs no fleet" } ] ] } ], "chain": true, "note": "The trough is conformance, and the fix is cheap: reject within the hour so the redelivery still fits the calendar.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "07-layered-architecture", "title": "Layered Architecture", "layout": "bands", "layerHeaderWidth": 158, "bands": [ { "name": "Supply & intake", "nodes": [ { "id": "l-intake", "label": "Delivery intake", "kind": "integration" }, { "id": "l-verify", "label": "Integrity verify" }, { "id": "l-probe", "label": "Deep probe" }, { "id": "l-conf", "label": "Conformance policy", "kind": "decision" }, { "id": "l-remed", "label": "Remediation" } ] }, { "name": "Recipe plane", "accent": "#fff2cc", "nodes": [ { "id": "l-shot", "label": "Shot & complexity analysis" }, { "id": "l-solve", "label": "Ladder solver", "icon": "gcp_cloudrun" }, { "id": "l-codec", "label": "Codec & device policy" }, { "id": "l-rec", "label": "Recipe store", "sub": "immutable, pinned", "kind": "store", "icon": "gcp_cloudstorage" } ] }, { "name": "Control plane", "nodes": [ { "id": "l-job", "label": "Job & chunk-task store", "sub": "Spanner", "kind": "store" }, { "id": "l-lane", "label": "Lane scheduler", "kind": "platform", "icon": "gcp_gke" }, { "id": "l-quota", "label": "Quotas & spend ceilings", "kind": "platform" }, { "id": "l-lin", "label": "Progress & lineage API", "kind": "integration" } ] }, { "name": "Execution", "nodes": [ { "id": "l-batch", "label": "Batch encode fleet", "sub": "Spot VMs", "icon": "gcp_compute" }, { "id": "l-jit", "label": "On-demand fleet", "sub": "warm, latency-bound" }, { "id": "l-sand", "label": "Decoder sandbox", "kind": "security" }, { "id": "l-stitch", "label": "Stitch & boundary verify" } ] }, { "name": "Assurance", "nodes": [ { "id": "l-vmaf", "label": "Perceptual scoring", "icon": "gcp_gke" }, { "id": "l-struct", "label": "Structural checks" }, { "id": "l-dec", "label": "Decoder matrix" }, { "id": "l-gate", "label": "Blocking gate", "kind": "decision" }, { "id": "l-hum", "label": "Human sample queue", "kind": "actor" } ] }, { "name": "Package & protect", "nodes": [ { "id": "l-seg", "label": "Segmenter" }, { "id": "l-enc", "label": "Common encryption", "kind": "security" }, { "id": "l-kms", "label": "Key broker", "kind": "security", "icon": "gcp_secretmanager" }, { "id": "l-man", "label": "Manifest builder" } ] }, { "name": "Publication & lifecycle", "nodes": [ { "id": "l-swap", "label": "Atomic manifest swap", "kind": "integration" }, { "id": "l-tier", "label": "Demand tiering", "kind": "platform" }, { "id": "l-evict", "label": "Eviction & regeneration", "kind": "platform" }, { "id": "l-take", "label": "Takedown & revocation", "kind": "security" } ] }, { "name": "Storage & telemetry", "nodes": [ { "id": "l-mast", "label": "Master store", "sub": "dual-region archive", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "l-rend", "label": "Rendition store", "sub": "content-addressed", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "l-score", "label": "Score & telemetry store", "kind": "store" }, { "id": "l-aud", "label": "Audit log", "sub": "append-only", "kind": "store" } ] } ], "edges": [ { "from": "l-conf", "to": "l-shot", "label": "accepted source" }, { "from": "l-rec", "to": "l-job", "label": "recipe digest" }, { "from": "l-lane", "to": "l-batch", "label": "chunk tasks" }, { "from": "l-stitch", "to": "l-gate", "label": "renditions" }, { "from": "l-gate", "to": "l-seg", "label": "gate-passed" }, { "from": "l-man", "to": "l-swap", "label": "candidate" }, { "from": "l-evict", "to": "l-jit", "label": "regenerate", "kind": "async" } ], "note": "The recipe plane is highlighted because it, not the rendition store, holds the authoritative derived artefact.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "08-platform-components", "title": "Platform Components (C4 Container View)", "layout": "nested", "canvas": { "width": 1760 }, "boxes": [ { "title": "Google Cloud — encode region", "kind": "cloud", "dir": "col", "children": [ { "title": "Recipe plane", "kind": "trust", "nodes": [ { "id": "c-analyse", "label": "Complexity analyser", "sub": "Cloud Run jobs", "icon": "gcp_cloudrun" }, { "id": "c-solver", "label": "Ladder solver", "sub": "Cloud Run", "icon": "gcp_cloudrun" }, { "id": "c-recipe", "label": "Recipe store", "sub": "GCS + Spanner index", "kind": "store", "icon": "gcp_cloudstorage" } ] }, { "title": "Control plane", "kind": "boundary", "nodes": [ { "id": "c-api", "label": "Job API", "sub": "Cloud Run", "kind": "integration", "icon": "gcp_cloudrun" }, { "id": "c-state", "label": "Job & task store", "sub": "Spanner", "kind": "store" }, { "id": "c-sched", "label": "Lane scheduler", "sub": "GKE, deadline-ordered", "kind": "platform", "icon": "gcp_gke" }, { "id": "c-disp", "label": "Task dispatch", "sub": "Pub/Sub", "kind": "queue", "icon": "gcp_pubsub" }, { "id": "c-gov", "label": "Quota & spend governor", "sub": "GKE", "kind": "platform" } ] }, { "title": "Execution", "kind": "boundary", "dir": "row", "nodes": [ { "id": "c-batch", "label": "Batch encode workers", "sub": "Spot MIG + gVisor", "icon": "gcp_compute" }, { "id": "c-jit", "label": "On-demand encoders", "sub": "GKE warm pool", "icon": "gcp_gke" }, { "id": "c-stitch", "label": "Stitcher", "sub": "Cloud Run jobs", "icon": "gcp_cloudrun" } ] }, { "title": "Assurance", "kind": "boundary", "nodes": [ { "id": "c-score", "label": "Perceptual scorer", "sub": "VMAF on GKE", "icon": "gcp_gke" }, { "id": "c-checks", "label": "Structural checker" }, { "id": "c-matrix", "label": "Decoder matrix runner", "sub": "device profiles" }, { "id": "c-gate", "label": "Gate decision service", "kind": "decision" } ] }, { "title": "Package, publish and protect", "kind": "trust", "nodes": [ { "id": "c-pack", "label": "Packager", "sub": "Shaka on GKE", "icon": "gcp_gke" }, { "id": "c-keys", "label": "Key broker", "sub": "Cloud KMS client", "kind": "security", "icon": "gcp_secretmanager" }, { "id": "c-pub", "label": "Manifest publisher", "sub": "Cloud Run", "kind": "integration", "icon": "gcp_cloudrun" }, { "id": "c-life", "label": "Lifecycle controller", "sub": "tiering, eviction", "kind": "platform" } ] }, { "title": "Storage", "kind": "boundary", "dir": "row", "nodes": [ { "id": "c-mast", "label": "Master bucket", "sub": "dual-region archive", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "c-rend", "label": "Rendition bucket", "sub": "regional, addressed", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "c-proj", "label": "Catalogue projection", "sub": "Spanner + Memorystore", "kind": "store" }, { "id": "c-tele", "label": "Score & telemetry", "sub": "BigQuery", "kind": "store", "icon": "gcp_bigquery" }, { "id": "c-audit", "label": "Audit log", "sub": "append-only", "kind": "store" } ] } ] } ], "outside": [ { "id": "c-kms", "label": "Cloud KMS", "kind": "external", "sub": "CMEK + content keys", "icon": "gcp_secretmanager" }, { "id": "c-drm", "label": "DRM licence service", "kind": "external", "sub": "3 systems" }, { "id": "c-cdn", "label": "Media CDN", "kind": "external", "sub": "out of scope", "icon": "gcp_cloudcdn" }, { "id": "c-rights", "label": "Rights & windows", "kind": "external" } ], "edges": [ { "from": "c-recipe", "to": "c-api", "label": "digest" }, { "from": "c-sched", "to": "c-disp", "label": "tasks" }, { "from": "c-disp", "to": "c-batch", "label": "claim" }, { "from": "c-stitch", "to": "c-score", "label": "renditions" }, { "from": "c-gate", "to": "c-pack", "label": "gate-passed" }, { "from": "c-keys", "to": "c-kms", "label": "wrap/unwrap", "kind": "bidirectional" }, { "from": "c-pub", "to": "c-proj" }, { "from": "c-life", "to": "c-rend", "label": "tiering", "kind": "batch" } ], "note": "Thirteen of the edges are omitted: every component writes to the audit log and emits to telemetry.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "09-integration-surface", "title": "Integration Surface — Interfaces In and Out", "layout": "hub", "left": { "title": "Inbound", "nodes": [ { "id": "i-up", "label": "Resumable upload", "sub": "signed, per-supplier", "kind": "integration", "rel": "mezzanine" }, { "id": "i-pull", "label": "Supplier pull", "sub": "scheduled fetch", "kind": "integration", "rel": "mezzanine", "kind2": "batch" }, { "id": "i-ops", "label": "Operations console", "kind": "actor", "rel": "overrides" }, { "id": "i-rights", "label": "Rights & calendar", "kind": "external", "sub": "windows, deadlines", "rel": "policy", "kind2": "batch" }, { "id": "i-tel", "label": "Playback telemetry", "kind": "external", "sub": "rebuffer, rung, errors", "rel": "feedback", "kind2": "async" } ] }, "centre": { "title": "Encoding & packaging platform", "nodes": [ { "id": "h-api", "label": "Job & lineage API", "sub": "gRPC + REST", "kind": "integration", "icon": "gcp_cloudrun" }, { "id": "h-core", "label": "Pipeline core" }, { "id": "h-evt", "label": "Event stream", "sub": "Pub/Sub", "kind": "queue", "icon": "gcp_pubsub" } ] }, "right": { "title": "Outbound", "nodes": [ { "id": "o-origin", "label": "Packaging origin", "sub": "manifests + segments", "kind": "integration", "rel": "pull-through", "dir": "out", "icon": "gcp_cloudrun" }, { "id": "o-cat", "label": "Catalogue service", "kind": "external", "rel": "playability", "dir": "out" }, { "id": "o-drm", "label": "DRM licence service", "kind": "external", "rel": "key ids", "dir": "out" }, { "id": "o-fin", "label": "FinOps reporting", "kind": "external", "rel": "cost", "dir": "out", "kind2": "batch" } ] }, "note": "Playback telemetry is an inbound interface, not an export — it is what makes the post-hoc gate possible. Four outbound surfaces, five inbound.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "10-encode-data-flow", "title": "Data Flow — One Master Through the Pipeline", "layout": "flow", "chain": true, "align": "top", "nodeWidth": 198, "stages": [ { "title": "Source", "nodes": [ { "id": "d-mez", "label": "Mezzanine", "sub": "~36 GB/source hour", "kind": "store" }, { "id": "d-man", "label": "Delivery manifest", "kind": "store" } ] }, { "title": "Landing", "nodes": [ { "id": "d-imm", "label": "Immutable landing", "sub": "write-once, locked", "kind": "store" }, { "id": "d-prof", "label": "Technical profile", "kind": "store" } ] }, { "title": "Derived decision", "nodes": [ { "id": "d-cplx", "label": "Complexity profile", "kind": "store" }, { "id": "d-recipe", "label": "Recipe", "sub": "content-addressed", "kind": "store", "icon": "gcp_cloudstorage" } ] }, { "title": "Work", "nodes": [ { "id": "d-task", "label": "Chunk tasks", "sub": "220 k/day", "kind": "queue" }, { "id": "d-chunk", "label": "Chunk outputs", "sub": "idempotent", "kind": "store" } ] }, { "title": "Rendition", "nodes": [ { "id": "d-rend", "label": "Elementary streams", "kind": "store" }, { "id": "d-sc", "label": "Quality scores", "kind": "store" } ] }, { "title": "Protected", "nodes": [ { "id": "d-seg", "label": "Encrypted segments", "sub": "immutable, 30 d cache", "kind": "store" }, { "id": "d-mf", "label": "Manifests", "sub": "mutable, 30 s cache", "kind": "store" } ] }, { "title": "Consumers", "nodes": [ { "id": "d-cdn", "label": "CDN + players", "kind": "external", "icon": "gcp_cloudcdn" }, { "id": "d-bq", "label": "Cost & quality analytics", "sub": "BigQuery", "kind": "store", "icon": "gcp_bigquery" } ] } ], "edges": [ { "from": "d-man", "to": "d-imm", "label": "verify then accept" }, { "from": "d-recipe", "to": "d-task", "label": "digest per chunk" }, { "from": "d-sc", "to": "d-bq", "label": "gate decisions", "kind": "batch", "route": "gutter" }, { "from": "d-cdn", "to": "d-bq", "label": "playback", "kind": "async" } ], "note": "Only the first three columns are irreplaceable. Everything from chunk outputs rightwards is derivable from them.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "11-storage-zones", "title": "Storage Zones — Ownership, Durability and Rebuildability", "layout": "nested", "canvas": { "width": 1700 }, "boxes": [ { "title": "Irreplaceable — replicated, locked, never derived", "kind": "trust", "dir": "row", "children": [ { "title": "Master custody", "kind": "boundary", "nodes": [ { "id": "z-mast", "label": "Mezzanine masters", "sub": "dual-region archive", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "z-lock", "label": "Retention lock", "sub": "7 years", "kind": "security" } ] }, { "title": "Recipe custody", "kind": "boundary", "nodes": [ { "id": "z-rec", "label": "Recipes", "sub": "immutable, addressed", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "z-build", "label": "Encoder build registry", "sub": "pinned digests", "kind": "store" } ] }, { "title": "Evidence", "kind": "boundary", "nodes": [ { "id": "z-aud", "label": "Audit log", "sub": "append-only, 7 y", "kind": "store" }, { "id": "z-sc", "label": "Gate scores", "sub": "BigQuery, 7 y", "kind": "store" } ] } ] }, { "title": "Recoverable — RPO 5 s, rebuildable from the zone above", "kind": "boundary", "dir": "row", "children": [ { "title": "Control state", "kind": "plain", "nodes": [ { "id": "z-job", "label": "Job & chunk tasks", "sub": "Spanner", "kind": "store" }, { "id": "z-lane", "label": "Lane accounting", "kind": "store" } ] }, { "title": "Serving projection", "kind": "plain", "nodes": [ { "id": "z-proj", "label": "Catalogue projection", "sub": "rebuildable", "kind": "store" }, { "id": "z-cache", "label": "Manifest cache", "sub": "Memorystore, 30 s", "kind": "store" } ] } ] }, { "title": "Disposable — no RPO by design, regenerated not restored", "kind": "lane", "dir": "row", "children": [ { "title": "Hot renditions", "kind": "plain", "nodes": [ { "id": "z-hot", "label": "Top-demand renditions", "sub": "standard class", "kind": "store" }, { "id": "z-seg", "label": "Encrypted segments", "sub": "content-addressed", "kind": "store" } ] }, { "title": "Cooling and absent", "kind": "plain", "nodes": [ { "id": "z-cool", "label": "Demoted renditions", "sub": "nearline / coldline", "kind": "store" }, { "id": "z-gone", "label": "Evicted renditions", "sub": "regenerate on demand", "kind": "risk" }, { "id": "z-tmp", "label": "Chunk scratch", "sub": "TTL 48 h", "kind": "store" } ] } ] } ], "edges": [ { "from": "z-rec", "to": "z-gone", "label": "regenerate", "kind": "async", "route": "gutter" }, { "from": "z-job", "to": "z-proj", "label": "rebuild", "kind": "batch" } ], "note": "Zones are ordered by what loss costs. Only the top zone is backed up; the bottom zone is deliberately not.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "12-data-model", "title": "Data Model — Lineage from Master to Published Byte", "layout": "er", "canvas": { "width": 1680, "cols": 4 }, "rowGap": 238, "entities": [ { "id": "title", "name": "title", "kind": "store", "row": 0, "col": 0, "attrs": [ "title_id PK", "owner_id FK -> owner", "release_at", "demand_tier" ] }, { "id": "source", "name": "source_version", "kind": "store", "row": 0, "col": 1, "attrs": [ "source_id PK", "title_id FK -> title", "checksum UQ", "profile_json", "conformance_verdict", "accepted_at" ] }, { "id": "recipe", "name": "recipe", "kind": "store", "row": 0, "col": 2, "attrs": [ "recipe_digest PK", "source_id FK -> source_version", "encoder_build FK -> encoder_build", "ladder_json", "codec_set", "override_actor" ] }, { "id": "build", "name": "encoder_build", "kind": "store", "row": 0, "col": 3, "attrs": [ "encoder_build PK", "encoder", "version", "image_digest", "approved_at" ] }, { "id": "job", "name": "encode_job", "row": 1, "col": 1, "attrs": [ "job_id PK", "recipe_digest FK -> recipe", "lane", "deadline_at", "chunk_seconds_done", "spend_cents" ] }, { "id": "task", "name": "chunk_task", "row": 2, "col": 0, "attrs": [ "task_id PK", "job_id FK -> encode_job", "chunk_index", "start_pts / end_pts", "attempt_no", "output_digest" ] }, { "id": "rend", "name": "rendition", "kind": "store", "row": 1, "col": 2, "attrs": [ "rendition_digest PK", "recipe_digest FK -> recipe", "rung", "codec", "residency_class", "generated_by" ] }, { "id": "score", "name": "quality_score", "kind": "store", "row": 1, "col": 3, "attrs": [ "score_id PK", "rendition_digest FK -> rendition", "metric", "aggregate", "worst_window", "gate_verdict" ] }, { "id": "seg", "name": "segment", "kind": "store", "row": 2, "col": 2, "attrs": [ "segment_digest PK", "rendition_digest FK -> rendition", "sequence_no", "key_id FK -> content_key" ] }, { "id": "man", "name": "manifest", "kind": "store", "row": 2, "col": 1, "attrs": [ "manifest_id PK", "title_id FK -> title", "territory", "device_class", "revision", "published_at" ] }, { "id": "key", "name": "content_key", "kind": "security", "row": 2, "col": 3, "attrs": [ "key_id PK", "title_id FK -> title", "scheme", "rotated_at", "revoked_at" ] } ], "relations": [ { "from": "title", "to": "source", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "source", "to": "recipe", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "build", "to": "recipe", "label": "1 : N", "from_side": "w", "to_side": "e" }, { "from": "recipe", "to": "job", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "job", "to": "task", "label": "1 : N", "from_side": "w", "to_side": "n" }, { "from": "recipe", "to": "rend", "label": "1 : N", "from_side": "s3", "to_side": "n" }, { "from": "rend", "to": "score", "label": "1 : N", "from_side": "e", "to_side": "w" }, { "from": "rend", "to": "seg", "label": "1 : N", "from_side": "s", "to_side": "n" }, { "from": "title", "to": "man", "label": "1 : N", "from_side": "s1", "to_side": "n1" }, { "from": "man", "to": "seg", "label": "N : M", "from_side": "e", "to_side": "w" }, { "from": "key", "to": "seg", "label": "1 : N", "from_side": "w", "to_side": "e" } ], "note": "recipe_digest is the join key the whole model turns on: delete every rendition and the lineage still resolves. The audit log keys off the same digests and is omitted here.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "13-critical-flow-master-to-playable", "title": "Critical Flow — Accepted Master to Published Manifest", "layout": "sequence", "lifelines": [ { "id": "s-ops", "label": "Content ops", "kind": "actor" }, { "id": "s-api", "label": "Job API", "kind": "integration", "icon": "gcp_cloudrun" }, { "id": "s-rec", "label": "Recipe plane" }, { "id": "s-sch", "label": "Lane scheduler", "kind": "platform", "icon": "gcp_gke" }, { "id": "s-wrk", "label": "Encode worker" }, { "id": "s-gate", "label": "QC gate", "kind": "decision" }, { "id": "s-pack", "label": "Packager", "icon": "gcp_gke" }, { "id": "s-pub", "label": "Publisher", "kind": "integration" } ], "messages": [ { "from": "s-ops", "to": "s-api", "label": "submit (source, lane, deadline)", "kind": "call" }, { "from": "s-api", "to": "s-rec", "label": "analyse + solve ladder", "kind": "call" }, { "from": "s-rec", "to": "s-api", "label": "recipe digest", "kind": "return" }, { "from": "s-api", "to": "s-sch", "label": "admit job, check spend ceiling", "kind": "call" }, { "from": "s-sch", "to": "s-sch", "label": "order by deadline and work left", "kind": "self" }, { "from": "s-sch", "to": "s-wrk", "label": "chunk task + build digest", "kind": "async" }, { "from": "s-wrk", "to": "s-wrk", "label": "refuse on build mismatch", "kind": "self" }, { "from": "s-wrk", "to": "s-sch", "label": "reclaimed: one chunk lost", "kind": "error" }, { "from": "s-sch", "to": "s-wrk", "label": "reschedule same chunk", "kind": "async" }, { "from": "s-wrk", "to": "s-gate", "label": "stitched rendition", "kind": "call" }, { "from": "s-gate", "to": "s-gate", "label": "score, structure, decoder matrix", "kind": "self" }, { "from": "s-gate", "to": "s-rec", "label": "below floor: new recipe version", "kind": "error" }, { "from": "s-gate", "to": "s-pack", "label": "gate passed", "kind": "call" }, { "from": "s-pack", "to": "s-pack", "label": "segment, encrypt, build manifest", "kind": "self" }, { "from": "s-pack", "to": "s-pub", "label": "manifest candidate", "kind": "call" }, { "from": "s-pub", "to": "s-pub", "label": "verify every segment durable", "kind": "self" }, { "from": "s-pub", "to": "s-ops", "label": "published, atomically", "kind": "return" } ], "note": "Two of the seventeen messages are failures, and both are normal: a reclaim and a gate rejection.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "14-chunked-encode-and-reclaim", "title": "Chunked Encode and Reclaim — What Each Layer Does", "layout": "swimlane", "laneHeaderWidth": 164, "stages": [ "Split", "Dispatch", "Encode", "Reclaim", "Stitch", "Verify" ], "lanes": [ { "title": "Recipe plane", "cells": [ [ { "label": "Shot-boundary chunk plan", "kind": "app" } ], [], [ { "label": "Parameters per chunk", "kind": "store" } ], [], [], [ { "label": "Expected digest", "kind": "store" } ] ] }, { "title": "Control plane", "cells": [ [ { "label": "Materialise chunk tasks", "kind": "app" } ], [ { "label": "Lane capacity floor", "kind": "platform" }, { "label": "Pub/Sub dispatch", "kind": "queue" } ], [ { "label": "Claim with lease", "kind": "app" } ], [ { "label": "Lease expiry detected", "kind": "risk" }, { "label": "Requeue one chunk", "kind": "queue" } ], [ { "label": "All chunks present?", "kind": "decision" } ], [ { "label": "Monotonic progress", "kind": "platform" } ] ] }, { "title": "Encode fleet", "cells": [ [], [ { "label": "Spot worker starts", "kind": "app" } ], [ { "label": "Sandboxed decode", "kind": "security" }, { "label": "Deterministic encode", "kind": "app" } ], [ { "label": "Preemption signal", "kind": "risk" } ], [ { "label": "Stitcher joins chunks", "kind": "app" } ], [ { "label": "Boundary frames checked", "kind": "app" } ] ] }, { "title": "Storage", "cells": [ [ { "label": "Master range reads", "kind": "store" } ], [], [ { "label": "Chunk scratch, 48 h TTL", "kind": "store" } ], [ { "label": "Partial output discarded", "kind": "store" } ], [ { "label": "Elementary streams", "kind": "store" } ], [ { "label": "Scores recorded", "kind": "store" } ] ] }, { "title": "Cost", "cells": [ [ { "label": "Chunk size sets loss", "kind": "opportunity" } ], [ { "label": "Spot, per-second billed", "kind": "opportunity" } ], [], [ { "label": "≤1 chunk-minute lost", "kind": "opportunity" } ], [], [ { "label": "Cost per source hour", "kind": "opportunity" } ] ] } ], "note": "The reclaim column is the architecture: it is a column, not an exception handler.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" }, "canvas": { "width": 1820 } }, { "id": "15-on-demand-regeneration", "title": "On-Demand Regeneration — Serving a Rendition That Does Not Exist", "layout": "flow", "chain": true, "align": "middle", "nodeWidth": 204, "stages": [ { "title": "Request", "nodes": [ { "id": "r-player", "label": "Player", "kind": "external", "sub": "asks for a rung" }, { "id": "r-cdn", "label": "CDN edge", "kind": "external", "sub": "miss", "icon": "gcp_cloudcdn" } ] }, { "title": "Origin decision", "nodes": [ { "id": "r-orig", "label": "Packaging origin", "kind": "integration", "icon": "gcp_cloudrun" }, { "id": "r-res", "label": "Resident?", "kind": "decision" } ] }, { "title": "Admission", "nodes": [ { "id": "r-coal", "label": "Request coalescer", "sub": "one encode per digest", "kind": "queue" }, { "id": "r-budget", "label": "Budget check", "sub": "≤6% of encode spend", "kind": "platform" } ] }, { "title": "Generate", "nodes": [ { "id": "r-warm", "label": "Warm encoder pool", "sub": "GKE, reserved headroom", "icon": "gcp_gke" }, { "id": "r-recipe", "label": "Recipe lookup", "sub": "by digest", "kind": "store" }, { "id": "r-mast", "label": "Master range read", "sub": "archive restore", "kind": "store" } ] }, { "title": "Serve", "nodes": [ { "id": "r-seg", "label": "Segment written", "sub": "content-addressed", "kind": "store" }, { "id": "r-first", "label": "First byte", "sub": "p95 ≤ 1.5 s" } ] }, { "title": "Or degrade", "nodes": [ { "id": "r-near", "label": "Nearest resident rung", "kind": "risk", "sub": "budget exceeded" }, { "id": "r-tier", "label": "Demand tier promoted", "kind": "platform" } ] } ], "edges": [ { "from": "r-res", "to": "r-coal", "kind": "error" }, { "from": "r-budget", "to": "r-near", "label": "over budget", "kind": "error", "route": "gutter" }, { "from": "r-recipe", "to": "r-warm", "label": "parameters" }, { "from": "r-mast", "to": "r-warm", "label": "source range" }, { "from": "r-seg", "to": "r-tier", "kind": "async" } ], "note": "The fallback is the contract: a viewer is never held past the budget, they are served a lower rung and the tier is promoted.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "16-encoder-build-rollout", "title": "Encoder Build Rollout and Environments", "layout": "flow", "chain": true, "align": "top", "nodeWidth": 196, "stages": [ { "title": "Source", "nodes": [ { "id": "b-src", "label": "Encoder wrapper repo", "kind": "store" }, { "id": "b-up", "label": "Upstream encoder", "kind": "external", "sub": "open source release" } ] }, { "title": "Build", "nodes": [ { "id": "b-img", "label": "Reproducible image", "sub": "pinned digest" }, { "id": "b-sbom", "label": "SBOM + signature", "kind": "security" } ] }, { "title": "Determinism gate", "nodes": [ { "id": "b-det", "label": "Byte-identical replay", "sub": "golden corpus", "kind": "decision" }, { "id": "b-reg", "label": "Quality regression", "sub": "vs current build", "kind": "decision" } ] }, { "title": "Shadow", "nodes": [ { "id": "b-shadow", "label": "Shadow encode", "sub": "1% of new titles" }, { "id": "b-cmp", "label": "Score comparison" } ] }, { "title": "Canary", "nodes": [ { "id": "b-can", "label": "Canary lane", "sub": "one tenant, 48 h" }, { "id": "b-tele", "label": "Playback telemetry", "kind": "platform" } ] }, { "title": "Promote", "nodes": [ { "id": "b-reg2", "label": "Build registry", "sub": "approved digests", "kind": "store" }, { "id": "b-new", "label": "New recipes pin it", "sub": "old recipes unchanged" } ] } ], "edges": [ { "from": "b-det", "to": "b-src", "label": "non-deterministic: reject", "kind": "error", "route": "gutter" }, { "from": "b-tele", "to": "b-can", "label": "decode errors: halt", "kind": "error" }, { "from": "b-reg2", "to": "b-new", "label": "promotion" } ], "note": "A promoted build never re-encodes the catalogue. It is pinned by new recipes only; existing renditions keep their old digest until demand asks for a new one.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "17-deployment-architecture", "title": "Deployment Architecture", "layout": "nested", "canvas": { "width": 1740 }, "boxes": [ { "title": "Google Cloud — primary encode region (europe-west4)", "kind": "cloud", "dir": "col", "children": [ { "title": "Control plane — regional, multi-zone", "kind": "boundary", "dir": "row", "nodes": [ { "id": "p-run", "label": "Job & lineage API", "sub": "Cloud Run, 3 zones", "kind": "integration", "icon": "gcp_cloudrun" }, { "id": "p-gke", "label": "Scheduler & governor", "sub": "GKE regional", "kind": "platform", "icon": "gcp_gke" }, { "id": "p-span", "label": "Job state", "sub": "Spanner regional", "kind": "store" }, { "id": "p-ps", "label": "Task dispatch", "sub": "Pub/Sub", "kind": "queue", "icon": "gcp_pubsub" } ] }, { "title": "Batch encode fleet — zonal MIGs, Spot, no HA", "kind": "lane", "dir": "row", "children": [ { "title": "zone a", "kind": "plain", "nodes": [ { "id": "p-a", "label": "Spot workers", "sub": "~4,700 slots", "icon": "gcp_compute" } ] }, { "title": "zone b", "kind": "plain", "nodes": [ { "id": "p-b", "label": "Spot workers", "sub": "~4,700 slots", "icon": "gcp_compute" } ] }, { "title": "zone c", "kind": "plain", "nodes": [ { "id": "p-c", "label": "Spot workers", "sub": "~4,600 slots", "icon": "gcp_compute" } ] } ] }, { "title": "Latency-bound tier — reserved, multi-zone", "kind": "boundary", "dir": "row", "nodes": [ { "id": "p-warm", "label": "On-demand encoders", "sub": "GKE, warm pool", "icon": "gcp_gke" }, { "id": "p-orig", "label": "Packaging origin", "sub": "Cloud Run", "kind": "integration", "icon": "gcp_cloudrun" }, { "id": "p-mem", "label": "Manifest cache", "sub": "Memorystore", "kind": "store" } ] }, { "title": "Regional storage", "kind": "boundary", "dir": "row", "nodes": [ { "id": "p-rend", "label": "Rendition bucket", "sub": "regional + lifecycle", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "p-scratch", "label": "Chunk scratch", "sub": "48 h TTL", "kind": "store" } ] } ] }, { "title": "Dual-region custody (europe-west4 + europe-west1)", "kind": "trust", "dir": "row", "nodes": [ { "id": "p-mast", "label": "Master bucket", "sub": "dual-region archive, locked", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "p-recipe", "label": "Recipe store", "sub": "dual-region", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "p-aud", "label": "Audit log", "sub": "dual-region, append-only", "kind": "store" } ] }, { "title": "Standby encode region (europe-west1) — control plane warm, no fleet", "kind": "cloud", "dir": "row", "nodes": [ { "id": "p-sb", "label": "Control plane standby", "sub": "RTO 15 min", "kind": "platform" }, { "id": "p-sbo", "label": "Origin standby", "sub": "degraded rung set", "kind": "integration" } ] } ], "outside": [ { "id": "p-kms", "label": "Cloud KMS", "kind": "external", "sub": "multi-region", "icon": "gcp_secretmanager" }, { "id": "p-cdn", "label": "Media CDN", "kind": "external", "sub": "global anycast", "icon": "gcp_cloudcdn" }, { "id": "p-drm", "label": "DRM licence service", "kind": "external" } ], "edges": [ { "from": "p-mast", "to": "p-sb", "label": "regenerate from custody", "kind": "async", "route": "gutter" }, { "from": "p-orig", "to": "p-cdn", "label": "pull-through" }, { "from": "p-warm", "to": "p-rend", "label": "writes segments" } ], "note": "The standby region carries no encode fleet on purpose: recovery is regeneration from custody, not replication of 1.2 M rendition sets.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "18-observability", "title": "Observability — Signal by Pipeline Stage", "layout": "grid", "laneHeaderWidth": 152, "columns": [ "Intake", "Recipe", "Encode", "Gate", "Package", "Publish & serve" ], "rows": [ { "title": "Metrics", "cells": [ [ { "label": "Accept / reject rate", "kind": "platform" } ], [ { "label": "Analysis cost / hour", "kind": "platform" } ], [ { "label": "Chunk-seconds / s", "kind": "platform" }, { "label": "Reclaim rate", "kind": "platform" } ], [ { "label": "Pass rate by rung", "kind": "platform" } ], [ { "label": "Key fetch latency", "kind": "platform" } ], [ { "label": "Manifest p99", "kind": "platform" }, { "label": "JIT first-byte p95", "kind": "platform" } ] ] }, { "title": "Logs & audit", "cells": [ [ { "label": "Conformance verdicts", "kind": "store" } ], [ { "label": "Override actor + reason", "kind": "store" } ], [ { "label": "Build-mismatch refusals", "kind": "store" } ], [ { "label": "Waivers", "kind": "store" } ], [ { "label": "Key id, never key", "kind": "security" } ], [ { "label": "Swaps and rollbacks", "kind": "store" } ] ] }, { "title": "Traces", "cells": [ [ { "label": "Upload to accept" } ], [ { "label": "Analyse to digest" } ], [ { "label": "Task claim to output" } ], [ { "label": "Score fan-in" } ], [ { "label": "Segment to manifest" } ], [ { "label": "Request to first byte" } ] ] }, { "title": "SLOs", "cells": [ [ { "label": "Verdict within 1 h", "kind": "opportunity" } ], [], [ { "label": "p95 ≤ 0.80× duration", "kind": "opportunity" } ], [ { "label": "False pass ≤ 0.2%", "kind": "opportunity" } ], [], [ { "label": "Control plane 99.95%", "kind": "opportunity" }, { "label": "Rollback ≤ 120 s", "kind": "opportunity" } ] ] }, { "title": "Cost signals", "cells": [ [ { "label": "TB landed / day", "kind": "store" } ], [ { "label": "Compute vs egress saved", "kind": "decision" } ], [ { "label": "Cost per source hour", "kind": "store" } ], [ { "label": "Re-encode cost of failures", "kind": "store" } ], [], [ { "label": "Cost per delivered hour", "kind": "store" }, { "label": "JIT share of spend", "kind": "store" } ] ] }, { "title": "Alarms", "cells": [ [ { "label": "Supplier silent 24 h", "kind": "risk" } ], [], [ { "label": "Progress stalled", "kind": "risk" }, { "label": "Spend ceiling hit", "kind": "risk" } ], [ { "label": "Quarantine opened", "kind": "risk" } ], [ { "label": "Key service degraded", "kind": "risk" } ], [ { "label": "Nearest-rung fallback rate", "kind": "risk" } ] ] } ], "note": "Cost is a signal row, not a monthly report: at 40 PB/day of egress the ladder decision is an operational control.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" }, "canvas": { "width": 1840 } }, { "id": "19-rendition-lifecycle", "title": "Rendition Lifecycle — The Loop That Closes", "layout": "cycle", "centre": { "label": "Rendition as cache" }, "rx": 440, "ry": 218, "nodes": [ { "label": "Planned", "sub": "a rung in a recipe", "id": "planned" }, { "label": "Generated", "sub": "batch or on demand", "id": "generated" }, { "label": "Gate-passed", "sub": "scored, structural, decoder", "kind": "decision", "id": "passed" }, { "label": "Published", "sub": "in a manifest revision", "kind": "integration", "id": "published" }, { "label": "Hot", "sub": "top demand tier", "kind": "store", "id": "hot", "icon": "gcp_cloudstorage" }, { "label": "Demoted", "sub": "nearline / coldline", "kind": "store", "id": "demoted", "icon": "gcp_cloudstorage" }, { "label": "Evicted", "sub": "regeneration beats retention", "kind": "risk", "id": "evicted" } ], "ringLabels": [ "digest", "scores", "swap", "demand", "decay", "horizon", "demand returns" ], "note": "The ring closes because eviction is not an end state: a request for an evicted digest re-enters at Generated.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "20-security-trust-zones", "title": "Security and Trust Zones", "layout": "zones", "zones": [ { "title": "Public internet", "kind": "trust", "nodes": [ { "id": "t-player", "label": "Players", "kind": "external" }, { "id": "t-cdn", "label": "Media CDN", "kind": "external", "icon": "gcp_cloudcdn" }, { "id": "t-sup", "label": "Rights holders", "kind": "external" } ] }, { "title": "Perimeter — authenticated, no content", "kind": "trust", "nodes": [ { "id": "t-up", "label": "Signed upload endpoint", "kind": "integration" }, { "id": "t-orig", "label": "Packaging origin", "sub": "encrypted segments only", "kind": "integration", "icon": "gcp_cloudrun" }, { "id": "t-api", "label": "Job & lineage API", "sub": "workload identity", "kind": "integration", "icon": "gcp_cloudrun" } ] }, { "title": "Untrusted execution — the fleet decodes hostile input", "kind": "trust", "nodes": [ { "id": "t-sand", "label": "Decoder sandbox", "sub": "gVisor, no egress", "kind": "security" }, { "id": "t-wrk", "label": "Encode workers", "sub": "read one prefix, write one", "icon": "gcp_compute" }, { "id": "t-scr", "label": "Chunk scratch", "sub": "48 h TTL", "kind": "store" } ] }, { "title": "Trusted platform", "kind": "trust", "nodes": [ { "id": "t-sch", "label": "Scheduler & governor", "kind": "platform", "icon": "gcp_gke" }, { "id": "t-gate", "label": "Gate decision service", "kind": "decision" }, { "id": "t-pack", "label": "Packager", "kind": "app", "icon": "gcp_gke" }, { "id": "t-pub", "label": "Manifest publisher", "kind": "integration", "icon": "gcp_cloudrun" } ] }, { "title": "Custody — highest sensitivity", "kind": "trust", "nodes": [ { "id": "t-mast", "label": "Pre-release masters", "sub": "CMEK, per-object audit", "kind": "store" }, { "id": "t-kms", "label": "Key broker + KMS", "sub": "keys never persisted here", "kind": "security", "icon": "gcp_secretmanager" }, { "id": "t-aud", "label": "Audit log", "sub": "append-only", "kind": "store" }, { "id": "t-pre", "label": "Pre-release origin", "sub": "not publicly routable", "kind": "security" } ] } ], "edges": [ { "from": "t-sup", "to": "t-up", "label": "mTLS + signed URL" }, { "from": "t-up", "to": "t-mast", "label": "write-once" }, { "from": "t-wrk", "to": "t-mast", "label": "range read" }, { "from": "t-pack", "to": "t-kms", "label": "key per job, in memory" }, { "from": "t-pub", "to": "t-orig", "label": "manifest pointer" }, { "from": "t-orig", "to": "t-cdn", "label": "ciphertext only" }, { "from": "t-cdn", "to": "t-player", "label": "HTTPS" } ], "note": "The fleet sits in its own zone because a codec parser is the assumed attack surface, not the API. Two intra-zone writes are omitted: sandbox to scratch, and gate to the pre-release origin.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "21-key-and-licence-flow", "title": "Identity, Keys and Licence Flow", "layout": "sequence", "lifelines": [ { "id": "k-pack", "label": "Packager", "kind": "app", "icon": "gcp_gke" }, { "id": "k-id", "label": "Workload identity", "kind": "security" }, { "id": "k-brk", "label": "Key broker", "kind": "security", "icon": "gcp_secretmanager" }, { "id": "k-kms", "label": "Cloud KMS", "kind": "external", "icon": "gcp_secretmanager" }, { "id": "k-rend", "label": "Rendition store", "kind": "store", "icon": "gcp_cloudstorage" }, { "id": "k-play", "label": "Player", "kind": "external" }, { "id": "k-drm", "label": "DRM licence service", "kind": "external" } ], "messages": [ { "from": "k-pack", "to": "k-id", "label": "attest workload", "kind": "call" }, { "from": "k-id", "to": "k-pack", "label": "short-lived token", "kind": "return" }, { "from": "k-pack", "to": "k-brk", "label": "request key for (title, territory)", "kind": "call" }, { "from": "k-brk", "to": "k-kms", "label": "generate + wrap content key", "kind": "call" }, { "from": "k-kms", "to": "k-brk", "label": "key + key_id", "kind": "return" }, { "from": "k-brk", "to": "k-pack", "label": "key in memory, key_id to record", "kind": "return" }, { "from": "k-pack", "to": "k-pack", "label": "encrypt once, common encryption", "kind": "self" }, { "from": "k-pack", "to": "k-rend", "label": "segments + key_id reference", "kind": "call" }, { "from": "k-pack", "to": "k-drm", "label": "register key_id and policy", "kind": "async" }, { "from": "k-play", "to": "k-rend", "label": "fetch segment (via CDN)", "kind": "call" }, { "from": "k-play", "to": "k-drm", "label": "licence request for key_id", "kind": "call" }, { "from": "k-drm", "to": "k-play", "label": "licence, or refusal on revoked key", "kind": "return" }, { "from": "k-brk", "to": "k-drm", "label": "takedown: revoke key_id", "kind": "error" } ], "note": "One encrypted segment set serves three DRM systems, and the pipeline never holds a key past the packaging job.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } }, { "id": "22-failure-classes", "title": "Failure Classes and Their Handling", "layout": "grid", "laneHeaderWidth": 196, "columns": [ "Assumed failure", "Detected by", "Immediate handling", "Residual exposure" ], "rows": [ { "title": "Capacity reclaim", "cells": [ [ { "label": "60% of fleet in 10 min", "kind": "risk" } ], [ { "label": "Lease expiry", "kind": "platform" } ], [ { "label": "Requeue one chunk", "kind": "opportunity" } ], [ { "label": "None — expected state", "kind": "opportunity" } ] ] }, { "title": "Poison source", "cells": [ [ { "label": "Decoder hangs or crashes", "kind": "risk" } ], [ { "label": "Chunk timeout + memory cap", "kind": "platform" } ], [ { "label": "Quarantine with diagnosis", "kind": "app" } ], [ { "label": "Title misses its deadline", "kind": "risk" } ] ] }, { "title": "Encoder build drift", "cells": [ [ { "label": "Worker ≠ recipe build", "kind": "risk" } ], [ { "label": "Admission check", "kind": "decision" } ], [ { "label": "Worker refuses task", "kind": "opportunity" } ], [ { "label": "Throughput dip on rollout", "kind": "risk" } ] ] }, { "title": "Stitch defect", "cells": [ [ { "label": "Visible pumping at a join", "kind": "risk" } ], [ { "label": "Boundary verification", "kind": "app" } ], [ { "label": "Re-encode chunk pair", "kind": "app" } ], [ { "label": "Subtle artefacts below threshold", "kind": "risk" } ] ] }, { "title": "Gate false pass", "cells": [ [ { "label": "Scores well, plays badly", "kind": "risk" } ], [ { "label": "Playback telemetry", "kind": "platform" } ], [ { "label": "Quarantine + manifest revision", "kind": "integration" } ], [ { "label": "≤0.2% of titles per quarter", "kind": "risk" } ] ] }, { "title": "Key service outage", "cells": [ [ { "label": "KMS or DRM unavailable", "kind": "risk" } ], [ { "label": "Key fetch latency alarm", "kind": "platform" } ], [ { "label": "Stall publish, retry", "kind": "queue" } ], [ { "label": "Premiere slips; never unencrypted", "kind": "security" } ] ] }, { "title": "Partial publish", "cells": [ [ { "label": "Some renditions absent", "kind": "risk" } ], [ { "label": "Durability pre-check", "kind": "decision" } ], [ { "label": "Publish narrower manifest", "kind": "integration" } ], [ { "label": "Lower top rung on day one", "kind": "risk" } ] ] }, { "title": "Cold-title demand spike", "cells": [ [ { "label": "Evicted rungs, sudden load", "kind": "risk" } ], [ { "label": "Miss rate by title", "kind": "platform" } ], [ { "label": "Coalesce + promote tier", "kind": "platform" } ], [ { "label": "First viewers get a lower rung", "kind": "risk" } ] ] }, { "title": "Region loss", "cells": [ [ { "label": "Encode region unavailable", "kind": "risk" } ], [ { "label": "Control-plane health", "kind": "platform" } ], [ { "label": "Fail over, regenerate", "kind": "app" } ], [ { "label": "Degraded rung set until refilled", "kind": "risk" } ] ] }, { "title": "Cost runaway", "cells": [ [ { "label": "Retry loop or bad recipe", "kind": "risk" } ], [ { "label": "Spend ceiling at admission", "kind": "decision" } ], [ { "label": "Hard stop, not an alert", "kind": "security" } ], [ { "label": "Blocked job needs a human", "kind": "actor" } ] ] } ], "note": "The residual column is the honest one: every row leaves something, and in eight of ten rows what it leaves is quality or lateness, never protection.", "meta": { "v": "1.0", "owner": "Media Platform Architecture" } } ]