diff --git a/packages/producer/README.md b/packages/producer/README.md index 8e5bc6f48..c77691732 100644 --- a/packages/producer/README.md +++ b/packages/producer/README.md @@ -106,6 +106,38 @@ This is not chroma keying. There is no green/blue background to remove and no "k Don't paint a fullscreen background in your HTML. The default body background is overridden to transparent automatically — any `body { background: ... }`, `#root { background: ... }`, or `[data-composition-id] { background: ... }` rule is force-overridden during alpha rendering. Backgrounds on inner elements (cards, scenes, components) are kept. +## Distributed rendering + +For renders too large for a single machine, the producer ships a public set of distributed-render primitives. They are pure functions over local file paths — networking and orchestration live in adapter packages (Temporal, AWS Lambda + Step Functions, Cloud Run Jobs, K8s Jobs). + +```typescript +import { plan, renderChunk, assemble } from "@hyperframes/producer/distributed"; + +// Controller-side: produce a self-contained planDir + content-addressed planHash. +const planResult = await plan( + projectDir, + { fps: 30, width: 1920, height: 1080, format: "mp4" }, + "/tmp/plan", +); + +// Worker-side: render one chunk. Byte-identical retries on the same +// `(planDir, chunkIndex)` — Temporal / Step Functions retry policies are safe +// to point at this. +const chunk = await renderChunk("/tmp/plan", 0, "/tmp/chunks/0.mp4"); + +// Controller-side: stitch chunks into the final deliverable. +await assemble( + "/tmp/plan", + ["/tmp/chunks/0.mp4", "/tmp/chunks/1.mp4"], + "/tmp/plan/audio.aac", + "/tmp/output.mp4", +); +``` + +The three activity functions plus their result types are also re-exported from `@hyperframes/producer` so callers that pin the main package don't need a separate subpath import. Supported formats: `mp4` SDR, `mov` ProRes 4444, and `png-sequence`. webm and HDR mp4 trip a typed `FormatNotSupportedInDistributedError` — use the in-process renderer (`executeRenderJob`) for those. + +See [`DISTRIBUTED-RENDERING-PLAN.md`](../../DISTRIBUTED-RENDERING-PLAN.md) for the full architecture. + ## How it works 1. **Serve** — spins up a local file server for the HTML composition diff --git a/packages/producer/build.mjs b/packages/producer/build.mjs index 7b205f140..b2f4d4fb8 100644 --- a/packages/producer/build.mjs +++ b/packages/producer/build.mjs @@ -93,6 +93,23 @@ await Promise.all([ entryPoints: ["src/services/shaderTransitionWorker.ts"], outfile: "dist/services/shaderTransitionWorker.js", }), + // `@hyperframes/producer/distributed` subpath — the public distributed + // render primitives (plan / renderChunk / assemble). Bundled as a + // separate entry so adopters that don't need the in-process renderer + // (Lambda chunk workers, CDK constructs, thin orchestrators) can import + // only this surface and skip the rest of the producer's dependency tree. + build({ + bundle: true, + platform: "node", + target: "node22", + format: "esm", + external: ["puppeteer", "esbuild", "postcss"], + plugins: [workspaceAliasPlugin], + minify: false, + sourcemap: true, + entryPoints: ["src/distributed.ts"], + outfile: "dist/distributed.js", + }), ]); // Copy core runtime artifacts so the producer can find them at dist/ diff --git a/packages/producer/package.json b/packages/producer/package.json index abf019f44..b93b35ebe 100644 --- a/packages/producer/package.json +++ b/packages/producer/package.json @@ -20,6 +20,10 @@ }, "./server": { "import": "./dist/public-server.js" + }, + "./distributed": { + "import": "./dist/distributed.js", + "types": "./dist/distributed.d.ts" } }, "publishConfig": { diff --git a/packages/producer/src/distributed.ts b/packages/producer/src/distributed.ts new file mode 100644 index 000000000..8a402187d --- /dev/null +++ b/packages/producer/src/distributed.ts @@ -0,0 +1,78 @@ +/** + * `@hyperframes/producer/distributed` — the distributed render primitives. + * + * See `DISTRIBUTED-RENDERING-PLAN.md` for the full architecture. The three + * activities (`plan` → `renderChunk` × N → `assemble`) are pure functions + * over local file paths; networking + orchestration live in adapters. + * + * Adopters (AWS Lambda, Cloud Run Jobs, Temporal, K8s Jobs, plain SSH): + * + * ```ts + * import { + * plan, + * renderChunk, + * assemble, + * } from "@hyperframes/producer/distributed"; + * + * // Controller-side: produce a self-contained planDir + content-addressed planHash. + * const planResult = await plan(projectDir, config, planDir); + * + * // Worker-side: render one chunk. Byte-identical retries on the same + * // (planDir, chunkIndex) — Temporal / Step Functions retry policies are + * // safe to point at this. + * const chunk = await renderChunk(planDir, chunkIndex, outputChunkPath); + * + * // Controller-side: stitch chunks into the final deliverable. + * await assemble(planDir, chunkPaths, audioPath, outputPath); + * ``` + * + * No networking, no AWS SDK, no Temporal SDK — those live in adapter + * packages. This module is library code only. + */ + +// ── Plan (Activity A) ─────────────────────────────────────────────────────── +export { + // Functions + buildChunkSlices, + measurePlanDirBytes, + plan, + rejectUnsupportedDistributedFormat, + resolveChunkPlan, + // Types + type DistributedRenderConfig, + type PlanResult, + // Constants + DEFAULT_CHUNK_SIZE, + DEFAULT_MAX_PARALLEL_CHUNKS, + PLAN_DIR_SIZE_LIMIT_BYTES, + // Error codes + classes + FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED, + FormatNotSupportedInDistributedError, + PLAN_TOO_LARGE, + PlanTooLargeError, +} from "./services/distributed/plan.js"; + +// ── RenderChunk (Activity B) ──────────────────────────────────────────────── +export { + applyRuntimeEnvSnapshot, + readWebGlVendorInfoFromCanvas, + renderChunk, + // Types + type ChunkResult, + // Error codes + classes + FFMPEG_VERSION_MISMATCH, + PLAN_HASH_MISMATCH, + RenderChunkValidationError, +} from "./services/distributed/renderChunk.js"; + +// ── Assemble (Activity C) ─────────────────────────────────────────────────── +export { assemble, type AssembleResult } from "./services/distributed/assemble.js"; + +// ── Plan-time shared types from `freezePlan` ─────────────────────────────── +// Re-exported so adopters that deserialize a planDir's `meta/encoder.json` +// or `meta/chunks.json` see the same shapes the producer wrote them as. +export type { + ChunkSliceJson, + CompositionMetadataJson, + LockedRenderConfig, +} from "./services/render/stages/freezePlan.js"; diff --git a/packages/producer/src/index.ts b/packages/producer/src/index.ts index 91eb8cb82..692931e86 100644 --- a/packages/producer/src/index.ts +++ b/packages/producer/src/index.ts @@ -75,3 +75,18 @@ export { runHyperframeLint, type PreparedHyperframeLintInput, } from "./services/hyperframeLint.js"; + +// ── Distributed render primitives ─────────────────────────────────────────── +// The full surface lives at `@hyperframes/producer/distributed`; we +// additionally re-export the three activity functions + their result +// types here so callers that pin `@hyperframes/producer` don't need a +// separate subpath import. +export { + assemble, + plan, + renderChunk, + type AssembleResult, + type ChunkResult, + type DistributedRenderConfig, + type PlanResult, +} from "./distributed.js"; diff --git a/packages/producer/src/services/distributed/publicExports.test.ts b/packages/producer/src/services/distributed/publicExports.test.ts new file mode 100644 index 000000000..08ea9d350 --- /dev/null +++ b/packages/producer/src/services/distributed/publicExports.test.ts @@ -0,0 +1,79 @@ +/** + * Unit tests for the public-export surface of the distributed primitives. + * + * Two import paths must work for adopters: + * + * 1. `import { plan, renderChunk, assemble } from "@hyperframes/producer"` + * — the canonical package entry. Includes the three activity functions + * and their result types. + * + * 2. `import { plan, renderChunk, assemble } from "@hyperframes/producer/distributed"` + * — the focused subpath, suitable for Lambda chunk-runner images that + * don't pull in the in-process renderer's transitive deps. + * + * These tests are surface-only: they assert the symbols exist and have the + * expected shapes. The functional behaviour is covered by `plan.test.ts` / + * `renderChunk.test.ts` / `assemble.test.ts`. + * + * We import via the workspace-relative `../../distributed.js` / + * `../../index.js` paths rather than `"@hyperframes/producer"` because the + * package resolver inside the workspace points back at `src/index.ts` — + * either form exercises the same surface. + */ + +import { describe, expect, it } from "bun:test"; +import * as distributedSubpath from "../../distributed.js"; +import * as producerIndex from "../../index.js"; + +describe("@hyperframes/producer/distributed (subpath)", () => { + it("exports the three activity functions", () => { + expect(typeof distributedSubpath.plan).toBe("function"); + expect(typeof distributedSubpath.renderChunk).toBe("function"); + expect(typeof distributedSubpath.assemble).toBe("function"); + }); + + it("exports the chunking helpers + constants", () => { + expect(typeof distributedSubpath.resolveChunkPlan).toBe("function"); + expect(typeof distributedSubpath.buildChunkSlices).toBe("function"); + expect(typeof distributedSubpath.measurePlanDirBytes).toBe("function"); + expect(distributedSubpath.DEFAULT_CHUNK_SIZE).toBe(240); + expect(distributedSubpath.DEFAULT_MAX_PARALLEL_CHUNKS).toBe(16); + expect(distributedSubpath.PLAN_DIR_SIZE_LIMIT_BYTES).toBe(2 * 1024 * 1024 * 1024); + }); + + it("exports the non-retryable error codes + classes", () => { + expect(distributedSubpath.PLAN_TOO_LARGE).toBe("PLAN_TOO_LARGE"); + expect(distributedSubpath.FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED).toBe( + "FORMAT_NOT_SUPPORTED_IN_DISTRIBUTED", + ); + expect(distributedSubpath.FFMPEG_VERSION_MISMATCH).toBe("FFMPEG_VERSION_MISMATCH"); + expect(distributedSubpath.PLAN_HASH_MISMATCH).toBe("PLAN_HASH_MISMATCH"); + + expect(typeof distributedSubpath.PlanTooLargeError).toBe("function"); + expect(typeof distributedSubpath.FormatNotSupportedInDistributedError).toBe("function"); + expect(typeof distributedSubpath.RenderChunkValidationError).toBe("function"); + }); + + it("exports the input-validation helpers", () => { + expect(typeof distributedSubpath.rejectUnsupportedDistributedFormat).toBe("function"); + expect(typeof distributedSubpath.applyRuntimeEnvSnapshot).toBe("function"); + expect(typeof distributedSubpath.readWebGlVendorInfoFromCanvas).toBe("function"); + }); +}); + +describe("@hyperframes/producer (main entry)", () => { + it("re-exports the three activity functions", () => { + expect(typeof producerIndex.plan).toBe("function"); + expect(typeof producerIndex.renderChunk).toBe("function"); + expect(typeof producerIndex.assemble).toBe("function"); + }); + + it("preserves the existing in-process exports (executeRenderJob unchanged)", () => { + // The distributed primitives must NOT break the in-process surface; + // spot-check the load-bearing exports the in-process callers rely on. + expect(typeof producerIndex.executeRenderJob).toBe("function"); + expect(typeof producerIndex.createRenderJob).toBe("function"); + expect(typeof producerIndex.createCaptureSession).toBe("function"); + expect(typeof producerIndex.createFileServer).toBe("function"); + }); +});