import { defineSchema, type PromptContext } from "@json-render/core"; /** * Prompt template for Remotion timeline generation * * Uses JSONL patch format (same as React) but builds up a timeline spec structure. */ function remotionPromptTemplate(context: PromptContext): string { const { catalog, options } = context; const { system = "You are a video timeline generator.", customRules = [] } = options; const lines: string[] = []; lines.push(system); lines.push(""); // Output format - JSONL patches lines.push("OUTPUT FORMAT:"); lines.push( "Output JSONL (one JSON object per line) with patches to build a timeline spec.", ); lines.push( "Each line is a JSON patch operation. Build the timeline incrementally.", ); lines.push(""); lines.push("Example output (each line is a separate JSON object):"); lines.push(""); lines.push(`{"op":"add","path":"/composition","value":{"id":"intro","fps":30,"width":1920,"height":1080,"durationInFrames":300}} {"op":"add","path":"/tracks","value":[{"id":"main","name":"Main","type":"video","enabled":true},{"id":"overlay","name":"Overlay","type":"overlay","enabled":true}]} {"op":"add","path":"/clips","value":[]} {"op":"add","path":"/clips/-","value":{"id":"clip-1","trackId":"main","component":"TitleCard","props":{"title":"Welcome","subtitle":"Getting Started"},"from":0,"durationInFrames":90,"transitionIn":{"type":"fade","durationInFrames":15},"transitionOut":{"type":"fade","durationInFrames":15},"motion":{"enter":{"opacity":0,"y":50,"scale":0.9,"duration":25},"spring":{"damping":15}}}} {"op":"add","path":"/clips/-","value":{"id":"clip-2","trackId":"main","component":"TitleCard","props":{"title":"Features"},"from":90,"durationInFrames":90,"motion":{"enter":{"opacity":0,"x":-100,"duration":20},"exit":{"opacity":0,"x":100,"duration":15}}}} {"op":"add","path":"/audio","value":{"tracks":[]}}`); lines.push(""); // Components const catalogData = catalog as { components?: Record< string, { description?: string; defaultDuration?: number } >; transitions?: Record; effects?: Record; }; if (catalogData.components) { lines.push( `AVAILABLE COMPONENTS (${Object.keys(catalogData.components).length}):`, ); lines.push(""); for (const [name, def] of Object.entries(catalogData.components)) { const duration = def.defaultDuration ? ` [default: ${def.defaultDuration} frames]` : ""; lines.push( `- ${name}: ${def.description || "No description"}${duration}`, ); } lines.push(""); } // Transitions if ( catalogData.transitions && Object.keys(catalogData.transitions).length > 0 ) { lines.push("AVAILABLE TRANSITIONS:"); lines.push(""); for (const [name, def] of Object.entries(catalogData.transitions)) { lines.push(`- ${name}: ${def.description || "No description"}`); } lines.push(""); } // Motion system documentation lines.push("MOTION SYSTEM:"); lines.push( "Clips can have a 'motion' field for declarative animations (optional, use for dynamic/engaging videos):", ); lines.push(""); lines.push( "- enter: {opacity?, scale?, x?, y?, rotate?, duration?} - animate FROM these values TO normal when clip starts", ); lines.push( "- exit: {opacity?, scale?, x?, y?, rotate?, duration?} - animate FROM normal TO these values when clip ends", ); lines.push( "- spring: {damping?, stiffness?, mass?} - physics config (lower damping = more bounce)", ); lines.push( '- loop: {property, from, to, duration, easing?} - continuous animation (property: "scale"|"rotate"|"x"|"y"|"opacity")', ); lines.push(""); lines.push("Example motion configs:"); lines.push(' Fade up: {"enter":{"opacity":0,"y":30,"duration":20}}'); lines.push( ' Scale pop: {"enter":{"scale":0.5,"opacity":0,"duration":15},"spring":{"damping":10}}', ); lines.push( ' Slide in/out: {"enter":{"x":-100,"duration":20},"exit":{"x":100,"duration":15}}', ); lines.push( ' Gentle pulse: {"loop":{"property":"scale","from":1,"to":1.05,"duration":60,"easing":"ease"}}', ); lines.push(""); // Rules lines.push("RULES:"); const baseRules = [ "Output ONLY JSONL patches - one JSON object per line, no markdown, no code fences", 'First add /composition with {id, fps:30, width:1920, height:1080, durationInFrames}: {"op":"add","path":"/composition","value":{...}}', 'Then add /tracks array with video/overlay tracks: {"op":"add","path":"/tracks","value":[...]}', 'Then add each clip by appending to the array: {"op":"add","path":"/clips/-","value":{...}}', 'Finally add /audio with {tracks:[]}: {"op":"add","path":"/audio","value":{...}}', "ONLY use components listed above", "fps is always 30 (1 second = 30 frames, 10 seconds = 300 frames)", 'Clips on "main" track flow sequentially (from = previous clip\'s from + durationInFrames)', 'Overlay clips (LowerThird, TextOverlay) go on "overlay" track', "Use motion.enter for engaging clip entrances, motion.exit for smooth departures", "Spring damping: 20=smooth, 10=bouncy, 5=very bouncy", ]; const allRules = [...baseRules, ...customRules]; allRules.forEach((rule, i) => { lines.push(`${i + 1}. ${rule}`); }); return lines.join("\n"); } /** * The schema for @json-render/remotion * * This schema is fundamentally different from the React element tree schema. * It's timeline-based, designed for video composition: * * - Spec: A composition with tracks containing timed clips * - Catalog: Video components (scenes, overlays, etc.) and effects * * This demonstrates that json-render is truly agnostic - different renderers * can have completely different spec formats. */ export const schema = defineSchema( (s) => ({ // What the AI-generated SPEC looks like (timeline-based) spec: s.object({ /** Composition settings */ composition: s.object({ /** Unique composition ID */ id: s.string(), /** Frames per second */ fps: s.number(), /** Width in pixels */ width: s.number(), /** Height in pixels */ height: s.number(), /** Total duration in frames */ durationInFrames: s.number(), }), /** Timeline tracks (like layers in video editing) */ tracks: s.array( s.object({ /** Unique track ID */ id: s.string(), /** Track name for organization */ name: s.string(), /** Track type: "video" | "audio" | "overlay" | "text" */ type: s.string(), /** Whether track is muted/hidden */ enabled: s.boolean(), }), ), /** Clips placed on the timeline */ clips: s.array( s.object({ /** Unique clip ID */ id: s.string(), /** Which track this clip belongs to */ trackId: s.string(), /** Component type from catalog */ component: s.ref("catalog.components"), /** Component props */ props: s.propsOf("catalog.components"), /** Start frame (when clip begins) */ from: s.number(), /** Duration in frames */ durationInFrames: s.number(), /** Transition in effect */ transitionIn: s.object({ type: s.ref("catalog.transitions"), durationInFrames: s.number(), }), /** Transition out effect */ transitionOut: s.object({ type: s.ref("catalog.transitions"), durationInFrames: s.number(), }), /** Declarative motion configuration for custom animations */ motion: s.object({ /** Enter animation - animates FROM these values TO neutral */ enter: s.object({ /** Starting opacity (0-1), animates to 1 */ opacity: s.number(), /** Starting scale (e.g., 0.8 = 80%), animates to 1 */ scale: s.number(), /** Starting X offset in pixels, animates to 0 */ x: s.number(), /** Starting Y offset in pixels, animates to 0 */ y: s.number(), /** Starting rotation in degrees, animates to 0 */ rotate: s.number(), /** Duration of enter animation in frames (default: 20) */ duration: s.number(), }), /** Exit animation - animates FROM neutral TO these values */ exit: s.object({ /** Ending opacity (0-1), animates from 1 */ opacity: s.number(), /** Ending scale, animates from 1 */ scale: s.number(), /** Ending X offset in pixels, animates from 0 */ x: s.number(), /** Ending Y offset in pixels, animates from 0 */ y: s.number(), /** Ending rotation in degrees, animates from 0 */ rotate: s.number(), /** Duration of exit animation in frames (default: 20) */ duration: s.number(), }), /** Spring physics configuration */ spring: s.object({ /** Damping coefficient (default: 20) */ damping: s.number(), /** Stiffness (default: 100) */ stiffness: s.number(), /** Mass (default: 1) */ mass: s.number(), }), /** Continuous looping animation */ loop: s.object({ /** Property to animate: "scale" | "rotate" | "x" | "y" | "opacity" */ property: s.string(), /** Starting value */ from: s.number(), /** Ending value */ to: s.number(), /** Duration of one cycle in frames */ duration: s.number(), /** Easing type: "linear" | "ease" | "spring" (default: "ease") */ easing: s.string(), }), }), }), ), /** Audio configuration */ audio: s.object({ /** Background music/audio clips */ tracks: s.array( s.object({ id: s.string(), src: s.string(), from: s.number(), durationInFrames: s.number(), volume: s.number(), }), ), }), }), // What the CATALOG must provide catalog: s.object({ /** Video component definitions (scenes, overlays, etc.) */ components: s.map({ /** Zod schema for component props */ props: s.zod(), /** Component type: "scene" | "overlay" | "text" | "image" | "video" */ type: s.string(), /** Default duration in frames (can be overridden per clip) */ defaultDuration: s.number(), /** Description for AI generation hints */ description: s.string(), }), /** Transition effect definitions */ transitions: s.map({ /** Default duration in frames */ defaultDuration: s.number(), /** Description for AI generation hints */ description: s.string(), }), /** Effect definitions (filters, animations, etc.) */ effects: s.map({ /** Zod schema for effect params */ params: s.zod(), /** Description for AI generation hints */ description: s.string(), }), }), }), { promptTemplate: remotionPromptTemplate, }, ); /** * Type for the Remotion schema */ export type RemotionSchema = typeof schema; /** * Infer the spec type from a catalog */ export type RemotionSpec = typeof schema extends { createCatalog: (catalog: TCatalog) => { _specType: infer S }; } ? S : never;