@
Build backend images / build content-svc (push) Failing after 19s
Build backend images / build file-svc (push) Failing after 1m53s
Build backend images / build gateway (push) Failing after 16s
Build backend images / build identity-svc (push) Failing after 7m1s
Build backend images / build notification-svc (push) Failing after 7m24s
Build backend images / build render-svc (push) Failing after 3m12s
Build backend images / build studio-svc (push) Failing after 43s

feat: AE template scanner + scene editor + AEP bundle pipeline

Scene editor (admin): per-project Scenes / Shared Colors / Color Presets
manager (ProjectScenes) reachable from each project.

AEP bundle pipeline: upload .aep or .zip → stored once per template at
templates/{project_id}/(bundle.zip|template.aep); render claim probes and
returns is_bundle+md5; node-agent extracts the bundle, locates the .aep
(zip-slip guarded), and caches by md5 so repeated renders extract once.

AE template scanner ("read scenes/colours/configs from the AEP"):
- content-svc importer: POST /v1/projects/{id}/scan/{preview,apply} —
  review-diff-then-merge into scenes/elements/colours (manual edits kept).
- render-svc Go quick-scan: stdlib RIFX parser extracts comp names+durations
  (no AE) → POST /v1/template-scans/{id}/quick.
- render-svc AE scan jobs + node-agent runner: queue → node runs scan.jsx
  (reverse of legacy JSXGenerator conventions: frfinal/frshare/frl_/frd_) →
  posts ScanResult back. Migration 26_render_scan_jobs.
- admin UI: "اسکن از افترافکت" with quick/full engines + diff-review modal.

Verified: importer preview/apply, Go quick-scan end-to-end (synthetic .aep →
scene imported), bundle extract unit tests, RIFX parser unit tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
@
This commit is contained in:
soroush.asadi
2026-06-04 10:39:45 +03:30
parent 264fccf21f
commit 1ff6e494c0
26 changed files with 2691 additions and 27 deletions
@@ -0,0 +1,275 @@
using FlatRender.ContentSvc.Domain.Entities;
using FlatRender.ContentSvc.Domain.Enums;
using FlatRender.ContentSvc.Infrastructure.Data;
using FlatRender.ContentSvc.Models;
using Microsoft.EntityFrameworkCore;
namespace FlatRender.ContentSvc.Application.Services;
/// <summary>
/// Imports a scanned After Effects project structure (scenes / frl_ frd_ elements /
/// frd_ &amp; frshare colours) into a project. Supports a dry-run <see cref="PreviewAsync"/>
/// that reports the diff without writing, and an <see cref="ApplyAsync"/> that merges:
/// matched items (by key) are refreshed, new items added, manual edits preserved.
/// </summary>
public class AepImportService(ContentDbContext db)
{
public Task<ImportDiff> PreviewAsync(Guid projectId, ScanResult scan) => RunAsync(projectId, scan, null);
public Task<ImportDiff> ApplyAsync(Guid projectId, ScanResult scan, ScanApplyOptions options) =>
RunAsync(projectId, scan, options);
private async Task<ImportDiff> RunAsync(Guid projectId, ScanResult scan, ScanApplyOptions? apply)
{
var existing = await db.Scenes
.Where(s => s.ProjectId == projectId && s.DeletedAt == null)
.Include(s => s.ContentElements)
.Include(s => s.ColorElements)
.ToListAsync();
var existingShared = await db.SharedColors.Where(c => c.ProjectId == projectId).ToListAsync();
var scanScenes = scan.Scenes ?? new List<ScanScene>();
var byKey = existing.ToDictionary(s => s.Key, StringComparer.Ordinal);
var scanKeys = new HashSet<string>(scanScenes.Select(s => s.Key), StringComparer.Ordinal);
int sAdded = 0, sChanged = 0, sUnchanged = 0;
var diffs = new List<SceneDiff>();
int sortBase = existing.Count;
foreach (var ss in scanScenes)
{
byKey.TryGetValue(ss.Key, out var scene);
bool isNew = scene is null;
var exElems = scene?.ContentElements.ToList() ?? new List<SceneContentElement>();
var exColors = scene?.ColorElements.ToList() ?? new List<SceneColorElement>();
var scElems = ss.Elements ?? new List<ScanElement>();
var scColors = ss.Colors ?? new List<ScanColor>();
var exElemByKey = exElems.ToDictionary(e => e.Key, StringComparer.Ordinal);
var exColorByKey = exColors.ToDictionary(c => c.ElementKey, StringComparer.Ordinal);
var scElemKeys = new HashSet<string>(scElems.Select(e => e.Key), StringComparer.Ordinal);
var scColorKeys = new HashSet<string>(scColors.Select(c => c.ElementKey), StringComparer.Ordinal);
int eAdded = scElems.Count(e => !exElemByKey.ContainsKey(e.Key));
int eChanged = scElems.Count(e => exElemByKey.TryGetValue(e.Key, out var ex) && ElementDiffers(e, ex));
int eRemoved = exElems.Count(e => !scElemKeys.Contains(e.Key));
int cAdded = scColors.Count(c => !exColorByKey.ContainsKey(c.ElementKey));
int cChanged = scColors.Count(c => exColorByKey.TryGetValue(c.ElementKey, out var ex) && ColorDiffers(c, ex));
int cRemoved = exColors.Count(c => !scColorKeys.Contains(c.ElementKey));
string status;
if (isNew) { status = "added"; sAdded++; }
else if (eAdded + eChanged + eRemoved + cAdded + cChanged + cRemoved > 0 || SceneHeaderDiffers(ss, scene!))
{ status = "changed"; sChanged++; }
else { status = "unchanged"; sUnchanged++; }
diffs.Add(new SceneDiff(ss.Key, ss.Title ?? ss.Key, status,
eAdded, eChanged, eRemoved, cAdded, cChanged, cRemoved));
if (apply is not null)
{
if (isNew)
{
scene = new Scene { ProjectId = projectId, Key = ss.Key, Sort = ss.Sort ?? sortBase++ };
db.Scenes.Add(scene);
}
ApplySceneHeader(scene!, ss, isNew, apply);
ApplyElements(scene!, scElems, exElemByKey, scElemKeys, apply);
ApplyColors(scene!, scColors, exColorByKey, scColorKeys, apply);
}
}
var orphans = existing.Where(s => !scanKeys.Contains(s.Key)).ToList();
if (apply is not null && apply.RemoveOrphanScenes)
foreach (var s in orphans) s.DeletedAt = DateTime.UtcNow;
var (shAdded, shChanged, shRemoved) = MergeShared(projectId, scan.SharedColors ?? new List<ScanColor>(), existingShared, apply);
if (apply is not null) await db.SaveChangesAsync();
return new ImportDiff(
apply is not null,
sAdded, sChanged, sUnchanged, orphans.Count,
shAdded, shChanged, shRemoved,
diffs, orphans.Select(o => o.Key).ToList());
}
// ── scene header ─────────────────────────────────────────────────────────
private static bool SceneHeaderDiffers(ScanScene ss, Scene e) =>
(ss.SceneType is not null && ParseScene(ss.SceneType) != e.SceneType)
|| (ss.DefaultDurationSec is not null && ss.DefaultDurationSec != e.DefaultDurationSec)
|| (ss.Title is not null && ss.Title != e.Title);
private static void ApplySceneHeader(Scene scene, ScanScene ss, bool isNew, ScanApplyOptions o)
{
if (isNew || o.OverwriteExisting)
{
scene.Title = ss.Title ?? scene.Title ?? ss.Key;
if (ss.SceneType is not null) scene.SceneType = ParseScene(ss.SceneType);
if (ss.DefaultDurationSec is not null) scene.DefaultDurationSec = ss.DefaultDurationSec;
if (ss.MinDurationSec is not null) scene.MinDurationSec = ss.MinDurationSec;
if (ss.MaxDurationSec is not null) scene.MaxDurationSec = ss.MaxDurationSec;
if (ss.Sort is not null) scene.Sort = ss.Sort.Value;
scene.UpdatedAt = DateTime.UtcNow;
}
if (isNew && string.IsNullOrEmpty(scene.Title)) scene.Title = ss.Key;
}
// ── content elements ──────────────────────────────────────────────────────
private static bool ElementDiffers(ScanElement s, SceneContentElement e) =>
(s.Type is not null && ParseElemType(s.Type) != e.Type)
|| (s.DefaultValue is not null && s.DefaultValue != e.DefaultValue)
|| (s.FontFace is not null && s.FontFace != e.FontFace)
|| (s.FontSize is not null && s.FontSize != e.FontSize)
|| (s.Justify is not null && ParseJustify(s.Justify) != e.Justify)
|| (s.IsHidden is not null && s.IsHidden != e.IsHidden);
private void ApplyElements(Scene scene, List<ScanElement> scElems,
Dictionary<string, SceneContentElement> exByKey, HashSet<string> scKeys, ScanApplyOptions o)
{
int sort = 0;
foreach (var se in scElems)
{
if (exByKey.TryGetValue(se.Key, out var e))
{
if (o.OverwriteExisting) WriteElement(e, se, scene.Id, sort);
}
else
{
var ne = new SceneContentElement { SceneId = scene.Id, Key = se.Key };
WriteElement(ne, se, scene.Id, sort);
db.SceneContentElements.Add(ne);
}
sort++;
}
if (o.RemoveOrphanElements)
foreach (var e in exByKey.Values.Where(e => !scKeys.Contains(e.Key)))
db.SceneContentElements.Remove(e);
}
private static void WriteElement(SceneContentElement e, ScanElement s, Guid sceneId, int sort)
{
e.SceneId = sceneId;
e.Title = s.Title ?? (string.IsNullOrEmpty(e.Title) ? s.Key : e.Title);
if (s.Type is not null) e.Type = ParseElemType(s.Type);
if (s.DefaultValue is not null) e.DefaultValue = s.DefaultValue;
if (s.FontFace is not null) { e.FontFace = s.FontFace; e.DefaultFontFace = s.FontFace; }
if (s.FontFaceName is not null) e.FontFaceName = s.FontFaceName;
if (s.FontSize is not null) { e.FontSize = s.FontSize; e.DefaultFontSize = s.FontSize; }
if (s.Justify is not null) e.Justify = ParseJustify(s.Justify);
if (s.PositionInContainer is not null) e.PositionInContainer = s.PositionInContainer.Value;
if (s.IsTextBox is not null) e.IsTextBox = s.IsTextBox.Value;
if (s.MaxSize is not null) e.MaxSize = s.MaxSize;
if (s.VideoSupport is not null) e.VideoSupport = s.VideoSupport.Value;
if (s.Width is not null) e.Width = s.Width;
if (s.Height is not null) e.Height = s.Height;
if (s.IsHidden is not null) e.IsHidden = s.IsHidden.Value;
if (s.DirectionLayerKey is not null) e.DirectionLayerKey = s.DirectionLayerKey;
e.Sort = s.Sort ?? sort;
e.UpdatedAt = DateTime.UtcNow;
}
// ── colour elements (per scene) ───────────────────────────────────────────
private static bool ColorDiffers(ScanColor s, SceneColorElement e) =>
(s.DefaultColor is not null && !ColorEq(s.DefaultColor, e.DefaultColor))
|| (s.AttrValue is not null && ParseAttr(s.AttrValue) != e.AttrValue)
|| (s.Title is not null && s.Title != e.Title);
private void ApplyColors(Scene scene, List<ScanColor> scColors,
Dictionary<string, SceneColorElement> exByKey, HashSet<string> scKeys, ScanApplyOptions o)
{
int sort = 0;
foreach (var sc in scColors)
{
if (exByKey.TryGetValue(sc.ElementKey, out var e))
{
if (o.OverwriteExisting) WriteColor(e, sc, scene.Id, sort);
}
else
{
var ne = new SceneColorElement { SceneId = scene.Id, ElementKey = sc.ElementKey, DefaultColor = sc.DefaultColor ?? "#000000" };
WriteColor(ne, sc, scene.Id, sort);
db.SceneColorElements.Add(ne);
}
sort++;
}
if (o.RemoveOrphanElements)
foreach (var e in exByKey.Values.Where(e => !scKeys.Contains(e.ElementKey)))
db.SceneColorElements.Remove(e);
}
private static void WriteColor(SceneColorElement e, ScanColor s, Guid sceneId, int sort)
{
e.SceneId = sceneId;
e.Title = s.Title ?? (string.IsNullOrEmpty(e.Title) ? s.ElementKey : e.Title);
if (s.AttrValue is not null) e.AttrValue = ParseAttr(s.AttrValue);
if (s.DefaultColor is not null) e.DefaultColor = s.DefaultColor;
e.Sort = s.Sort ?? sort;
e.UpdatedAt = DateTime.UtcNow;
}
// ── shared colours (project-level) ────────────────────────────────────────
private (int added, int changed, int removed) MergeShared(
Guid projectId, List<ScanColor> scan, List<SharedColor> existing, ScanApplyOptions? apply)
{
var byKey = existing.ToDictionary(c => c.ElementKey, StringComparer.Ordinal);
var scanKeys = new HashSet<string>(scan.Select(c => c.ElementKey), StringComparer.Ordinal);
int added = 0, changed = 0, removed = 0, sort = 0;
foreach (var sc in scan)
{
if (byKey.TryGetValue(sc.ElementKey, out var e))
{
if (ColorDiffersShared(sc, e)) changed++;
if (apply is not null && apply.OverwriteExisting)
{
e.Title = sc.Title ?? e.Title;
if (sc.AttrValue is not null) e.AttrValue = ParseAttr(sc.AttrValue);
if (sc.DefaultColor is not null) e.DefaultColor = sc.DefaultColor;
e.Sort = sc.Sort ?? e.Sort;
e.UpdatedAt = DateTime.UtcNow;
}
}
else
{
added++;
if (apply is not null)
db.SharedColors.Add(new SharedColor
{
ProjectId = projectId, ElementKey = sc.ElementKey, Title = sc.Title ?? sc.ElementKey,
AttrValue = ParseAttr(sc.AttrValue), DefaultColor = sc.DefaultColor ?? "#000000", Sort = sc.Sort ?? sort,
});
}
sort++;
}
var orphans = existing.Where(c => !scanKeys.Contains(c.ElementKey)).ToList();
removed = orphans.Count;
if (apply is not null && apply.RemoveOrphanElements)
foreach (var c in orphans) db.SharedColors.Remove(c);
return (added, changed, removed);
}
private static bool ColorDiffersShared(ScanColor s, SharedColor e) =>
(s.DefaultColor is not null && !ColorEq(s.DefaultColor, e.DefaultColor))
|| (s.AttrValue is not null && ParseAttr(s.AttrValue) != e.AttrValue)
|| (s.Title is not null && s.Title != e.Title);
// ── parsing helpers ───────────────────────────────────────────────────────
private static bool ColorEq(string? a, string? b) =>
string.Equals((a ?? "").Trim().TrimStart('#'), (b ?? "").Trim().TrimStart('#'), StringComparison.OrdinalIgnoreCase);
private static SceneKind ParseScene(string? v) =>
Enum.TryParse<SceneKind>(v, true, out var k) ? k : SceneKind.Normal;
private static ContentElementType ParseElemType(string? v) =>
Enum.TryParse<ContentElementType>(v, true, out var t) ? t : ContentElementType.Text;
private static JustifyKind ParseJustify(string? v) =>
Enum.TryParse<JustifyKind>(v, true, out var j) ? j : JustifyKind.CENTER_JUSTIFY;
private static AttrValueKind ParseAttr(string? v) =>
Enum.TryParse<AttrValueKind>(v, true, out var a) ? a : AttrValueKind.fill;
}
@@ -0,0 +1,30 @@
using FlatRender.ContentSvc.Application.Services;
using FlatRender.ContentSvc.Models;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
namespace FlatRender.ContentSvc.Controllers;
/// <summary>Import a scanned AE project structure into a project's scenes/elements/colours.</summary>
[ApiController]
[Route("v1/projects")]
public class AepImportController(AepImportService svc) : ControllerBase
{
/// <summary>Dry run — returns the diff (added/changed/removed) without writing.</summary>
[Authorize(Roles = "Admin")]
[HttpPost("{id:guid}/scan/preview")]
public async Task<IActionResult> Preview(Guid id, [FromBody] ScanImportRequest req)
{
if (req?.Scan is null) return BadRequest(new { message = "scan is required" });
return Ok(await svc.PreviewAsync(id, req.Scan));
}
/// <summary>Apply the scan, merging into the project (matched items refreshed, new added).</summary>
[Authorize(Roles = "Admin")]
[HttpPost("{id:guid}/scan/apply")]
public async Task<IActionResult> Apply(Guid id, [FromBody] ScanImportRequest req)
{
if (req?.Scan is null) return BadRequest(new { message = "scan is required" });
return Ok(await svc.ApplyAsync(id, req.Scan, req.Options ?? new ScanApplyOptions()));
}
}
@@ -0,0 +1,80 @@
namespace FlatRender.ContentSvc.Models;
// ── Canonical scan result ────────────────────────────────────────────────────
// Produced by either engine (the AE-JSX scanner on a render node, or the headless
// Go quick-scan) and consumed by AepImportService. Both engines emit this shape;
// the Go scan simply leaves colour/font/text fields null.
public record ScanResult(
string? Source, // "ae-jsx" | "go-parser"
string? RenderComp, // final render comp name, e.g. "frfinal"
List<ScanScene>? Scenes,
List<ScanColor>? SharedColors
);
public record ScanScene(
string Key, // AE comp name
string? Title,
string? SceneType, // Normal | Config | DesignStart | DesignEnd
decimal? DefaultDurationSec,
decimal? MinDurationSec,
decimal? MaxDurationSec,
int? Sort,
List<ScanElement>? Elements, // frl_/frd_ editable layers
List<ScanColor>? Colors // per-scene frd_ colour zones
);
public record ScanElement(
string Key, // AE layer name (frl_… / frd_…)
string? Title,
string? Type, // Text | TextArea | Media | Audio | … (content_element_type)
string? DefaultValue,
string? FontFace,
string? FontFaceName,
int? FontSize,
string? Justify, // LEFT_JUSTIFY | CENTER_JUSTIFY | RIGHT_JUSTIFY | FULL_JUSTIFY
int? PositionInContainer,
bool? IsTextBox,
int? MaxSize,
bool? VideoSupport,
int? Width,
int? Height,
bool? IsHidden,
string? DirectionLayerKey,
int? Sort
);
public record ScanColor(
string ElementKey, // AE frd_ layer name / shared colour key
string? Title,
string? AttrValue, // fill | stroke | tracking | dropshadow
string? DefaultColor, // #RRGGBB
int? Sort
);
// ── Import request + diff ─────────────────────────────────────────────────────
public record ScanApplyOptions(
bool RemoveOrphanScenes = false, // soft-delete scenes not present in the scan
bool RemoveOrphanElements = false, // delete elements/colours not present in the scan
bool OverwriteExisting = true // refresh matched items from the scan (null scan fields are kept)
);
public record ScanImportRequest(
ScanResult Scan,
ScanApplyOptions? Options
);
public record SceneDiff(
string Key, string Title, string Status, // added | changed | unchanged | orphan
int ElementsAdded, int ElementsChanged, int ElementsRemoved,
int ColorsAdded, int ColorsChanged, int ColorsRemoved
);
public record ImportDiff(
bool Applied,
int ScenesAdded, int ScenesChanged, int ScenesUnchanged, int ScenesOrphan,
int SharedColorsAdded, int SharedColorsChanged, int SharedColorsRemoved,
List<SceneDiff> Scenes,
List<string> OrphanSceneKeys
);
@@ -65,6 +65,7 @@ builder.Services.AddScoped<TemplateService>();
builder.Services.AddScoped<CmsService>();
builder.Services.AddScoped<AiContentService>();
builder.Services.AddScoped<SceneColorService>();
builder.Services.AddScoped<AepImportService>();
// HTTP client for the OpenAI-compatible AI provider (base URL is per-tenant config).
builder.Services.AddHttpClient("openai");
+3
View File
@@ -138,6 +138,9 @@ func main() {
v1.Any("/node-fonts/*path", apiRL, auth, render.Handler())
v1.Any("/admin-exports/*path", apiRL, auth, render.Handler())
v1.Any("/admin-renders", apiRL, auth, render.Handler())
v1.Any("/template-bundles/*path", apiRL, auth, render.Handler())
v1.Any("/template-scans/*path", apiRL, auth, render.Handler())
v1.Any("/template-scan-jobs/*path", apiRL, auth, render.Handler())
v1.Any("/node-updates/*path", apiRL, auth, render.Handler())
// ── Notification Service ──────────────────────────────────────────────────
+110 -9
View File
@@ -46,6 +46,7 @@ type Agent struct {
orch *client.Client
mu sync.Mutex
currentJob *client.ClaimedJob
scanning bool // true while running an AE scan job (shares the AE app)
status string // "Ready" | "Busy"
}
@@ -75,9 +76,25 @@ func (a *Agent) getStatus() (string, *string) {
jobID := a.currentJob.JobID
return a.status, &jobID
}
if a.scanning {
return "Busy", nil
}
return a.status, nil
}
func (a *Agent) setScanning(v bool) {
a.mu.Lock()
a.scanning = v
a.mu.Unlock()
}
// isBusy reports whether the AE app is in use (rendering or scanning).
func (a *Agent) isBusy() bool {
a.mu.Lock()
defer a.mu.Unlock()
return a.currentJob != nil || a.scanning
}
// ── Main ──────────────────────────────────────────────────────────────────────
func main() {
@@ -121,10 +138,11 @@ func main() {
// Main loops
var wg sync.WaitGroup
wg.Add(3)
wg.Add(4)
go func() { defer wg.Done(); agent.heartbeatLoop(ctx) }()
go func() { defer wg.Done(); agent.pollLoop(ctx) }()
go func() { defer wg.Done(); agent.fontSyncLoop(ctx) }()
go func() { defer wg.Done(); agent.scanLoop(ctx) }()
wg.Wait()
log.Printf("shutdown complete")
}
@@ -183,6 +201,83 @@ func (a *Agent) syncFonts(ctx context.Context) {
}
}
// ── Scan loop ──────────────────────────────────────────────────────────────────
// Claims AE scan jobs and runs the template scanner (scan.jsx) via afterfx.exe,
// posting the resulting ScanResult JSON back to the orchestrator. Requires the AE
// app — skipped entirely in mock/dev (no AE_PATH).
func (a *Agent) scanLoop(ctx context.Context) {
interval := time.Duration(a.cfg.PollIntervalSec) * time.Second
if interval < 5*time.Second {
interval = 5 * time.Second
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
a.pollScanOnce(ctx)
}
}
}
func (a *Agent) pollScanOnce(ctx context.Context) {
if a.cfg.AfterFxPath == "" || a.cfg.AEPath == "" {
return // scanning needs the real AE app
}
if a.isBusy() {
return // don't contend with a render (or another scan) for the AE app
}
claim, err := a.orch.ClaimScan(ctx, a.cfg.NodeID, a.cfg.Region)
if err != nil {
log.Printf("scan claim error: %v", err)
return
}
if claim == nil {
return // nothing queued
}
a.setScanning(true)
defer a.setScanning(false)
log.Printf("[scan %s] claimed (project %s)", claim.ScanJobID, claim.ProjectID)
// Reuse the template prepare/cache pipeline (download + extract bundle by md5).
prepCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
aepPath, perr := runner.PrepareTemplate(prepCtx, claim.AEPDownloadURL, claim.IsBundle, claim.BundleMD5, a.cfg.WorkDir, "scan-"+claim.ScanJobID)
cancel()
if perr != nil {
a.failScan(claim.ScanJobID, "prepare template: "+perr.Error())
return
}
outPath := filepath.Join(a.cfg.WorkDir, "scans", claim.ScanJobID, "scan.json")
scanCtx, cancel2 := context.WithTimeout(ctx, 10*time.Minute)
result, serr := runner.RunScan(scanCtx, a.cfg.AfterFxPath, aepPath, a.cfg.WorkDir, outPath)
cancel2()
if serr != nil {
a.failScan(claim.ScanJobID, serr.Error())
return
}
rptCtx, cancel3 := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel3()
if err := a.orch.ReportScanResult(rptCtx, claim.ScanJobID, result); err != nil {
log.Printf("[scan %s] report result error: %v", claim.ScanJobID, err)
return
}
log.Printf("[scan %s] done (%d bytes)", claim.ScanJobID, len(result))
}
func (a *Agent) failScan(id, reason string) {
log.Printf("[scan %s] failed: %s", id, reason)
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
_ = a.orch.ReportScanFail(ctx, id, reason)
}
// ── Heartbeat loop ────────────────────────────────────────────────────────────
func (a *Agent) heartbeatLoop(ctx context.Context) {
@@ -267,18 +362,24 @@ func (a *Agent) tryClaimAndRun(ctx context.Context) {
func (a *Agent) runJob(ctx context.Context, job *client.ClaimedJob) {
log.Printf("[job %s] starting render", job.JobID)
// ── Step 1: Download .aep template ───────────────────────────────────────
// ── Step 1: Fetch + prepare the .aep template ────────────────────────────
// PrepareTemplate downloads the project file; if it's a .zip bundle it is
// extracted and the .aep inside located. Prepared templates are cached by md5
// so repeated renders of the same template skip the download + extraction.
aepPath := ""
if job.AEPDownloadURL != "" && a.cfg.AEPath != "" {
localAEP := filepath.Join(a.cfg.WorkDir, "templates", job.JobID, "template.aep")
dlCtx, dlCancel := context.WithTimeout(ctx, 10*time.Minute)
n, dlErr := runner.DownloadFile(dlCtx, job.AEPDownloadURL, localAEP)
dlCtx, dlCancel := context.WithTimeout(ctx, 15*time.Minute)
p, prepErr := runner.PrepareTemplate(dlCtx, job.AEPDownloadURL, job.IsBundle, job.BundleMD5, a.cfg.WorkDir, job.JobID)
dlCancel()
if dlErr != nil {
log.Printf("[job %s] AEP download failed (%v) — falling back to mock", job.JobID, dlErr)
if prepErr != nil {
log.Printf("[job %s] template prepare failed (%v) — falling back to mock", job.JobID, prepErr)
} else {
log.Printf("[job %s] AEP downloaded (%d bytes) → %s", job.JobID, n, localAEP)
aepPath = localAEP
kind := "aep"
if job.IsBundle {
kind = "bundle"
}
log.Printf("[job %s] template ready (%s) → %s", job.JobID, kind, p)
aepPath = p
}
}
+68 -2
View File
@@ -148,9 +148,15 @@ type ClaimedJob struct {
FrameRate int `json:"frame_rate"`
HasMusic bool `json:"has_music"`
HasVoiceover bool `json:"has_voiceover"`
// AEPDownloadURL is a presigned MinIO GET URL for the .aep template file.
// Empty when the template has not been uploaded yet — triggers mock render.
// AEPDownloadURL is a presigned MinIO GET URL for the .aep template file
// (or .zip bundle). Empty when the template has not been uploaded yet — triggers mock render.
AEPDownloadURL string `json:"aep_download_url,omitempty"`
// IsBundle is true when AEPDownloadURL points to a .zip bundle (.aep + footage/fonts)
// that must be extracted before rendering.
IsBundle bool `json:"is_bundle,omitempty"`
// BundleMD5 identifies the bundle content; used as a local cache key so repeated
// renders of the same template download + extract it only once.
BundleMD5 string `json:"bundle_md5,omitempty"`
}
// OutputUploadURLResponse is returned by GetOutputUploadURL.
@@ -240,6 +246,66 @@ func (c *Client) ClaimJob(ctx context.Context, nodeID, region string) (*ClaimedJ
return &job, nil
}
// ScanClaim is returned when an AE scan job is claimed.
type ScanClaim struct {
ScanJobID string `json:"scan_job_id"`
ProjectID string `json:"project_id"`
AEPDownloadURL string `json:"aep_download_url"`
IsBundle bool `json:"is_bundle"`
BundleMD5 string `json:"bundle_md5"`
}
// ClaimScan atomically claims the next queued AE scan job.
// Returns (nil, nil) when nothing is queued (204 No Content).
func (c *Client) ClaimScan(ctx context.Context, nodeID, region string) (*ScanClaim, error) {
resp, err := c.do(ctx, http.MethodPost, "/v1/internal/scan/claim",
ClaimJobRequest{NodeID: nodeID, Region: region})
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNoContent {
return nil, nil
}
if resp.StatusCode >= 300 {
return nil, fmt.Errorf("scan claim: HTTP %d", resp.StatusCode)
}
var sc ScanClaim
if err := json.NewDecoder(resp.Body).Decode(&sc); err != nil {
return nil, fmt.Errorf("scan claim decode: %w", err)
}
return &sc, nil
}
// ReportScanResult posts the raw ScanResult JSON produced by scan.jsx.
func (c *Client) ReportScanResult(ctx context.Context, scanJobID string, resultJSON []byte) error {
// json.RawMessage marshals to its raw bytes, so the body is the JSON verbatim.
resp, err := c.do(ctx, http.MethodPost,
fmt.Sprintf("/v1/internal/scan/%s/result", scanJobID), json.RawMessage(resultJSON))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
return fmt.Errorf("scan result: HTTP %d", resp.StatusCode)
}
return nil
}
// ReportScanFail marks a scan job as failed.
func (c *Client) ReportScanFail(ctx context.Context, scanJobID, reason string) error {
resp, err := c.do(ctx, http.MethodPost,
fmt.Sprintf("/v1/internal/scan/%s/fail", scanJobID), FailRequest{Reason: reason})
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
return fmt.Errorf("scan fail: HTTP %d", resp.StatusCode)
}
return nil
}
// UpdatePreview sends a base64-encoded preview frame to the orchestrator.
// Errors are non-fatal — the UI simply won't update the preview image.
func (c *Client) UpdatePreview(ctx context.Context, jobID, imageB64 string) error {
@@ -4,6 +4,7 @@ package config
import (
"fmt"
"os"
"path/filepath"
"strconv"
)
@@ -30,6 +31,11 @@ type Config struct {
// Leave empty to use mock rendering (for development / testing without AE).
AEPath string
// AfterFxPath is the full path to afterfx.exe (the AE app, used to run the
// template scanner script). Defaults to afterfx.exe alongside aerender.exe.
// Leave AEPath empty too to disable scanning (dev/mock).
AfterFxPath string
// WorkDir is the scratch directory for render temp files and AE project copies.
WorkDir string
@@ -59,6 +65,7 @@ func Load() (*Config, error) {
NodeHMACSecret: getEnv("NODE_HMAC_SECRET", "node-secret-change-me"),
Region: getEnv("NODE_REGION", ""),
AEPath: getEnv("AE_PATH", ""),
AfterFxPath: getEnv("AFTERFX_PATH", ""),
WorkDir: getEnv("WORK_DIR", os.TempDir()),
AgentVersion: getEnv("AGENT_VERSION", "0.1.0"),
AEVersion: getEnv("AE_VERSION", "2024"),
@@ -69,6 +76,10 @@ func Load() (*Config, error) {
if c.NodeID == "" {
return nil, fmt.Errorf("NODE_ID environment variable is required")
}
// Derive afterfx.exe next to aerender.exe when not explicitly set.
if c.AfterFxPath == "" && c.AEPath != "" {
c.AfterFxPath = filepath.Join(filepath.Dir(c.AEPath), "afterfx.exe")
}
return c, nil
}
@@ -0,0 +1,204 @@
// bundle.go prepares an After Effects template for rendering. A template may be a
// plain .aep/.aepx file, or a .zip bundle containing the .aep plus its footage and
// fonts. Bundles are extracted so aerender can resolve relative footage paths.
//
// Prepared templates are cached on disk keyed by the bundle's MD5, so when many
// renders use the same template the node downloads + extracts it only once.
package runner
import (
"archive/zip"
"context"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
)
// PrepareTemplate ensures the template referenced by url is available locally and
// returns the absolute path to the .aep/.aepx file aerender should open.
//
// - isBundle=false: url is a raw project file; it is downloaded as-is.
// - isBundle=true: url is a .zip; it is downloaded and extracted, then the .aep
// inside is located.
//
// md5 (when non-empty) is the cache key: a prepared template with the same md5 is
// reused without re-downloading. baseDir is the agent work dir; jobID is the
// fallback cache key when md5 is empty.
func PrepareTemplate(ctx context.Context, url string, isBundle bool, md5, baseDir, jobID string) (string, error) {
cacheKey := md5
if cacheKey == "" {
cacheKey = jobID // no md5 → per-job, no reuse
}
cacheDir := filepath.Join(baseDir, "templates", "cache", sanitize(cacheKey))
if err := os.MkdirAll(cacheDir, 0o755); err != nil {
return "", fmt.Errorf("mkdir cache: %w", err)
}
if !isBundle {
aepPath := filepath.Join(cacheDir, "template.aep")
if fileExists(aepPath) {
return aepPath, nil // cache hit
}
if _, err := DownloadFile(ctx, url, aepPath); err != nil {
return "", fmt.Errorf("download aep: %w", err)
}
return aepPath, nil
}
// Bundle: the resolved .aep path is remembered in a marker so cache hits skip
// both download and extraction.
marker := filepath.Join(cacheDir, "aep_path.txt")
if b, err := os.ReadFile(marker); err == nil {
if p := strings.TrimSpace(string(b)); p != "" && fileExists(p) {
return p, nil // cache hit
}
}
zipPath := filepath.Join(cacheDir, "bundle.zip")
if _, err := DownloadFile(ctx, url, zipPath); err != nil {
return "", fmt.Errorf("download bundle: %w", err)
}
extractDir := filepath.Join(cacheDir, "extracted")
if err := os.RemoveAll(extractDir); err != nil {
return "", fmt.Errorf("clean extract dir: %w", err)
}
if err := ExtractZip(zipPath, extractDir); err != nil {
return "", fmt.Errorf("extract bundle: %w", err)
}
aepPath, err := FindAEP(extractDir)
if err != nil {
return "", err
}
_ = os.WriteFile(marker, []byte(aepPath), 0o644)
_ = os.Remove(zipPath) // free space — the extracted tree is what we render from
return aepPath, nil
}
// ExtractZip unpacks the zip at zipPath into destDir, guarding against path
// traversal ("zip slip"). Directories and parents are created as needed.
func ExtractZip(zipPath, destDir string) error {
zr, err := zip.OpenReader(zipPath)
if err != nil {
return fmt.Errorf("open zip: %w", err)
}
defer zr.Close()
if err := os.MkdirAll(destDir, 0o755); err != nil {
return err
}
destAbs, err := filepath.Abs(destDir)
if err != nil {
return err
}
for _, f := range zr.File {
// Clean and confine the target path to destDir.
target := filepath.Join(destDir, f.Name) // #nosec G305 — validated below
targetAbs, aerr := filepath.Abs(target)
if aerr != nil {
return aerr
}
if targetAbs != destAbs && !strings.HasPrefix(targetAbs, destAbs+string(os.PathSeparator)) {
return fmt.Errorf("illegal path in zip: %q", f.Name)
}
if f.FileInfo().IsDir() {
if err := os.MkdirAll(targetAbs, 0o755); err != nil {
return err
}
continue
}
if err := os.MkdirAll(filepath.Dir(targetAbs), 0o755); err != nil {
return err
}
if err := extractOne(f, targetAbs); err != nil {
return err
}
}
return nil
}
func extractOne(f *zip.File, target string) error {
rc, err := f.Open()
if err != nil {
return err
}
defer rc.Close()
out, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
if err != nil {
return err
}
defer out.Close()
// #nosec G110 — template bundles are admin-uploaded, not untrusted user input.
if _, err := io.Copy(out, rc); err != nil {
return err
}
return nil
}
// FindAEP locates the After Effects project file within an extracted bundle. It
// prefers the shallowest file (fewest path segments), .aep over .aepx, and ignores
// macOS resource-fork siblings ("._name") and __MACOSX entries.
func FindAEP(root string) (string, error) {
var matches []string
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
if info.Name() == "__MACOSX" {
return filepath.SkipDir
}
return nil
}
name := info.Name()
if strings.HasPrefix(name, "._") {
return nil
}
ext := strings.ToLower(filepath.Ext(name))
if ext == ".aep" || ext == ".aepx" {
matches = append(matches, p)
}
return nil
})
if err != nil {
return "", fmt.Errorf("walk bundle: %w", err)
}
if len(matches) == 0 {
return "", fmt.Errorf("no .aep file found in bundle")
}
sort.Slice(matches, func(i, j int) bool {
di, dj := strings.Count(matches[i], string(os.PathSeparator)), strings.Count(matches[j], string(os.PathSeparator))
if di != dj {
return di < dj // shallower first
}
ei, ej := strings.ToLower(filepath.Ext(matches[i])), strings.ToLower(filepath.Ext(matches[j]))
if ei != ej {
return ei == ".aep" // .aep before .aepx
}
return matches[i] < matches[j]
})
return matches[0], nil
}
func fileExists(p string) bool {
info, err := os.Stat(p)
return err == nil && !info.IsDir()
}
// sanitize keeps a cache key safe for use as a directory name.
func sanitize(s string) string {
return strings.Map(func(r rune) rune {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_':
return r
default:
return '_'
}
}, s)
}
@@ -0,0 +1,117 @@
package runner
import (
"archive/zip"
"os"
"path/filepath"
"strings"
"testing"
)
// writeZip builds a zip at path from the given name→content entries (dirs implied
// by trailing slash). Returns the path.
func writeZip(t *testing.T, path string, entries map[string]string) {
t.Helper()
f, err := os.Create(path)
if err != nil {
t.Fatalf("create zip: %v", err)
}
defer f.Close()
zw := zip.NewWriter(f)
for name, content := range entries {
w, err := zw.Create(name)
if err != nil {
t.Fatalf("zip create entry %s: %v", name, err)
}
if _, err := w.Write([]byte(content)); err != nil {
t.Fatalf("zip write %s: %v", name, err)
}
}
if err := zw.Close(); err != nil {
t.Fatalf("close zip: %v", err)
}
}
func TestExtractAndFindAEP(t *testing.T) {
dir := t.TempDir()
zipPath := filepath.Join(dir, "bundle.zip")
// Realistic bundle: project + footage, plus a macOS resource-fork decoy that
// must NOT be chosen as the project file.
writeZip(t, zipPath, map[string]string{
"MyTemplate/template.aep": "AEP-DATA",
"MyTemplate/footage/clip.mp4": "VIDEO",
"MyTemplate/fonts/Vazir.ttf": "FONT",
"__MACOSX/MyTemplate/._template.aep": "RESOURCE-FORK",
})
dest := filepath.Join(dir, "extracted")
if err := ExtractZip(zipPath, dest); err != nil {
t.Fatalf("ExtractZip: %v", err)
}
// footage + font extracted
if _, err := os.Stat(filepath.Join(dest, "MyTemplate", "footage", "clip.mp4")); err != nil {
t.Errorf("footage not extracted: %v", err)
}
if _, err := os.Stat(filepath.Join(dest, "MyTemplate", "fonts", "Vazir.ttf")); err != nil {
t.Errorf("font not extracted: %v", err)
}
aep, err := FindAEP(dest)
if err != nil {
t.Fatalf("FindAEP: %v", err)
}
if filepath.Base(aep) != "template.aep" {
t.Errorf("expected template.aep, got %s", aep)
}
if strings.Contains(aep, "__MACOSX") || strings.Contains(filepath.Base(aep), "._") {
t.Errorf("FindAEP picked a macOS resource-fork decoy: %s", aep)
}
}
func TestFindAEPPrefersShallowAndAepOverAepx(t *testing.T) {
dir := t.TempDir()
// deep .aepx + shallow .aep — shallow .aep must win.
mustWrite(t, filepath.Join(dir, "a", "b", "deep.aepx"), "x")
mustWrite(t, filepath.Join(dir, "root.aep"), "x")
aep, err := FindAEP(dir)
if err != nil {
t.Fatalf("FindAEP: %v", err)
}
if filepath.Base(aep) != "root.aep" {
t.Errorf("expected root.aep (shallow, .aep), got %s", aep)
}
}
func TestFindAEPNoneFound(t *testing.T) {
dir := t.TempDir()
mustWrite(t, filepath.Join(dir, "readme.txt"), "no project here")
if _, err := FindAEP(dir); err == nil {
t.Error("expected error when no .aep present, got nil")
}
}
func TestExtractZipRejectsZipSlip(t *testing.T) {
dir := t.TempDir()
zipPath := filepath.Join(dir, "evil.zip")
writeZip(t, zipPath, map[string]string{
"../escape.txt": "pwned",
})
dest := filepath.Join(dir, "extracted")
if err := ExtractZip(zipPath, dest); err == nil {
t.Error("expected zip-slip to be rejected, got nil error")
}
if _, err := os.Stat(filepath.Join(dir, "escape.txt")); err == nil {
t.Error("zip-slip wrote a file outside the destination dir")
}
}
func mustWrite(t *testing.T, path, content string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
@@ -111,6 +111,9 @@ func aeRender(ctx context.Context, aePath string, job *Job, outputPath string, o
log.Printf("[ae] running: %s %v", aePath, args)
cmd := exec.CommandContext(ctx, aePath, args...)
// Run from the project's folder so a .zip bundle's relative footage/font paths
// resolve correctly (the .aep sits alongside its assets after extraction).
cmd.Dir = filepath.Dir(job.AEPFilePath)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
@@ -0,0 +1,67 @@
package runner
import (
"context"
_ "embed"
"fmt"
"os"
"os/exec"
"path/filepath"
"time"
)
//go:embed scan.jsx
var scanScript []byte
// WriteScanScript writes the embedded scanner JSX into workDir and returns its path.
func WriteScanScript(workDir string) (string, error) {
dir := filepath.Join(workDir, "scripts")
if err := os.MkdirAll(dir, 0o755); err != nil {
return "", err
}
p := filepath.Join(dir, "scan.jsx")
if err := os.WriteFile(p, scanScript, 0o644); err != nil {
return "", err
}
return p, nil
}
// RunScan runs the After Effects template scanner against aepPath and returns the
// JSON the script writes to outPath. afterfxPath must be afterfx.exe (the AE app,
// not aerender.exe — only the app can run ScriptUI/ExtendScript).
//
// afterfx -r runs the script and the script calls app.quit(); we still poll for the
// output file because afterfx can return before the file is flushed.
func RunScan(ctx context.Context, afterfxPath, aepPath, workDir, outPath string) ([]byte, error) {
scriptPath, err := WriteScanScript(workDir)
if err != nil {
return nil, fmt.Errorf("write scan script: %w", err)
}
if err := os.MkdirAll(filepath.Dir(outPath), 0o755); err != nil {
return nil, err
}
_ = os.Remove(outPath)
_ = os.Remove(outPath + ".error")
cmd := exec.CommandContext(ctx, afterfxPath, "-r", scriptPath)
cmd.Env = append(os.Environ(), "FR_SCAN_AEP="+aepPath, "FR_SCAN_OUT="+outPath)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
// afterfx may exit non-zero on app.quit() — don't treat the exit code as fatal;
// success is determined by the presence of the output JSON file.
_ = cmd.Run()
for {
if eb, rerr := os.ReadFile(outPath + ".error"); rerr == nil && len(eb) > 0 {
return nil, fmt.Errorf("scan script error: %s", string(eb))
}
if b, rerr := os.ReadFile(outPath); rerr == nil && len(b) > 0 {
return b, nil
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("scan timed out waiting for %s", outPath)
case <-time.After(2 * time.Second):
}
}
}
@@ -0,0 +1,186 @@
/*
* FlatRender — After Effects template SCANNER (read-only / ExtendScript).
*
* Walks an opened .aep project and emits its structure as JSON for the content
* service to import (scenes, frl_/frd_ elements, frd_/frshare colours).
*
* Launch (headless), with the node agent setting env vars first:
* SET FR_SCAN_AEP=C:\work\templates\cache\<md5>\extracted\proj\template.aep
* SET FR_SCAN_OUT=C:\work\scans\<job>\scan.json
* "...\Adobe After Effects <ver>\Support Files\afterfx.exe" -r "...\scan.jsx"
*
* Conventions (mirrors the legacy NewBrain/JSX binder, in reverse):
* comp "frfinal" → final render comp (recorded, not a scene)
* comp "frshare" → shared colours: each layer name = key, its "Source Text" = colour value
* layer frl_* → editable element: TextLayer→Text (font/size/justify/text), else Media/Audio
* layer frd_* → data/direction layer; if its text looks like a colour → per-scene colour zone
* any comp containing frl_/frd_ layers → a scene (key = comp name, duration = comp.duration)
*
* Output shape == content-svc ScanResult:
* { source, render_comp, scenes:[{key,title,scene_type,default_duration_sec,sort,
* elements:[{key,title,type,default_value,font_face,font_size,justify,width,height,video_support,is_hidden,sort}],
* colors:[{element_key,title,attr_value,default_color,sort}] }],
* shared_colors:[{element_key,title,attr_value,default_color,sort}] }
*/
(function () {
function getenv(name) { try { return $.getenv(name); } catch (e) { return null; } }
var aepPath = getenv("FR_SCAN_AEP");
var outPath = getenv("FR_SCAN_OUT") || (Folder.temp.fsName + "/fr_scan.json");
// ── minimal JSON serializer (older AE has no JSON.stringify) ──────────────
function esc(s) {
s = String(s); var out = "", i, c;
for (i = 0; i < s.length; i++) {
c = s.charAt(i);
if (c === '"') out += '\\"';
else if (c === '\\') out += '\\\\';
else if (c === '\n') out += '\\n';
else if (c === '\r') out += '\\r';
else if (c === '\t') out += '\\t';
else if (c < ' ') out += '\\u' + ('0000' + c.charCodeAt(0).toString(16)).slice(-4);
else out += c;
}
return out;
}
function jstr(v) {
if (v === null || v === undefined) return "null";
var t = typeof v;
if (t === "number") return isFinite(v) ? String(v) : "null";
if (t === "boolean") return v ? "true" : "false";
if (t === "string") return '"' + esc(v) + '"';
if (v instanceof Array) {
var a = [], i; for (i = 0; i < v.length; i++) a.push(jstr(v[i]));
return "[" + a.join(",") + "]";
}
var props = [], k;
for (k in v) { if (v.hasOwnProperty(k)) props.push('"' + esc(k) + '":' + jstr(v[k])); }
return "{" + props.join(",") + "}";
}
// ── colour helpers ────────────────────────────────────────────────────────
function trim(s) { return String(s).replace(/^\s+|\s+$/g, ""); }
function isColor(s) {
if (!s) return false; s = trim(s);
return /^#?[0-9a-fA-F]{6}$/.test(s) || /^\d{1,3}\s*,\s*\d{1,3}\s*,\s*\d{1,3}$/.test(s);
}
function normColor(s) {
s = trim(s);
if (/^[0-9a-fA-F]{6}$/.test(s)) return "#" + s; // bare hex → add #
var m = s.match(/^(\d{1,3})\s*,\s*(\d{1,3})\s*,\s*(\d{1,3})$/); // r,g,b → #hex
if (m) {
function h(n) { n = Math.max(0, Math.min(255, parseInt(n, 10))); var x = n.toString(16); return x.length === 1 ? "0" + x : x; }
return "#" + h(m[1]) + h(m[2]) + h(m[3]);
}
return s;
}
function justifyName(j) {
try {
if (j === ParagraphJustification.LEFT_JUSTIFY) return "LEFT_JUSTIFY";
if (j === ParagraphJustification.RIGHT_JUSTIFY) return "RIGHT_JUSTIFY";
if (j === ParagraphJustification.FULL_JUSTIFY) return "FULL_JUSTIFY";
} catch (e) {}
return "CENTER_JUSTIFY";
}
function readText(layer) {
try {
var st = layer.property("Source Text");
if (!st) return null;
var td = st.value; // TextDocument
return { text: td.text, font: td.font, fontSize: Math.round(td.fontSize), justify: justifyName(td.justification) };
} catch (e) { return null; }
}
function pre3(name) { return String(name).substring(0, 3); }
function scanScene(comp) {
var elements = [], colors = [];
for (var i = 1; i <= comp.numLayers; i++) {
var layer = comp.layer(i), name = layer.name, p = pre3(name);
if (p === "frl") {
var el = { key: name, title: name, sort: i };
var txt = readText(layer);
if (txt) {
el.type = "Text"; el.default_value = txt.text;
el.font_face = txt.font; el.font_size = txt.fontSize; el.justify = txt.justify;
} else {
var src = null; try { src = layer.source; } catch (e) {}
if (src && src.hasVideo === false && src.hasAudio === true) {
el.type = "Audio";
} else {
el.type = "Media";
if (src) { try { el.width = src.width; el.height = src.height; } catch (e) {} }
try { el.video_support = !!(src && src.hasVideo); } catch (e) {}
}
}
elements.push(el);
} else if (p === "frd") {
var t2 = readText(layer);
if (t2 && isColor(t2.text)) {
colors.push({ element_key: name, title: name, attr_value: "fill", default_color: normColor(t2.text), sort: i });
} else {
var del = { key: name, title: name, type: "Text", is_hidden: true, sort: i };
if (t2) { del.default_value = t2.text; del.font_face = t2.font; del.font_size = t2.fontSize; }
elements.push(del);
}
}
}
return { elements: elements, colors: colors };
}
function compHasEditable(comp) {
for (var k = 1; k <= comp.numLayers; k++) {
var pp = pre3(comp.layer(k).name);
if (pp === "frl" || pp === "frd") return true;
}
return false;
}
function run() {
if (aepPath) app.open(new File(aepPath));
var proj = app.project;
var result = { source: "ae-jsx", render_comp: null, scenes: [], shared_colors: [] };
for (var i = 1; i <= proj.numItems; i++) {
var item = proj.item(i);
if (!(item instanceof CompItem)) continue;
var nm = item.name;
if (nm === "frfinal") { result.render_comp = "frfinal"; continue; }
if (nm === "frshare") {
for (var j = 1; j <= item.numLayers; j++) {
var cl = item.layer(j), ct = readText(cl);
result.shared_colors.push({
element_key: cl.name, title: cl.name, attr_value: "fill",
default_color: (ct && isColor(ct.text)) ? normColor(ct.text) : "#000000", sort: j
});
}
continue;
}
if (!compHasEditable(item)) continue;
var s = scanScene(item);
result.scenes.push({
key: nm, title: nm, scene_type: "Normal",
default_duration_sec: Math.round(item.duration * 100) / 100,
sort: result.scenes.length, elements: s.elements, colors: s.colors
});
}
var out = new File(outPath);
out.encoding = "UTF-8";
out.open("w");
out.write(jstr(result));
out.close();
}
try {
run();
} catch (e) {
try { var ef = new File(outPath + ".error"); ef.open("w"); ef.write("scan error: " + e.toString()); ef.close(); } catch (e2) {}
}
// Quit AE so the headless afterfx process returns (set FR_SCAN_QUIT=0 to keep open while debugging).
try { if (getenv("FR_SCAN_QUIT") !== "0") app.quit(); } catch (e) {}
})();
+15
View File
@@ -68,6 +68,8 @@ func main() {
exportH := handlers.NewExportHandler(store, mc, minioBucket)
nodeH := handlers.NewNodeHandler(store)
fontH := handlers.NewFontHandler(store)
bundleH := handlers.NewTemplateBundleHandler(mc, minioTemplatesBucket)
scanH := handlers.NewScanHandler(store, mc, minioTemplatesBucket)
internalH := handlers.NewInternalHandler(store, notifyClient, mc, minioTemplatesBucket, minioBucket)
// ── Router ────────────────────────────────────────────────────────────────
@@ -148,6 +150,14 @@ func main() {
// ── Render queue (admin: all users' jobs) ─────────────────────────────────
v1.GET("/admin-renders", auth, admin, renderH.AdminList)
// ── Template bundles (admin: store the canonical .aep/.zip per project) ────
v1.POST("/template-bundles/:project_id", auth, admin, bundleH.Set)
// ── Template scans (admin: read scenes/colours/configs from the AEP) ───────
v1.POST("/template-scans/:project_id/quick", auth, admin, scanH.QuickScan) // headless Go quick-scan
v1.POST("/template-scans/:project_id/jobs", auth, admin, scanH.CreateJob) // queue an AE full scan
v1.GET("/template-scan-jobs/:id", auth, admin, scanH.GetJob)
// ── Exports management (admin: all users' rendered videos) ────────────────
adminExports := v1.Group("/admin-exports", auth, admin)
{
@@ -172,6 +182,11 @@ func main() {
internal.POST("/render/jobs/:job_id/fail", internalH.Fail)
internal.POST("/render/jobs/:job_id/crash", internalH.Crash)
internal.POST("/render/jobs/:job_id/replica-ready", internalH.ReplicaReady)
// AE scan jobs (node claims, runs scan.jsx, posts the ScanResult back)
internal.POST("/scan/claim", scanH.Claim)
internal.POST("/scan/:id/result", scanH.Result)
internal.POST("/scan/:id/fail", scanH.Fail)
}
log.Printf("render-svc listening on :%s", port)
+127
View File
@@ -0,0 +1,127 @@
// Package aep is a minimal, stdlib-only reader for After Effects project (.aep)
// files. AEP is a RIFX container (big-endian RIFF). This reader does NOT fully
// decode the project — it walks the chunk tree to extract composition names and
// (best-effort) durations, which is enough for a headless "quick scan" that
// scaffolds scenes without After Effects. Full fidelity (layers, colours, fonts)
// requires the AE-JSX scanner running on a node.
package aep
import (
"encoding/binary"
"errors"
"strings"
)
// Comp is a composition discovered in the project.
type Comp struct {
Name string
DurationSec float64 // 0 when it could not be derived
}
// ParseComps walks a RIFX (.aep) buffer and returns its compositions.
//
// A composition is an "Item" LIST that contains a "cdta" (composition data)
// chunk; its name is the Item's "Utf8" chunk. Folders and footage items lack
// "cdta" and are skipped. This rule is robust across AE versions because only
// comps carry cdta.
func ParseComps(data []byte) ([]Comp, error) {
if len(data) < 12 || string(data[0:4]) != "RIFX" {
return nil, errors.New("not a RIFX/.aep file")
}
// data[4:8] = file size (BE), data[8:12] = form type ("Egg!"). Body follows.
var comps []Comp
seen := map[string]bool{}
walk(data[12:], &comps, seen)
return comps, nil
}
// walk iterates the chunks in buf (a LIST body or the file root), recursing into
// every LIST so nested items inside folders are found.
func walk(buf []byte, comps *[]Comp, seen map[string]bool) {
off := 0
for off+8 <= len(buf) {
id := string(buf[off : off+4])
size := int(binary.BigEndian.Uint32(buf[off+4 : off+8]))
ds := off + 8
de := ds + size
if size < 0 || de > len(buf) {
break
}
if id == "LIST" && size >= 4 {
listType := string(buf[ds : ds+4])
body := buf[ds+4 : de]
if listType == "Item" {
handleItem(body, comps, seen)
}
walk(body, comps, seen) // recurse (folders hold nested Item LISTs)
}
adv := 8 + size
if size%2 == 1 {
adv++ // chunks are word-aligned
}
off += adv
}
}
// handleItem inspects the DIRECT children of an "Item" LIST: a "cdta" marks it as
// a composition, the first "Utf8" is its name.
func handleItem(body []byte, comps *[]Comp, seen map[string]bool) {
var name string
var cdta []byte
hasCdta := false
off := 0
for off+8 <= len(body) {
id := string(body[off : off+4])
size := int(binary.BigEndian.Uint32(body[off+4 : off+8]))
ds := off + 8
de := ds + size
if size < 0 || de > len(body) {
break
}
switch id {
case "cdta":
hasCdta = true
cdta = body[ds:de]
case "Utf8":
if name == "" {
name = strings.TrimSpace(strings.TrimRight(string(body[ds:de]), "\x00"))
}
}
adv := 8 + size
if size%2 == 1 {
adv++
}
off += adv
}
if hasCdta && name != "" && !seen[name] {
seen[name] = true
*comps = append(*comps, Comp{Name: name, DurationSec: durationFromCdta(cdta)})
}
}
// durationFromCdta makes a best-effort attempt to read the comp duration from the
// cdta chunk. The cdta layout varies by AE version; we read the frame-duration /
// time-scale pair at well-known offsets and fall back to 0 (unknown) on any doubt.
// Returning 0 is safe — the importer treats it as "leave duration unset".
func durationFromCdta(cdta []byte) float64 {
// cdta encodes time as rational values. Two uint32 BE commonly hold the comp
// duration (in frames at the comp's time scale) and the time scale (fps base).
// Offsets 0x20 (duration) and 0x24 (scale) are the most consistent across
// recent versions; guard heavily and bail to 0 if the numbers look invalid.
if len(cdta) < 0x28 {
return 0
}
durFrames := binary.BigEndian.Uint32(cdta[0x20:0x24])
scale := binary.BigEndian.Uint32(cdta[0x24:0x28])
if scale == 0 || durFrames == 0 || scale > 100000 || durFrames > 100000000 {
return 0
}
sec := float64(durFrames) / float64(scale)
if sec <= 0 || sec > 36000 { // > 10h is nonsense → treat as unknown
return 0
}
// round to 2 dp
return float64(int(sec*100+0.5)) / 100
}
@@ -0,0 +1,76 @@
package aep
import (
"bytes"
"encoding/binary"
"testing"
)
// chunk builds a RIFX chunk: 4-byte id + BE32 size + data (+ pad to even).
func chunk(id string, data []byte) []byte {
b := new(bytes.Buffer)
b.WriteString(id)
_ = binary.Write(b, binary.BigEndian, uint32(len(data)))
b.Write(data)
if len(data)%2 == 1 {
b.WriteByte(0)
}
return b.Bytes()
}
// list builds a LIST chunk of the given form type wrapping the body chunks.
func list(formType string, body []byte) []byte {
return chunk("LIST", append([]byte(formType), body...))
}
func cdtaWithDuration(durFrames, scale uint32) []byte {
b := make([]byte, 0x28)
binary.BigEndian.PutUint32(b[0x20:0x24], durFrames)
binary.BigEndian.PutUint32(b[0x24:0x28], scale)
return b
}
func TestParseComps(t *testing.T) {
// One composition (has cdta), one footage item (no cdta), inside a folder.
comp := list("Item", bytes.Join([][]byte{
chunk("cdta", cdtaWithDuration(150, 30)), // 150 frames @ 30 → 5.0s
chunk("Utf8", []byte("scene_intro")),
}, nil))
footage := list("Item", bytes.Join([][]byte{
chunk("Utf8", []byte("clip.mp4")),
chunk("sspc", make([]byte, 8)),
}, nil))
folder := list("Item", bytes.Join([][]byte{
chunk("Utf8", []byte("My Folder")),
comp,
footage,
}, nil))
body := bytes.Join([][]byte{folder}, nil)
// RIFX header: "RIFX" + size + "Egg!" + body
file := new(bytes.Buffer)
file.WriteString("RIFX")
_ = binary.Write(file, binary.BigEndian, uint32(len(body)+4))
file.WriteString("Egg!")
file.Write(body)
comps, err := ParseComps(file.Bytes())
if err != nil {
t.Fatalf("ParseComps: %v", err)
}
if len(comps) != 1 {
t.Fatalf("expected 1 comp, got %d: %+v", len(comps), comps)
}
if comps[0].Name != "scene_intro" {
t.Errorf("name = %q, want scene_intro", comps[0].Name)
}
if comps[0].DurationSec != 5.0 {
t.Errorf("duration = %v, want 5.0", comps[0].DurationSec)
}
}
func TestParseCompsRejectsNonRifx(t *testing.T) {
if _, err := ParseComps([]byte("not an aep file at all")); err == nil {
t.Error("expected error for non-RIFX input")
}
}
+82
View File
@@ -0,0 +1,82 @@
package db
import (
"context"
"encoding/json"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
// ScanJob is an async "scan this project's AE template" job.
type ScanJob struct {
ID uuid.UUID `json:"id"`
ProjectID uuid.UUID `json:"project_id"`
Status string `json:"status"` // queued | running | done | error
Engine string `json:"engine"`
Result json.RawMessage `json:"result,omitempty"` // ScanResult JSON, present when done
Error *string `json:"error,omitempty"`
}
// ScanClaim is the minimal info a node needs to run a claimed scan.
type ScanClaim struct {
ID uuid.UUID
ProjectID uuid.UUID
}
func (s *Store) CreateScanJob(ctx context.Context, projectID uuid.UUID, engine string) (uuid.UUID, error) {
var id uuid.UUID
err := s.pool.QueryRow(ctx,
`INSERT INTO render.scan_jobs (project_id, engine, status) VALUES ($1, $2, 'queued') RETURNING id`,
projectID, engine).Scan(&id)
return id, err
}
// ClaimScanJob atomically grabs the oldest queued ae-jsx scan for a node.
// Returns nil when the queue is empty.
func (s *Store) ClaimScanJob(ctx context.Context, nodeID uuid.UUID) (*ScanClaim, error) {
var c ScanClaim
err := s.pool.QueryRow(ctx, `
UPDATE render.scan_jobs SET status = 'running', node_id = $1, updated_at = NOW()
WHERE id = (
SELECT id FROM render.scan_jobs
WHERE status = 'queued' AND engine = 'ae-jsx'
ORDER BY created_at
LIMIT 1 FOR UPDATE SKIP LOCKED
)
RETURNING id, project_id`, nodeID).Scan(&c.ID, &c.ProjectID)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
}
return nil, err
}
return &c, nil
}
func (s *Store) SetScanResult(ctx context.Context, id uuid.UUID, resultJSON string) error {
_, err := s.pool.Exec(ctx,
`UPDATE render.scan_jobs SET status = 'done', result = $2::jsonb, error = NULL, updated_at = NOW() WHERE id = $1`,
id, resultJSON)
return err
}
func (s *Store) SetScanError(ctx context.Context, id uuid.UUID, msg string) error {
_, err := s.pool.Exec(ctx,
`UPDATE render.scan_jobs SET status = 'error', error = $2, updated_at = NOW() WHERE id = $1`, id, msg)
return err
}
func (s *Store) GetScanJob(ctx context.Context, id uuid.UUID) (*ScanJob, error) {
var j ScanJob
err := s.pool.QueryRow(ctx,
`SELECT id, project_id, status, engine, result, error FROM render.scan_jobs WHERE id = $1`,
id).Scan(&j.ID, &j.ProjectID, &j.Status, &j.Engine, &j.Result, &j.Error)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
}
return nil, err
}
return &j, nil
}
+31 -9
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net/http"
"strings"
"time"
"github.com/flatrender/render-svc/internal/db"
@@ -254,18 +255,37 @@ func (h *InternalHandler) Claim(c *gin.Context) {
return
}
// Generate presigned AEP download URL. AEP files are stored at
// templates/{original_project_id}/template.aep in the templates bucket.
// Errors are non-fatal — node agent falls back to mock render when URL is empty.
// Resolve the canonical template object. Each template is stored once, per
// project id, at templates/{original_project_id}/<name> in the templates bucket
// and reused by every render of that template. A .zip is a full AE project
// bundle (.aep + footage/fonts) the node must extract before rendering.
// Errors are non-fatal — the node agent falls back to mock render when URL is empty.
aepURL := ""
isBundle := false
bundleMD5 := ""
if h.minio != nil {
objectKey := fmt.Sprintf("templates/%s/template.aep", job.OriginalProjectID)
purl, perr := h.minio.PresignedGetObject(
context.Background(), h.templatesBucket, objectKey,
2*time.Hour, nil,
)
if perr == nil {
candidates := []struct {
name string
bundle bool
}{
{"bundle.zip", true},
{"template.aep", false},
{"template.aepx", false},
}
for _, cand := range candidates {
objectKey := fmt.Sprintf("templates/%s/%s", job.OriginalProjectID, cand.name)
info, serr := h.minio.StatObject(context.Background(), h.templatesBucket, objectKey, minio.StatObjectOptions{})
if serr != nil {
continue // not this format
}
purl, perr := h.minio.PresignedGetObject(context.Background(), h.templatesBucket, objectKey, 2*time.Hour, nil)
if perr != nil {
continue
}
aepURL = purl.String()
isBundle = cand.bundle
bundleMD5 = strings.Trim(info.ETag, "\"")
break
}
}
@@ -278,6 +298,8 @@ func (h *InternalHandler) Claim(c *gin.Context) {
HasMusic: job.HasMusic,
HasVoiceover: job.HasVoiceover,
AEPDownloadURL: aepURL,
IsBundle: isBundle,
BundleMD5: bundleMD5,
})
}
+302
View File
@@ -0,0 +1,302 @@
package handlers
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"path"
"strings"
"time"
"github.com/flatrender/render-svc/internal/aep"
"github.com/flatrender/render-svc/internal/db"
"github.com/flatrender/render-svc/internal/models"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/minio/minio-go/v7"
)
// ScanHandler exposes the headless Go "quick scan" and the async AE scan-job
// lifecycle. Both produce the same ScanResult shape the content importer consumes.
type ScanHandler struct {
store *db.Store
minio *minio.Client
templatesBucket string
}
func NewScanHandler(store *db.Store, mc *minio.Client, templatesBucket string) *ScanHandler {
return &ScanHandler{store: store, minio: mc, templatesBucket: templatesBucket}
}
// ── ScanResult shape (matches content-svc) ────────────────────────────────────
type scanColor struct {
ElementKey string `json:"element_key"`
Title string `json:"title"`
AttrValue string `json:"attr_value"`
DefaultColor string `json:"default_color"`
Sort int `json:"sort"`
}
type scanScene struct {
Key string `json:"key"`
Title string `json:"title"`
SceneType string `json:"scene_type"`
DefaultDurationSec *float64 `json:"default_duration_sec,omitempty"`
Sort int `json:"sort"`
Elements []interface{} `json:"elements"`
Colors []scanColor `json:"colors"`
}
type scanResult struct {
Source string `json:"source"`
RenderComp string `json:"render_comp,omitempty"`
Scenes []scanScene `json:"scenes"`
SharedColors []scanColor `json:"shared_colors"`
}
// reserved comp names that are not scenes.
func isReservedComp(name string) bool {
switch strings.ToLower(name) {
case "frfinal", "frshare", "all":
return true
}
return false
}
// ── Quick scan (headless, no AE) ──────────────────────────────────────────────
// POST /v1/template-scans/:project_id/quick
func (h *ScanHandler) QuickScan(c *gin.Context) {
pid, err := uuid.Parse(c.Param("project_id"))
if err != nil {
c.JSON(http.StatusBadRequest, models.APIError{Code: "bad_request", Message: "invalid project_id"})
return
}
if h.minio == nil {
c.JSON(http.StatusServiceUnavailable, models.APIError{Code: "no_storage", Message: "object storage not configured"})
return
}
ctx, cancel := context.WithTimeout(c.Request.Context(), 60*time.Second)
defer cancel()
data, err := h.loadAep(ctx, pid)
if err != nil {
c.JSON(http.StatusNotFound, models.APIError{Code: "no_template", Message: err.Error()})
return
}
comps, err := aep.ParseComps(data)
if err != nil {
c.JSON(http.StatusUnprocessableEntity, models.APIError{Code: "parse_failed", Message: err.Error()})
return
}
res := scanResult{Source: "go-parser", Scenes: []scanScene{}, SharedColors: []scanColor{}}
for _, comp := range comps {
if strings.EqualFold(comp.Name, "frfinal") {
res.RenderComp = "frfinal"
}
if isReservedComp(comp.Name) {
continue
}
sc := scanScene{
Key: comp.Name, Title: comp.Name, SceneType: "Normal",
Sort: len(res.Scenes), Elements: []interface{}{}, Colors: []scanColor{},
}
if comp.DurationSec > 0 {
d := comp.DurationSec
sc.DefaultDurationSec = &d
}
res.Scenes = append(res.Scenes, sc)
}
c.JSON(http.StatusOK, res)
}
// loadAep fetches the project's template .aep bytes from MinIO — directly when a
// raw .aep was uploaded, or by extracting the .aep from the .zip bundle.
func (h *ScanHandler) loadAep(ctx context.Context, pid uuid.UUID) ([]byte, error) {
tryGet := func(key string) ([]byte, bool) {
if _, err := h.minio.StatObject(ctx, h.templatesBucket, key, minio.StatObjectOptions{}); err != nil {
return nil, false
}
obj, err := h.minio.GetObject(ctx, h.templatesBucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, false
}
defer obj.Close()
b, rerr := io.ReadAll(obj)
if rerr != nil || len(b) == 0 {
return nil, false
}
return b, true
}
if b, ok := tryGet(fmt.Sprintf("templates/%s/template.aep", pid)); ok {
return b, nil
}
if zb, ok := tryGet(fmt.Sprintf("templates/%s/bundle.zip", pid)); ok {
return extractAepFromZip(zb)
}
return nil, errors.New("no .aep template uploaded for this project (a raw .aepx can only be read by the AE scan)")
}
func extractAepFromZip(zb []byte) ([]byte, error) {
zr, err := zip.NewReader(bytes.NewReader(zb), int64(len(zb)))
if err != nil {
return nil, fmt.Errorf("open bundle: %w", err)
}
var best *zip.File
bestDepth := 1 << 30
for _, f := range zr.File {
base := path.Base(f.Name)
if strings.HasPrefix(base, "._") || strings.Contains(f.Name, "__MACOSX") {
continue
}
if strings.ToLower(path.Ext(f.Name)) == ".aep" {
depth := strings.Count(f.Name, "/")
if depth < bestDepth {
bestDepth = depth
best = f
}
}
}
if best == nil {
return nil, errors.New("no .aep file found inside the bundle")
}
rc, err := best.Open()
if err != nil {
return nil, err
}
defer rc.Close()
return io.ReadAll(rc)
}
// ── AE scan jobs (async, full fidelity) ───────────────────────────────────────
// POST /v1/template-scans/:project_id/jobs (admin)
func (h *ScanHandler) CreateJob(c *gin.Context) {
pid, err := uuid.Parse(c.Param("project_id"))
if err != nil {
c.JSON(http.StatusBadRequest, models.APIError{Code: "bad_request", Message: "invalid project_id"})
return
}
id, err := h.store.CreateScanJob(c.Request.Context(), pid, "ae-jsx")
if err != nil {
c.JSON(http.StatusInternalServerError, models.APIError{Code: "internal_error", Message: err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"id": id, "status": "queued"})
}
// GET /v1/template-scan-jobs/:id (admin)
func (h *ScanHandler) GetJob(c *gin.Context) {
id, err := uuid.Parse(c.Param("id"))
if err != nil {
c.JSON(http.StatusBadRequest, models.APIError{Code: "bad_request", Message: "invalid id"})
return
}
j, err := h.store.GetScanJob(c.Request.Context(), id)
if err != nil {
c.JSON(http.StatusInternalServerError, models.APIError{Code: "internal_error", Message: err.Error()})
return
}
if j == nil {
c.JSON(http.StatusNotFound, models.APIError{Code: "not_found", Message: "scan job not found"})
return
}
c.JSON(http.StatusOK, j)
}
// ── internal (node agents, HMAC) ──────────────────────────────────────────────
// POST /v1/internal/scan/claim
func (h *ScanHandler) Claim(c *gin.Context) {
var req models.ClaimJobRequest
_ = c.ShouldBindJSON(&req)
claim, err := h.store.ClaimScanJob(c.Request.Context(), req.NodeID)
if err != nil {
c.JSON(http.StatusInternalServerError, models.APIError{Code: "internal_error", Message: err.Error()})
return
}
if claim == nil {
c.Status(http.StatusNoContent)
return
}
url, isBundle, md5 := resolveTemplateObject(h.minio, h.templatesBucket, claim.ProjectID)
c.JSON(http.StatusOK, gin.H{
"scan_job_id": claim.ID,
"project_id": claim.ProjectID,
"aep_download_url": url,
"is_bundle": isBundle,
"bundle_md5": md5,
})
}
// POST /v1/internal/scan/:id/result body = ScanResult JSON
func (h *ScanHandler) Result(c *gin.Context) {
id, err := uuid.Parse(c.Param("id"))
if err != nil {
c.JSON(http.StatusBadRequest, models.APIError{Code: "bad_request", Message: "invalid id"})
return
}
body, err := io.ReadAll(c.Request.Body)
if err != nil || !json.Valid(body) {
c.JSON(http.StatusBadRequest, models.APIError{Code: "bad_request", Message: "result must be valid JSON"})
return
}
if err := h.store.SetScanResult(c.Request.Context(), id, string(body)); err != nil {
c.JSON(http.StatusInternalServerError, models.APIError{Code: "internal_error", Message: err.Error()})
return
}
c.Status(http.StatusNoContent)
}
// POST /v1/internal/scan/:id/fail body {reason}
func (h *ScanHandler) Fail(c *gin.Context) {
id, err := uuid.Parse(c.Param("id"))
if err != nil {
c.JSON(http.StatusBadRequest, models.APIError{Code: "bad_request", Message: "invalid id"})
return
}
var req struct {
Reason string `json:"reason"`
}
_ = c.ShouldBindJSON(&req)
if req.Reason == "" {
req.Reason = "scan failed"
}
if err := h.store.SetScanError(c.Request.Context(), id, req.Reason); err != nil {
c.JSON(http.StatusInternalServerError, models.APIError{Code: "internal_error", Message: err.Error()})
return
}
c.Status(http.StatusNoContent)
}
// resolveTemplateObject presigns the canonical template object for a project,
// probing bundle.zip → template.aep → template.aepx (same order as render claim).
func resolveTemplateObject(mc *minio.Client, bucket string, projectID uuid.UUID) (url string, isBundle bool, md5 string) {
if mc == nil {
return "", false, ""
}
candidates := []struct {
name string
bundle bool
}{
{"bundle.zip", true},
{"template.aep", false},
{"template.aepx", false},
}
for _, cand := range candidates {
key := fmt.Sprintf("templates/%s/%s", projectID, cand.name)
info, serr := mc.StatObject(context.Background(), bucket, key, minio.StatObjectOptions{})
if serr != nil {
continue
}
purl, perr := mc.PresignedGetObject(context.Background(), bucket, key, 2*time.Hour, nil)
if perr != nil {
continue
}
return purl.String(), cand.bundle, strings.Trim(info.ETag, "\"")
}
return "", false, ""
}
@@ -0,0 +1,151 @@
package handlers
import (
"context"
"fmt"
"net/http"
"net/url"
"path"
"strings"
"time"
"github.com/flatrender/render-svc/internal/models"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/minio/minio-go/v7"
)
// TemplateBundleHandler stores the canonical After Effects template object for a
// project. Each project has ONE template at a deterministic per-project key, and
// every render of that project reuses it. The source may be a plain .aep/.aepx or
// a .zip bundle (the .aep plus its footage/fonts) — the node agent extracts zips.
type TemplateBundleHandler struct {
minio *minio.Client
templatesBucket string
}
func NewTemplateBundleHandler(mc *minio.Client, templatesBucket string) *TemplateBundleHandler {
return &TemplateBundleHandler{minio: mc, templatesBucket: templatesBucket}
}
type setBundleRequest struct {
// SourceURL is the public/path-style MinIO URL of the freshly uploaded file,
// e.g. http://host:9000/user-uploads/uploads/<uid>/<uuid>.zip
SourceURL string `json:"source_url" binding:"required"`
}
type setBundleResponse struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
MD5 string `json:"md5"`
Size int64 `json:"size_bytes"`
IsBundle bool `json:"is_bundle"`
}
// POST /v1/template-bundles/:project_id
// Copies an uploaded .aep/.aepx/.zip into templates/{project_id}/(bundle.zip|
// template.aep|template.aepx) via a server-side MinIO copy (same backend, no
// proxy through this service), then returns the stored key + md5 so the caller
// can record it on the project. Replacing a template overwrites the same key, so
// in-flight and future renders pick up the new bundle (md5 changes → nodes refetch).
func (h *TemplateBundleHandler) Set(c *gin.Context) {
pid, err := uuid.Parse(c.Param("project_id"))
if err != nil {
c.JSON(http.StatusBadRequest, models.APIError{Code: "bad_request", Message: "invalid project_id"})
return
}
var req setBundleRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, models.APIError{Code: "bad_request", Message: err.Error()})
return
}
if h.minio == nil {
c.JSON(http.StatusServiceUnavailable, models.APIError{Code: "no_storage", Message: "object storage not configured"})
return
}
srcBucket, srcKey, err := parseObjectURL(req.SourceURL)
if err != nil {
c.JSON(http.StatusBadRequest, models.APIError{Code: "bad_source", Message: err.Error()})
return
}
ext := strings.ToLower(path.Ext(srcKey))
isBundle := ext == ".zip"
destName := "template.aep"
switch ext {
case ".zip":
destName = "bundle.zip"
case ".aepx":
destName = "template.aepx"
default: // .aep or unknown — treat as a raw project file
destName = "template.aep"
}
destKey := fmt.Sprintf("templates/%s/%s", pid, destName)
ctx, cancel := context.WithTimeout(c.Request.Context(), 10*time.Minute)
defer cancel()
// The templates bucket may not exist on a fresh deployment — create it (private;
// the node downloads via presigned URL, so no public policy is needed).
if exists, berr := h.minio.BucketExists(ctx, h.templatesBucket); berr == nil && !exists {
if merr := h.minio.MakeBucket(ctx, h.templatesBucket, minio.MakeBucketOptions{}); merr != nil {
// A concurrent request may have created it — only fail if it's still missing.
if ex2, _ := h.minio.BucketExists(ctx, h.templatesBucket); !ex2 {
c.JSON(http.StatusBadGateway, models.APIError{Code: "bucket_error", Message: merr.Error()})
return
}
}
}
// Drop any stale sibling in the other format so the claim probe (which checks
// bundle.zip → template.aep → template.aepx in order) can't return the old file.
for _, n := range []string{"bundle.zip", "template.aep", "template.aepx"} {
if n != destName {
_ = h.minio.RemoveObject(ctx, h.templatesBucket, fmt.Sprintf("templates/%s/%s", pid, n), minio.RemoveObjectOptions{})
}
}
src := minio.CopySrcOptions{Bucket: srcBucket, Object: srcKey}
dst := minio.CopyDestOptions{Bucket: h.templatesBucket, Object: destKey}
if _, err := h.minio.CopyObject(ctx, dst, src); err != nil {
c.JSON(http.StatusBadGateway, models.APIError{Code: "copy_failed", Message: err.Error()})
return
}
info, err := h.minio.StatObject(ctx, h.templatesBucket, destKey, minio.StatObjectOptions{})
if err != nil {
c.JSON(http.StatusBadGateway, models.APIError{Code: "stat_failed", Message: err.Error()})
return
}
c.JSON(http.StatusOK, setBundleResponse{
Bucket: h.templatesBucket,
Key: destKey,
MD5: strings.Trim(info.ETag, "\""),
Size: info.Size,
IsBundle: isBundle,
})
}
// parseObjectURL extracts the bucket and object key from a path-style storage URL
// such as http://host:9000/<bucket>/<key...>. Query/fragment are ignored.
func parseObjectURL(raw string) (bucket, key string, err error) {
u, perr := url.Parse(strings.TrimSpace(raw))
if perr != nil {
return "", "", fmt.Errorf("parse url: %w", perr)
}
p := strings.TrimPrefix(u.Path, "/")
if p == "" {
return "", "", fmt.Errorf("url has no path: %q", raw)
}
parts := strings.SplitN(p, "/", 2)
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return "", "", fmt.Errorf("cannot derive bucket/key from %q", raw)
}
k, uerr := url.PathUnescape(parts[1])
if uerr != nil {
k = parts[1]
}
return parts[0], k, nil
}
+8 -2
View File
@@ -415,9 +415,15 @@ type ClaimedJob struct {
FrameRate int `json:"frame_rate"`
HasMusic bool `json:"has_music"`
HasVoiceover bool `json:"has_voiceover"`
// AEPDownloadURL is a presigned MinIO GET URL for the .aep project file.
// Valid for 2 hours. Empty when the template is not yet uploaded.
// AEPDownloadURL is a presigned MinIO GET URL for the .aep project file
// (or .zip bundle). Valid for 2 hours. Empty when the template is not yet uploaded.
AEPDownloadURL string `json:"aep_download_url,omitempty"`
// IsBundle is true when AEPDownloadURL points to a .zip bundle (the .aep plus
// footage/fonts) that the node agent must extract before rendering.
IsBundle bool `json:"is_bundle,omitempty"`
// BundleMD5 is the stored object's ETag/MD5 — the node uses it as a cache key so
// repeated renders of the same template download + extract the bundle only once.
BundleMD5 string `json:"bundle_md5,omitempty"`
}
// OutputUploadURLResponse is returned by POST .../output-upload-url.