feat(scan): binary FIX scan reads frl_/frd_ names from .aep (no AE, never hangs)
Build backend images / build content-svc (push) Failing after 15s
Build backend images / build file-svc (push) Failing after 1m51s
Build backend images / build gateway (push) Failing after 51s
Build backend images / build identity-svc (push) Failing after 57s
Build backend images / build notification-svc (push) Failing after 52s
Build backend images / build render-svc (push) Failing after 56s
Build backend images / build studio-svc (push) Failing after 57s

Root cause of 'stuck on AE': heavy expression-driven projects take >10min for AE
to open, exceeding the scan timeout → job dies → admin UI stuck 'scanning'.

Fix: extend the stdlib .aep RIFX parser to collect every Utf8 name (ParseNames),
since FIX media placeholders are renamed footage ITEMS (frl_c1m1), not layers, and
text are layer names (frl_c1t1) — both are Utf8 chunks. QuickScan now branches on
?mode= (or auto-detects frl_ names) and scaffolds FIX scenes/elements + frd_*color
slots directly from the binary. Verified on the real final.aep that timed out in AE:
1 scene, 6 elements, 4 colors in 0.5s vs 10-min AE timeout.

Admin 'Quick scan (no AE)' is now the recommended path and passes the project mode.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
soroush.asadi
2026-06-04 22:48:53 +03:30
parent f0ce286527
commit 718564bce4
5 changed files with 240 additions and 13 deletions
+85 -5
View File
@@ -16,6 +16,7 @@ import (
type Comp struct {
Name string
DurationSec float64 // 0 when it could not be derived
Layers []string // layer names in file order (used by FIX-mode scanning)
}
// ParseComps walks a RIFX (.aep) buffer and returns its compositions.
@@ -35,6 +36,48 @@ func ParseComps(data []byte) ([]Comp, error) {
return comps, nil
}
// ParseNames returns every non-empty "Utf8" string in the project tree — layer
// names, item/footage names, comp names (and incidentally some text/expression
// strings). FIX-mode scanning filters this for the frl_/frd_ naming convention,
// which is reliable because those names are globally unique and distinctive.
//
// This is needed because FIX media placeholders are renamed *footage items*
// (e.g. frl_c1m1), not layers, so a layers-only walk misses them. Order is file
// order; duplicates are preserved (callers dedup).
func ParseNames(data []byte) ([]string, error) {
if len(data) < 12 || string(data[0:4]) != "RIFX" {
return nil, errors.New("not a RIFX/.aep file")
}
var names []string
collectUtf8(data[12:], &names)
return names, nil
}
func collectUtf8(buf []byte, out *[]string) {
off := 0
for off+8 <= len(buf) {
id := string(buf[off : off+4])
size := int(binary.BigEndian.Uint32(buf[off+4 : off+8]))
ds := off + 8
de := ds + size
if size < 0 || de > len(buf) {
break
}
if id == "Utf8" {
if s := clean(buf[ds:de]); s != "" {
*out = append(*out, s)
}
} else if id == "LIST" && size >= 4 {
collectUtf8(buf[ds+4:de], out)
}
adv := 8 + size
if size%2 == 1 {
adv++
}
off += adv
}
}
// walk iterates the chunks in buf (a LIST body or the file root), recursing into
// every LIST so nested items inside folders are found.
func walk(buf []byte, comps *[]Comp, seen map[string]bool) {
@@ -69,6 +112,7 @@ func handleItem(body []byte, comps *[]Comp, seen map[string]bool) {
var name string
var cdta []byte
hasCdta := false
var layers []string
off := 0
for off+8 <= len(body) {
@@ -79,13 +123,20 @@ func handleItem(body []byte, comps *[]Comp, seen map[string]bool) {
if size < 0 || de > len(body) {
break
}
switch id {
case "cdta":
switch {
case id == "cdta":
hasCdta = true
cdta = body[ds:de]
case "Utf8":
case id == "Utf8":
if name == "" {
name = strings.TrimSpace(strings.TrimRight(string(body[ds:de]), "\x00"))
name = clean(body[ds:de])
}
case id == "LIST" && size >= 4 && string(body[ds:ds+4]) == "Layr":
// A composition's layers are nested "Layr" LISTs; the layer's display
// name is the first "Utf8" inside it (FIX templates encode scenes in
// these names, e.g. frl_c1t1). Empty/unnamed layers are skipped.
if ln := layerName(body[ds+4 : de]); ln != "" {
layers = append(layers, ln)
}
}
adv := 8 + size
@@ -97,10 +148,39 @@ func handleItem(body []byte, comps *[]Comp, seen map[string]bool) {
if hasCdta && name != "" && !seen[name] {
seen[name] = true
*comps = append(*comps, Comp{Name: name, DurationSec: durationFromCdta(cdta)})
*comps = append(*comps, Comp{Name: name, DurationSec: durationFromCdta(cdta), Layers: layers})
}
}
// layerName returns the first "Utf8" string among the DIRECT children of a "Layr"
// LIST body — that is the layer's display name. It does not recurse, so nested
// property/effect names (also stored as Utf8) cannot be mistaken for the name.
func layerName(body []byte) string {
off := 0
for off+8 <= len(body) {
id := string(body[off : off+4])
size := int(binary.BigEndian.Uint32(body[off+4 : off+8]))
ds := off + 8
de := ds + size
if size < 0 || de > len(body) {
break
}
if id == "Utf8" {
return clean(body[ds:de])
}
adv := 8 + size
if size%2 == 1 {
adv++
}
off += adv
}
return ""
}
func clean(b []byte) string {
return strings.TrimSpace(strings.TrimRight(string(b), "\x00"))
}
// durationFromCdta makes a best-effort attempt to read the comp duration from the
// cdta chunk. The cdta layout varies by AE version; we read the frame-duration /
// time-scale pair at well-known offsets and fall back to 0 (unknown) on any doubt.
@@ -0,0 +1,28 @@
package aep
import (
"os"
"testing"
)
// TestDumpRealAEP is a manual harness: set FR_TEST_AEP to a .aep path to print the
// comps + layer names the parser extracts. Skipped in normal CI runs.
func TestDumpRealAEP(t *testing.T) {
path := os.Getenv("FR_TEST_AEP")
if path == "" {
t.Skip("set FR_TEST_AEP to a .aep path")
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
comps, err := ParseComps(data)
if err != nil {
t.Fatal(err)
}
names, _ := ParseNames(data)
t.Logf("parsed %d comps, %d names from %s (%d bytes)", len(comps), len(names), path, len(data))
for _, c := range comps {
t.Logf("COMP %q dur=%.2fs layers=%d", c.Name, c.DurationSec, len(c.Layers))
}
}
+82
View File
@@ -10,6 +10,9 @@ import (
"io"
"net/http"
"path"
"regexp"
"sort"
"strconv"
"strings"
"time"
@@ -86,6 +89,17 @@ func (h *ScanHandler) QuickScan(c *gin.Context) {
c.JSON(http.StatusNotFound, models.APIError{Code: "no_template", Message: err.Error()})
return
}
// FIX / MusicVisualizer projects encode scenes in layer/item NAMES
// (frl_c{scene}{t|m}{idx}) rather than one-comp-per-scene. Detect them by the
// ?mode= query (or auto: any frl_c name present) and parse names instead of comps.
mode := strings.ToLower(c.Query("mode"))
names, nerr := aep.ParseNames(data)
if nerr == nil && (mode == "fix" || mode == "musicvisualizer" || (mode == "" && hasFrlNames(names))) {
c.JSON(http.StatusOK, buildFixResult(names))
return
}
comps, err := aep.ParseComps(data)
if err != nil {
c.JSON(http.StatusUnprocessableEntity, models.APIError{Code: "parse_failed", Message: err.Error()})
@@ -113,6 +127,74 @@ func (h *ScanHandler) QuickScan(c *gin.Context) {
c.JSON(http.StatusOK, res)
}
var frlRe = regexp.MustCompile(`^frl_c(\d+)([tm])(\d+)$`)
func hasFrlNames(names []string) bool {
for _, n := range names {
if frlRe.MatchString(n) {
return true
}
}
return false
}
// buildFixResult turns the flat list of project names into FIX-mode scenes.
// frl_c{scene}{t|m}{idx} → element grouped by scene (t=Text, m=Media); frd_*color
// → shared colour slot (value left empty for the binder/admin to fill, since the
// binary parser cannot read the RGBA value).
func buildFixResult(names []string) scanResult {
res := scanResult{Source: "go-parser", RenderComp: "frfinal", Scenes: []scanScene{}, SharedColors: []scanColor{}}
byScene := map[int]*scanScene{}
order := []int{}
seen := map[string]bool{}
for _, n := range names {
m := frlRe.FindStringSubmatch(n)
if m == nil || seen[n] {
continue
}
seen[n] = true
sceneNum, _ := strconv.Atoi(m[1])
idx, _ := strconv.Atoi(m[3])
sc := byScene[sceneNum]
if sc == nil {
sc = &scanScene{
Key: fmt.Sprintf("c%d", sceneNum), Title: fmt.Sprintf("Scene %d", sceneNum),
SceneType: "Normal", Elements: []interface{}{}, Colors: []scanColor{},
}
byScene[sceneNum] = sc
order = append(order, sceneNum)
}
elemType := "Text"
if m[2] == "m" {
elemType = "Media"
}
sc.Elements = append(sc.Elements, map[string]interface{}{
"key": n, "title": n, "type": elemType, "sort": idx,
})
}
sort.Ints(order)
for i, sn := range order {
sc := byScene[sn]
sc.Sort = i
res.Scenes = append(res.Scenes, *sc)
}
// shared colours — frd_*color names (controls like frd_bgstyle are skipped;
// they are render-time switches, not scene colours).
seenColor := map[string]bool{}
for _, n := range names {
if !strings.HasPrefix(n, "frd_") || !strings.Contains(strings.ToLower(n), "color") || seenColor[n] {
continue
}
seenColor[n] = true
res.SharedColors = append(res.SharedColors, scanColor{
ElementKey: n, Title: n, AttrValue: n, Sort: len(res.SharedColors),
})
}
return res
}
// loadAep fetches the project's template .aep bytes from MinIO — directly when a
// raw .aep was uploaded, or by extracting the .aep from the .zip bundle.
func (h *ScanHandler) loadAep(ctx context.Context, pid uuid.UUID) ([]byte, error) {
@@ -0,0 +1,37 @@
package handlers
import (
"encoding/json"
"os"
"testing"
"github.com/flatrender/render-svc/internal/aep"
)
// TestBuildFixResultRealAEP: set FR_TEST_AEP to a FIX .aep to see the scenes the
// binary parser scaffolds (no AE). Skipped in normal runs.
func TestBuildFixResultRealAEP(t *testing.T) {
path := os.Getenv("FR_TEST_AEP")
if path == "" {
t.Skip("set FR_TEST_AEP")
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
names, err := aep.ParseNames(data)
if err != nil {
t.Fatal(err)
}
res := buildFixResult(names)
b, _ := json.MarshalIndent(res, "", " ")
t.Logf("FIX scan result:\n%s", string(b))
if len(res.Scenes) == 0 {
t.Fatalf("expected at least one scene")
}
total := 0
for _, s := range res.Scenes {
total += len(s.Elements)
}
t.Logf("scenes=%d total elements=%d shared_colors=%d", len(res.Scenes), total, len(res.SharedColors))
}
+7 -7
View File
@@ -80,7 +80,7 @@ export function ProjectScanImport({ projectId, onClose, onApplied }: {
const runQuick = async () => {
jobRef.current = null; // quick scan is synchronous — no timer/cancel
setStep("scanning"); setErr(null); setStatusMsg("در حال خواندن ساختار پروژه از فایل AEP…");
const r = await fetch(`/api/admin/resource/template-scans/${projectId}/quick`, { method: "POST", headers: { "Content-Type": "application/json" }, body: "{}" });
const r = await fetch(`/api/admin/resource/template-scans/${projectId}/quick?mode=${encodeURIComponent(mode)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: "{}" });
const d = await r.json().catch(() => null);
if (!r.ok) { fail(d?.error ?? "اسکن سریع ناموفق بود (آیا فایل AEP آپلود شده؟)"); return; }
await preview(d);
@@ -135,12 +135,8 @@ export function ProjectScanImport({ projectId, onClose, onApplied }: {
{step === "idle" && (
<div className="space-y-3">
<p className="text-sm text-gray-400">ساختار قالب را مستقیماً از فایل افترافکت بخوانید. ابتدا یک پیشنمایش از تغییرات میبینید و سپس اعمال میکنید (ویرایشهای دستی حفظ میشوند).</p>
<button className="w-full rounded-lg border border-[#262b40] p-3 text-right hover:bg-[#161a2e]" onClick={runQuick}>
<div className="text-sm font-medium text-white">اسکن سریع (بدون افترافکت)</div>
<div className="mt-0.5 text-xs text-gray-500">فقط نام صحنهها و مدتها را از فایل AEP میخواند. فوری، بدون نیاز به نود. رنگها/فونتها بعداً با اسکن کامل پر میشوند.</div>
</button>
<div className="rounded-lg border border-[#262b40] p-3">
<label className="mb-1 block text-xs text-gray-400">نوع پروژه (برای اسکن کامل)</label>
<label className="mb-1 block text-xs text-gray-400">نوع پروژه</label>
<select className="w-full rounded-lg border border-[#262b40] bg-[#0c0e1a] px-2.5 py-1.5 text-sm text-gray-100 outline-none focus:border-indigo-500" value={mode} onChange={(e) => setMode(e.target.value)}>
<option value="fix">Fix صحنهها از نام لایهها (frl_c1t1)</option>
<option value="flexible">Flexible هر صحنه یک کامپوزیشن</option>
@@ -148,9 +144,13 @@ export function ProjectScanImport({ projectId, onClose, onApplied }: {
<option value="musicvisualizer">Music Visualizer</option>
</select>
</div>
<button className="w-full rounded-lg border border-emerald-500/30 bg-emerald-500/5 p-3 text-right hover:bg-emerald-500/10" onClick={runQuick}>
<div className="text-sm font-medium text-white">اسکن سریع (بدون افترافکت) پیشنهادی</div>
<div className="mt-0.5 text-xs text-gray-500">صحنهها و عناصر (متن/مدیا) را مستقیماً از فایل AEP میخواند فوری و بدون نیاز به نود، هیچوقت گیر نمیکند. مقادیر رنگ/فونت بعداً با اسکن کامل پر میشوند.</div>
</button>
<button className="w-full rounded-lg border border-[#262b40] p-3 text-right hover:bg-[#161a2e]" onClick={runFull}>
<div className="text-sm font-medium text-white">اسکن کامل (روی نود افترافکت)</div>
<div className="mt-0.5 text-xs text-gray-500">صحنهها، عناصر، فونتها، چینش و رنگها (frshare) را کامل میخواند. نیازمند یک نود افترافکت آنلاین است.</div>
<div className="mt-0.5 text-xs text-gray-500">علاوه بر ساختار، مقدار فونتها، چینش و رنگها (frshare) را هم میخواند. کندتر و نیازمند یک نود افترافکت آنلاین است.</div>
</button>
</div>
)}