feat(scan): binary FIX scan reads frl_/frd_ names from .aep (no AE, never hangs)
Build backend images / build content-svc (push) Failing after 15s
Build backend images / build file-svc (push) Failing after 1m51s
Build backend images / build gateway (push) Failing after 51s
Build backend images / build identity-svc (push) Failing after 57s
Build backend images / build notification-svc (push) Failing after 52s
Build backend images / build render-svc (push) Failing after 56s
Build backend images / build studio-svc (push) Failing after 57s
Build backend images / build content-svc (push) Failing after 15s
Build backend images / build file-svc (push) Failing after 1m51s
Build backend images / build gateway (push) Failing after 51s
Build backend images / build identity-svc (push) Failing after 57s
Build backend images / build notification-svc (push) Failing after 52s
Build backend images / build render-svc (push) Failing after 56s
Build backend images / build studio-svc (push) Failing after 57s
Root cause of 'stuck on AE': heavy expression-driven projects take >10min for AE to open, exceeding the scan timeout → job dies → admin UI stuck 'scanning'. Fix: extend the stdlib .aep RIFX parser to collect every Utf8 name (ParseNames), since FIX media placeholders are renamed footage ITEMS (frl_c1m1), not layers, and text are layer names (frl_c1t1) — both are Utf8 chunks. QuickScan now branches on ?mode= (or auto-detects frl_ names) and scaffolds FIX scenes/elements + frd_*color slots directly from the binary. Verified on the real final.aep that timed out in AE: 1 scene, 6 elements, 4 colors in 0.5s vs 10-min AE timeout. Admin 'Quick scan (no AE)' is now the recommended path and passes the project mode. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,7 @@ import (
|
||||
type Comp struct {
|
||||
Name string
|
||||
DurationSec float64 // 0 when it could not be derived
|
||||
Layers []string // layer names in file order (used by FIX-mode scanning)
|
||||
}
|
||||
|
||||
// ParseComps walks a RIFX (.aep) buffer and returns its compositions.
|
||||
@@ -35,6 +36,48 @@ func ParseComps(data []byte) ([]Comp, error) {
|
||||
return comps, nil
|
||||
}
|
||||
|
||||
// ParseNames returns every non-empty "Utf8" string in the project tree — layer
|
||||
// names, item/footage names, comp names (and incidentally some text/expression
|
||||
// strings). FIX-mode scanning filters this for the frl_/frd_ naming convention,
|
||||
// which is reliable because those names are globally unique and distinctive.
|
||||
//
|
||||
// This is needed because FIX media placeholders are renamed *footage items*
|
||||
// (e.g. frl_c1m1), not layers, so a layers-only walk misses them. Order is file
|
||||
// order; duplicates are preserved (callers dedup).
|
||||
func ParseNames(data []byte) ([]string, error) {
|
||||
if len(data) < 12 || string(data[0:4]) != "RIFX" {
|
||||
return nil, errors.New("not a RIFX/.aep file")
|
||||
}
|
||||
var names []string
|
||||
collectUtf8(data[12:], &names)
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func collectUtf8(buf []byte, out *[]string) {
|
||||
off := 0
|
||||
for off+8 <= len(buf) {
|
||||
id := string(buf[off : off+4])
|
||||
size := int(binary.BigEndian.Uint32(buf[off+4 : off+8]))
|
||||
ds := off + 8
|
||||
de := ds + size
|
||||
if size < 0 || de > len(buf) {
|
||||
break
|
||||
}
|
||||
if id == "Utf8" {
|
||||
if s := clean(buf[ds:de]); s != "" {
|
||||
*out = append(*out, s)
|
||||
}
|
||||
} else if id == "LIST" && size >= 4 {
|
||||
collectUtf8(buf[ds+4:de], out)
|
||||
}
|
||||
adv := 8 + size
|
||||
if size%2 == 1 {
|
||||
adv++
|
||||
}
|
||||
off += adv
|
||||
}
|
||||
}
|
||||
|
||||
// walk iterates the chunks in buf (a LIST body or the file root), recursing into
|
||||
// every LIST so nested items inside folders are found.
|
||||
func walk(buf []byte, comps *[]Comp, seen map[string]bool) {
|
||||
@@ -69,6 +112,7 @@ func handleItem(body []byte, comps *[]Comp, seen map[string]bool) {
|
||||
var name string
|
||||
var cdta []byte
|
||||
hasCdta := false
|
||||
var layers []string
|
||||
|
||||
off := 0
|
||||
for off+8 <= len(body) {
|
||||
@@ -79,13 +123,20 @@ func handleItem(body []byte, comps *[]Comp, seen map[string]bool) {
|
||||
if size < 0 || de > len(body) {
|
||||
break
|
||||
}
|
||||
switch id {
|
||||
case "cdta":
|
||||
switch {
|
||||
case id == "cdta":
|
||||
hasCdta = true
|
||||
cdta = body[ds:de]
|
||||
case "Utf8":
|
||||
case id == "Utf8":
|
||||
if name == "" {
|
||||
name = strings.TrimSpace(strings.TrimRight(string(body[ds:de]), "\x00"))
|
||||
name = clean(body[ds:de])
|
||||
}
|
||||
case id == "LIST" && size >= 4 && string(body[ds:ds+4]) == "Layr":
|
||||
// A composition's layers are nested "Layr" LISTs; the layer's display
|
||||
// name is the first "Utf8" inside it (FIX templates encode scenes in
|
||||
// these names, e.g. frl_c1t1). Empty/unnamed layers are skipped.
|
||||
if ln := layerName(body[ds+4 : de]); ln != "" {
|
||||
layers = append(layers, ln)
|
||||
}
|
||||
}
|
||||
adv := 8 + size
|
||||
@@ -97,10 +148,39 @@ func handleItem(body []byte, comps *[]Comp, seen map[string]bool) {
|
||||
|
||||
if hasCdta && name != "" && !seen[name] {
|
||||
seen[name] = true
|
||||
*comps = append(*comps, Comp{Name: name, DurationSec: durationFromCdta(cdta)})
|
||||
*comps = append(*comps, Comp{Name: name, DurationSec: durationFromCdta(cdta), Layers: layers})
|
||||
}
|
||||
}
|
||||
|
||||
// layerName returns the first "Utf8" string among the DIRECT children of a "Layr"
|
||||
// LIST body — that is the layer's display name. It does not recurse, so nested
|
||||
// property/effect names (also stored as Utf8) cannot be mistaken for the name.
|
||||
func layerName(body []byte) string {
|
||||
off := 0
|
||||
for off+8 <= len(body) {
|
||||
id := string(body[off : off+4])
|
||||
size := int(binary.BigEndian.Uint32(body[off+4 : off+8]))
|
||||
ds := off + 8
|
||||
de := ds + size
|
||||
if size < 0 || de > len(body) {
|
||||
break
|
||||
}
|
||||
if id == "Utf8" {
|
||||
return clean(body[ds:de])
|
||||
}
|
||||
adv := 8 + size
|
||||
if size%2 == 1 {
|
||||
adv++
|
||||
}
|
||||
off += adv
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func clean(b []byte) string {
|
||||
return strings.TrimSpace(strings.TrimRight(string(b), "\x00"))
|
||||
}
|
||||
|
||||
// durationFromCdta makes a best-effort attempt to read the comp duration from the
|
||||
// cdta chunk. The cdta layout varies by AE version; we read the frame-duration /
|
||||
// time-scale pair at well-known offsets and fall back to 0 (unknown) on any doubt.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package aep
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDumpRealAEP is a manual harness: set FR_TEST_AEP to a .aep path to print the
|
||||
// comps + layer names the parser extracts. Skipped in normal CI runs.
|
||||
func TestDumpRealAEP(t *testing.T) {
|
||||
path := os.Getenv("FR_TEST_AEP")
|
||||
if path == "" {
|
||||
t.Skip("set FR_TEST_AEP to a .aep path")
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
comps, err := ParseComps(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names, _ := ParseNames(data)
|
||||
t.Logf("parsed %d comps, %d names from %s (%d bytes)", len(comps), len(names), path, len(data))
|
||||
for _, c := range comps {
|
||||
t.Logf("COMP %q dur=%.2fs layers=%d", c.Name, c.DurationSec, len(c.Layers))
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,9 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -86,6 +89,17 @@ func (h *ScanHandler) QuickScan(c *gin.Context) {
|
||||
c.JSON(http.StatusNotFound, models.APIError{Code: "no_template", Message: err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// FIX / MusicVisualizer projects encode scenes in layer/item NAMES
|
||||
// (frl_c{scene}{t|m}{idx}) rather than one-comp-per-scene. Detect them by the
|
||||
// ?mode= query (or auto: any frl_c name present) and parse names instead of comps.
|
||||
mode := strings.ToLower(c.Query("mode"))
|
||||
names, nerr := aep.ParseNames(data)
|
||||
if nerr == nil && (mode == "fix" || mode == "musicvisualizer" || (mode == "" && hasFrlNames(names))) {
|
||||
c.JSON(http.StatusOK, buildFixResult(names))
|
||||
return
|
||||
}
|
||||
|
||||
comps, err := aep.ParseComps(data)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusUnprocessableEntity, models.APIError{Code: "parse_failed", Message: err.Error()})
|
||||
@@ -113,6 +127,74 @@ func (h *ScanHandler) QuickScan(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, res)
|
||||
}
|
||||
|
||||
var frlRe = regexp.MustCompile(`^frl_c(\d+)([tm])(\d+)$`)
|
||||
|
||||
func hasFrlNames(names []string) bool {
|
||||
for _, n := range names {
|
||||
if frlRe.MatchString(n) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// buildFixResult turns the flat list of project names into FIX-mode scenes.
|
||||
// frl_c{scene}{t|m}{idx} → element grouped by scene (t=Text, m=Media); frd_*color
|
||||
// → shared colour slot (value left empty for the binder/admin to fill, since the
|
||||
// binary parser cannot read the RGBA value).
|
||||
func buildFixResult(names []string) scanResult {
|
||||
res := scanResult{Source: "go-parser", RenderComp: "frfinal", Scenes: []scanScene{}, SharedColors: []scanColor{}}
|
||||
|
||||
byScene := map[int]*scanScene{}
|
||||
order := []int{}
|
||||
seen := map[string]bool{}
|
||||
for _, n := range names {
|
||||
m := frlRe.FindStringSubmatch(n)
|
||||
if m == nil || seen[n] {
|
||||
continue
|
||||
}
|
||||
seen[n] = true
|
||||
sceneNum, _ := strconv.Atoi(m[1])
|
||||
idx, _ := strconv.Atoi(m[3])
|
||||
sc := byScene[sceneNum]
|
||||
if sc == nil {
|
||||
sc = &scanScene{
|
||||
Key: fmt.Sprintf("c%d", sceneNum), Title: fmt.Sprintf("Scene %d", sceneNum),
|
||||
SceneType: "Normal", Elements: []interface{}{}, Colors: []scanColor{},
|
||||
}
|
||||
byScene[sceneNum] = sc
|
||||
order = append(order, sceneNum)
|
||||
}
|
||||
elemType := "Text"
|
||||
if m[2] == "m" {
|
||||
elemType = "Media"
|
||||
}
|
||||
sc.Elements = append(sc.Elements, map[string]interface{}{
|
||||
"key": n, "title": n, "type": elemType, "sort": idx,
|
||||
})
|
||||
}
|
||||
sort.Ints(order)
|
||||
for i, sn := range order {
|
||||
sc := byScene[sn]
|
||||
sc.Sort = i
|
||||
res.Scenes = append(res.Scenes, *sc)
|
||||
}
|
||||
|
||||
// shared colours — frd_*color names (controls like frd_bgstyle are skipped;
|
||||
// they are render-time switches, not scene colours).
|
||||
seenColor := map[string]bool{}
|
||||
for _, n := range names {
|
||||
if !strings.HasPrefix(n, "frd_") || !strings.Contains(strings.ToLower(n), "color") || seenColor[n] {
|
||||
continue
|
||||
}
|
||||
seenColor[n] = true
|
||||
res.SharedColors = append(res.SharedColors, scanColor{
|
||||
ElementKey: n, Title: n, AttrValue: n, Sort: len(res.SharedColors),
|
||||
})
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// loadAep fetches the project's template .aep bytes from MinIO — directly when a
|
||||
// raw .aep was uploaded, or by extracting the .aep from the .zip bundle.
|
||||
func (h *ScanHandler) loadAep(ctx context.Context, pid uuid.UUID) ([]byte, error) {
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/flatrender/render-svc/internal/aep"
|
||||
)
|
||||
|
||||
// TestBuildFixResultRealAEP: set FR_TEST_AEP to a FIX .aep to see the scenes the
|
||||
// binary parser scaffolds (no AE). Skipped in normal runs.
|
||||
func TestBuildFixResultRealAEP(t *testing.T) {
|
||||
path := os.Getenv("FR_TEST_AEP")
|
||||
if path == "" {
|
||||
t.Skip("set FR_TEST_AEP")
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names, err := aep.ParseNames(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res := buildFixResult(names)
|
||||
b, _ := json.MarshalIndent(res, "", " ")
|
||||
t.Logf("FIX scan result:\n%s", string(b))
|
||||
if len(res.Scenes) == 0 {
|
||||
t.Fatalf("expected at least one scene")
|
||||
}
|
||||
total := 0
|
||||
for _, s := range res.Scenes {
|
||||
total += len(s.Elements)
|
||||
}
|
||||
t.Logf("scenes=%d total elements=%d shared_colors=%d", len(res.Scenes), total, len(res.SharedColors))
|
||||
}
|
||||
@@ -80,7 +80,7 @@ export function ProjectScanImport({ projectId, onClose, onApplied }: {
|
||||
const runQuick = async () => {
|
||||
jobRef.current = null; // quick scan is synchronous — no timer/cancel
|
||||
setStep("scanning"); setErr(null); setStatusMsg("در حال خواندن ساختار پروژه از فایل AEP…");
|
||||
const r = await fetch(`/api/admin/resource/template-scans/${projectId}/quick`, { method: "POST", headers: { "Content-Type": "application/json" }, body: "{}" });
|
||||
const r = await fetch(`/api/admin/resource/template-scans/${projectId}/quick?mode=${encodeURIComponent(mode)}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: "{}" });
|
||||
const d = await r.json().catch(() => null);
|
||||
if (!r.ok) { fail(d?.error ?? "اسکن سریع ناموفق بود (آیا فایل AEP آپلود شده؟)"); return; }
|
||||
await preview(d);
|
||||
@@ -135,12 +135,8 @@ export function ProjectScanImport({ projectId, onClose, onApplied }: {
|
||||
{step === "idle" && (
|
||||
<div className="space-y-3">
|
||||
<p className="text-sm text-gray-400">ساختار قالب را مستقیماً از فایل افترافکت بخوانید. ابتدا یک پیشنمایش از تغییرات میبینید و سپس اعمال میکنید (ویرایشهای دستی حفظ میشوند).</p>
|
||||
<button className="w-full rounded-lg border border-[#262b40] p-3 text-right hover:bg-[#161a2e]" onClick={runQuick}>
|
||||
<div className="text-sm font-medium text-white">اسکن سریع (بدون افترافکت)</div>
|
||||
<div className="mt-0.5 text-xs text-gray-500">فقط نام صحنهها و مدتها را از فایل AEP میخواند. فوری، بدون نیاز به نود. رنگها/فونتها بعداً با اسکن کامل پر میشوند.</div>
|
||||
</button>
|
||||
<div className="rounded-lg border border-[#262b40] p-3">
|
||||
<label className="mb-1 block text-xs text-gray-400">نوع پروژه (برای اسکن کامل)</label>
|
||||
<label className="mb-1 block text-xs text-gray-400">نوع پروژه</label>
|
||||
<select className="w-full rounded-lg border border-[#262b40] bg-[#0c0e1a] px-2.5 py-1.5 text-sm text-gray-100 outline-none focus:border-indigo-500" value={mode} onChange={(e) => setMode(e.target.value)}>
|
||||
<option value="fix">Fix — صحنهها از نام لایهها (frl_c1t1)</option>
|
||||
<option value="flexible">Flexible — هر صحنه یک کامپوزیشن</option>
|
||||
@@ -148,9 +144,13 @@ export function ProjectScanImport({ projectId, onClose, onApplied }: {
|
||||
<option value="musicvisualizer">Music Visualizer</option>
|
||||
</select>
|
||||
</div>
|
||||
<button className="w-full rounded-lg border border-emerald-500/30 bg-emerald-500/5 p-3 text-right hover:bg-emerald-500/10" onClick={runQuick}>
|
||||
<div className="text-sm font-medium text-white">اسکن سریع (بدون افترافکت) — پیشنهادی</div>
|
||||
<div className="mt-0.5 text-xs text-gray-500">صحنهها و عناصر (متن/مدیا) را مستقیماً از فایل AEP میخواند — فوری و بدون نیاز به نود، هیچوقت گیر نمیکند. مقادیر رنگ/فونت بعداً با اسکن کامل پر میشوند.</div>
|
||||
</button>
|
||||
<button className="w-full rounded-lg border border-[#262b40] p-3 text-right hover:bg-[#161a2e]" onClick={runFull}>
|
||||
<div className="text-sm font-medium text-white">اسکن کامل (روی نود افترافکت)</div>
|
||||
<div className="mt-0.5 text-xs text-gray-500">صحنهها، عناصر، فونتها، چینش و رنگها (frshare) را کامل میخواند. نیازمند یک نود افترافکت آنلاین است.</div>
|
||||
<div className="mt-0.5 text-xs text-gray-500">علاوه بر ساختار، مقدار فونتها، چینش و رنگها (frshare) را هم میخواند. کندتر و نیازمند یک نود افترافکت آنلاین است.</div>
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
Reference in New Issue
Block a user