From 21b6a30f08f796ced811be884f3ac7ea7fdc71e3 Mon Sep 17 00:00:00 2001 From: "soroush.asadi" Date: Thu, 25 Jun 2026 10:09:41 +0330 Subject: [PATCH] feat(scripts): portable template import/export (bundles) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move a template fully between environments (local → live): container, projects, all scenes + editable fields, shared colours/layers, its categories & tags, and the asset files. - export_template.py → a self-contained bundle (template.json + assets/). One SQL query captures the whole tree as JSON; assets are resolved from template-media references and copied in. Source DB via PSQL env (default = local docker). - import_template.py → idempotent SQL (pipe to target psql). Replaces by slug via one cascading delete (all content.* FKs are ON DELETE CASCADE), recreates rows verbatim (UUIDs preserved → FKs intact), merges categories/tags BY SLUG so they line up across DBs. --assets-to copies media; docker cp / mc cp hints for remote. - TEMPLATE_BUNDLES.md documents it. Round-trip tested on fr-instagram-promo: DB → bundle → DB restores identical 3 projects / 15 scenes / 138 fields and field values. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + scripts/TEMPLATE_BUNDLES.md | 67 ++++++++++++++++++++++++++ scripts/export_template.py | 93 +++++++++++++++++++++++++++++++++++++ scripts/import_template.py | 86 ++++++++++++++++++++++++++++++++++ 4 files changed, 247 insertions(+) create mode 100644 scripts/TEMPLATE_BUNDLES.md create mode 100644 scripts/export_template.py create mode 100644 scripts/import_template.py diff --git a/.gitignore b/.gitignore index 1ba30bb..596efd1 100644 --- a/.gitignore +++ b/.gitignore @@ -63,3 +63,4 @@ services/remotion/out/ # local scratch / agent work /-w /.agent-work/ +dist/ diff --git a/scripts/TEMPLATE_BUNDLES.md b/scripts/TEMPLATE_BUNDLES.md new file mode 100644 index 0000000..de85f09 --- /dev/null +++ b/scripts/TEMPLATE_BUNDLES.md @@ -0,0 +1,67 @@ +# Template import / export (portable bundles) + +Move a template — **everything**: container, projects, all scenes + editable fields, +shared colours/layers, the categories & tags it belongs to, and the asset files — +between environments (e.g. local → live). + +A **bundle** is a self-contained directory: + +``` +dist/template-bundles// +├── template.json # every DB row related to the template +└── assets/ # the media it references (thumbnails, scene stills, preview…) +``` + +## Export (from the source DB) + +```bash +PYTHONIOENCODING=utf-8 python scripts/export_template.py +# → dist/template-bundles// + +# non-default source DB: +PSQL="psql 'postgresql://user:pass@host:5432/flatrender'" \ + python scripts/export_template.py +``` + +## Import (into ANY target DB) + +The importer prints idempotent SQL — pipe it to the target's `psql`. It **replaces** +any template with the same slug (one cascading delete), recreates rows verbatim +(original UUIDs, so foreign keys stay intact), and **merges categories & tags by +slug** so they line up with whatever the target already calls them. + +```bash +# local: +python scripts/import_template.py dist/template-bundles/ \ + | docker exec -i fr2-postgres psql -U postgres -d flatrender + +# live: +python scripts/import_template.py dist/template-bundles/ \ + | psql "postgresql://user:pass@live-host:5432/flatrender" +``` + +### Assets +The SQL only moves DB rows. Place the bundle's `assets/` into the target's +`template-media`: + +```bash +# copy locally while importing: +python scripts/import_template.py --assets-to ./public/template-media | ... + +# live docker frontend: +for f in /assets/*; do docker cp "$f" :/app/public/template-media/; done + +# live MinIO/object store: +mc cp --recursive /assets/ myminio//template-media/ +``` + +## Guarantees & notes +- **Idempotent** — re-importing the same bundle re-creates the template cleanly. +- **Cross-DB safe** — UUIDs are preserved; categories/tags merge by their unique slug. +- **Round-trip tested** — DB → bundle → DB restores identical counts and values. +- **Not exported** (intentionally): user data — comments, favourites, view/use counts. +- **External (http) media** referenced by a template is flagged on export but not + bundled — host it on the target yourself. +- Scope today covers FlexStory/Remotion + standard fields. AEP-specific child tables + (characters, colour presets) are an easy extension — add them to the export query + and the importer's insert list (both are data-driven). diff --git a/scripts/export_template.py b/scripts/export_template.py new file mode 100644 index 0000000..0975895 --- /dev/null +++ b/scripts/export_template.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Export a FlatRender template to a portable, self-contained bundle. + +A bundle = a directory holding `template.json` (every DB row related to the template +— container, projects, scenes, content-elements, shared colours/layers, plus the +categories & tags it links to, merged by slug on import) and `assets/` (the actual +media files it references). Move the directory to any environment and run +`import_template.py` to load it into that DB. + +Usage: + python scripts/export_template.py [out_dir] + # source DB override (default = local docker): + PSQL="psql 'postgresql://user:pass@host:5432/db'" python scripts/export_template.py + +What it does NOT export: user-generated data (comments, favourites, view counts are +reset on import) — only the template definition + assets travel. +""" +import os, sys, json, subprocess, shutil, re, shlex + +if len(sys.argv) < 2: + print("usage: export_template.py [out_dir]", file=sys.stderr); sys.exit(1) +SLUG = sys.argv[1] +OUT = sys.argv[2] if len(sys.argv) > 2 else os.path.join("dist", "template-bundles", SLUG) +PSQL = shlex.split(os.environ.get("PSQL", "docker exec -i fr2-postgres psql -U postgres -d flatrender")) +PUBLIC = os.environ.get("TEMPLATE_MEDIA", "public/template-media") +S = SLUG.replace("'", "''") + +QUERY = f""" +WITH c AS (SELECT id FROM content.project_containers WHERE slug = '{S}' AND deleted_at IS NULL) +SELECT json_build_object( + 'version', 1, + 'slug', '{S}', + 'container', (SELECT row_to_json(x) FROM content.project_containers x WHERE x.id=(SELECT id FROM c)), + 'projects', (SELECT coalesce(json_agg(row_to_json(x) ORDER BY x.sort),'[]') FROM content.projects x WHERE x.container_id=(SELECT id FROM c)), + 'scenes', (SELECT coalesce(json_agg(row_to_json(x) ORDER BY x.sort),'[]') FROM content.scenes x WHERE x.project_id IN (SELECT id FROM content.projects WHERE container_id=(SELECT id FROM c))), + 'content_elements', (SELECT coalesce(json_agg(row_to_json(x) ORDER BY x.position_in_container),'[]') FROM content.scene_content_elements x WHERE x.scene_id IN (SELECT s.id FROM content.scenes s JOIN content.projects p ON p.id=s.project_id WHERE p.container_id=(SELECT id FROM c))), + 'shared_colors', (SELECT coalesce(json_agg(row_to_json(x) ORDER BY x.sort),'[]') FROM content.shared_colors x WHERE x.project_id IN (SELECT id FROM content.projects WHERE container_id=(SELECT id FROM c))), + 'shared_layers', (SELECT coalesce(json_agg(row_to_json(x)),'[]') FROM content.shared_layers x WHERE x.project_id IN (SELECT id FROM content.projects WHERE container_id=(SELECT id FROM c))), + 'categories', (SELECT coalesce(json_agg(row_to_json(x)),'[]') FROM content.categories x WHERE x.id IN (SELECT category_id FROM content.container_categories WHERE container_id=(SELECT id FROM c))), + 'category_links', (SELECT coalesce(json_agg(json_build_object('category_slug',cat.slug,'sort',cc.sort)),'[]') FROM content.container_categories cc JOIN content.categories cat ON cat.id=cc.category_id WHERE cc.container_id=(SELECT id FROM c)), + 'tags', (SELECT coalesce(json_agg(row_to_json(x)),'[]') FROM content.tags x WHERE x.id IN (SELECT tag_id FROM content.container_tags WHERE container_id=(SELECT id FROM c))), + 'tag_links', (SELECT coalesce(json_agg(json_build_object('tag_slug',tg.slug)),'[]') FROM content.container_tags ct JOIN content.tags tg ON tg.id=ct.tag_id WHERE ct.container_id=(SELECT id FROM c)) +); +""" + +res = subprocess.run(PSQL + ["-t", "-A"], input=QUERY, capture_output=True, text=True, encoding="utf-8") +if res.returncode != 0: + print("psql error:\n" + res.stderr, file=sys.stderr); sys.exit(1) +raw = res.stdout.strip() +if not raw or raw == "": + print(f"no output from DB for slug '{SLUG}'", file=sys.stderr); sys.exit(1) +bundle = json.loads(raw) +if not bundle.get("container"): + print(f"template '{SLUG}' not found in source DB", file=sys.stderr); sys.exit(1) + +# Collect asset filenames referenced anywhere in the bundle (template-media paths). +assets, externals = set(), set() +def scan(v): + if isinstance(v, str): + for m in re.findall(r"/?template-media/([^\"'\s]+)", v): + assets.add(m) + if re.match(r"^https?://", v) and "template-media" not in v: + externals.add(v) + elif isinstance(v, dict): + for x in v.values(): scan(x) + elif isinstance(v, list): + for x in v: scan(x) +scan(bundle) + +os.makedirs(os.path.join(OUT, "assets"), exist_ok=True) +copied, missing = [], [] +for a in sorted(assets): + src = os.path.join(PUBLIC, a) + if os.path.isfile(src): + shutil.copy2(src, os.path.join(OUT, "assets", os.path.basename(a))) + copied.append(a) + else: + missing.append(a) +bundle["assets"] = sorted(os.path.basename(a) for a in copied) + +with open(os.path.join(OUT, "template.json"), "w", encoding="utf-8") as f: + json.dump(bundle, f, ensure_ascii=False, indent=2) + +n_proj = len(bundle.get("projects") or []) +n_scene = len(bundle.get("scenes") or []) +n_el = len(bundle.get("content_elements") or []) +print(f"✓ exported '{SLUG}' → {OUT}") +print(f" {n_proj} projects · {n_scene} scenes · {n_el} fields · {len(copied)} assets · " + f"{len(bundle.get('categories') or [])} categories · {len(bundle.get('tags') or [])} tags") +if missing: + print(f" ⚠ {len(missing)} asset(s) referenced but not found in {PUBLIC}: {', '.join(missing[:5])}") +if externals: + print(f" ⚠ {len(externals)} external URL asset(s) NOT bundled (host them on the target): {list(externals)[:3]}") diff --git a/scripts/import_template.py b/scripts/import_template.py new file mode 100644 index 0000000..479e20e --- /dev/null +++ b/scripts/import_template.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Import a template bundle (from export_template.py) into ANY FlatRender DB. + +Emits idempotent SQL to STDOUT — pipe it to the target's psql. It replaces any +existing template with the same slug (one cascading delete), recreates the rows +verbatim (original UUIDs, so FKs stay intact), and merges categories & tags BY SLUG +(so they line up with whatever the target DB already calls them). + +Usage: + python scripts/import_template.py | + # e.g. local: + python scripts/import_template.py dist/template-bundles/fr-instagram-promo \\ + | docker exec -i fr2-postgres psql -U postgres -d flatrender + # e.g. live: + python scripts/import_template.py dist/template-bundles/fr-instagram-promo \\ + | psql "postgresql://user:pass@live-host:5432/flatrender" + +Assets: copy the bundle's assets/ into the target's template-media (the SQL only +moves DB rows). Pass --assets-to to copy them locally; for a remote box use +`docker cp` / `mc cp` (printed below). +""" +import os, sys, json, shutil + +args = [a for a in sys.argv[1:] if not a.startswith("--")] +if not args: + print("usage: import_template.py [--assets-to ] | ", file=sys.stderr); sys.exit(1) +BUNDLE = args[0] +ASSETS_TO = None +if "--assets-to" in sys.argv: + ASSETS_TO = sys.argv[sys.argv.index("--assets-to") + 1] + +with open(os.path.join(BUNDLE, "template.json"), encoding="utf-8") as f: + b = json.load(f) +SLUG = b["slug"] + +def qv(v): + if v is None: return "NULL" + if isinstance(v, bool): return "TRUE" if v else "FALSE" + if isinstance(v, (int, float)): return str(v) + if isinstance(v, (dict, list)): return "'" + json.dumps(v, ensure_ascii=False).replace("'", "''") + "'" + return "'" + str(v).replace("'", "''") + "'" + +def ins(table, row, conflict=None): + cols = list(row.keys()) + s = f"INSERT INTO content.{table} ({','.join(cols)}) VALUES ({','.join(qv(row[c]) for c in cols)})" + if conflict: s += f" ON CONFLICT ({conflict}) DO NOTHING" + return s + ";" + +out = ["BEGIN;"] +# categories & tags first (merge by slug — never clobber the target's own) +for cat in b.get("categories") or []: out.append(ins("categories", cat, "slug")) +for tg in b.get("tags") or []: out.append(ins("tags", tg, "slug")) +# replace any existing template with this slug (cascades to all children) +out.append(f"DELETE FROM content.project_containers WHERE slug={qv(SLUG)};") +# rows verbatim (original UUIDs keep the FK tree intact) +out.append(ins("project_containers", b["container"])) +for p in b.get("projects") or []: out.append(ins("projects", p)) +for s in b.get("scenes") or []: out.append(ins("scenes", s)) +for e in b.get("content_elements") or []: out.append(ins("scene_content_elements", e)) +for sc in b.get("shared_colors") or []: out.append(ins("shared_colors", sc)) +for sl in b.get("shared_layers") or []: out.append(ins("shared_layers", sl)) +# links resolve the category/tag id BY SLUG in the target DB +cid = b["container"]["id"] +for link in b.get("category_links") or []: + out.append(f"INSERT INTO content.container_categories (container_id,category_id,sort) " + f"SELECT {qv(cid)}, id, {qv(link.get('sort', 0))} FROM content.categories WHERE slug={qv(link['category_slug'])} ON CONFLICT DO NOTHING;") +for link in b.get("tag_links") or []: + out.append(f"INSERT INTO content.container_tags (container_id,tag_id) " + f"SELECT {qv(cid)}, id FROM content.tags WHERE slug={qv(link['tag_slug'])} ON CONFLICT DO NOTHING;") +out.append("COMMIT;") +out.append(f"SELECT slug, name, primary_mode FROM content.project_containers WHERE slug={qv(SLUG)};") +print("\n".join(out)) + +# Assets: copy locally if asked; otherwise print placement guidance to stderr. +assets = b.get("assets") or [] +if ASSETS_TO and assets: + os.makedirs(ASSETS_TO, exist_ok=True) + for a in assets: + src = os.path.join(BUNDLE, "assets", a) + if os.path.isfile(src): shutil.copy2(src, os.path.join(ASSETS_TO, a)) + sys.stderr.write(f"-- copied {len(assets)} asset(s) → {ASSETS_TO}\n") +elif assets: + sys.stderr.write(f"\n-- {len(assets)} asset(s) in {BUNDLE}/assets/ — place them in the target's template-media, e.g.:\n") + sys.stderr.write(f"-- local docker: for f in {BUNDLE}/assets/*; do docker cp \"$f\" fr2-frontend:/app/public/template-media/; done\n") + sys.stderr.write(f"-- live (MinIO): mc cp --recursive {BUNDLE}/assets/ myminio//template-media/\n") + sys.stderr.write(f"-- or just copy {BUNDLE}/assets/* into the live app's public/template-media/\n")