From 6bb7622b69e6b1a876ca963f93c2f69ea660b33f Mon Sep 17 00:00:00 2001 From: Don Kingdon Date: Sun, 19 Apr 2026 16:20:35 +0000 Subject: [PATCH] OTB Cloud v1.1.0-alpha3: complete alpha3-a video workshop pipeline --- PROJECT_STATE.md | 150 +++++++------------- README.md | 18 +++ VERSION | 2 +- app/main/routes.py | 38 +++++ app/services/gpu_select.py | 31 +++- app/services/video_jobs.py | 153 ++++++++++++++++++-- app/services/video_worker.py | 197 +++++++++++++++++++++++++- app/templates/cloud/device_files.html | 53 ++++++- app/templates/cloud/workshop.html | 137 ++++++++++++++++++ 9 files changed, 656 insertions(+), 123 deletions(-) create mode 100644 app/templates/cloud/workshop.html diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 05d5c2f..8813e62 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -1,103 +1,51 @@ # PROJECT_STATE.md -## Project -OTB Cloud - -## Current version -v0.2.3 - -## Build date -2026-04-12 - -## Host -vault3 - -## App path -/opt/otb_cloud - -## Purpose -Portal-authenticated secure backup and storage platform for customer files, including images, videos, documents, and other uploaded data. - -## Current implemented scaffold -- Portal handoff from OTB Billing -- Branded OTB portal shell styling -- User-created devices -- Device add/remove -- Browser upload to device originals -- Device file browser -- Selection actions -- Soft-delete to deleted folder -- Recover from deleted folder -- Zip workspace staging and zip export -- Deleted files page with hard delete -- Exports page - -## Retention and safety notes -- Original files are stored as immutable originals -- Deleted files are retained in the deleted area for up to 24 hours -- Deleted files can be recovered during that hold window -- Deleted files can also be hard-deleted immediately by the user -- Recovered files return to originals with `-recovered` appended to filename -- Zip staging copies are temporary working copies -- Successful zip creation clears staged copies but does not affect original source files - -## Immediate next tasks -1. Add basename-only rename flow -2. Add searchable file listing -3. Add bulk folder upload -4. Add media processing jobs -5. Add derived/original filtering -6. Add better single-file actions in browser - - -## Current update: v0.2.5 -- Added inline image serving route for browser previews -- Added device browser view toggle: list or gallery -- Added gallery cards with thumbnails, preview modal, rename, download, and checkbox actions -- Existing bulk delete, download, and zip staging continue to work in both views - -## v0.2.5 — Gallery View + Image Preview - -### Added -- Gallery view toggle for device file browser -- Image thumbnail rendering (inline file route) -- Click-to-preview full image modal -- Gallery cards with: - - checkbox selection - - rename input - - download button - - preview button - -### Improved -- File browsing now supports both: - - list (management) - - gallery (visual) -- Bulk actions work in both views -- Display filename system fully integrated across UI - -### Notes -- Originals remain immutable -- Thumbnails currently use original images (no derived images yet) -- Foundation ready for future media processing pipeline - - - -## Current update: v0.2.8 -- Added folder-tree browser scoped by current path -- Added clickable breadcrumbs for direct jumps to any parent folder -- Added folders-first navigation while preserving list/gallery modes for files in the current folder -- Browser now reflects preserved backup folder structure instead of flattening all files into one device-wide listing - -## v1.1.0-alpha1 — Video System Foundation -- Added video_jobs table (processing queue) -- Added tenant_usage_metrics table (dashboard metrics) -- Added video service scaffolding (jobs, metrics, gpu select, profiles) -- Extended device structure to include: - - video - - video-workshop - - archive - - lts -- Prepared system for background worker architecture - -Next step: -- Build video worker processing engine +Project: OTB Cloud +Version: v1.1.0-alpha3 +Updated: 2026-04-19 +Location: /opt/otb_cloud + +## Current State +OTB Cloud now has a functioning workshop-driven video processing pipeline. + +### Confirmed Working +- Portal and branded UI shell +- Device browser +- File selection flow into Video Workshop +- Video Workshop page +- Enqueue API +- Jobs API +- MariaDB-backed video_jobs integration +- Tenant/device path resolution for queued jobs +- Worker service startup and queue pickup +- Worker-side absolute path resolution from tenant storage_root +- Intel iGPU processing path +- Successful completed output for device 27 (ripper) + +### Latest Proven Result +A queued workshop job for: +- source file: 05142013003.mp4 +- device: 27 (ripper) + +completed successfully with: +- assigned_processor: intel +- status: complete +- progress_percent: 100 +- output_relative_path: + devices/ripper/originals/20260413T210325474049Z__05142013003_processed.mp4 + +## Known Remaining Improvements +- Jobs panel is still raw JSON instead of a polished table/cards view +- Failed jobs do not yet surface log_excerpt nicely in UI +- No direct preview/download button for completed outputs in workshop +- No health/storage/GPU dashboard panel yet +- No explicit processor chooser in UI +- Output placement may later deserve a dedicated derived/video output area +- Existing patch helper scripts were moved out of repo to keep git clean + +## Recommended Next Step +Proceed to alpha3-b: +- replace raw JSON jobs output with styled job cards/table +- add output links for completed jobs +- add visible failure details from log_excerpt +- add storage/GPU/worker health panel diff --git a/README.md b/README.md index a508cd1..eed819e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,21 @@ +# OTB Cloud + +## v1.1.0-alpha3 - 2026-04-19 + +- Added Video Workshop UI for queued processing +- Added workshop selection flow from device browser +- Added enqueue and jobs API integration +- Fixed MariaDB-backed video job insert and listing logic +- Fixed tenant/device path resolution for queued workshop jobs +- Fixed worker-side absolute source path resolution +- Confirmed successful Intel iGPU processing path +- Output files now complete successfully from workshop-triggered jobs + +## v1.1.0-alpha2 — Video Worker Engine +- Background worker implemented +- GPU processing enabled (Intel primary, AMD secondary) +- Video job queue now active + ## v1.1.0-alpha1 — Video System Foundation - Introduced video job queue system - Introduced tenant usage metrics diff --git a/VERSION b/VERSION index 1710bc8..51ec1c5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.1.0-alpha1 +v1.1.0-alpha3 diff --git a/app/main/routes.py b/app/main/routes.py index 2d45bb8..a2a5849 100644 --- a/app/main/routes.py +++ b/app/main/routes.py @@ -1606,3 +1606,41 @@ def lts_view(): tenant_slug=session.get("otb_tenant_slug"), lts_files=lts_files, ) + +# ========================= +# VIDEO WORKSHOP (alpha3-a) +# ========================= + +from app.services.video_jobs import create_video_job, list_jobs_for_tenant + +@bp.route("/workshop/") +def workshop(device_id): + return render_template("cloud/workshop.html", device_id=device_id) + +@bp.route("/api/video/enqueue", methods=["POST"]) +def video_enqueue(): + data = request.json + tenant = session.get("tenant") or 'def' + device_id = data.get("device_id") + files = data.get("files", []) + profile = data.get("profile", "default") + + job_ids = [] + + for f in files: + job_id = create_video_job( + tenant=tenant, + device_id=device_id, + input_filename=f, + profile=profile + ) + job_ids.append(job_id) + + return jsonify({"status": "ok", "jobs": job_ids}) + +@bp.route("/api/video/jobs") +def video_jobs(): + tenant = session.get("tenant") or 'def' + jobs = list_jobs_for_tenant(tenant) + return jsonify(jobs) + diff --git a/app/services/gpu_select.py b/app/services/gpu_select.py index 176ced0..de98330 100644 --- a/app/services/gpu_select.py +++ b/app/services/gpu_select.py @@ -1,3 +1,30 @@ +import os + +LOCK_DIR = "/var/lib/otbcloud/locks" + +def _lock_path(name): + return os.path.join(LOCK_DIR, f"{name}.lock") + +def is_locked(name): + return os.path.exists(_lock_path(name)) + +def acquire(name): + os.makedirs(LOCK_DIR, exist_ok=True) + path = _lock_path(name) + if os.path.exists(path): + return False + with open(path, "w") as f: + f.write(str(os.getpid())) + return True + +def release(name): + path = _lock_path(name) + if os.path.exists(path): + os.remove(path) + def select_processor(): - # v1.1.0 logic placeholder - return "intel" + if acquire("intel"): + return "intel" + if acquire("amd"): + return "amd" + return "cpu" diff --git a/app/services/video_jobs.py b/app/services/video_jobs.py index c2bfabf..0d56f62 100644 --- a/app/services/video_jobs.py +++ b/app/services/video_jobs.py @@ -1,9 +1,144 @@ -def create_job(db, tenant_id, device_id, source_path, filename, profile): - return { - "tenant_id": tenant_id, - "device_id": device_id, - "source_path": source_path, - "filename": filename, - "profile": profile, - "status": "queued" - } +from app.db import get_db +from pathlib import Path + +def get_tenant_row(db, tenant): + cur = db.cursor() + cur.execute( + "SELECT id, storage_root FROM tenants WHERE slug = %s LIMIT 1", + (tenant,) + ) + row = cur.fetchone() + if not row: + return None + return row + +def get_device_row(db, device_id): + cur = db.cursor() + cur.execute( + "SELECT id, device_name, relative_path FROM devices WHERE id = %s LIMIT 1", + (device_id,) + ) + row = cur.fetchone() + if not row: + return None + return row + +def resolve_source_relative_path(storage_root, device_relative_path, input_filename): + base = Path(storage_root) / device_relative_path + if not base.exists(): + raise FileNotFoundError(f"Device base path not found: {base}") + + candidates = [] + + for p in base.rglob("*"): + if not p.is_file(): + continue + name = p.name + if name == input_filename or name.endswith("__" + input_filename): + candidates.append(p) + + if not candidates: + raise FileNotFoundError( + f"Could not locate source file for {input_filename} under {base}" + ) + + candidates.sort(key=lambda p: p.stat().st_mtime, reverse=True) + chosen = candidates[0] + + rel = chosen.relative_to(Path(storage_root)) + return str(rel) + +def create_video_job(tenant, device_id, input_filename, profile="default"): + db = get_db() + + tenant_row = get_tenant_row(db, tenant) + if not tenant_row: + raise Exception(f"Tenant not found: {tenant}") + + device_row = get_device_row(db, device_id) + if not device_row: + raise Exception(f"Device not found: {device_id}") + + tenant_id = tenant_row["id"] + storage_root = tenant_row["storage_root"] + device_relative_path = device_row["relative_path"] + + source_relative_path = resolve_source_relative_path( + storage_root, + device_relative_path, + input_filename + ) + + cur = db.cursor() + cur.execute( + """ + INSERT INTO video_jobs ( + tenant_id, + device_id, + source_file_id, + source_relative_path, + source_original_filename, + requested_profile, + requested_gpu_preference, + status, + progress_percent + ) VALUES (%s, %s, NULL, %s, %s, %s, 'auto', 'queued', 0) + """, + (tenant_id, device_id, source_relative_path, input_filename, profile) + ) + db.commit() + return cur.lastrowid + +def list_jobs_for_tenant(tenant): + db = get_db() + + tenant_row = get_tenant_row(db, tenant) + if not tenant_row: + return [] + + tenant_id = tenant_row["id"] + + cur = db.cursor() + cur.execute( + """ + SELECT + id, + device_id, + source_original_filename, + requested_profile, + status, + progress_percent, + assigned_processor, + output_relative_path, + error_message, + created_at, + started_at, + completed_at + FROM video_jobs + WHERE tenant_id = %s + ORDER BY id DESC + LIMIT 100 + """, + (tenant_id,) + ) + + rows = cur.fetchall() + + out = [] + for r in rows: + out.append({ + "id": r["id"], + "device_id": r["device_id"], + "filename": r["source_original_filename"], + "profile": r["requested_profile"], + "status": r["status"], + "progress_percent": r["progress_percent"], + "assigned_processor": r["assigned_processor"], + "output_relative_path": r["output_relative_path"], + "error_message": r["error_message"], + "created_at": str(r["created_at"]) if r["created_at"] is not None else None, + "started_at": str(r["started_at"]) if r["started_at"] is not None else None, + "completed_at": str(r["completed_at"]) if r["completed_at"] is not None else None, + }) + + return out diff --git a/app/services/video_worker.py b/app/services/video_worker.py index d94f22d..56a2089 100644 --- a/app/services/video_worker.py +++ b/app/services/video_worker.py @@ -1,6 +1,197 @@ import time +import subprocess +from datetime import datetime +from pathlib import Path + +from app import create_app +from app.db import get_db +from app.services.gpu_select import select_processor, release + +INTEL_DEV = "/dev/dri/renderD129" +AMD_DEV = "/dev/dri/renderD128" + +def run_ffmpeg(cmd): + return subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + +def build_absolute_source_path(db, job): + with db.cursor() as cur: + cur.execute( + "SELECT storage_root FROM tenants WHERE id = %s", + (job["tenant_id"],) + ) + tenant_row = cur.fetchone() + + if not tenant_row: + raise RuntimeError(f"Tenant id {job['tenant_id']} not found") + + storage_root = tenant_row["storage_root"] + return str(Path(storage_root) / job["source_relative_path"]) + +def process_job(db, job): + job_id = job["id"] + src = build_absolute_source_path(db, job) + profile = job["requested_profile"] + + processor = select_processor() + device = INTEL_DEV if processor == "intel" else AMD_DEV + + output = str(Path(src).with_name(Path(src).stem + "_processed.mp4")) + + if profile == "portrait_web": + vf = "format=nv12,hwupload,scale_vaapi=w=720:h=1280:force_original_aspect_ratio=decrease" + else: + vf = "format=nv12,hwupload,scale_vaapi=w=1280:h=720:force_original_aspect_ratio=decrease" + + if processor in ("intel", "amd"): + cmd = [ + "ffmpeg", "-hide_banner", "-y", + "-vaapi_device", device, + "-i", src, + "-vf", vf, + "-c:v", "h264_vaapi", + "-b:v", "3M", + "-maxrate", "3M", + "-bufsize", "6M", + "-c:a", "aac", "-b:a", "128k", + output + ] + else: + cmd = [ + "ffmpeg", "-hide_banner", "-y", + "-i", src, + "-c:v", "libx264", + "-preset", "medium", + "-crf", "23", + "-c:a", "aac", "-b:a", "128k", + output + ] + + start = datetime.utcnow() + + try: + result = run_ffmpeg(cmd) + end = datetime.utcnow() + + with db.cursor() as cur: + if result.returncode == 0: + rel_output = None + try: + with db.cursor() as cur2: + cur2.execute( + "SELECT storage_root FROM tenants WHERE id = %s", + (job["tenant_id"],) + ) + tenant_row = cur2.fetchone() + if tenant_row: + rel_output = str(Path(output).relative_to(Path(tenant_row["storage_root"]))) + except Exception: + rel_output = output + + cur.execute( + """ + UPDATE video_jobs + SET status='complete', + assigned_processor=%s, + output_relative_path=%s, + progress_percent=100, + started_at=COALESCE(started_at, %s), + completed_at=%s, + log_excerpt=%s, + error_message=NULL + WHERE id=%s + """, + ( + processor, + rel_output or output, + start, + end, + (result.stderr or "")[:1000], + job_id, + ), + ) + else: + cur.execute( + """ + UPDATE video_jobs + SET status='failed', + error_message=%s, + log_excerpt=%s, + completed_at=%s + WHERE id=%s + """, + ( + "ffmpeg failed", + (result.stderr or "")[:4000], + end, + job_id, + ), + ) + db.commit() + + except Exception as e: + with db.cursor() as cur: + cur.execute( + """ + UPDATE video_jobs + SET status='failed', + error_message=%s, + completed_at=UTC_TIMESTAMP() + WHERE id=%s + """, + (str(e)[:1000], job_id), + ) + db.commit() + finally: + if processor in ("intel", "amd"): + release(processor) def run_worker(): - print("video worker starting (stub)") - while True: - time.sleep(10) + app = create_app() + + with app.app_context(): + print("video worker started", flush=True) + + while True: + try: + db = get_db() + + try: + db.rollback() + except Exception: + pass + + with db.cursor() as cur: + cur.execute( + """ + SELECT * + FROM video_jobs + WHERE status='queued' + ORDER BY id ASC + LIMIT 1 + """ + ) + job = cur.fetchone() + + if job: + print( + f"worker picked job id={job['id']} source={job['source_relative_path']}", + flush=True + ) + cur.execute( + """ + UPDATE video_jobs + SET status='processing', + started_at=COALESCE(started_at, UTC_TIMESTAMP()), + progress_percent=5 + WHERE id=%s + """, + (job["id"],), + ) + db.commit() + + process_job(db, job) + + except Exception as e: + print(f"worker loop error: {e}", flush=True) + + time.sleep(5) diff --git a/app/templates/cloud/device_files.html b/app/templates/cloud/device_files.html index 9b31f3a..b6eee2c 100644 --- a/app/templates/cloud/device_files.html +++ b/app/templates/cloud/device_files.html @@ -1,6 +1,13 @@ {% extends "portal_base.html" %} -{% block title %}Device Files - OTB Cloud{% endblock %} +{% block title %}Device Files - OTB Cloud + + + + + + +{% endblock %} {% block portal_content %} + +
+
+

Video Workshop

+

Device ID: {{ device_id }}

+
+
+ Back to Device Files + Back to Portal +
+
+ +
+
+
+

Queue Video Jobs

+

Selected files from the device browser are staged in your browser and can now be queued for processing.

+
+
+ alpha3-a +
+
+ +
+
+
+ +
+ +
+ Selected items + 

+    

+
+    

+      
+      
+      
+    

+  

+

+
+

+  

+    

+      
Jobs

+      
Live queue/status feed for this tenant.

+    

+  

+  

+    

+  

+

+
+
+{% endblock %}