feat: reconstruction survives service restarts + phone-friendly glb + network-first SW

Owner's video 'stopped' on his phone but actually FINISHED server-side — his client
just lost the view and, on an old cached build, couldn't reattach. Three fixes:

- Detached restart-surviving runner: run_job.py runs each job in its own session,
  serialized on the GPU by an flock; unit gets KillMode=process; app startup
  re-attaches to a live runner or relaunches a dead one. Verified: a
  'systemctl restart genrecon-app' mid-COLMAP left the same runner PID alive and the
  job progressing. Nothing kills a job now.
- Phone-first glb: default simplify 0.12->0.04 (~900k->~300k tris) and textures
  2048->1024, so the walkthrough doesn't exhaust mobile WebGL memory (blank scene).
- SW: app shell (HTML/JS/CSS) is now network-first so code updates land immediately
  online (a stale cache-first app.js is why ?job= showed nothing); heavy vendor/glb
  stay cache-first.

backend/app.py

@@ -13,8 +13,9 @@ One serial worker thread (single GPU). Job state persists to
 from __future__ import annotations
 
 import json
-import queue
-import threading
+import os
+import subprocess
+import sys
 import time
 import uuid
 from pathlib import Path
@@ -27,7 +28,7 @@ import config
 import pipeline
 
 app = FastAPI(title="GenRecon", version="1.0")
-_work: "queue.Queue[str]" = queue.Queue()
+RUN_JOB = str(Path(__file__).resolve().parent / "run_job.py")
 
 
 def _job_dir(job_id: str) -> Path:
@@ -61,42 +62,51 @@ def _log_line(job_id: str, line: str) -> None:
         f.write(line + "\n")
 
 
-def _worker() -> None:
-    while True:
-        job_id = _work.get()
-        try:
-            _write(job_id, status="running", stage=config.STAGES[0][0], progress=1, message="start", error=None)
-            pipeline.run_pipeline(
-                _job_dir(job_id),
-                log=lambda ln, j=job_id: _log_line(j, ln),
-                set_progress=lambda stage, pct, msg, j=job_id: _write(j, status="running", stage=stage, progress=pct, message=msg),
-            )
-            _write(job_id, status="done", stage="glb", progress=100, message="ready")
-        except Exception as e:  # noqa: BLE001 — surface any stage failure to the client
-            _log_line(job_id, f"ERROR: {e}")
-            _write(job_id, status="error", message="failed", error=str(e))
-        finally:
-            _work.task_done()
+def _pid_alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+        return True
+    except (OSError, ValueError):
+        return False
+
+
+def _runner_alive(job_id: str) -> bool:
+    try:
+        return _pid_alive(int((_job_dir(job_id) / "worker.pid").read_text().strip()))
+    except Exception:
+        return False
+
+
+def _launch_runner(job_id: str) -> None:
+    """Run the reconstruction in a DETACHED process that outlives this web service
+    (KillMode=process on the unit), self-serialized on the GPU by an flock. A
+    redeploy / restart of genrecon-app therefore never kills an in-flight scan."""
+    subprocess.Popen(
+        [sys.executable, RUN_JOB, job_id],
+        cwd=str(Path(RUN_JOB).parent),
+        stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+        start_new_session=True, close_fds=True,
+    )
 
 
 @app.on_event("startup")
 def _startup() -> None:
-    # Re-queue jobs that were queued, fail jobs that were mid-flight at shutdown.
+    # Re-attach: any job that isn't finished gets a live runner. If its detached
+    # runner survived our restart (KillMode=process) we leave it alone; otherwise
+    # we (re)launch — so a service restart never strands a reconstruction.
     for d in sorted(config.JOBS_DIR.glob("*/")):
         st = _read(d.name)
-        if not st:
+        if not st or st.get("status") in ("done", "error"):
             continue
-        if st.get("status") == "queued":
-            _work.put(d.name)
-        elif st.get("status") == "running":
-            _write(d.name, status="error", message="interrupted by restart",
-                   error="server restarted while this job was running")
-    threading.Thread(target=_worker, daemon=True).start()
+        if not _runner_alive(d.name):
+            _launch_runner(d.name)
 
 
 @app.get("/api/health")
 def health() -> dict:
-    return {"ok": True, "mock": config.MOCK, "queue": _work.qsize()}
+    active = sum(1 for d in config.JOBS_DIR.glob("*/")
+                 if (_read(d.name) or {}).get("status") in ("queued", "running"))
+    return {"ok": True, "mock": config.MOCK, "active": active}
 
 
 @app.post("/api/jobs")
@@ -119,7 +129,7 @@ async def create_job(video: UploadFile = File(...)) -> dict:
             f.write(chunk)
     _write(job_id, status="queued", stage=None, progress=0, message="queued",
            created=time.time(), filename=video.filename, error=None)
-    _work.put(job_id)
+    _launch_runner(job_id)
     return {"id": job_id, "status": "queued"}
 
 

backend/genrecon-app.service

@@ -4,6 +4,10 @@ After=network.target
 
 [Service]
 Type=simple
+# Only kill the web process on stop/restart — detached reconstruction runners
+# (run_job.py, own session) keep going so a redeploy never strands an in-flight
+# scan; the next startup re-attaches to them.
+KillMode=process
 WorkingDirectory=/opt/genrecon/app/backend
 Environment=GENRECON_JOBS_DIR=/srv/jobs
 Environment=GENRECON_FRONTEND_DIR=/opt/genrecon/app/frontend

backend/optimize_glb.sh

@@ -7,8 +7,11 @@
 # (a split @gltf-transform/core between separate globals dies with null.getLogger).
 set -uo pipefail
 IN="$1"; OUT="$2"
-RATIO="${GENRECON_SIMPLIFY_RATIO:-0.12}"
-TEXSZ="${GENRECON_TEX_SIZE:-2048}"
+# Aggressive by default: the walkthrough runs on PHONES, where a multi-hundred-k
+# triangle mesh + big textures exhausts mobile WebGL memory (blank scene). ~0.04
+# vertex ratio ≈ a few hundred k triangles; 1024px textures. Tune up on desktop.
+RATIO="${GENRECON_SIMPLIFY_RATIO:-0.04}"
+TEXSZ="${GENRECON_TEX_SIZE:-1024}"
 export NODE_OPTIONS="--max-old-space-size=${GENRECON_GLTF_HEAP_MB:-49152}"
 GT="/opt/genrecon/gltftool/node_modules/.bin/gltf-transform"
 [ -x "$GT" ] || GT="$(command -v gltf-transform || true)"

backend/run_job.py

@@ -0,0 +1,78 @@
+"""Detached, restart-surviving reconstruction runner.
+
+Launched by app.py with start_new_session=True and, with `KillMode=process` on the
+systemd unit, it OUTLIVES a `systemctl restart genrecon-app` — so redeploying or
+restarting the web service never kills an in-flight reconstruction. Concurrent
+uploads are serialized on the single GPU by an flock. app.py's startup re-attaches
+to a live runner (worker.pid) or relaunches a dead one, instead of failing the job.
+"""
+from __future__ import annotations
+
+import fcntl
+import json
+import os
+import sys
+import time
+import traceback
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import config          # noqa: E402
+import pipeline        # noqa: E402
+
+
+def main(job_id: str) -> None:
+    job = config.JOBS_DIR / job_id
+    status = job / "status.json"
+    pid_file = job / "worker.pid"
+
+    def write(**patch) -> None:
+        st: dict = {"id": job_id}
+        try:
+            st = json.loads(status.read_text())
+        except Exception:
+            pass
+        st.update(patch)
+        st["updated"] = time.time()
+        tmp = status.with_suffix(".json.tmp")
+        tmp.write_text(json.dumps(st))
+        tmp.replace(status)                       # atomic — a reader never sees a half-written status
+
+    def log_line(line: str) -> None:
+        with open(job / "pipeline.log", "a") as f:
+            f.write(line + "\n")
+
+    # Claim the job immediately (before the GPU flock) so app.py's startup sees a
+    # live runner and re-attaches rather than relaunching.
+    pid_file.write_text(str(os.getpid()))
+    lock = open(config.JOBS_DIR / ".gpu.lock", "w")
+    try:
+        write(status="running", stage=None, progress=1, message="w kolejce (GPU)…", error=None)
+        fcntl.flock(lock, fcntl.LOCK_EX)          # one reconstruction at a time on the single GPU
+        write(status="running", stage=config.STAGES[0][0], progress=1, message="start", error=None)
+        pipeline.run_pipeline(
+            job,
+            log=log_line,
+            set_progress=lambda stage, pct, msg: write(status="running", stage=stage, progress=pct, message=msg),
+        )
+        write(status="done", stage="glb", progress=100, message="ready")
+    except Exception as e:  # noqa: BLE001 — surface any stage failure to the client
+        log_line(f"ERROR: {e}")
+        traceback.print_exc()
+        write(status="error", message="failed", error=str(e))
+    finally:
+        try:
+            fcntl.flock(lock, fcntl.LOCK_UN)
+        except OSError:
+            pass
+        try:
+            pid_file.unlink()
+        except OSError:
+            pass
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("usage: run_job.py <job_id>", file=sys.stderr)
+        sys.exit(2)
+    main(sys.argv[1])

frontend/sw.js

@@ -1,5 +1,8 @@
-// Offline app-shell cache. API calls are never cached.
-const CACHE = 'genrecon-v3';   // bump on shell change (app.js/viewer.js) so clients refresh
+// Offline app-shell cache. API calls are never cached. The shell (HTML/JS/CSS)
+// is NETWORK-FIRST so code updates land immediately for online clients (a stale
+// cache-first app.js is why "?job=" showed no model on an old install); heavy,
+// stable assets (three.js vendor, demo.glb, icons) stay cache-first.
+const CACHE = 'genrecon-v4';   // bump on shell change so clients refresh
 const SHELL = [
   './', './index.html', './styles.css', './app.js', './viewer.js', './manifest.webmanifest',
   './demo.glb', './icons/icon-192.png', './icons/icon-512.png',
@@ -21,7 +24,20 @@ self.addEventListener('activate', (e) => {
 });
 
 self.addEventListener('fetch', (e) => {
-  const url = new URL(e.request.url);
-  if (url.pathname.startsWith('/api/')) return;            // dynamic — straight to network
-  e.respondWith(caches.match(e.request).then((r) => r || fetch(e.request)));
+  const req = e.request;
+  const url = new URL(req.url);
+  if (url.pathname.startsWith('/api/')) return;            // dynamic — straight to network, never cached
+  const isVendor = url.pathname.includes('/vendor/') || url.pathname.endsWith('.glb') || url.pathname.includes('/icons/');
+  const isShell = !isVendor && (req.mode === 'navigate' || /\.(?:js|css|webmanifest)$/.test(url.pathname));
+  if (isShell) {
+    // Network-first: always try the latest code, cache it, fall back to cache offline.
+    e.respondWith(
+      fetch(req)
+        .then((r) => { const copy = r.clone(); caches.open(CACHE).then((c) => c.put(req, copy)); return r; })
+        .catch(() => caches.match(req).then((r) => r || caches.match('./index.html'))),
+    );
+  } else {
+    // Cache-first for heavy, stable assets (three.js, demo.glb, icons).
+    e.respondWith(caches.match(req).then((r) => r || fetch(req)));
+  }
 });