fabric/e2e-test.sh

#!/bin/bash
exec 2>&1
echo "===== FABRIC EPIC i-zmaudd54 — FINAL E2E TEST SUITE ====="
echo

echo "=== TEST 0: Unit tests (schemas / agent / registry / vram-capacity) ==="
if ( cd /srv/fabric && PYTHONPATH=/srv venv/bin/python -m pytest tests/ -q ); then
  echo "UNIT TESTS: PASS"
else
  echo "UNIT TESTS: FAIL (falling back to standalone runner)"
  ( cd /srv/fabric && PYTHONPATH=/srv venv/bin/python tests/test_vram_capacity.py )
fi

echo
echo "=== TEST 1: Control plane health (v0) ==="
curl -s http://127.0.0.1:7700/health | python3 -m json.tool

echo
echo "=== TEST 2: Multi-node registry ==="
curl -s http://127.0.0.1:7700/registry/nodes | python3 -c "import sys,json; d=json.load(sys.stdin); print(f'Total nodes: {len(d[\"nodes\"])}'); [print(f'  {n[\"node_id\"]:12s} {n[\"node_ip\"]:15s} hyp={n[\"hypervisor\"]:5s} mods={len(n[\"modules\"])} alive={n[\"alive\"]}') for n in d['nodes']]"

echo
echo "=== TEST 3: Module discovery ==="
curl -s http://127.0.0.1:7700/registry/models | python3 -c "import sys,json; d=json.load(sys.stdin); print(f'Modules: {len(d[\"models\"])} across {d[\"nodes_count\"]} nodes, total VRAM {d[\"total_gpu_vram_mb\"]}MB'); [print(f'  {m[\"name\"]:18s} state={m[\"state\"]:10s} vram={m[\"vram_mb\"]:>6}MB modality={m[\"modality\"]:10s}') for m in d['models']]"

echo
echo "=== TEST 3b: VRAM capacity reporting (free/total) ==="
curl -s http://127.0.0.1:7700/registry/models | python3 -c "import sys,json; d=json.load(sys.stdin); print(f'capacity={d.get(\"total_gpu_capacity_mb\")}MB free={d.get(\"total_gpu_free_mb\")}MB used={d.get(\"total_gpu_vram_mb\")}MB'); assert d.get('total_gpu_capacity_mb',0) > 0, 'no capacity reported'; print('  capacity OK')"

echo
echo "=== TEST 4: Multi-hypervisor (v2.1) ==="
curl -s http://127.0.0.1:7700/registry/hypervisors | python3 -c "import sys,json; d=json.load(sys.stdin); print(f'Hypervisors: {d[\"total_hypervisors\"]}'); [print(f'  hyp={hyp}: {len(nodes)} nodes') for hyp, nodes in d['hypervisors'].items()]"

echo
echo "=== TEST 5: Placement & alias resolution ==="
PASS=0
FAIL=0
for m in echo-canary echo ollama qwen acestep music tts comfyui sd ltx lipsync stt whisper trellis 3d dreamstack seedream audioface dirty pipecat voice; do
  result=$(curl -s -X POST http://127.0.0.1:7700/place -H 'Content-Type: application/json' -d "{\"model\":\"$m\"}")
  if echo "$result" | grep -q "node_id"; then
    node=$(echo "$result" | python3 -c "import sys,json; d=json.load(sys.stdin); print(f'{d[\"module_name\"]}@{d[\"node_id\"]}')")
    echo "  $m -> $node OK"
    PASS=$((PASS+1))
  else
    echo "  $m -> NOT FOUND FAIL"
    FAIL=$((FAIL+1))
  fi
done
echo "Placement: $PASS passed, $FAIL failed"

echo
echo "=== TEST 6: Batch fan-out (v1.5) ==="
JOB=$(curl -s -X POST "http://127.0.0.1:7700/batch?model=echo-canary&priority=normal" -H 'Content-Type: application/json' -d '[{"i":1},{"i":2},{"i":3},{"i":4},{"i":5},{"i":6},{"i":7},{"i":8},{"i":9},{"i":10},{"i":11},{"i":12},{"i":13},{"i":14},{"i":15},{"i":16}]' | python3 -c "import sys,json; print(json.load(sys.stdin)['job_id'])")
sleep 2
curl -s "http://127.0.0.1:7700/batch/$JOB" | python3 -m json.tool

echo
echo "=== TEST 7: Recipes (v2.4) ==="
curl -s http://127.0.0.1:7700/install/recipes | python3 -m json.tool

echo
echo "=== TEST 8: Installer jobs (v2.2) ==="
curl -s http://127.0.0.1:7700/install | python3 -c "import sys,json; d=json.load(sys.stdin); print(f'Total install jobs: {len(d[\"jobs\"])}'); [print(f'  {j[\"job_id\"]} {j[\"name\"]:15s} state={j[\"state\"]:10s} steps={len(j[\"steps\"])}') for j in d['jobs']]"

echo
echo "=== TEST 9: CUDA-checkpoint availability (v2.3) ==="
curl -s http://127.0.0.1:7700/checkpoint/availability | python3 -m json.tool

echo
echo "=== TEST 10: Scheduler utilization (v1) ==="
curl -s http://127.0.0.1:7700/scheduler/utilization | python3 -m json.tool

echo
echo "=== TEST 11: Per-PID NVML accounting (red-team mitigation) ==="
curl -s http://10.0.2.162:7701/gpu-status | python3 -m json.tool

echo
echo "=== TEST 12: Module lifecycle (load/unload) ==="
echo "BEFORE:"
curl -s http://10.0.2.162:7701/modules | python3 -c "import sys,json; m=[x for x in json.load(sys.stdin)['modules'] if x['name']=='echo-canary'][0]; print(f'  echo-canary state={m[\"state\"]}')"
echo "Unloading..."
curl -s -X POST http://10.0.2.162:7701/unload/echo-canary | python3 -m json.tool
sleep 1
curl -s http://10.0.2.162:7701/modules | python3 -c "import sys,json; m=[x for x in json.load(sys.stdin)['modules'] if x['name']=='echo-canary'][0]; print(f'  AFTER unload: state={m[\"state\"]}')"
echo "Loading..."
curl -s -X POST http://10.0.2.162:7701/load/echo-canary | python3 -m json.tool
sleep 2
curl -s http://10.0.2.162:7701/modules | python3 -c "import sys,json; m=[x for x in json.load(sys.stdin)['modules'] if x['name']=='echo-canary'][0]; print(f'  AFTER reload: state={m[\"state\"]}')"

echo
echo "===== ALL TESTS COMPLETE ====="