Bläddra i källkod

fix(fabric): dedupe GPU capacity per hypervisor (passthrough double-count)

Nodes on a hypervisor that passes ALL physical GPUs through to every LXC each
report the full GPU set via nvidia-smi; summing across nodes multiplied one GPU
set by node count (11 nodes x 8 GPUs => ~4.3 TB phantom). total_gpu_capacity_mb
and total_gpu_free_mb now count each hypervisor once (max over its nodes);
unknown-hypervisor nodes are counted individually. Real total now ~417 GB. +2 tests.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude 1 månad sedan
förälder
incheckning
f24af670a3
2 ändrade filer med 52 tillägg och 4 borttagningar
  1. 25 4
      control/registry.py
  2. 27 0
      tests/test_vram_capacity.py

+ 25 - 4
control/registry.py

@@ -109,10 +109,31 @@ class Registry:
                 total += g.vram_mb
         return total
 
+    def _capacity_by_hypervisor(self, attr: str) -> int:
+        """Sum `attr` over alive nodes, deduplicated per hypervisor.
+
+        On hypervisors that pass ALL physical GPUs through to every LXC node,
+        each node's nvidia-smi reports the SAME physical GPUs. Summing across
+        nodes would multiply one GPU set by the node count (e.g. 11 nodes x 8
+        GPUs => ~4.3 TB phantom capacity). We therefore count each hypervisor
+        ONCE (max over its nodes). Nodes whose hypervisor is unknown can't be
+        proven to share GPUs, so each is counted on its own (keyed by node_id).
+        """
+        best: dict[str, int] = {}
+        for n in self.nodes.values():
+            if not n.alive:
+                continue
+            val = getattr(n, attr, 0) or 0
+            hv = (n.hypervisor or "").strip().lower()
+            key = hv if hv and hv not in ("unknown", "?") else f"__node__{n.node_id}"
+            if val > best.get(key, 0):
+                best[key] = val
+        return sum(best.values())
+
     def total_gpu_capacity_mb(self) -> int:
-        """Sum of physical-GPU total VRAM across alive nodes (hardware capacity)."""
-        return sum(n.vram_total_mb for n in self.nodes.values() if n.alive)
+        """Total physical-GPU VRAM (hardware capacity), deduplicated per hypervisor."""
+        return self._capacity_by_hypervisor("vram_total_mb")
 
     def total_gpu_free_mb(self) -> int:
-        """Sum of physical-GPU free VRAM across alive nodes (placement headroom)."""
-        return sum(n.vram_free_mb for n in self.nodes.values() if n.alive)
+        """Free physical-GPU VRAM (placement headroom), deduplicated per hypervisor."""
+        return self._capacity_by_hypervisor("vram_free_mb")

+ 27 - 0
tests/test_vram_capacity.py

@@ -115,6 +115,33 @@ def test_registry_module_carries_required_vram():
     assert cands and cands[0][1].required_vram_mb == 20480
 
 
+def test_capacity_deduplicates_per_hypervisor():
+    """Nodes on the SAME hypervisor see the same passthrough GPUs -> counted once."""
+    reg = Registry()
+    # Two nodes on hypervisor "a" each see all 8 GPUs (same physical hardware).
+    reg.handle_heartbeat(Heartbeat(node_id="a1", node_ip="10.0.0.1", hypervisor="a",
+                                   vram_total_mb=393120, vram_free_mb=200000))
+    reg.handle_heartbeat(Heartbeat(node_id="a2", node_ip="10.0.0.2", hypervisor="a",
+                                   vram_total_mb=393120, vram_free_mb=190000))
+    # One node on a different hypervisor with its own single GPU.
+    reg.handle_heartbeat(Heartbeat(node_id="g1", node_ip="10.0.1.1", hypervisor="gfx2",
+                                   vram_total_mb=24564, vram_free_mb=12000))
+    # Capacity: max per hypervisor, summed -> 393120 (a) + 24564 (gfx2), NOT x2.
+    assert reg.total_gpu_capacity_mb() == 393120 + 24564
+    # Free: same dedup (max per hypervisor).
+    assert reg.total_gpu_free_mb() == 200000 + 12000
+
+
+def test_capacity_counts_unknown_hypervisor_nodes_individually():
+    """Unknown-hypervisor nodes can't be proven to share GPUs -> counted each."""
+    reg = Registry()
+    reg.handle_heartbeat(Heartbeat(node_id="u1", node_ip="10.0.0.1",
+                                   vram_total_mb=49140, vram_free_mb=20000))
+    reg.handle_heartbeat(Heartbeat(node_id="u2", node_ip="10.0.0.2",
+                                   vram_total_mb=24576, vram_free_mb=10000))
+    assert reg.total_gpu_capacity_mb() == 49140 + 24576
+
+
 if __name__ == "__main__":
     fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
     failed = 0