Only fail when 10 consecutive socket errors occur (#1928)

2025-10-19 00:04:41 +08:00 · 2025-03-05 13:16:19 -08:00
parent fd0d63ba5b
commit 0792ff02ff
2 changed files with 16 additions and 3 deletions
--- a/python/mlx/distributed_run.py
+++ b/python/mlx/distributed_run.py
@@ -112,7 +112,12 @@ def extract_rings(hosts, index):
                    break
            if not ring:
                break
-            rings.append(normalize(concretize(ring, used_ports)))
+            try:
+                rings.append(normalize(concretize(ring, used_ports)))
+            except RuntimeError:
+                if len(rings) > 0:
+                    return rings
+                raise

    return rings