zhangyiss/mlx
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx.git synced 2025-09-18 01:50:16 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Only fail when 10 consecutive socket errors occur (#1928)

Browse Source
This commit is contained in:
Angelos Katharopoulos
2025-03-05 13:16:19 -08:00
committed by GitHub
parent fd0d63ba5b
commit 0792ff02ff
2 changed files with 16 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
mlx/distributed/ring/ring.cpp
View File

@@ -199,6 +199,7 @@ class SocketThread {
}
void worker() {
int error_count = 0;
bool delete_recv = false;
bool delete_send = false;
while (true) {
@@ -235,10 +236,11 @@ class SocketThread {
task.buffer = static_cast<char*>(task.buffer) + r;
task.size -= r;
delete_recv = task.size == 0;
error_count = 0;
} else if (errno != EAGAIN) {
error_count++;
log_info(
true, "Receiving from socket", fd_, "failed with errno", errno);
return;
}
}
if (!sends_.empty()) {
@@ -248,11 +250,17 @@ class SocketThread {
task.buffer = static_cast<char*>(task.buffer) + r;
task.size -= r;
delete_send = task.size == 0;
error_count = 0;
} else if (errno != EAGAIN) {
error_count++;
log_info(true, "Sending to socket", fd_, "failed with errno", errno);
return;
}
}
if (error_count >= 10) {
log_info(true, "Too many send/recv errors. Aborting...");
return;
}
}
}