fix(daemon): ask the OS for enough open files, and stop spinning when it says no

A search fans out to hundreds of peers, each holding a socket. A shell hands
its children a soft limit of 256 open files on macOS, which one busy search
blows straight through; a one-shot command never noticed because it exited
before it mattered, but a resident daemon hits it and then cannot accept peers
or control connections. From a client that looks like 'cannot reach the
daemon: Connection closed', intermittently, for no visible reason.

The daemon now raises its own soft limit to 8192 at startup, before anything
opens a socket, and says so plainly when the hard limit will not allow it.
macOS rejects an unlimited hard limit for open files, so both fields are named.

Both accept loops also slept zero on failure. Out of descriptors does not clear
by the next instruction, so one exhausted moment became a spinning core and
thousands of identical log lines — which is what the report looked like. They
now back off.
This commit is contained in:
Michel de Graaf
2026-07-27 20:16:41 +02:00
parent 4836934881
commit 220beb4e41
5 changed files with 104 additions and 3 deletions
Generated
+3 -2
View File
@@ -1465,9 +1465,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.186" version = "0.2.189"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]] [[package]]
name = "libdbus-sys" name = "libdbus-sys"
@@ -2556,6 +2556,7 @@ dependencies = [
"getrandom 0.4.3", "getrandom 0.4.3",
"igd-next", "igd-next",
"keyring", "keyring",
"libc",
"ratatui", "ratatui",
"schemars", "schemars",
"serde", "serde",
+12
View File
@@ -13,6 +13,13 @@ use crate::types::Download;
use crate::utils::lock::RwLockExt; use crate::utils::lock::RwLockExt;
use crate::{DownloadStatus, debug, error, info, trace}; use crate::{DownloadStatus, debug, error, info, trace};
/// How long to wait before accepting again after a failure.
///
/// The failure worth pausing for is running out of file descriptors, which a
/// busy search can reach: it does not clear by the next instruction, and
/// retrying flat out spins a core and floods the log.
const ACCEPT_BACKOFF: Duration = Duration::from_millis(100);
const PEER_INIT_MESSAGE_CODE: u8 = 1; const PEER_INIT_MESSAGE_CODE: u8 = 1;
/// How long an accepted peer gets to send its peer-init handshake. /// How long an accepted peer gets to send its peer-init handshake.
@@ -406,6 +413,11 @@ impl Listen {
"[listener] Failed to accept connection: {}", "[listener] Failed to accept connection: {}",
stream.unwrap_err() stream.unwrap_err()
); );
// Running out of file descriptors does not clear by the next
// instruction, and retrying flat out turns one exhausted
// moment into a spinning core and thousands of identical log
// lines. Pause long enough for something to be released.
std::thread::sleep(ACCEPT_BACKOFF);
continue; continue;
}; };
+3
View File
@@ -47,3 +47,6 @@ getrandom = "0.4.3"
[dev-dependencies] [dev-dependencies]
schemars = "1.2.1" schemars = "1.2.1"
tempfile = "3" tempfile = "3"
[target."cfg(unix)".dependencies]
libc = "0.2.189"
+77
View File
@@ -44,6 +44,15 @@ pub const TOKEN_NAME: &str = "daemon.token";
/// loses little, rare enough not to churn the disk on a busy queue. /// loses little, rare enough not to churn the disk on a busy queue.
const SAVE_INTERVAL: Duration = Duration::from_secs(5); const SAVE_INTERVAL: Duration = Duration::from_secs(5);
/// Open files the daemon asks the OS for at startup.
///
/// The client holds a socket per peer and will talk to a thousand of them
/// during a busy search, plus transfers, the share index, and control
/// connections. A shell hands its children a soft limit of 256 on macOS, which
/// a single search blows through — and a one-shot command never noticed
/// because it exited before it mattered. A resident process has to ask.
const WANTED_OPEN_FILES: u64 = 8192;
#[must_use] #[must_use]
pub fn socket_path() -> Option<PathBuf> { pub fn socket_path() -> Option<PathBuf> {
crate::persist::paths::state_dir().map(|dir| dir.join(SOCKET_NAME)) crate::persist::paths::state_dir().map(|dir| dir.join(SOCKET_NAME))
@@ -152,6 +161,21 @@ pub fn run(
let token = load_or_create_token(&state_dir.join(TOKEN_NAME)) let token = load_or_create_token(&state_dir.join(TOKEN_NAME))
.map_err(|e| CliError::usage(format!("cannot read the token: {e}")))?; .map_err(|e| CliError::usage(format!("cannot read the token: {e}")))?;
// Before anything opens a socket: running out of file descriptors mid
// search leaves the client unable to accept peers *or* control
// connections, and the only symptom is everything quietly failing.
match raise_open_file_limit() {
Ok(limit) if limit < WANTED_OPEN_FILES => ctx.out.warn(&format!(
"only {limit} open files allowed; a busy search wants \
{WANTED_OPEN_FILES}. Raise it with `ulimit -n {WANTED_OPEN_FILES}` \
before starting the daemon."
)),
Ok(_) => {}
Err(e) => ctx
.out
.warn(&format!("could not check the open-file limit: {e}")),
}
// Bind before logging in: a second daemon should fail in a millisecond, // Bind before logging in: a second daemon should fail in a millisecond,
// not after a full login handshake against the server. // not after a full login handshake against the server.
let mut listeners = Vec::new(); let mut listeners = Vec::new();
@@ -247,6 +271,59 @@ pub fn run(
Ok(()) Ok(())
} }
/// Ask the OS for enough open files to run a busy session, and report what we
/// actually got.
///
/// Raising our own soft limit is what a service does: the limit a shell
/// inherits is sized for a shell. The hard limit is the ceiling, and asking
/// past it fails outright rather than clamping, so this asks for the smaller
/// of what we want and what we are allowed.
#[cfg(unix)]
fn raise_open_file_limit() -> std::io::Result<u64> {
// SAFETY: every call takes a pointer to a `rlimit` this function owns, and
// none of them retains it.
unsafe {
let mut limit = std::mem::zeroed::<libc::rlimit>();
if libc::getrlimit(libc::RLIMIT_NOFILE, &raw mut limit) != 0 {
return Err(std::io::Error::last_os_error());
}
if limit.rlim_cur >= WANTED_OPEN_FILES {
return Ok(limit.rlim_cur);
}
let previous = limit.rlim_cur;
let ceiling = if limit.rlim_max == libc::RLIM_INFINITY {
WANTED_OPEN_FILES
} else {
limit.rlim_max.min(WANTED_OPEN_FILES)
};
// macOS rejects an unlimited *hard* limit for open files, so raising
// the soft one while leaving `rlim_max` as it came back fails with
// EINVAL. Naming both is what it will accept.
for candidate in [
libc::rlimit {
rlim_cur: ceiling,
rlim_max: limit.rlim_max,
},
libc::rlimit {
rlim_cur: ceiling,
rlim_max: ceiling,
},
] {
if libc::setrlimit(libc::RLIMIT_NOFILE, &raw const candidate) == 0 {
return Ok(ceiling);
}
}
Ok(previous)
}
}
#[cfg(not(unix))]
const fn raise_open_file_limit() -> std::io::Result<u64> {
Ok(WANTED_OPEN_FILES)
}
/// Say what is now possible, once, on stderr. /// Say what is now possible, once, on stderr.
/// ///
/// The token goes here rather than only into a file because the moment you /// The token goes here rather than only into a file because the moment you
+9 -1
View File
@@ -38,6 +38,9 @@ const MAX_LINE: u64 = 1 << 20;
/// connection open for events without sending anything. /// connection open for events without sending anything.
const AUTH_TIMEOUT: Duration = Duration::from_secs(30); const AUTH_TIMEOUT: Duration = Duration::from_secs(30);
/// How long to wait before accepting again after a failure.
const ACCEPT_BACKOFF: Duration = Duration::from_millis(100);
/// How many control connections may be open at once. Far above any real use /// How many control connections may be open at once. Far above any real use
/// (a TUI, a few scripts, an agent), and low enough that an unauthenticated /// (a TUI, a few scripts, an agent), and low enough that an unauthenticated
/// peer cannot exhaust the daemon's threads. /// peer cannot exhaust the daemon's threads.
@@ -132,7 +135,12 @@ impl Listener {
loop { loop {
match self.accept() { match self.accept() {
Ok(stream) => spawn_connection(stream, Arc::clone(&daemon)), Ok(stream) => spawn_connection(stream, Arc::clone(&daemon)),
Err(e) => soulseek_rs::warn!("control accept failed: {e}"), Err(e) => {
soulseek_rs::warn!("control accept failed: {e}");
// Out of file descriptors does not clear by the next
// instruction; retrying flat out spins a core.
std::thread::sleep(ACCEPT_BACKOFF);
}
} }
} }
} }