fix(daemon): reconnect after the server session drops

A long-running daemon that lost its TCP connection to the server reported
session_loss "disconnected" and answered every command from the dead session
until a manual restart. Two causes:

- the daemon owned one session for its lifetime with no reconnect at all; and
- the lib's ServerActor queued a Login sent while disconnected instead of
  dialing, so even asking it to log in again never re-established a connection.

Reconnect on a dedicated thread with 1s-60s capped backoff, only for a dropped
connection to a session this process owns (never a displaced or borrowed one,
which would start a login war or fight another daemon). Let a Login through the
disconnected guard so a parked actor dials again, and bound that dial with
connect_timeout so a blackholed route cannot park the actor thread. Reset the
message reader, the queued backlog and the login verdict on disconnect so a
partial frame or stale Some(false) cannot wedge the reconnected session.
Announce the drop once per outage and re-join rooms and re-watch users on
recovery; the daemon main loop is back to save-and-stop so `daemon stop` stays
responsive during an outage. Update the session_loss event summary, which no
longer means the session is gone for good.
This commit is contained in:
Michel de Graaf
2026-09-02 11:21:19 +02:00
parent 597f378f17
commit 67d979b809
8 changed files with 436 additions and 63 deletions
+71 -6
View File
@@ -29,7 +29,7 @@ struct Daemon {
child: Child,
state: TempDir,
port: u16,
_server: Soulfind,
server: Soulfind,
}
/// The binary with a scrubbed environment: no test may see the developer's
@@ -78,7 +78,7 @@ impl Daemon {
child,
state,
port,
_server: server,
server,
};
daemon.wait_until_ready()?;
Some(daemon)
@@ -130,6 +130,17 @@ impl Daemon {
let read = stream.try_clone().expect("a second handle");
Some(Connection::from_parts(Box::new(read), Box::new(stream)))
}
fn status(&self) -> Option<serde_json::Value> {
let mut connection = self.connect_tcp()?;
connection.authenticate(Some(&self.token()))?;
let reply = connection
.call(r#"{"jsonrpc":"2.0","id":1,"method":"daemon.status"}"#)?;
serde_json::from_str::<serde_json::Value>(&reply)
.ok()?
.get_mut("result")
.map(serde_json::Value::take)
}
}
impl Drop for Daemon {
@@ -428,10 +439,15 @@ struct Soulfind {
impl Soulfind {
fn start() -> Option<Self> {
let bin = soulfind_binary()?;
let port = free_port()?;
let db = std::env::temp_dir().join(format!("soulfind-proto-{port}.db"));
let _ = std::fs::remove_file(&db);
let child = Self::spawn(port, &db)?;
Some(Self { child, db, port })
}
fn spawn(port: u16, db: &Path) -> Option<Child> {
let bin = soulfind_binary()?;
let mut child = Command::new(bin)
.args(["-d", db.to_str()?, "-p", &port.to_string()])
@@ -443,7 +459,7 @@ impl Soulfind {
let deadline = Instant::now() + Duration::from_secs(10);
while Instant::now() < deadline {
if TcpStream::connect(("127.0.0.1", port)).is_ok() {
return Some(Self { child, db, port });
return Some(child);
}
std::thread::sleep(Duration::from_millis(100));
}
@@ -455,16 +471,65 @@ impl Soulfind {
fn address(&self) -> String {
format!("127.0.0.1:{}", self.port)
}
fn stop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
}
fn restart(&mut self) -> Option<()> {
self.stop();
self.child = Self::spawn(self.port, &self.db)?;
Some(())
}
}
impl Drop for Soulfind {
fn drop(&mut self) {
let _ = self.child.kill();
let _ = self.child.wait();
self.stop();
let _ = std::fs::remove_file(&self.db);
}
}
fn wait_until(timeout: Duration, mut f: impl FnMut() -> bool) -> bool {
let deadline = Instant::now() + timeout;
while Instant::now() < deadline {
if f() {
return true;
}
std::thread::sleep(Duration::from_millis(250));
}
false
}
#[test]
fn the_daemon_logs_back_in_after_the_server_drops_it() {
let mut daemon = daemon_or_skip!();
assert!(
daemon.status().expect("a status")["session_loss"].is_null(),
"the daemon starts with a live session"
);
daemon.server.stop();
assert!(
wait_until(Duration::from_secs(15), || daemon
.status()
.is_some_and(|status| status["session_loss"] == "disconnected")),
"the daemon never noticed the server going away"
);
daemon
.server
.restart()
.expect("soulfind comes back on the same port");
assert!(
wait_until(Duration::from_secs(45), || daemon
.status()
.is_some_and(|status| status["session_loss"].is_null())),
"the daemon never logged back in once the server was reachable again"
);
}
/// A client configured for a remote daemon reports the *daemon's* world.
///
/// This is the download-box workflow: the token lives in the client's config