fix(daemon): reconnect after the server session drops
A long-running daemon that lost its TCP connection to the server reported session_loss "disconnected" and answered every command from the dead session until a manual restart. Two causes: - the daemon owned one session for its lifetime with no reconnect at all; and - the lib's ServerActor queued a Login sent while disconnected instead of dialing, so even asking it to log in again never re-established a connection. Reconnect on a dedicated thread with 1s-60s capped backoff, only for a dropped connection to a session this process owns (never a displaced or borrowed one, which would start a login war or fight another daemon). Let a Login through the disconnected guard so a parked actor dials again, and bound that dial with connect_timeout so a blackholed route cannot park the actor thread. Reset the message reader, the queued backlog and the login verdict on disconnect so a partial frame or stale Some(false) cannot wedge the reconnected session. Announce the drop once per outage and re-join rooms and re-watch users on recovery; the daemon main loop is back to save-and-stop so `daemon stop` stays responsive during an outage. Update the session_loss event summary, which no longer means the session is gone for good.
This commit is contained in:
@@ -29,7 +29,7 @@ struct Daemon {
|
||||
child: Child,
|
||||
state: TempDir,
|
||||
port: u16,
|
||||
_server: Soulfind,
|
||||
server: Soulfind,
|
||||
}
|
||||
|
||||
/// The binary with a scrubbed environment: no test may see the developer's
|
||||
@@ -78,7 +78,7 @@ impl Daemon {
|
||||
child,
|
||||
state,
|
||||
port,
|
||||
_server: server,
|
||||
server,
|
||||
};
|
||||
daemon.wait_until_ready()?;
|
||||
Some(daemon)
|
||||
@@ -130,6 +130,17 @@ impl Daemon {
|
||||
let read = stream.try_clone().expect("a second handle");
|
||||
Some(Connection::from_parts(Box::new(read), Box::new(stream)))
|
||||
}
|
||||
|
||||
fn status(&self) -> Option<serde_json::Value> {
|
||||
let mut connection = self.connect_tcp()?;
|
||||
connection.authenticate(Some(&self.token()))?;
|
||||
let reply = connection
|
||||
.call(r#"{"jsonrpc":"2.0","id":1,"method":"daemon.status"}"#)?;
|
||||
serde_json::from_str::<serde_json::Value>(&reply)
|
||||
.ok()?
|
||||
.get_mut("result")
|
||||
.map(serde_json::Value::take)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Daemon {
|
||||
@@ -428,10 +439,15 @@ struct Soulfind {
|
||||
|
||||
impl Soulfind {
|
||||
fn start() -> Option<Self> {
|
||||
let bin = soulfind_binary()?;
|
||||
let port = free_port()?;
|
||||
let db = std::env::temp_dir().join(format!("soulfind-proto-{port}.db"));
|
||||
let _ = std::fs::remove_file(&db);
|
||||
let child = Self::spawn(port, &db)?;
|
||||
Some(Self { child, db, port })
|
||||
}
|
||||
|
||||
fn spawn(port: u16, db: &Path) -> Option<Child> {
|
||||
let bin = soulfind_binary()?;
|
||||
|
||||
let mut child = Command::new(bin)
|
||||
.args(["-d", db.to_str()?, "-p", &port.to_string()])
|
||||
@@ -443,7 +459,7 @@ impl Soulfind {
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
while Instant::now() < deadline {
|
||||
if TcpStream::connect(("127.0.0.1", port)).is_ok() {
|
||||
return Some(Self { child, db, port });
|
||||
return Some(child);
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
@@ -455,16 +471,65 @@ impl Soulfind {
|
||||
fn address(&self) -> String {
|
||||
format!("127.0.0.1:{}", self.port)
|
||||
}
|
||||
|
||||
fn stop(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
}
|
||||
|
||||
fn restart(&mut self) -> Option<()> {
|
||||
self.stop();
|
||||
self.child = Self::spawn(self.port, &self.db)?;
|
||||
Some(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Soulfind {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
self.stop();
|
||||
let _ = std::fs::remove_file(&self.db);
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_until(timeout: Duration, mut f: impl FnMut() -> bool) -> bool {
|
||||
let deadline = Instant::now() + timeout;
|
||||
while Instant::now() < deadline {
|
||||
if f() {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(250));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_daemon_logs_back_in_after_the_server_drops_it() {
|
||||
let mut daemon = daemon_or_skip!();
|
||||
assert!(
|
||||
daemon.status().expect("a status")["session_loss"].is_null(),
|
||||
"the daemon starts with a live session"
|
||||
);
|
||||
|
||||
daemon.server.stop();
|
||||
assert!(
|
||||
wait_until(Duration::from_secs(15), || daemon
|
||||
.status()
|
||||
.is_some_and(|status| status["session_loss"] == "disconnected")),
|
||||
"the daemon never noticed the server going away"
|
||||
);
|
||||
|
||||
daemon
|
||||
.server
|
||||
.restart()
|
||||
.expect("soulfind comes back on the same port");
|
||||
assert!(
|
||||
wait_until(Duration::from_secs(45), || daemon
|
||||
.status()
|
||||
.is_some_and(|status| status["session_loss"].is_null())),
|
||||
"the daemon never logged back in once the server was reachable again"
|
||||
);
|
||||
}
|
||||
|
||||
/// A client configured for a remote daemon reports the *daemon's* world.
|
||||
///
|
||||
/// This is the download-box workflow: the token lives in the client's config
|
||||
|
||||
Reference in New Issue
Block a user