fix: identity-aware peer eviction so a replaced actor can't evict its successor
When a healthy peer actor was replaced under the same username (both sides connecting at once, or two overlapping downloads to the same not-yet-connected peer), its graceful shutdown still emitted PeerDisconnected, and the client evicted by username — removing and stopping the NEWER actor now occupying the slot, tearing down the live connection and stalling that peer's downloads. Give each peer actor a unique id (assigned by the registry) and echo it in PeerDisconnected/PeerConnectFailed. The client now evicts via remove_peer_if(username, id), which removes the entry only when the reporting actor still owns the slot, so a stale actor's shutdown is a no-op. The brokered timeout no longer removes an actor by username either, avoiding the same aliasing. Add a registry unit test for the identity guard.
This commit is contained in:
@@ -61,6 +61,9 @@ pub struct PeerActor {
|
|||||||
/// replaced actor is later stopped — which would otherwise alias and evict
|
/// replaced actor is later stopped — which would otherwise alias and evict
|
||||||
/// a newer actor registered under the same username.
|
/// a newer actor registered under the same username.
|
||||||
disconnect_reported: bool,
|
disconnect_reported: bool,
|
||||||
|
/// Registry-assigned unique id, echoed in terminal notifications so the
|
||||||
|
/// client evicts only this actor and never a newer namesake.
|
||||||
|
id: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PeerActor {
|
impl PeerActor {
|
||||||
@@ -71,6 +74,7 @@ impl PeerActor {
|
|||||||
reader: Option<MessageReader>,
|
reader: Option<MessageReader>,
|
||||||
client_channel: Sender<ClientOperation>,
|
client_channel: Sender<ClientOperation>,
|
||||||
own_username: String,
|
own_username: String,
|
||||||
|
id: u64,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let outbound = stream.is_none();
|
let outbound = stream.is_none();
|
||||||
let connection_state = if stream.is_some() {
|
let connection_state = if stream.is_some() {
|
||||||
@@ -93,6 +97,7 @@ impl PeerActor {
|
|||||||
outbound,
|
outbound,
|
||||||
established: false,
|
established: false,
|
||||||
disconnect_reported: false,
|
disconnect_reported: false,
|
||||||
|
id,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -449,9 +454,13 @@ impl PeerActor {
|
|||||||
// client can fall back to server-brokered connect. Anything else is a
|
// client can fall back to server-brokered connect. Anything else is a
|
||||||
// normal disconnect.
|
// normal disconnect.
|
||||||
let op = if self.outbound && !self.established {
|
let op = if self.outbound && !self.established {
|
||||||
ClientOperation::PeerConnectFailed(username)
|
ClientOperation::PeerConnectFailed(self.id, username)
|
||||||
} else {
|
} else {
|
||||||
ClientOperation::PeerDisconnected(username, Some(error.into()))
|
ClientOperation::PeerDisconnected(
|
||||||
|
self.id,
|
||||||
|
username,
|
||||||
|
Some(error.into()),
|
||||||
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Err(e) = self.client_channel.send(op) {
|
if let Err(e) = self.client_channel.send(op) {
|
||||||
@@ -471,7 +480,7 @@ impl PeerActor {
|
|||||||
|
|
||||||
if let Err(e) = self
|
if let Err(e) = self
|
||||||
.client_channel
|
.client_channel
|
||||||
.send(ClientOperation::PeerDisconnected(username, None))
|
.send(ClientOperation::PeerDisconnected(self.id, username, None))
|
||||||
{
|
{
|
||||||
error!("Failed to send disconnect notification: {}", e);
|
error!("Failed to send disconnect notification: {}", e);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,11 +8,20 @@ use crate::{debug, error};
|
|||||||
|
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::net::TcpStream;
|
use std::net::TcpStream;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::sync::mpsc::Sender;
|
use std::sync::mpsc::Sender;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
/// Source of unique per-actor ids so terminal-outcome eviction can be made
|
||||||
|
/// identity-aware (a replaced actor must not evict its replacement).
|
||||||
|
static NEXT_PEER_ID: AtomicU64 = AtomicU64::new(1);
|
||||||
|
|
||||||
|
/// Registered peers keyed by username, each stored with the unique id of the
|
||||||
|
/// actor currently occupying the slot.
|
||||||
|
type PeerMap = HashMap<String, (u64, ActorHandle<PeerMessage>)>;
|
||||||
|
|
||||||
pub struct PeerRegistry {
|
pub struct PeerRegistry {
|
||||||
peers: Arc<Mutex<HashMap<String, ActorHandle<PeerMessage>>>>,
|
peers: Arc<Mutex<PeerMap>>,
|
||||||
actor_system: Arc<ActorSystem>,
|
actor_system: Arc<ActorSystem>,
|
||||||
client_channel: Sender<ClientOperation>,
|
client_channel: Sender<ClientOperation>,
|
||||||
own_username: String,
|
own_username: String,
|
||||||
@@ -40,6 +49,7 @@ impl PeerRegistry {
|
|||||||
reader: Option<MessageReader>,
|
reader: Option<MessageReader>,
|
||||||
) -> Result<ActorHandle<PeerMessage>, String> {
|
) -> Result<ActorHandle<PeerMessage>, String> {
|
||||||
let username = peer.username.clone();
|
let username = peer.username.clone();
|
||||||
|
let id = NEXT_PEER_ID.fetch_add(1, Ordering::Relaxed);
|
||||||
|
|
||||||
let actor = PeerActor::new(
|
let actor = PeerActor::new(
|
||||||
peer,
|
peer,
|
||||||
@@ -47,6 +57,7 @@ impl PeerRegistry {
|
|||||||
reader,
|
reader,
|
||||||
self.client_channel.clone(),
|
self.client_channel.clone(),
|
||||||
self.own_username.clone(),
|
self.own_username.clone(),
|
||||||
|
id,
|
||||||
);
|
);
|
||||||
|
|
||||||
let handle =
|
let handle =
|
||||||
@@ -59,9 +70,11 @@ impl PeerRegistry {
|
|||||||
.lock_safe()
|
.lock_safe()
|
||||||
.map_err(|e| format!("peer registry lock poisoned: {e}"))?;
|
.map_err(|e| format!("peer registry lock poisoned: {e}"))?;
|
||||||
// Stop any actor already registered under this username so it does not
|
// Stop any actor already registered under this username so it does not
|
||||||
// become an orphan pinning a pool worker forever (and so its later
|
// become an orphan pinning a pool worker forever. Eviction on the
|
||||||
// shutdown cannot remove this new connection from the registry).
|
// replaced actor's later shutdown is identity-aware (keyed on its id),
|
||||||
if let Some(old_handle) = peers.insert(username.clone(), handle.clone())
|
// so stopping it here cannot evict this new connection.
|
||||||
|
if let Some((_, old_handle)) =
|
||||||
|
peers.insert(username.clone(), (id, handle.clone()))
|
||||||
{
|
{
|
||||||
let _ = old_handle.stop();
|
let _ = old_handle.stop();
|
||||||
debug!(
|
debug!(
|
||||||
@@ -76,7 +89,7 @@ impl PeerRegistry {
|
|||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn get_peer(&self, username: &str) -> Option<ActorHandle<PeerMessage>> {
|
pub fn get_peer(&self, username: &str) -> Option<ActorHandle<PeerMessage>> {
|
||||||
match self.peers.lock_safe() {
|
match self.peers.lock_safe() {
|
||||||
Ok(peers) => peers.get(username).cloned(),
|
Ok(peers) => peers.get(username).map(|(_, handle)| handle.clone()),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("[peer_registry] get_peer: {}", e);
|
error!("[peer_registry] get_peer: {}", e);
|
||||||
None
|
None
|
||||||
@@ -96,13 +109,40 @@ impl PeerRegistry {
|
|||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let handle = peers.remove(username);
|
let removed = peers.remove(username);
|
||||||
|
|
||||||
if handle.is_some() {
|
if removed.is_some() {
|
||||||
debug!("[peer_registry] Removed peer actor for {}", username);
|
debug!("[peer_registry] Removed peer actor for {}", username);
|
||||||
}
|
}
|
||||||
|
|
||||||
handle
|
removed.map(|(_, handle)| handle)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove and return the actor for `username` only if it is still the actor
|
||||||
|
/// with `id`. A stale (replaced) actor's terminal notification therefore
|
||||||
|
/// cannot evict the newer actor that now occupies the slot.
|
||||||
|
#[must_use]
|
||||||
|
pub fn remove_peer_if(
|
||||||
|
&self,
|
||||||
|
username: &str,
|
||||||
|
id: u64,
|
||||||
|
) -> Option<ActorHandle<PeerMessage>> {
|
||||||
|
let mut peers = match self.peers.lock_safe() {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(e) => {
|
||||||
|
error!("[peer_registry] remove_peer_if: {}", e);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if peers.get(username).is_some_and(|(stored, _)| *stored == id) {
|
||||||
|
let removed = peers.remove(username).map(|(_, handle)| handle);
|
||||||
|
debug!(
|
||||||
|
"[peer_registry] Removed peer actor {} for {}",
|
||||||
|
id, username
|
||||||
|
);
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
@@ -169,3 +209,52 @@ impl Clone for PeerRegistry {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::PeerRegistry;
|
||||||
|
use crate::actor::ActorSystem;
|
||||||
|
use crate::peer::{ConnectionType, Peer};
|
||||||
|
use crate::utils::thread_pool::ThreadPool;
|
||||||
|
use std::net::{TcpListener, TcpStream};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn remove_peer_if_respects_actor_identity() {
|
||||||
|
let pool = Arc::new(ThreadPool::new(2));
|
||||||
|
let system = Arc::new(ActorSystem::new(pool));
|
||||||
|
let (tx, _rx) = std::sync::mpsc::channel();
|
||||||
|
let registry = PeerRegistry::new(system, tx, "me".to_string());
|
||||||
|
|
||||||
|
// A real loopback connection makes the actor inbound (no dial-out);
|
||||||
|
// non-blocking so it can process Stop promptly on teardown.
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let stream = TcpStream::connect(addr).unwrap();
|
||||||
|
stream.set_nonblocking(true).unwrap();
|
||||||
|
let _server_side = listener.accept().unwrap().0;
|
||||||
|
|
||||||
|
let peer = Peer::new(
|
||||||
|
"bob".to_string(),
|
||||||
|
ConnectionType::P,
|
||||||
|
"127.0.0.1".to_string(),
|
||||||
|
u32::from(addr.port()),
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
registry.register_peer(peer, Some(stream), None).unwrap();
|
||||||
|
assert!(registry.contains("bob"));
|
||||||
|
|
||||||
|
// A stale / wrong id must not evict the live actor.
|
||||||
|
assert!(registry.remove_peer_if("bob", u64::MAX).is_none());
|
||||||
|
assert!(registry.contains("bob"));
|
||||||
|
|
||||||
|
// Unconditional removal still works (and stops the actor).
|
||||||
|
let handle = registry.remove_peer("bob");
|
||||||
|
assert!(handle.is_some());
|
||||||
|
let _ = handle.unwrap().stop();
|
||||||
|
assert!(!registry.contains("bob"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -89,7 +89,7 @@ pub enum ClientOperation {
|
|||||||
NewPeer(NewPeer),
|
NewPeer(NewPeer),
|
||||||
ConnectToPeer(Peer),
|
ConnectToPeer(Peer),
|
||||||
SearchResult(SearchResult),
|
SearchResult(SearchResult),
|
||||||
PeerDisconnected(String, Option<SoulseekRs>),
|
PeerDisconnected(u64, String, Option<SoulseekRs>),
|
||||||
PierceFireWall(Peer),
|
PierceFireWall(Peer),
|
||||||
DownloadFromPeer(u32, Peer, bool),
|
DownloadFromPeer(u32, Peer, bool),
|
||||||
UpdateDownloadTokens(Transfer, String),
|
UpdateDownloadTokens(Transfer, String),
|
||||||
@@ -111,8 +111,8 @@ pub enum ClientOperation {
|
|||||||
PeerConnected(String),
|
PeerConnected(String),
|
||||||
/// A direct outbound connection to this peer failed before it was
|
/// A direct outbound connection to this peer failed before it was
|
||||||
/// established — the peer is likely firewalled, so fall back to asking the
|
/// established — the peer is likely firewalled, so fall back to asking the
|
||||||
/// server to broker the connection.
|
/// server to broker the connection. Carries the reporting actor's id.
|
||||||
PeerConnectFailed(String),
|
PeerConnectFailed(u64, String),
|
||||||
}
|
}
|
||||||
pub struct ClientContext {
|
pub struct ClientContext {
|
||||||
pub peer_registry: Option<PeerRegistry>,
|
pub peer_registry: Option<PeerRegistry>,
|
||||||
@@ -934,6 +934,7 @@ impl Client {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
ClientOperation::PeerDisconnected(
|
ClientOperation::PeerDisconnected(
|
||||||
|
id,
|
||||||
username,
|
username,
|
||||||
error,
|
error,
|
||||||
) => {
|
) => {
|
||||||
@@ -941,7 +942,9 @@ impl Client {
|
|||||||
// below acquires a write lock on the same
|
// below acquires a write lock on the same
|
||||||
// RwLock, which would self-deadlock the entire
|
// RwLock, which would self-deadlock the entire
|
||||||
// client ops loop if this read guard were still
|
// client ops loop if this read guard were still
|
||||||
// held on this thread.
|
// held on this thread. Evict only if this exact
|
||||||
|
// actor still occupies the slot, so a replaced
|
||||||
|
// actor's shutdown can't remove its successor.
|
||||||
{
|
{
|
||||||
let context = match client_context
|
let context = match client_context
|
||||||
.read_safe()
|
.read_safe()
|
||||||
@@ -957,8 +960,8 @@ impl Client {
|
|||||||
};
|
};
|
||||||
if let Some(ref registry) =
|
if let Some(ref registry) =
|
||||||
context.peer_registry
|
context.peer_registry
|
||||||
&& let Some(handle) =
|
&& let Some(handle) = registry
|
||||||
registry.remove_peer(&username)
|
.remove_peer_if(&username, id)
|
||||||
{
|
{
|
||||||
let _ = handle.stop();
|
let _ = handle.stop();
|
||||||
}
|
}
|
||||||
@@ -1373,7 +1376,10 @@ impl Client {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ClientOperation::PeerConnectFailed(username) => {
|
ClientOperation::PeerConnectFailed(
|
||||||
|
id,
|
||||||
|
username,
|
||||||
|
) => {
|
||||||
// Direct connect failed: ask the server to
|
// Direct connect failed: ask the server to
|
||||||
// broker it. Register a correlation token, then
|
// broker it. Register a correlation token, then
|
||||||
// send ConnectToPeer so the (firewalled) peer
|
// send ConnectToPeer so the (firewalled) peer
|
||||||
@@ -1388,11 +1394,14 @@ impl Client {
|
|||||||
// longer shadows the brokered reconnect
|
// longer shadows the brokered reconnect
|
||||||
// (a stale registry entry would make
|
// (a stale registry entry would make
|
||||||
// later downloads queue into a dead,
|
// later downloads queue into a dead,
|
||||||
// streamless actor and hang).
|
// streamless actor and hang). Identity-
|
||||||
if let Some(handle) =
|
// aware so a newer namesake is untouched.
|
||||||
ctx.peer_registry.as_ref().and_then(
|
if let Some(handle) = ctx
|
||||||
|r| r.remove_peer(&username),
|
.peer_registry
|
||||||
)
|
.as_ref()
|
||||||
|
.and_then(|r| {
|
||||||
|
r.remove_peer_if(&username, id)
|
||||||
|
})
|
||||||
{
|
{
|
||||||
let _ = handle.stop();
|
let _ = handle.stop();
|
||||||
}
|
}
|
||||||
@@ -1422,10 +1431,10 @@ impl Client {
|
|||||||
.send(ServerMessage::SendMessage(msg));
|
.send(ServerMessage::SendMessage(msg));
|
||||||
|
|
||||||
// Bound the brokered attempt: if no PierceFirewall
|
// Bound the brokered attempt: if no PierceFirewall
|
||||||
// arrives, fail the peer's queued downloads (so
|
// consumes the token, fail the peer's queued
|
||||||
// the caller's Receiver unblocks) and reclaim the
|
// downloads (so the caller's Receiver unblocks)
|
||||||
// token. A successful pierce consumes the token
|
// and reclaim the token. A successful pierce
|
||||||
// first, making this a no-op.
|
// takes the token first, making this a no-op.
|
||||||
let timeout_ctx = client_context.clone();
|
let timeout_ctx = client_context.clone();
|
||||||
let timeout_user = username.clone();
|
let timeout_user = username.clone();
|
||||||
thread::spawn(move || {
|
thread::spawn(move || {
|
||||||
@@ -1437,21 +1446,6 @@ impl Client {
|
|||||||
.is_some()
|
.is_some()
|
||||||
});
|
});
|
||||||
if still_pending {
|
if still_pending {
|
||||||
if let Some(handle) = timeout_ctx
|
|
||||||
.read_safe()
|
|
||||||
.ok()
|
|
||||||
.and_then(|c| {
|
|
||||||
c.peer_registry
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|r| {
|
|
||||||
r.remove_peer(
|
|
||||||
&timeout_user,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
{
|
|
||||||
let _ = handle.stop();
|
|
||||||
}
|
|
||||||
Self::fail_queued_downloads(
|
Self::fail_queued_downloads(
|
||||||
&timeout_ctx,
|
&timeout_ctx,
|
||||||
&timeout_user,
|
&timeout_user,
|
||||||
|
|||||||
Reference in New Issue
Block a user