fix(lib): cap what a search collects
Counting cheaply was not enough: on the live network peers answer a popular query for minutes after the window closes, and nothing ever stopped the pile-up. One 'aphex twin' search grew to 867,990 files and 2.1 GB of daemon memory, and a single search.results reply serialized 134 MB of JSON over 104 seconds - an attached TUI that touched the search froze for minutes, which read as the whole application being forever slow. A search now stops collecting at 500 responses or 10,000 files, whichever comes first, the same shortlist stance other clients take: a result set is something to pick from, not an archive of everyone who answered. The same search now plateaus at ~10k files, a 1.55 MB reply in 23 ms, and 231 MB of daemon memory; the attached TUI idles at zero CPU with the search open.
This commit is contained in:
@@ -44,7 +44,7 @@ impl Client {
|
|||||||
// Find the search with matching token
|
// Find the search with matching token
|
||||||
for search in context.searches.values_mut() {
|
for search in context.searches.values_mut() {
|
||||||
if search.token == result_token {
|
if search.token == result_token {
|
||||||
search.results.push(search_result);
|
search.accept(search_result);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -174,27 +174,27 @@ fn build_search_response_matches_shares_and_echoes_token() {
|
|||||||
let _ = std::fs::remove_dir_all(dir);
|
let _ = std::fs::remove_dir_all(dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn peer_files(count: usize) -> SearchResult {
|
||||||
|
SearchResult {
|
||||||
|
token: 1,
|
||||||
|
files: (0..count)
|
||||||
|
.map(|i| crate::types::File {
|
||||||
|
username: "bob".to_string(),
|
||||||
|
name: format!("song-{i}.mp3"),
|
||||||
|
size: 1,
|
||||||
|
attribs: std::collections::HashMap::new(),
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
slots: 1,
|
||||||
|
speed: 0,
|
||||||
|
username: "bob".to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn search_file_counts_cover_every_search_without_the_results() {
|
fn search_file_counts_cover_every_search_without_the_results() {
|
||||||
// The daemon polls these counts many times a second; they must come from
|
// The daemon polls these counts many times a second; they must come from
|
||||||
// a walk of the cache, never a copy of it.
|
// a walk of the cache, never a copy of it.
|
||||||
fn peer_files(count: usize) -> SearchResult {
|
|
||||||
SearchResult {
|
|
||||||
token: 1,
|
|
||||||
files: (0..count)
|
|
||||||
.map(|i| crate::types::File {
|
|
||||||
username: "bob".to_string(),
|
|
||||||
name: format!("song-{i}.mp3"),
|
|
||||||
size: 1,
|
|
||||||
attribs: std::collections::HashMap::new(),
|
|
||||||
})
|
|
||||||
.collect(),
|
|
||||||
slots: 1,
|
|
||||||
speed: 0,
|
|
||||||
username: "bob".to_string(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
let client = Client::new("u", "p");
|
let client = Client::new("u", "p");
|
||||||
{
|
{
|
||||||
let mut context = client.context.write().unwrap();
|
let mut context = client.context.write().unwrap();
|
||||||
@@ -225,6 +225,39 @@ fn search_file_counts_cover_every_search_without_the_results() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_search_stops_collecting_once_it_has_enough_responses() {
|
||||||
|
// A popular query on the live network draws answers for minutes —
|
||||||
|
// nearly a million files and gigabytes of memory for one search.
|
||||||
|
// Surplus responders are dropped, not archived.
|
||||||
|
let mut search = Search {
|
||||||
|
token: 1,
|
||||||
|
results: Vec::new(),
|
||||||
|
};
|
||||||
|
for _ in 0..(crate::types::MAX_SEARCH_RESPONSES + 50) {
|
||||||
|
search.accept(peer_files(1));
|
||||||
|
}
|
||||||
|
assert_eq!(search.results.len(), crate::types::MAX_SEARCH_RESPONSES);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_flood_of_files_fills_a_search_before_the_response_cap() {
|
||||||
|
// A handful of whales with huge matching collections must not add up
|
||||||
|
// to an unbounded set just because the responses are few.
|
||||||
|
let mut search = Search {
|
||||||
|
token: 1,
|
||||||
|
results: Vec::new(),
|
||||||
|
};
|
||||||
|
for _ in 0..10 {
|
||||||
|
search.accept(peer_files(crate::types::MAX_SEARCH_FILES / 2));
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
search.results.len(),
|
||||||
|
2,
|
||||||
|
"two half-cap responses fill the search; the rest are dropped"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_client_removes_only_queued_downloads() {
|
fn test_client_removes_only_queued_downloads() {
|
||||||
let client = Client::new("test-user", "test-password");
|
let client = Client::new("test-user", "test-password");
|
||||||
|
|||||||
@@ -59,6 +59,36 @@ pub struct Search {
|
|||||||
pub results: Vec<SearchResult>,
|
pub results: Vec<SearchResult>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Peer responses a search keeps before it stops collecting.
|
||||||
|
///
|
||||||
|
/// Peers answer a popular query for minutes after the window closes; left
|
||||||
|
/// unbounded, one live-network search grew to nearly a million files and
|
||||||
|
/// gigabytes of memory, and every consumer of the results paid for the
|
||||||
|
/// whole set. A result set is a shortlist to pick from, not an archive of
|
||||||
|
/// everyone who answered.
|
||||||
|
pub const MAX_SEARCH_RESPONSES: usize = 500;
|
||||||
|
|
||||||
|
/// Files a search keeps across all responses, so a few peers with huge
|
||||||
|
/// matching collections cannot blow past the response cap.
|
||||||
|
pub const MAX_SEARCH_FILES: usize = 10_000;
|
||||||
|
|
||||||
|
impl Search {
|
||||||
|
/// Store one peer's response, unless this search already holds enough.
|
||||||
|
///
|
||||||
|
/// The last accepted response may carry the total past
|
||||||
|
/// [`MAX_SEARCH_FILES`]; responses are kept whole rather than truncated.
|
||||||
|
pub fn accept(&mut self, result: SearchResult) {
|
||||||
|
// ponytail: counts files linearly per call — bounded by the response
|
||||||
|
// cap; a running total would mean a new pub field on Search.
|
||||||
|
let files: usize =
|
||||||
|
self.results.iter().map(|result| result.files.len()).sum();
|
||||||
|
if self.results.len() < MAX_SEARCH_RESPONSES && files < MAX_SEARCH_FILES
|
||||||
|
{
|
||||||
|
self.results.push(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl SearchResult {
|
impl SearchResult {
|
||||||
pub fn new_from_message(message: &mut Message) -> Result<Self> {
|
pub fn new_from_message(message: &mut Message) -> Result<Self> {
|
||||||
let pointer = message.get_pointer();
|
let pointer = message.get_pointer();
|
||||||
|
|||||||
Reference in New Issue
Block a user