fix(lib): cap what a search collects
Counting cheaply was not enough: on the live network peers answer a popular query for minutes after the window closes, and nothing ever stopped the pile-up. One 'aphex twin' search grew to 867,990 files and 2.1 GB of daemon memory, and a single search.results reply serialized 134 MB of JSON over 104 seconds - an attached TUI that touched the search froze for minutes, which read as the whole application being forever slow. A search now stops collecting at 500 responses or 10,000 files, whichever comes first, the same shortlist stance other clients take: a result set is something to pick from, not an archive of everyone who answered. The same search now plateaus at ~10k files, a 1.55 MB reply in 23 ms, and 231 MB of daemon memory; the attached TUI idles at zero CPU with the search open.
This commit is contained in:
@@ -44,7 +44,7 @@ impl Client {
|
||||
// Find the search with matching token
|
||||
for search in context.searches.values_mut() {
|
||||
if search.token == result_token {
|
||||
search.results.push(search_result);
|
||||
search.accept(search_result);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -174,27 +174,27 @@ fn build_search_response_matches_shares_and_echoes_token() {
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
fn peer_files(count: usize) -> SearchResult {
|
||||
SearchResult {
|
||||
token: 1,
|
||||
files: (0..count)
|
||||
.map(|i| crate::types::File {
|
||||
username: "bob".to_string(),
|
||||
name: format!("song-{i}.mp3"),
|
||||
size: 1,
|
||||
attribs: std::collections::HashMap::new(),
|
||||
})
|
||||
.collect(),
|
||||
slots: 1,
|
||||
speed: 0,
|
||||
username: "bob".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_file_counts_cover_every_search_without_the_results() {
|
||||
// The daemon polls these counts many times a second; they must come from
|
||||
// a walk of the cache, never a copy of it.
|
||||
fn peer_files(count: usize) -> SearchResult {
|
||||
SearchResult {
|
||||
token: 1,
|
||||
files: (0..count)
|
||||
.map(|i| crate::types::File {
|
||||
username: "bob".to_string(),
|
||||
name: format!("song-{i}.mp3"),
|
||||
size: 1,
|
||||
attribs: std::collections::HashMap::new(),
|
||||
})
|
||||
.collect(),
|
||||
slots: 1,
|
||||
speed: 0,
|
||||
username: "bob".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
let client = Client::new("u", "p");
|
||||
{
|
||||
let mut context = client.context.write().unwrap();
|
||||
@@ -225,6 +225,39 @@ fn search_file_counts_cover_every_search_without_the_results() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_search_stops_collecting_once_it_has_enough_responses() {
|
||||
// A popular query on the live network draws answers for minutes —
|
||||
// nearly a million files and gigabytes of memory for one search.
|
||||
// Surplus responders are dropped, not archived.
|
||||
let mut search = Search {
|
||||
token: 1,
|
||||
results: Vec::new(),
|
||||
};
|
||||
for _ in 0..(crate::types::MAX_SEARCH_RESPONSES + 50) {
|
||||
search.accept(peer_files(1));
|
||||
}
|
||||
assert_eq!(search.results.len(), crate::types::MAX_SEARCH_RESPONSES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_flood_of_files_fills_a_search_before_the_response_cap() {
|
||||
// A handful of whales with huge matching collections must not add up
|
||||
// to an unbounded set just because the responses are few.
|
||||
let mut search = Search {
|
||||
token: 1,
|
||||
results: Vec::new(),
|
||||
};
|
||||
for _ in 0..10 {
|
||||
search.accept(peer_files(crate::types::MAX_SEARCH_FILES / 2));
|
||||
}
|
||||
assert_eq!(
|
||||
search.results.len(),
|
||||
2,
|
||||
"two half-cap responses fill the search; the rest are dropped"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_client_removes_only_queued_downloads() {
|
||||
let client = Client::new("test-user", "test-password");
|
||||
|
||||
@@ -59,6 +59,36 @@ pub struct Search {
|
||||
pub results: Vec<SearchResult>,
|
||||
}
|
||||
|
||||
/// Peer responses a search keeps before it stops collecting.
|
||||
///
|
||||
/// Peers answer a popular query for minutes after the window closes; left
|
||||
/// unbounded, one live-network search grew to nearly a million files and
|
||||
/// gigabytes of memory, and every consumer of the results paid for the
|
||||
/// whole set. A result set is a shortlist to pick from, not an archive of
|
||||
/// everyone who answered.
|
||||
pub const MAX_SEARCH_RESPONSES: usize = 500;
|
||||
|
||||
/// Files a search keeps across all responses, so a few peers with huge
|
||||
/// matching collections cannot blow past the response cap.
|
||||
pub const MAX_SEARCH_FILES: usize = 10_000;
|
||||
|
||||
impl Search {
|
||||
/// Store one peer's response, unless this search already holds enough.
|
||||
///
|
||||
/// The last accepted response may carry the total past
|
||||
/// [`MAX_SEARCH_FILES`]; responses are kept whole rather than truncated.
|
||||
pub fn accept(&mut self, result: SearchResult) {
|
||||
// ponytail: counts files linearly per call — bounded by the response
|
||||
// cap; a running total would mean a new pub field on Search.
|
||||
let files: usize =
|
||||
self.results.iter().map(|result| result.files.len()).sum();
|
||||
if self.results.len() < MAX_SEARCH_RESPONSES && files < MAX_SEARCH_FILES
|
||||
{
|
||||
self.results.push(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SearchResult {
|
||||
pub fn new_from_message(message: &mut Message) -> Result<Self> {
|
||||
let pointer = message.get_pointer();
|
||||
|
||||
Reference in New Issue
Block a user