From e163e7f22975d52dd7d341b8cae566cbd13075d5 Mon Sep 17 00:00:00 2001 From: nazeh Date: Thu, 8 Aug 2024 12:02:46 +0300 Subject: [PATCH] test(pubky): thorough testing for list options --- .../src/database/tables/entries.rs | 154 +++++++------- pubky/src/shared/public.rs | 199 ++++++++++++++++-- 2 files changed, 263 insertions(+), 90 deletions(-) diff --git a/pubky-homeserver/src/database/tables/entries.rs b/pubky-homeserver/src/database/tables/entries.rs index af26828..7c61bf6 100644 --- a/pubky-homeserver/src/database/tables/entries.rs +++ b/pubky-homeserver/src/database/tables/entries.rs @@ -103,85 +103,93 @@ impl DB { let limit = limit.unwrap_or(MAX_LIST_LIMIT).min(MAX_LIST_LIMIT); - // Remove directories from the cursor; - let cursor = cursor + // TODO: make this more performant than split and allocations? + let mut threshold = cursor .as_deref() - .and_then(|mut cursor| cursor.rsplit('/').next()) - .unwrap_or(if reverse { "~" } else { "" }); - - let mut cursor = format!("{path}{cursor}"); - - // Fetch data based on direction - if reverse { - for _ in 0..limit { - if let Some((key, _)) = self.tables.entries.get_lower_than(txn, &cursor)? { - if !key.starts_with(path) { - break; - } - - if shallow { - let mut split = key[path.len()..].split('/'); - let item = split.next().expect("should not be reachable"); - - let is_directory = split.next().is_some(); - - cursor = format!( - "{}{}", - &key[..(path.len() + item.len())], - // `.` is immediately lower than `/` - if is_directory { "." } else { "" } - ); - - let url = format!( - "pubky://{path}{item}{}", - if is_directory { "/" } else { "" } - ); - - results.push(url); - } else { - cursor = key.to_string(); - results.push(format!("pubky://{}", key)) - } - }; - } - } else { - for _ in 0..limit { - if let Some((key, _)) = self.tables.entries.get_greater_than(txn, &cursor)? { - if !key.starts_with(path) { - break; - } - - if shallow { - let mut split = key[path.len()..].split('/'); - let item = split.next().expect("should not be reachable"); - - let is_directory = split.next().is_some(); - - cursor = format!( - "{}{}", - &key[..(path.len() + item.len())], - // `0` is immediately higher than `/` - if is_directory { "0" } else { "" } - ); - - let url = format!( - "pubky://{path}{item}{}", - if is_directory { "/" } else { "" } - ); - - results.push(url); - } else { - cursor = key.to_string(); - results.push(format!("pubky://{}", key)) - } - }; - } - }; + .map(|mut cursor| { + // Get the name of the file or directory + // Similar to Path::new(cursor).file_name + let is_directory = cursor.ends_with('/'); + + let mut split = cursor.rsplit('/'); + + if is_directory { + // Move one step back + split.next(); + } + + let file_or_directory = split.next().expect("should not be reachable"); + + next_threshold(path, file_or_directory, is_directory, reverse, shallow) + }) + .unwrap_or(next_threshold(path, "", false, reverse, shallow)); + + for _ in 0..limit { + if let Some((key, _)) = (if reverse { + self.tables.entries.get_lower_than(txn, &threshold)? + } else { + self.tables.entries.get_greater_than(txn, &threshold)? + }) { + if !key.starts_with(path) { + break; + } + + if shallow { + let mut split = key[path.len()..].split('/'); + let file_or_directory = split.next().expect("should not be reachable"); + + let is_directory = split.next().is_some(); + + threshold = + next_threshold(path, file_or_directory, is_directory, reverse, shallow); + + results.push(format!( + "pubky://{path}{file_or_directory}{}", + if is_directory { "/" } else { "" } + )); + } else { + threshold = key.to_string(); + results.push(format!("pubky://{}", key)) + } + }; + } Ok(results) } } +/// Calculate the next threshold, only for flat (non-`shallow`) listing +fn next_threshold( + path: &str, + file_or_directory: &str, + is_directory: bool, + reverse: bool, + shallow: bool, +) -> String { + format!( + "{path}{file_or_directory}{}", + if file_or_directory.is_empty() { + // No file_or_directory, early return + if reverse { + // `path/to/dir/\x7f` to catch all paths than `path/to/dir/` + "\x7f" + } else { + "" + } + } else if shallow & is_directory { + if reverse { + // threshold = `path/to/dir\x2e`, since `\x2e` is lower than `/` + "\x2e" + } else { + //threshold = `path/to/dir\x7f`, since `\x7f` is greater than `/` + "\x7f" + } + } else { + "" + } + ) +} + #[derive(Clone, Default, Serialize, Deserialize, Debug, Eq, PartialEq)] pub struct Entry { /// Encoding version diff --git a/pubky/src/shared/public.rs b/pubky/src/shared/public.rs index 9bd099f..8e8f1c5 100644 --- a/pubky/src/shared/public.rs +++ b/pubky/src/shared/public.rs @@ -226,6 +226,10 @@ mod tests { format!("pubky://{}/pub/a.wrong/a.txt", keypair.public_key()), format!("pubky://{}/pub/example.com/a.txt", keypair.public_key()), format!("pubky://{}/pub/example.com/b.txt", keypair.public_key()), + format!( + "pubky://{}/pub/example.com/cc-nested/z.txt", + keypair.public_key() + ), format!("pubky://{}/pub/example.wrong/a.txt", keypair.public_key()), format!("pubky://{}/pub/example.com/c.txt", keypair.public_key()), format!("pubky://{}/pub/example.com/d.txt", keypair.public_key()), @@ -237,9 +241,10 @@ mod tests { } let url = format!("pubky://{}/pub/example.com/extra", keypair.public_key()); + let url = url.as_str(); { - let list = client.list(url.as_str()).unwrap().send().await.unwrap(); + let list = client.list(url).unwrap().send().await.unwrap(); assert_eq!( list, @@ -247,17 +252,35 @@ mod tests { format!("pubky://{}/pub/example.com/a.txt", keypair.public_key()), format!("pubky://{}/pub/example.com/b.txt", keypair.public_key()), format!("pubky://{}/pub/example.com/c.txt", keypair.public_key()), + format!( + "pubky://{}/pub/example.com/cc-nested/z.txt", + keypair.public_key() + ), format!("pubky://{}/pub/example.com/d.txt", keypair.public_key()), ], "normal list with no limit or cursor" ); } + { + let list = client.list(url).unwrap().limit(2).send().await.unwrap(); + + assert_eq!( + list, + vec![ + format!("pubky://{}/pub/example.com/a.txt", keypair.public_key()), + format!("pubky://{}/pub/example.com/b.txt", keypair.public_key()), + ], + "normal list with limit but no cursor" + ); + } + { let list = client - .list(url.as_str()) + .list(url) .unwrap() .limit(2) + .cursor("a.txt") .send() .await .unwrap(); @@ -265,19 +288,19 @@ mod tests { assert_eq!( list, vec![ - format!("pubky://{}/pub/example.com/a.txt", keypair.public_key()), format!("pubky://{}/pub/example.com/b.txt", keypair.public_key()), + format!("pubky://{}/pub/example.com/c.txt", keypair.public_key()), ], - "normal list with limit but no cursor" + "normal list with limit and a file cursor" ); } { let list = client - .list(url.as_str()) + .list(url) .unwrap() .limit(2) - .cursor("a.txt") + .cursor("cc-nested/") .send() .await .unwrap(); @@ -285,16 +308,19 @@ mod tests { assert_eq!( list, vec![ - format!("pubky://{}/pub/example.com/b.txt", keypair.public_key()), - format!("pubky://{}/pub/example.com/c.txt", keypair.public_key()), + format!( + "pubky://{}/pub/example.com/cc-nested/z.txt", + keypair.public_key() + ), + format!("pubky://{}/pub/example.com/d.txt", keypair.public_key()), ], - "normal list with limit and a suffix cursor" + "normal list with limit and a directory cursor" ); } { let list = client - .list(url.as_str()) + .list(url) .unwrap() .limit(2) .cursor(&format!( @@ -317,7 +343,7 @@ mod tests { { let list = client - .list(url.as_str()) + .list(url) .unwrap() .reverse(true) .send() @@ -328,6 +354,10 @@ mod tests { list, vec![ format!("pubky://{}/pub/example.com/d.txt", keypair.public_key()), + format!( + "pubky://{}/pub/example.com/cc-nested/z.txt", + keypair.public_key() + ), format!("pubky://{}/pub/example.com/c.txt", keypair.public_key()), format!("pubky://{}/pub/example.com/b.txt", keypair.public_key()), format!("pubky://{}/pub/example.com/a.txt", keypair.public_key()), @@ -338,7 +368,7 @@ mod tests { { let list = client - .list(url.as_str()) + .list(url) .unwrap() .reverse(true) .limit(2) @@ -350,7 +380,10 @@ mod tests { list, vec![ format!("pubky://{}/pub/example.com/d.txt", keypair.public_key()), - format!("pubky://{}/pub/example.com/c.txt", keypair.public_key()), + format!( + "pubky://{}/pub/example.com/cc-nested/z.txt", + keypair.public_key() + ), ], "reverse list with limit but no cursor" ); @@ -358,7 +391,7 @@ mod tests { { let list = client - .list(url.as_str()) + .list(url) .unwrap() .reverse(true) .limit(2) @@ -370,8 +403,11 @@ mod tests { assert_eq!( list, vec![ + format!( + "pubky://{}/pub/example.com/cc-nested/z.txt", + keypair.public_key() + ), format!("pubky://{}/pub/example.com/c.txt", keypair.public_key()), - format!("pubky://{}/pub/example.com/b.txt", keypair.public_key()), ], "reverse list with limit and cursor" ); @@ -407,10 +443,11 @@ mod tests { } let url = format!("pubky://{}/pub/", keypair.public_key()); + let url = url.as_str(); { let list = client - .list(url.as_str()) + .list(url) .unwrap() .shallow(true) .send() @@ -434,10 +471,73 @@ mod tests { { let list = client - .list(url.as_str()) + .list(url) + .unwrap() + .shallow(true) + .limit(2) + .send() + .await + .unwrap(); + + assert_eq!( + list, + vec![ + format!("pubky://{}/pub/a.com/", keypair.public_key()), + format!("pubky://{}/pub/example.com/", keypair.public_key()), + ], + "normal list shallow with limit but no cursor" + ); + } + + { + let list = client + .list(url) + .unwrap() + .shallow(true) + .limit(2) + .cursor("example.com/a.txt") + .send() + .await + .unwrap(); + + assert_eq!( + list, + vec![ + format!("pubky://{}/pub/example.com/", keypair.public_key()), + format!("pubky://{}/pub/example.con", keypair.public_key()), + ], + "normal list shallow with limit and a file cursor" + ); + } + + { + let list = client + .list(url) .unwrap() .shallow(true) + .limit(3) + .cursor("example.com/") + .send() + .await + .unwrap(); + + assert_eq!( + list, + vec![ + format!("pubky://{}/pub/example.con", keypair.public_key()), + format!("pubky://{}/pub/example.con/", keypair.public_key()), + format!("pubky://{}/pub/file", keypair.public_key()), + ], + "normal list shallow with limit and a directory cursor" + ); + } + + { + let list = client + .list(url) + .unwrap() .reverse(true) + .shallow(true) .send() .await .unwrap(); @@ -456,5 +556,70 @@ mod tests { "reverse list shallow" ); } + + { + let list = client + .list(url) + .unwrap() + .reverse(true) + .shallow(true) + .limit(2) + .send() + .await + .unwrap(); + + assert_eq!( + list, + vec![ + format!("pubky://{}/pub/z.com/", keypair.public_key()), + format!("pubky://{}/pub/file2", keypair.public_key()), + ], + "reverse list shallow with limit but no cursor" + ); + } + + { + let list = client + .list(url) + .unwrap() + .shallow(true) + .reverse(true) + .limit(2) + .cursor("file2") + .send() + .await + .unwrap(); + + assert_eq!( + list, + vec![ + format!("pubky://{}/pub/file", keypair.public_key()), + format!("pubky://{}/pub/example.con/", keypair.public_key()), + ], + "reverse list shallow with limit and a file cursor" + ); + } + + { + let list = client + .list(url) + .unwrap() + .shallow(true) + .reverse(true) + .limit(2) + .cursor("example.con/") + .send() + .await + .unwrap(); + + assert_eq!( + list, + vec![ + format!("pubky://{}/pub/example.con", keypair.public_key()), + format!("pubky://{}/pub/example.com/", keypair.public_key()), + ], + "reverse list shallow with limit and a directory cursor" + ); + } } }