diff --git a/CHANGELOG.md b/CHANGELOG.md index 08c7c6b5..403223ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,34 @@ +## v0.8.0 (2020-08-10) + +* Added: + * If you create a file called `ludusavi.portable` in the same location as + the executable, then Ludusavi will store its config file and the manifest + there as well. +* Fixed: + * Read-only files could only be backed up once, since the original backup + could not be replaced by a newer copy, and you could not restore a backup + if the original file was read-only. Now, Ludusavi will try to unset the + read-only flag on backups before replacing them with newer backups, and + it will try to unset the flag on target files before restoring a backup. + * Invalid paths like `C:\Users\Foo\Documents\C:\Users\Foo` would be shortened + to just `C:\Users\Foo`, which could cause irrelevant files to be backed up. + Now, the extraneous `C:` will be converted to `C_` so that it simply won't + match any files or directories. + * When some games were deselected, the disk space display only showed units + for the total space, not the used space, which could lead to it showing + "1.42 of 1.56 GiB", where 1.42 was actually MiB and not GiB. + Units are now shown for both sides. +* Changed: + * When backing up or restoring a file, if it already exists with the correct + content, then Ludusavi won't re-copy it. + * In GUI mode, Ludusavi now tries to be smarter about when a full scan is + needed. Previously, every backup and backup preview would trigger a full + scan. Now, Ludusavi will remember which games it found and only re-scan + those games (until you change your roots, change the "other" settings, + or reopen the program). + * In CLI mode, `--try-update` will use a default, empty manifest if there is + no local copy of the manifest and it cannot be downloaded. + ## v0.7.0 (2020-08-01) **The backup structure has changed! Read below for more detail.** diff --git a/Cargo.lock b/Cargo.lock index 3d6785b2..e50b4a20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1685,7 +1685,7 @@ dependencies = [ [[package]] name = "ludusavi" -version = "0.7.0" +version = "0.8.0" dependencies = [ "base64 0.12.3", "byte-unit", diff --git a/Cargo.toml b/Cargo.toml index 8b32f8be..dd3a3720 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ludusavi" -version = "0.7.0" +version = "0.8.0" authors = ["mtkennerly "] edition = "2018" description = "Game save backup tool" diff --git a/README.md b/README.md index 0dfe1504..e3a879de 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,17 @@ If you are on Mac: ## Usage ### GUI #### Backup mode +
+Click to expand + * This is the default mode when you open the program. * You can press `preview` to see what the backup will include, without actually performing it. + + After you've done one preview or backup, Ludusavi will remember which games + it found and only re-scan those games the next time. If you change your root + configuration, change the "other" settings, or reopen the program, then + it will do another full scan. * You can press `back up` to perform the backup for real. * If the target folder already exists, it will be deleted first and recreated, unless you've enabled the merge option. @@ -126,7 +134,12 @@ If you are on Mac: one game is deselected) to quickly toggle all of them at once. Ludusavi will remember your most recent checkbox settings. +
+ #### Restore mode +
+Click to expand + * Switch to restore mode by clicking the `restore mode` button. * You can press `preview` to see what the restore will include, without actually performing it. @@ -152,7 +165,12 @@ If you are on Mac: * You can select/deselect specific games in restore mode just like you can in backup mode. The checkbox settings are remembered separately for both modes. +
+ #### Custom games +
+Click to expand + * Switch to this mode by clicking the `custom games` button. * You can click `add game` to add entries for as many games as you like. Within each game's entry, you can click the plus icons to add paths @@ -171,6 +189,8 @@ If you are on Mac: If the game name matches one from Ludusavi's primary data set, then your custom entry will override it. +
+ #### Other settings * Switch to this screen by clicking the `other` button. * This screen contains some additional settings that are less commonly used. @@ -178,6 +198,10 @@ If you are on Mac: ### CLI Run `ludusavi --help` for the full usage information. +#### API output +
+Click to expand + CLI mode defaults to a human-readable format, but you can switch to a machine-readable JSON format with the `--api` flag. In that case, the output will have the following structure: @@ -254,13 +278,24 @@ Example: } ``` +
+ ### Configuration Ludusavi stores its configuration in `~/.config/ludusavi` (Windows: `C:/Users//.config/ludusavi`). -If you're using the GUI, you don't need to worry about this at all, -since the GUI will automatically update the config file as needed. -However, if you're using the CLI exclusively, you'll need to edit `config.yaml`. +Alternatively, if you'd like Ludusavi to store its configuration in the same +place as the executable, then simply create a file called `ludusavi.portable` +in the directory that contains the executable file. You might want to do that +if you're going to run Ludusavi from a flash drive on multiple computers. + +if you're using the GUI, then it will automatically update the config file +as needed, so you don't need to worry about its content. However, if you're +using the CLI exclusively, then you'll need to edit `config.yaml` yourself. + Here are the available settings (all are required unless otherwise noted): +
+Click to expand + * `manifest` (map): * `url` (string): Where to download the primary manifest. * `etag` (string or null): An identifier for the current version of the manifest. @@ -315,6 +350,8 @@ restore: path: ~/ludusavi-backup ``` +
+ Ludusavi also stores `manifest.yaml` (info on what to back up) here. You should not modify that file, because Ludusavi will overwrite your changes whenever it downloads a new copy. diff --git a/src/cli.rs b/src/cli.rs index b3982475..65b11297 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -376,7 +376,13 @@ pub fn run_cli(sub: Subcommand) -> Result<(), Error> { let manifest = if try_update { match Manifest::load(&mut config, true) { Ok(x) => x, - Err(_) => Manifest::load(&mut config, false)?, + Err(e) => { + eprintln!("{}", translator.handle_error(&e)); + match Manifest::load(&mut config, false) { + Ok(y) => y, + Err(_) => Manifest::default(), + } + } } } else { Manifest::load(&mut config, update)? @@ -482,7 +488,7 @@ pub fn run_cli(sub: Subcommand) -> Result<(), Error> { let backup_info = if preview || ignored { crate::prelude::BackupInfo::default() } else { - back_up_game(&scan_info, &name, &layout) + back_up_game(&scan_info, &name, &layout, config.backup.merge) }; (name, scan_info, backup_info, decision) }) @@ -914,7 +920,7 @@ foo [100.00 KiB]: Overall: Games: 1 of 1 - Size: 100.00 of 150.00 KiB + Size: 100.00 KiB of 150.00 KiB Location: /dev/null "# .trim() diff --git a/src/gui.rs b/src/gui.rs index 0f0cd5ef..c306a307 100644 --- a/src/gui.rs +++ b/src/gui.rs @@ -949,6 +949,8 @@ struct BackupScreenComponent { backup_target_history: TextHistory, backup_target_browse_button: button::State, root_editor: RootEditor, + only_scan_recent_found_games: bool, + recent_found_games: std::collections::HashSet, } impl BackupScreenComponent { @@ -1447,6 +1449,14 @@ impl Application for App { all_games.insert(custom_game.name.clone(), Game::from(custom_game.to_owned())); } + if self.backup_screen.only_scan_recent_found_games { + all_games.retain(|k, _| { + self.backup_screen.recent_found_games.contains(k) + || self.config.custom_games.iter().any(|x| &x.name == k) + }); + } + self.backup_screen.recent_found_games.clear(); + self.backup_screen.status.clear(); self.backup_screen.log.entries.clear(); self.modal_theme = None; @@ -1471,6 +1481,7 @@ impl Application for App { let steam_id = game.steam.clone().unwrap_or(SteamMetadata { id: None }).id; let cancel_flag = self.operation_should_cancel.clone(); let ignored = !self.config.is_game_enabled_for_backup(&key); + let merge = self.config.backup.merge; commands.push(Command::perform( async move { if key.trim().is_empty() { @@ -1495,7 +1506,7 @@ impl Application for App { } let backup_info = if !preview { - Some(back_up_game(&scan_info, &key, &layout2)) + Some(back_up_game(&scan_info, &key, &layout2, merge)) } else { None }; @@ -1589,6 +1600,9 @@ impl Application for App { self.progress.current += 1.0; if let Some(scan_info) = scan_info { if scan_info.found_anything() { + self.backup_screen + .recent_found_games + .insert(scan_info.game_name.clone()); self.backup_screen.status.add_game( &scan_info, &backup_info, @@ -1664,6 +1678,7 @@ impl Application for App { } } } + self.backup_screen.only_scan_recent_found_games = !self.backup_screen.recent_found_games.is_empty(); Command::perform(async move {}, move |_| Message::Idle) } Message::RestoreComplete => { @@ -1715,11 +1730,13 @@ impl Application for App { } } self.config.save(); + self.backup_screen.only_scan_recent_found_games = false; Command::none() } Message::SelectedRootStore(index, store) => { self.config.roots[index].store = store; self.config.save(); + self.backup_screen.only_scan_recent_found_games = false; Command::none() } Message::EditedRedirect(action, field) => { @@ -1828,11 +1845,13 @@ impl Application for App { Message::EditedExcludeOtherOsData(enabled) => { self.config.backup.filter.exclude_other_os_data = enabled; self.config.save(); + self.backup_screen.only_scan_recent_found_games = false; Command::none() } Message::EditedExcludeStoreScreenshots(enabled) => { self.config.backup.filter.exclude_store_screenshots = enabled; self.config.save(); + self.backup_screen.only_scan_recent_found_games = false; Command::none() } Message::SwitchScreen(screen) => { diff --git a/src/lang.rs b/src/lang.rs index 88496591..f5027ddb 100644 --- a/src/lang.rs +++ b/src/lang.rs @@ -147,7 +147,7 @@ impl Translator { "\nOverall:\n Games: {} of {}\n Size: {} of {}\n Location: {}", status.processed_games, status.total_games, - self.adjusted_size_unlabelled(status.processed_bytes), + self.adjusted_size(status.processed_bytes), self.adjusted_size(status.total_bytes), location.render() ), @@ -347,12 +347,6 @@ impl Translator { adjusted_byte.to_string() } - pub fn adjusted_size_unlabelled(&self, bytes: u64) -> String { - let byte = Byte::from_bytes(bytes.into()); - let adjusted_byte = byte.get_appropriate_unit(true); - format!("{:.2}", adjusted_byte.get_value()) - } - pub fn processed_games(&self, status: &OperationStatus) -> String { if status.completed() { match self.language { @@ -368,7 +362,7 @@ impl Translator { "{} of {} games | {} of {}", status.processed_games, status.total_games, - self.adjusted_size_unlabelled(status.processed_bytes), + self.adjusted_size(status.processed_bytes), self.adjusted_size(status.total_bytes) ), } diff --git a/src/layout.rs b/src/layout.rs index df7786e4..2b77bd3a 100644 --- a/src/layout.rs +++ b/src/layout.rs @@ -74,7 +74,18 @@ impl IndividualMapping { } pub fn save(&self, file: &StrictPath) { - std::fs::write(file.interpret(), self.serialize().as_bytes()).unwrap(); + let new_content = serde_yaml::to_string(&self).unwrap(); + + if let Ok(old) = Self::load(&file) { + let old_content = serde_yaml::to_string(&old).unwrap(); + if old_content == new_content { + return; + } + } + + if file.create_parent_dir().is_ok() { + std::fs::write(file.interpret(), self.serialize().as_bytes()).unwrap(); + } } pub fn serialize(&self) -> String { @@ -238,6 +249,40 @@ impl BackupLayout { } files } + + fn find_irrelevant_backup_files(&self, game_folder: &StrictPath, relevant_files: &[StrictPath]) -> Vec { + let relevant_files: Vec<_> = relevant_files.iter().map(|x| x.interpret()).collect(); + let mut irrelevant_files = vec![]; + + for drive_dir in walkdir::WalkDir::new(game_folder.interpret()) + .max_depth(1) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|x| x.file_name().to_string_lossy().starts_with("drive-")) + { + for file in walkdir::WalkDir::new(drive_dir.path()) + .max_depth(100) + .follow_links(false) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|x| x.file_type().is_file()) + { + let backup_file = StrictPath::new(file.path().display().to_string()); + if !relevant_files.contains(&backup_file.interpret()) { + irrelevant_files.push(backup_file); + } + } + } + + irrelevant_files + } + + pub fn remove_irrelevant_backup_files(&self, game_folder: &StrictPath, relevant_files: &[StrictPath]) { + for file in self.find_irrelevant_backup_files(&game_folder, &relevant_files) { + let _ = file.remove(); + } + } } #[cfg(test)] @@ -342,5 +387,33 @@ mod tests { layout().game_folder("...") ); } + + #[test] + fn can_find_irrelevant_backup_files() { + assert_eq!( + vec![if cfg!(target_os = "windows") { + StrictPath::new(format!("\\\\?\\{}\\tests\\backup\\game1\\drive-X\\file2.txt", repo())) + } else { + StrictPath::new(format!("{}/tests/backup/game1/drive-X/file2.txt", repo())) + }], + layout().find_irrelevant_backup_files( + &StrictPath::new(format!("{}/tests/backup/game1", repo())), + &[StrictPath::new(format!( + "{}/tests/backup/game1/drive-X/file1.txt", + repo() + ))] + ) + ); + assert_eq!( + Vec::::new(), + layout().find_irrelevant_backup_files( + &StrictPath::new(format!("{}/tests/backup/game1", repo())), + &[ + StrictPath::new(format!("{}/tests/backup/game1/drive-X/file1.txt", repo())), + StrictPath::new(format!("{}/tests/backup/game1/drive-X/file2.txt", repo())), + ] + ) + ); + } } } diff --git a/src/path.rs b/src/path.rs index cf08b183..0939d23b 100644 --- a/src/path.rs +++ b/src/path.rs @@ -47,7 +47,19 @@ fn parse_dots(path: &str, basis: &str) -> String { ret.pop(); } std::path::Component::Normal(c) => { - ret.push(c); + let lossy = c.to_string_lossy(); + if lossy.contains(':') { + // This can happen if the manifest contains invalid paths, + // such as `/`. In this example, `` + // means we could try to push `C:` in the middle of the path, + // which would truncate the rest of the path up to that point, + // causing us to check the entire home folder. + // We escape it so that it (likely) just won't be found, + // rather than finding something irrelevant. + ret.push(lossy.replace(":", "_")); + } else { + ret.push(c); + } } } } @@ -213,6 +225,35 @@ impl StrictPath { }, ) } + + pub fn unset_readonly(&self) -> Result<(), ()> { + let interpreted = self.interpret(); + if self.is_file() { + let mut perms = std::fs::metadata(&interpreted).map_err(|_| ())?.permissions(); + if perms.readonly() { + perms.set_readonly(false); + std::fs::set_permissions(&interpreted, perms).map_err(|_| ())?; + } + } else { + for entry in walkdir::WalkDir::new(interpreted) + .max_depth(100) + .follow_links(false) + .into_iter() + .skip(1) // the base path itself + .filter_map(|e| e.ok()) + .filter(|x| x.file_type().is_file()) + { + let file = &mut entry.path().display().to_string(); + let mut perms = std::fs::metadata(&file).map_err(|_| ())?.permissions(); + if perms.readonly() { + perms.set_readonly(false); + std::fs::set_permissions(&file, perms).map_err(|_| ())?; + } + } + } + + Ok(()) + } } // Based on: @@ -423,6 +464,34 @@ mod tests { } } + #[test] + #[cfg(target_os = "windows")] + fn does_not_truncate_path_up_to_drive_letter_in_classic_path() { + // https://github.com/mtkennerly/ludusavi/issues/36 + // Test for: / + + let sp = StrictPath { + raw: "C:\\Users\\Foo\\Documents/C:\\Users\\Bar".to_string(), + basis: Some("\\\\?\\C:\\Users\\Foo\\.config\\ludusavi".to_string()), + }; + assert_eq!(r#"\\?\C:\Users\Foo\Documents\C_\Users\Bar"#, sp.interpret(),); + assert_eq!("C:/Users/Foo/Documents/C_/Users/Bar", sp.render(),); + } + + #[test] + #[cfg(target_os = "windows")] + fn does_not_truncate_path_up_to_drive_letter_in_unc_path() { + // https://github.com/mtkennerly/ludusavi/issues/36 + // Test for: / + + let sp = StrictPath { + raw: "\\\\?\\C:\\Users\\Foo\\Documents\\C:\\Users\\Bar".to_string(), + basis: Some("\\\\?\\C:\\Users\\Foo\\.config\\ludusavi".to_string()), + }; + assert_eq!(r#"\\?\C:\Users\Foo\Documents\C_\Users\Bar"#, sp.interpret(),); + assert_eq!("C:/Users/Foo/Documents/C_/Users/Bar", sp.render(),); + } + #[test] fn can_check_if_it_is_a_file() { assert!(StrictPath::new(format!("{}/README.md", repo())).is_file()); diff --git a/src/prelude.rs b/src/prelude.rs index 1a4d05be..a2681cfd 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -3,6 +3,7 @@ use crate::{ layout::{BackupLayout, IndividualMapping}, manifest::{Game, GameFileConstraint, Os, Store}, }; +use std::io::Read; pub use crate::path::StrictPath; @@ -146,6 +147,15 @@ fn reslashed(path: &str) -> String { } pub fn app_dir() -> std::path::PathBuf { + if let Ok(mut flag) = std::env::current_exe() { + flag.pop(); + flag.push("ludusavi.portable"); + if flag.exists() { + flag.pop(); + return flag; + } + } + let mut path = dirs::home_dir().unwrap(); path.push(".config"); path.push("ludusavi"); @@ -510,37 +520,64 @@ pub fn prepare_backup_target(target: &StrictPath, merge: bool) -> Result<(), Err Ok(()) } -pub fn back_up_game(info: &ScanInfo, name: &str, layout: &BackupLayout) -> BackupInfo { +fn are_files_identical(file1: &StrictPath, file2: &StrictPath) -> Result> { + let f1 = std::fs::File::open(file1.interpret())?; + let mut f1r = std::io::BufReader::new(f1); + let f2 = std::fs::File::open(file2.interpret())?; + let mut f2r = std::io::BufReader::new(f2); + + let mut f1b = [0; 1024]; + let mut f2b = [0; 1024]; + loop { + let f1n = f1r.read(&mut f1b[..])?; + let f2n = f2r.read(&mut f2b[..])?; + + if f1n != f2n || f1b.iter().zip(f2b.iter()).any(|(a, b)| a != b) { + return Ok(false); + } + if f1n == 0 || f2n == 0 { + break; + } + } + Ok(true) +} + +pub fn back_up_game(info: &ScanInfo, name: &str, layout: &BackupLayout, merge: bool) -> BackupInfo { let mut failed_files = std::collections::HashSet::new(); #[allow(unused_mut)] let mut failed_registry = std::collections::HashSet::new(); let target_game = layout.game_folder(&name); - // Since we delete the game folder first, we don't need to worry about - // loading its existing mapping: - let mut mapping = IndividualMapping::new(name.to_string()); - - let mut unable_to_prepare = false; - if info.found_anything() { - match target_game.remove() { - Ok(_) => { - if std::fs::create_dir(target_game.interpret()).is_err() { - unable_to_prepare = true; - } - } - Err(_) => { - unable_to_prepare = true; - } - } - } + let able_to_prepare = info.found_anything() + && (merge || (target_game.unset_readonly().is_ok() && target_game.remove().is_ok())) + && std::fs::create_dir_all(target_game.interpret()).is_ok(); + + let mut mapping = match IndividualMapping::load(&layout.game_mapping_file(&target_game)) { + Ok(x) => x, + Err(_) => IndividualMapping::new(name.to_string()), + }; + + let mut relevant_backup_files = Vec::::new(); for file in &info.found_files { - if unable_to_prepare { + if !able_to_prepare { failed_files.insert(file.clone()); continue; } let target_file = layout.game_file(&target_game, &file.path, &mut mapping); + relevant_backup_files.push(target_file.clone()); + + if target_file.exists() { + match are_files_identical(&file.path, &target_file) { + Ok(true) => continue, + Ok(false) => (), + Err(_) => { + failed_files.insert(file.clone()); + continue; + } + } + } if target_file.create_parent_dir().is_err() { failed_files.insert(file.clone()); continue; @@ -551,15 +588,21 @@ pub fn back_up_game(info: &ScanInfo, name: &str, layout: &BackupLayout) -> Backu } } + if able_to_prepare && merge { + layout.remove_irrelevant_backup_files(&target_game, &relevant_backup_files); + } + #[cfg(target_os = "windows")] { + let mut hives = crate::registry::Hives::default(); + let mut found_some_registry = false; + for reg_path in &info.found_registry_keys { - if unable_to_prepare { + if !able_to_prepare { failed_registry.insert(reg_path.to_string()); continue; } - let mut hives = crate::registry::Hives::default(); match hives.store_key_from_full_path(®_path) { Err(_) => { failed_registry.insert(reg_path.to_string()); @@ -568,13 +611,20 @@ pub fn back_up_game(info: &ScanInfo, name: &str, layout: &BackupLayout) -> Backu failed_registry.insert(reg_path.to_string()); } _ => { - hives.save(&layout.game_registry_file(&target_game)); + found_some_registry = true; } } } + + let target_registry_file = layout.game_registry_file(&target_game); + if found_some_registry { + hives.save(&target_registry_file); + } else { + let _ = target_registry_file.remove(); + } } - if info.found_anything() && !unable_to_prepare { + if info.found_anything() && able_to_prepare { mapping.save(&layout.game_mapping_file(&target_game)); } @@ -595,12 +645,23 @@ pub fn restore_game(info: &ScanInfo, redirects: &[RedirectConfig]) -> BackupInfo }; let (target, _) = game_file_restoration_target(&original_path, &redirects); + if target.exists() { + match are_files_identical(&file.path, &target) { + Ok(true) => continue, + Ok(false) => (), + Err(_) => { + failed_files.insert(file.clone()); + continue; + } + } + } + if target.create_parent_dir().is_err() { failed_files.insert(file.clone()); continue; } for i in 0..99 { - if std::fs::copy(&file.path.interpret(), &target.interpret()).is_ok() { + if target.unset_readonly().is_ok() && std::fs::copy(&file.path.interpret(), &target.interpret()).is_ok() { continue 'outer; } // File might be busy, especially if multiple games share a file, @@ -951,4 +1012,23 @@ mod tests { ); } } + + #[test] + fn checks_if_files_are_identical() { + assert!(are_files_identical( + &StrictPath::new(format!("{}/tests/root2/game1/file1.txt", repo())), + &StrictPath::new(format!("{}/tests/root2/game2/file1.txt", repo())), + ) + .unwrap()); + assert!(!are_files_identical( + &StrictPath::new(format!("{}/tests/root1/game1/subdir/file2.txt", repo())), + &StrictPath::new(format!("{}/tests/root2/game1/file1.txt", repo())), + ) + .unwrap()); + assert!(are_files_identical( + &StrictPath::new(format!("{}/tests/root1/game1/file1.txt", repo())), + &StrictPath::new(format!("{}/nonexistent.txt", repo())), + ) + .is_err()); + } } diff --git a/src/registry.rs b/src/registry.rs index c2a450f2..ddf43a2e 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -45,6 +45,15 @@ impl Hives { } pub fn save(&self, file: &StrictPath) { + let new_content = serde_yaml::to_string(&self).unwrap(); + + if let Some(old) = Self::load(&file) { + let old_content = serde_yaml::to_string(&old).unwrap(); + if old_content == new_content { + return; + } + } + if file.create_parent_dir().is_ok() { std::fs::write(file.interpret(), self.serialize().as_bytes()).unwrap(); }