diff --git a/CHANGELOG.md b/CHANGELOG.md index 502f36d05..6a6eceaaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ### Changelog +#### Version - 3.7.3.0 - 9/19/2024 +* Downloads now start in a random order, to reduce the amount of lag in the UI when initially starting +* Wabbajack may now redirect some downloads to mirrors hosted on Nexus Mods (by request of the mod authors) + #### Version - 3.7.2.1 - 9/1/2024 * Fixed a bug with the html reports when in a folder with a space in the name diff --git a/Wabbajack.CLI.Builder/CommandLineBuilder.cs b/Wabbajack.CLI.Builder/CommandLineBuilder.cs index a9037f23a..a927378ef 100644 --- a/Wabbajack.CLI.Builder/CommandLineBuilder.cs +++ b/Wabbajack.CLI.Builder/CommandLineBuilder.cs @@ -32,6 +32,10 @@ public async Task Run(string[] args) typeof(string), d => new Option(d.Aliases, description: d.Description) }, + { + typeof(int), + d => new Option(d.Aliases, description: d.Description) + }, { typeof(AbsolutePath), d => new Option(d.Aliases, description: d.Description, parseArgument: d => d.Tokens.Single().Value.ToAbsolutePath()) diff --git a/Wabbajack.CLI/Program.cs b/Wabbajack.CLI/Program.cs index 919a220be..94208892b 100644 --- a/Wabbajack.CLI/Program.cs +++ b/Wabbajack.CLI/Program.cs @@ -41,6 +41,7 @@ private static async Task Main(string[] args) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(s => new GitHubClient(new ProductHeaderValue("wabbajack"))); + services.AddSingleton(); services.AddOSIntegrated(); services.AddServerLib(); diff --git a/Wabbajack.CLI/VerbRegistration.cs b/Wabbajack.CLI/VerbRegistration.cs index 373b4b438..69634dde2 100644 --- a/Wabbajack.CLI/VerbRegistration.cs +++ b/Wabbajack.CLI/VerbRegistration.cs @@ -1,4 +1,4 @@ - + using Microsoft.Extensions.DependencyInjection; namespace Wabbajack.CLI; using Wabbajack.CLI.Verbs; @@ -27,6 +27,8 @@ public static void AddCLIVerbs(this IServiceCollection services) { services.AddSingleton(); CommandLineBuilder.RegisterCommand(HashUrlString.Definition, c => ((HashUrlString)c).Run); services.AddSingleton(); +CommandLineBuilder.RegisterCommand(IndexNexusMod.Definition, c => ((IndexNexusMod)c).Run); +services.AddSingleton(); CommandLineBuilder.RegisterCommand(Install.Definition, c => ((Install)c).Run); services.AddSingleton(); CommandLineBuilder.RegisterCommand(InstallCompileInstallVerify.Definition, c => ((InstallCompileInstallVerify)c).Run); diff --git a/Wabbajack.CLI/Verbs/IndexNexusMod.cs b/Wabbajack.CLI/Verbs/IndexNexusMod.cs new file mode 100644 index 000000000..6066dbfb4 --- /dev/null +++ b/Wabbajack.CLI/Verbs/IndexNexusMod.cs @@ -0,0 +1,93 @@ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Wabbajack.CLI.Builder; +using Wabbajack.Downloaders; +using Wabbajack.DTOs; +using Wabbajack.DTOs.DownloadStates; +using Wabbajack.DTOs.JsonConverters; +using Wabbajack.Hashing.xxHash64; +using Wabbajack.Networking.NexusApi; +using Wabbajack.Paths; +using Wabbajack.Paths.IO; +using AbsolutePath = Wabbajack.Paths.AbsolutePath; + +namespace Wabbajack.CLI.Verbs; + +public class IndexNexusMod +{ + private readonly ILogger _logger; + private readonly NexusApi _client; + private readonly TemporaryFileManager _manager; + private readonly DownloadDispatcher _downloadDispatcher; + private readonly DTOSerializer _serializer; + + public IndexNexusMod(ILogger logger, NexusApi nexusClient, TemporaryFileManager manager, DownloadDispatcher downloadDispatcher, DTOSerializer serializer) + { + _logger = logger; + _client = nexusClient; + _manager = manager; + _downloadDispatcher = downloadDispatcher; + _serializer = serializer; + } + + public static VerbDefinition Definition = new VerbDefinition("index-nexus-mod", + "Downloads all files for a mod and creates a mirror.json entry for the files", new[] + { + new OptionDefinition(typeof(string), "g", "game", "Game Domain"), + new OptionDefinition(typeof(int), "m", "mod-id", "Nexus mod ID"), + new OptionDefinition(typeof(AbsolutePath), "o", "output", "Output mirror.json file") + }); + + public async Task Run(string game, int modId, AbsolutePath output, CancellationToken token) + { + var gameInstance = GameRegistry.GetByFuzzyName(game); + var modFiles = await _client.ModFiles(game, modId, token); + _logger.LogInformation("Found {Count} files", modFiles.info.Files.Length); + + var files = new List(); + foreach (var file in modFiles.info.Files) + { + _logger.LogInformation("Downloading {File}", file.FileName); + await using var path = _manager.CreateFile(); + var archive = new Archive() + { + Name = file.FileName, + State = new Nexus + { + FileID = file.FileId, + Game = gameInstance.Game, + ModID = modId, + Description = file.Description, + Name = file.Name, + Version = file.Version + } + }; + + Hash hash; + try + { + hash = await _downloadDispatcher.Download(archive, path.Path.ToString().ToAbsolutePath(), token); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download {File}", file.FileName); + continue; + } + + _logger.LogInformation("Downloaded {File} with hash {Hash}", file.FileName, hash); + archive.Hash = hash; + archive.Size = file.SizeInBytes!.Value; + files.Add(archive); + } + + await using var stream = output.Open(FileMode.Create, FileAccess.Write, FileShare.None); + await _serializer.Serialize(files, stream, true); + return 0; + } +} \ No newline at end of file diff --git a/Wabbajack.Compiler/MO2Compiler.cs b/Wabbajack.Compiler/MO2Compiler.cs index 24e7bc2e0..e6b86452e 100644 --- a/Wabbajack.Compiler/MO2Compiler.cs +++ b/Wabbajack.Compiler/MO2Compiler.cs @@ -206,9 +206,18 @@ private async Task RunValidation(ModList modList) { NextStep("Finalizing", "Validating Archives", modList.Archives.Length); var allowList = await _wjClient.LoadDownloadAllowList(); + var mirrors = (await _wjClient.LoadMirrors()).ToLookup(a => a.Hash); foreach (var archive in modList.Archives) { UpdateProgress(1); + var matchedHashes = mirrors[archive.Hash].ToArray(); + if (matchedHashes.Any()) + { + _logger.LogInformation("Replacing {name}, {primaryKeyString} with {mirror}", archive.Name, + archive.State.PrimaryKeyString, matchedHashes.First().Name); + archive.State = matchedHashes.First().State; + } + if (!_dispatcher.IsAllowed(archive, allowList)) { _logger.LogCritical("Archive {name}, {primaryKeyString} is not allowed", archive.Name, diff --git a/Wabbajack.Installer/AInstaller.cs b/Wabbajack.Installer/AInstaller.cs index 8f6c64823..c15fb76f9 100644 --- a/Wabbajack.Installer/AInstaller.cs +++ b/Wabbajack.Installer/AInstaller.cs @@ -324,8 +324,22 @@ public async Task DownloadArchives(CancellationToken token) _logger.LogInformation("Downloading validation data"); var validationData = await _wjClient.LoadDownloadAllowList(); + var mirrors = (await _wjClient.LoadMirrors()).ToLookup(m => m.Hash); _logger.LogInformation("Validating Archives"); + + foreach (var archive in missing) + { + var matches = mirrors[archive.Hash].ToArray(); + if (!matches.Any()) continue; + + archive.State = matches.First().State; + _ = _wjClient.SendMetric("rerouted", archive.Hash.ToString()); + _logger.LogInformation("Rerouted {Archive} to {Mirror}", archive.Name, + matches.First().State.PrimaryKeyString); + } + + foreach (var archive in missing.Where(archive => !_downloadDispatcher.Downloader(archive).IsAllowed(validationData, archive.State))) { @@ -358,7 +372,7 @@ public async Task DownloadMissingArchives(List missing, CancellationTok } await missing - .OrderBy(a => a.Size) + .Shuffle() .Where(a => a.State is not Manual) .PDoAll(async archive => { diff --git a/Wabbajack.Networking.WabbajackClientApi/Client.cs b/Wabbajack.Networking.WabbajackClientApi/Client.cs index d32d7d9f1..272221021 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Client.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Client.cs @@ -106,6 +106,12 @@ public async Task LoadDownloadAllowList() return d.Deserialize(str); } + public async Task LoadMirrors() + { + var str = await _client.GetStringAsync(_configuration.MirrorList); + return JsonSerializer.Deserialize(str, _dtos.Options) ?? []; + } + public async Task LoadMirrorAllowList() { var str = await _client.GetStringAsync(_configuration.MirrorAllowList); diff --git a/Wabbajack.Networking.WabbajackClientApi/Configuration.cs b/Wabbajack.Networking.WabbajackClientApi/Configuration.cs index a21e7c8c7..586997838 100644 --- a/Wabbajack.Networking.WabbajackClientApi/Configuration.cs +++ b/Wabbajack.Networking.WabbajackClientApi/Configuration.cs @@ -15,6 +15,9 @@ public class Configuration public Uri ServerAllowList { get; set; } = new("https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/ServerWhitelist.yml"); + + public Uri MirrorList { get; set; } = + new("https://raw.githubusercontent.com/wabbajack-tools/opt-out-lists/master/mirrors.json"); public Uri MirrorAllowList { get; set; } = new("https://raw.githubusercontent.com/wabbajack-tools/allow-lists/main/allowed-mirrors.yaml");