diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..454c2ae --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,18 @@ +name: Build PokéData Solution + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + build: + name: Build PokéData Solution + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Build Docker Image + run: docker build . -t francispion.azurecr.io/pokedata:${{ github.sha }} -f src/PokeData.ETL/Dockerfile diff --git a/.gitignore b/.gitignore index ba22a70..7a9bff2 100644 --- a/.gitignore +++ b/.gitignore @@ -360,4 +360,9 @@ MigrationBackup/ .ionide/ # Fody - auto-generated XML schema -FodyWeavers.xsd \ No newline at end of file +FodyWeavers.xsd + +# Output files +resources.json +species.csv +species.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a38163..ae11221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,4 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased -Nothing yet. +### Added + +- Implemented species extraction to JSON and CSV. diff --git a/PokeData.sln b/PokeData.sln index ba61be9..1a266f8 100644 --- a/PokeData.sln +++ b/PokeData.sln @@ -14,7 +14,19 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PokeData.ETL", "src\PokeData.ETL\PokeData.ETL.csproj", "{FC44EF38-D8A6-4A84-84AB-7597501F54BF}" +EndProject Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {FC44EF38-D8A6-4A84-84AB-7597501F54BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FC44EF38-D8A6-4A84-84AB-7597501F54BF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FC44EF38-D8A6-4A84-84AB-7597501F54BF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FC44EF38-D8A6-4A84-84AB-7597501F54BF}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection diff --git a/src/PokeData.ETL/Dockerfile b/src/PokeData.ETL/Dockerfile new file mode 100644 index 0000000..8cba8ec --- /dev/null +++ b/src/PokeData.ETL/Dockerfile @@ -0,0 +1,23 @@ +#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. + +FROM mcr.microsoft.com/dotnet/runtime:8.0 AS base +USER app +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +ARG BUILD_CONFIGURATION=Release +WORKDIR /src +COPY ["src/PokeData.ETL/PokeData.ETL.csproj", "src/PokeData.ETL/"] +RUN dotnet restore "./src/PokeData.ETL/./PokeData.ETL.csproj" +COPY . . +WORKDIR "/src/src/PokeData.ETL" +RUN dotnet build "./PokeData.ETL.csproj" -c $BUILD_CONFIGURATION -o /app/build + +FROM build AS publish +ARG BUILD_CONFIGURATION=Release +RUN dotnet publish "./PokeData.ETL.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . +ENTRYPOINT ["dotnet", "PokeData.ETL.dll"] \ No newline at end of file diff --git a/src/PokeData.ETL/Entities/Generation.cs b/src/PokeData.ETL/Entities/Generation.cs new file mode 100644 index 0000000..86d29f2 --- /dev/null +++ b/src/PokeData.ETL/Entities/Generation.cs @@ -0,0 +1,23 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Entities; + +internal class Generation +{ + public int Number { get; set; } + + public string UniqueName { get; set; } = string.Empty; + public string? DisplayName { get; set; } + + public Region? Region { get; set; } + + [JsonIgnore] + public List Species { get; set; } = []; + + public static Generation FromModel(Models.Generation model, string languageName) => new() + { + Number = model.Id, + UniqueName = model.Name, + DisplayName = model.Names.SingleOrDefault(name => name.Language?.Name == languageName)?.Value + }; +} diff --git a/src/PokeData.ETL/Entities/PokemonSpecies.cs b/src/PokeData.ETL/Entities/PokemonSpecies.cs new file mode 100644 index 0000000..35621f0 --- /dev/null +++ b/src/PokeData.ETL/Entities/PokemonSpecies.cs @@ -0,0 +1,45 @@ +namespace PokeData.ETL.Entities; + +internal class PokemonSpecies +{ + public ushort Number { get; set; } + public int Order { get; set; } + + public string UniqueName { get; set; } = string.Empty; + public string? DisplayName { get; set; } + public string? Category { get; set; } + + public double? GenderRatio { get; set; } + public byte CatchRate { get; set; } + + public byte HatchTime { get; set; } + + public byte BaseFriendship { get; set; } + + public bool IsBaby { get; set; } + public bool IsLegendary { get; set; } + public bool IsMythical { get; set; } + public bool HasGenderDifferences { get; set; } + public bool CanSwitchForm { get; set; } + + public Generation? Generation { get; set; } + public List Varieties { get; set; } = []; + + public static PokemonSpecies FromModel(Models.PokemonSpecies model, string languageName) => new() + { + Number = (ushort)model.Id, + Order = model.Order, + UniqueName = model.Name, + DisplayName = model.Names.SingleOrDefault(name => name.Language?.Name == languageName)?.Value, + Category = model.Genera.SingleOrDefault(genus => genus.Language?.Name == languageName)?.Value, + GenderRatio = model.GenderRate == -1 ? null : (1 - model.GenderRate / 8.0), + CatchRate = (byte)model.CaptureRate, + HatchTime = (byte?)model.HatchCounter ?? 0, + BaseFriendship = (byte?)model.BaseHappiness ?? 0, + IsBaby = model.IsBaby, + IsLegendary = model.IsLegendary, + IsMythical = model.IsMythical, + HasGenderDifferences = model.HasGenderDifferences, + CanSwitchForm = model.FormsSwitchable + }; +} diff --git a/src/PokeData.ETL/Entities/PokemonType.cs b/src/PokeData.ETL/Entities/PokemonType.cs new file mode 100644 index 0000000..7e9a7c1 --- /dev/null +++ b/src/PokeData.ETL/Entities/PokemonType.cs @@ -0,0 +1,16 @@ +namespace PokeData.ETL.Entities; + +internal class PokemonType +{ + public int Number { get; set; } + + public string UniqueName { get; set; } = string.Empty; + public string? DisplayName { get; set; } + + public static PokemonType FromModel(Models.Type model, string languageName) => new() + { + Number = model.Id, + UniqueName = model.Name, + DisplayName = model.Names.SingleOrDefault(name => name.Language?.Name == languageName)?.Value + }; +} diff --git a/src/PokeData.ETL/Entities/PokemonVariety.cs b/src/PokeData.ETL/Entities/PokemonVariety.cs new file mode 100644 index 0000000..049bb0d --- /dev/null +++ b/src/PokeData.ETL/Entities/PokemonVariety.cs @@ -0,0 +1,34 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Entities; + +internal class PokemonVariety +{ + public int Number { get; set; } + public int Order { get; set; } + + public bool IsDefault { get; set; } + + public string UniqueName { get; set; } = string.Empty; + + public double Height { get; set; } + public double Weight { get; set; } + + public ushort BaseExperienceYield { get; set; } + + public List Types { get; set; } = []; + + [JsonIgnore] + public PokemonSpecies? Species { get; set; } + + public static PokemonVariety FromModel(Models.Pokemon model) => new() + { + Number = model.Id, + Order = model.Order, + IsDefault = model.IsDefault, + UniqueName = model.Name, + Height = model.Height / 10.0, + Weight = model.Weight / 10.0, + BaseExperienceYield = (ushort?)model.BaseExperience ?? 0 + }; +} diff --git a/src/PokeData.ETL/Entities/Region.cs b/src/PokeData.ETL/Entities/Region.cs new file mode 100644 index 0000000..4fdfc08 --- /dev/null +++ b/src/PokeData.ETL/Entities/Region.cs @@ -0,0 +1,21 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Entities; + +internal class Region +{ + public int Number { get; set; } + + public string UniqueName { get; set; } = string.Empty; + public string? DisplayName { get; set; } + + [JsonIgnore] + public Generation? Generation { get; set; } + + public static Region FromModel(Models.Region model, string languageName) => new() + { + Number = model.Id, + UniqueName = model.Name, + DisplayName = model.Names.SingleOrDefault(name => name.Language?.Name == languageName)?.Value + }; +} diff --git a/src/PokeData.ETL/Models/APIResource.cs b/src/PokeData.ETL/Models/APIResource.cs new file mode 100644 index 0000000..f2205fe --- /dev/null +++ b/src/PokeData.ETL/Models/APIResource.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal record APIResource +{ + [JsonPropertyName("url")] + public string Url { get; set; } = string.Empty; +} diff --git a/src/PokeData.ETL/Models/Generation.cs b/src/PokeData.ETL/Models/Generation.cs new file mode 100644 index 0000000..eb84b77 --- /dev/null +++ b/src/PokeData.ETL/Models/Generation.cs @@ -0,0 +1,25 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal record Generation +{ + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + // TODO(fpion): abilities + + [JsonPropertyName("names")] + public List Names { get; set; } = []; + + [JsonPropertyName("main_region")] + public NamedAPIResource? MainRegion { get; set; } + + // TODO(fpion): moves + // TODO(fpion): pokemon_species + // TODO(fpion): types + // TODO(fpion): version_groups +} diff --git a/src/PokeData.ETL/Models/Genus.cs b/src/PokeData.ETL/Models/Genus.cs new file mode 100644 index 0000000..ac5d553 --- /dev/null +++ b/src/PokeData.ETL/Models/Genus.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal record Genus +{ + [JsonPropertyName("genus")] + public string Value { get; set; } = string.Empty; + + [JsonPropertyName("language")] + public NamedAPIResource? Language { get; set; } +} diff --git a/src/PokeData.ETL/Models/Name.cs b/src/PokeData.ETL/Models/Name.cs new file mode 100644 index 0000000..e9e7744 --- /dev/null +++ b/src/PokeData.ETL/Models/Name.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal record Name +{ + [JsonPropertyName("name")] + public string Value { get; set; } = string.Empty; + + [JsonPropertyName("language")] + public NamedAPIResource? Language { get; set; } +} diff --git a/src/PokeData.ETL/Models/NamedAPIResource.cs b/src/PokeData.ETL/Models/NamedAPIResource.cs new file mode 100644 index 0000000..bcec5f2 --- /dev/null +++ b/src/PokeData.ETL/Models/NamedAPIResource.cs @@ -0,0 +1,9 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal record NamedAPIResource : APIResource +{ + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; +} diff --git a/src/PokeData.ETL/Models/Pokemon.cs b/src/PokeData.ETL/Models/Pokemon.cs new file mode 100644 index 0000000..53a1133 --- /dev/null +++ b/src/PokeData.ETL/Models/Pokemon.cs @@ -0,0 +1,44 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal class Pokemon +{ + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("base_experience")] + public int? BaseExperience { get; set; } + + [JsonPropertyName("height")] + public int Height { get; set; } + + [JsonPropertyName("is_default")] + public bool IsDefault { get; set; } + + [JsonPropertyName("order")] + public int Order { get; set; } + + [JsonPropertyName("weight")] + public int Weight { get; set; } + + // TODO(fpion): abilities + // TODO(fpion): forms + // TODO(fpion): game_indices + // TODO(fpion): held_items + // TODO(fpion): location_area_encounters + // TODO(fpion): moves + // TODO(fpion): past_types + // TODO(fpion): sprites + + [JsonPropertyName("species")] + public NamedAPIResource? Species { get; set; } + + // TODO(fpion): stats + + [JsonPropertyName("types")] + public List Types { get; set; } = []; +} diff --git a/src/PokeData.ETL/Models/PokemonSpecies.cs b/src/PokeData.ETL/Models/PokemonSpecies.cs new file mode 100644 index 0000000..b57cdbf --- /dev/null +++ b/src/PokeData.ETL/Models/PokemonSpecies.cs @@ -0,0 +1,67 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal class PokemonSpecies +{ + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("order")] + public int Order { get; set; } + + [JsonPropertyName("gender_rate")] + public int GenderRate { get; set; } + + [JsonPropertyName("capture_rate")] + public int CaptureRate { get; set; } + + [JsonPropertyName("base_happiness")] + public int? BaseHappiness { get; set; } + + [JsonPropertyName("is_baby")] + public bool IsBaby { get; set; } + + [JsonPropertyName("is_legendary")] + public bool IsLegendary { get; set; } + + [JsonPropertyName("is_mythical")] + public bool IsMythical { get; set; } + + [JsonPropertyName("hatch_counter")] + public int? HatchCounter { get; set; } + + [JsonPropertyName("has_gender_differences")] + public bool HasGenderDifferences { get; set; } + + [JsonPropertyName("forms_switchable")] + public bool FormsSwitchable { get; set; } + + // TODO(fpion): growth_rate + // TODO(fpion): pokedex_numbers + // TODO(fpion): egg_groups + // TODO(fpion): color + // TODO(fpion): shape + // TODO(fpion): evolves_from_species + // TODO(fpion): evolution_chain + // TODO(fpion): habitat + + [JsonPropertyName("generation")] + public NamedAPIResource? Generation { get; set; } + + [JsonPropertyName("names")] + public List Names { get; set; } = []; + + // TODO(fpion): pal_park_encounters + // TODO(fpion): flavor_text_entries + // TODO(fpion): form_descriptions + + [JsonPropertyName("genera")] + public List Genera { get; set; } = []; + + [JsonPropertyName("varieties")] + public List Varieties { get; set; } = []; +} diff --git a/src/PokeData.ETL/Models/PokemonSpeciesVariety.cs b/src/PokeData.ETL/Models/PokemonSpeciesVariety.cs new file mode 100644 index 0000000..8b5c02d --- /dev/null +++ b/src/PokeData.ETL/Models/PokemonSpeciesVariety.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal record PokemonSpeciesVariety +{ + [JsonPropertyName("is_default")] + public bool IsDefault { get; set; } + + [JsonPropertyName("pokemon")] + public NamedAPIResource? Pokemon { get; set; } +} diff --git a/src/PokeData.ETL/Models/PokemonType.cs b/src/PokeData.ETL/Models/PokemonType.cs new file mode 100644 index 0000000..37aa69b --- /dev/null +++ b/src/PokeData.ETL/Models/PokemonType.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal record PokemonType +{ + [JsonPropertyName("slot")] + public int Slot { get; set; } + + [JsonPropertyName("type")] + public NamedAPIResource? Type { get; set; } +} diff --git a/src/PokeData.ETL/Models/Region.cs b/src/PokeData.ETL/Models/Region.cs new file mode 100644 index 0000000..66f6b85 --- /dev/null +++ b/src/PokeData.ETL/Models/Region.cs @@ -0,0 +1,27 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal class Region +{ + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("locations")] + public List Locations { get; set; } = []; + + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("names")] + public List Names { get; set; } = []; + + [JsonPropertyName("main_generation")] + public NamedAPIResource? MainGeneration { get; set; } + + [JsonPropertyName("pokedexes")] + public List Pokedexes { get; set; } = []; + + [JsonPropertyName("version_groups")] + public List VersionGroups { get; set; } = []; +} diff --git a/src/PokeData.ETL/Models/Type.cs b/src/PokeData.ETL/Models/Type.cs new file mode 100644 index 0000000..3e74fa8 --- /dev/null +++ b/src/PokeData.ETL/Models/Type.cs @@ -0,0 +1,27 @@ +using System.Text.Json.Serialization; + +namespace PokeData.ETL.Models; + +internal class Type +{ + [JsonPropertyName("id")] + public int Id { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + // TODO(fpion): damage_relations + // TODO(fpion): past_damage_relations + // TODO(fpion): game_indices + + [JsonPropertyName("generation")] + public NamedAPIResource? Generation { get; set; } + + // TODO(fpion): move_damage_class + + [JsonPropertyName("names")] + public List Names { get; set; } = []; + + // TODO(fpion): pokemon + // TODO(fpion): moves +} diff --git a/src/PokeData.ETL/PokeData.ETL.csproj b/src/PokeData.ETL/PokeData.ETL.csproj new file mode 100644 index 0000000..4a6982c --- /dev/null +++ b/src/PokeData.ETL/PokeData.ETL.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + enable + enable + Nullable + dotnet-PokeData.ETL-031fd880-0361-4a95-a7b9-7896d06e8524 + Linux + ..\.. + + + + + + + + + diff --git a/src/PokeData.ETL/Program.cs b/src/PokeData.ETL/Program.cs new file mode 100644 index 0000000..795d766 --- /dev/null +++ b/src/PokeData.ETL/Program.cs @@ -0,0 +1,14 @@ +namespace PokeData.ETL; + +public class Program +{ + public static void Main(string[] args) + { + HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); + builder.Services.AddHostedService(); + builder.Services.AddHttpClient(); + + IHost host = builder.Build(); + host.Run(); + } +} diff --git a/src/PokeData.ETL/Properties/launchSettings.json b/src/PokeData.ETL/Properties/launchSettings.json new file mode 100644 index 0000000..e85542c --- /dev/null +++ b/src/PokeData.ETL/Properties/launchSettings.json @@ -0,0 +1,15 @@ +{ + "profiles": { + "PokeData.ETL": { + "commandName": "Project", + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + }, + "dotnetRunMessages": true + }, + "Docker": { + "commandName": "Docker" + } + }, + "$schema": "http://json.schemastore.org/launchsettings.json" +} \ No newline at end of file diff --git a/src/PokeData.ETL/SpeciesLine.cs b/src/PokeData.ETL/SpeciesLine.cs new file mode 100644 index 0000000..aa06827 --- /dev/null +++ b/src/PokeData.ETL/SpeciesLine.cs @@ -0,0 +1,21 @@ +using CsvHelper.Configuration.Attributes; + +namespace PokeData.ETL; + +internal record SpeciesLine +{ + [Index(0)] + public string? Region { get; set; } + + [Index(1)] + public string? Number { get; set; } + + [Index(2)] + public string? Name { get; set; } + + [Index(3)] + public string? PrimaryType { get; set; } + + [Index(4)] + public string? SecondaryType { get; set; } +} diff --git a/src/PokeData.ETL/Worker.cs b/src/PokeData.ETL/Worker.cs new file mode 100644 index 0000000..20cdaab --- /dev/null +++ b/src/PokeData.ETL/Worker.cs @@ -0,0 +1,195 @@ +using CsvHelper; +using PokeData.ETL.Entities; +using System.Diagnostics; +using System.Globalization; +using System.Text; +using System.Text.Json; + +namespace PokeData.ETL; + +public class Worker : BackgroundService +{ + private const string ResourcesJsonPath = "resources.json"; + private const string SpeciesCsvPath = "species.csv"; + private const string SpeciesJsonPath = "species.json"; + + private static readonly CultureInfo Culture = CultureInfo.GetCultureInfo("en"); + private static readonly Encoding Encoding = Encoding.UTF8; + + private readonly HttpClient _client; + private readonly ILogger _logger; + + private readonly Random _random = new(); + private Dictionary _resources = []; + private Dictionary _species = []; + + public Worker(HttpClient client, ILogger logger) + { + _client = client; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken cancellationToken) + { + Stopwatch chrono = Stopwatch.StartNew(); + + await LoadAsync(cancellationToken); + + try + { + for (int id = 1; id <= 1025; id++) + { + string speciesUrl = $"https://pokeapi.co/api/v2/pokemon-species/{id}/"; + Models.PokemonSpecies? speciesModel = await GetResourceAsync(speciesUrl, cancellationToken); + if (speciesModel == null) + { + _logger.LogWarning("No species found at '{url}'.", speciesUrl); + } + else + { + PokemonSpecies species = PokemonSpecies.FromModel(speciesModel, Culture.Name); + string key = $"urn:pokemon:species:id:{species.Number}"; + _species[key] = species; + + if (speciesModel.Generation != null) + { + Models.Generation? generationModel = await GetResourceAsync(speciesModel.Generation, cancellationToken); + if (generationModel != null) + { + Generation generation = Generation.FromModel(generationModel, Culture.Name); + species.Generation = generation; + if (generationModel.MainRegion != null) + { + Models.Region? regionModel = await GetResourceAsync(generationModel.MainRegion, cancellationToken); + if (regionModel != null) + { + generation.Region = Region.FromModel(regionModel, Culture.Name); + } + } + } + } + + foreach (Models.PokemonSpeciesVariety speciesVariety in speciesModel.Varieties) + { + if (speciesVariety.Pokemon != null) + { + Models.Pokemon? varietyModel = await GetResourceAsync(speciesVariety.Pokemon, cancellationToken); + if (varietyModel != null) + { + PokemonVariety variety = PokemonVariety.FromModel(varietyModel); + species.Varieties.Add(variety); + + IOrderedEnumerable pokemonTypes = varietyModel.Types.OrderBy(type => type.Slot); + foreach (Models.PokemonType pokemonType in pokemonTypes) + { + if (pokemonType.Type != null) + { + Models.Type? typeModel = await GetResourceAsync(pokemonType.Type, cancellationToken); + if (typeModel != null) + { + PokemonType type = PokemonType.FromModel(typeModel, Culture.Name); + variety.Types.Add(type); + } + } + } + } + } + } + } + } + } + catch (Exception exception) + { + _logger.LogError(exception, "An unhandled exception has occurred."); + } + + await SaveAsync(cancellationToken); + + chrono.Stop(); + _logger.LogInformation("Operation completed in {elapsed}ms.", chrono.ElapsedMilliseconds); + } + + private async Task GetResourceAsync(Models.APIResource resource, CancellationToken cancellationToken) + => await GetResourceAsync(resource.Url, cancellationToken); + private async Task GetResourceAsync(string url, CancellationToken cancellationToken) + { + if (!_resources.TryGetValue(url, out string? json)) + { + int millisecondsDelay = _random.Next(1, 10 + 1) * 100; + await Task.Delay(millisecondsDelay, cancellationToken); + + Uri requestUri = new(url); + using HttpRequestMessage request = new(HttpMethod.Get, requestUri); + using HttpResponseMessage response = await _client.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); + + json = await response.Content.ReadAsStringAsync(cancellationToken); + _resources[url] = json; + } + + return JsonSerializer.Deserialize(json); + } + + private async Task LoadAsync(CancellationToken cancellationToken) + { + try + { + string resources = await File.ReadAllTextAsync(ResourcesJsonPath, Encoding, cancellationToken); + _resources = JsonSerializer.Deserialize>(resources) ?? []; + } + catch (Exception) + { + } + + try + { + string species = await File.ReadAllTextAsync(SpeciesJsonPath, Encoding, cancellationToken); + _species = JsonSerializer.Deserialize>(species) ?? []; + } + catch (Exception) + { + } + } + private async Task SaveAsync(CancellationToken cancellationToken) + { + string resources = JsonSerializer.Serialize(_resources); + await File.WriteAllTextAsync(ResourcesJsonPath, resources, Encoding, cancellationToken); + + string species = JsonSerializer.Serialize(_species); + await File.WriteAllTextAsync(SpeciesJsonPath, species, Encoding, cancellationToken); + + IEnumerable records = _species.Values.Select(species => + { + Region? region = species.Generation?.Region; + PokemonVariety? variety = species.Varieties.SingleOrDefault(variety => variety.IsDefault); + + SpeciesLine line = new() + { + Number = species.Number.ToString("0000"), + Name = species.DisplayName ?? species.UniqueName + }; + + if (region != null) + { + line.Region = region.DisplayName ?? region.UniqueName; + } + + if (variety != null) + { + PokemonType primaryType = variety.Types.First(); + line.PrimaryType = primaryType.DisplayName ?? primaryType.UniqueName; + + if (variety.Types.Count > 1) + { + PokemonType secondaryType = variety.Types.Skip(1).Single(); + line.SecondaryType = secondaryType.DisplayName ?? secondaryType.UniqueName; + } + } + + return line; + }); + using StreamWriter writer = new(SpeciesCsvPath); + using CsvWriter csv = new(writer, Culture); + csv.WriteRecords(records); + } +} diff --git a/src/PokeData.ETL/appsettings.Development.json b/src/PokeData.ETL/appsettings.Development.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/src/PokeData.ETL/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/src/PokeData.ETL/appsettings.json b/src/PokeData.ETL/appsettings.json new file mode 100644 index 0000000..b2dcdb6 --- /dev/null +++ b/src/PokeData.ETL/appsettings.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +}