Add Rule34 tag source

README.RU.md

@@ -3,7 +3,7 @@
 
 # Что такое ImoutoRebirth
 
-ImoutoRebirth это решение для организации медиа файлов с уклоном в аниме картинки. Вдохновленный и глубоко интегрированный с такими сайтами, как danbooru.donmai.us, yande.re, gelbooru.com.
+ImoutoRebirth это решение для организации медиа файлов с уклоном в аниме картинки. Вдохновленный и глубоко интегрированный с такими сайтами, как danbooru.donmai.us, yande.re, gelbooru.com, rule34.xxx.
 
 Писать я его начал для себя много лет назад, т.к. я сам еще тот сохранятель картинок. Со временем разные функции собрались в целостное решение, и я решил его опубликовать.
 

README.md

@@ -4,7 +4,7 @@ This repository is in process of open sourcing and isn't ready yet. I'm currentl
 # What is ImoutoRebirth
 ([ru version](https://github.com/ImoutoChan/ImoutoRebirth/blob/master/README.RU.md))
 
-ImoutoRebirth is a solution for organizing media files with a focus on anime images. It is inspired and deeply integrated with sites such as danbooru.donmai.us, yande.re, gelbooru.com.
+ImoutoRebirth is a solution for organizing media files with a focus on anime images. It is inspired and deeply integrated with sites such as danbooru.donmai.us, yande.re, gelbooru.com, rule34.xxx.
 
 I started writing it for myself many years ago because I am an avid image saver. Over time, different functions came together into a comprehensive solution, and I decided to publish it.
 
@@ -17,7 +17,7 @@ Key features:
     - jpg, jpeg, png, bmp, tiff, apng, gif, webp, jfif
     - mp4, mov, webm, wmv, mov, m2ts, mpg, mpeg, mkv, f4v, flv, avi, m4v, swf
     - zip as pixiv ugoira
-- Background search and updating of tags for your images and metadata based on file hash from danbooru, sankaku, yande.re, gelbooru
+- Background search and updating of tags for your images and metadata based on file hash from danbooru, sankaku, yande.re, gelbooru, rule34
 - Local search for saved files and their tags
 - Bulk tagging of files
 - Display of notes from boorus on local images

Source/Imouto.Extensions/chrome/src/manifest.json

@@ -30,7 +30,8 @@
         "*://donmai.us/*",
         "*://yande.re/*",
         "*://konachan.com/*",
-        "*://gelbooru.com/*"
+        "*://gelbooru.com/*",
+        "*://rule34.xxx/*"
       ],
       "css": [ "contentscript.css" ],
       "js": [ "contentscript.js" ]
@@ -46,4 +47,4 @@
   "permissions": [
     "storage"
   ]
-}
+}

Source/ImoutoRebirth.Arachne/ImoutoRebirth.Arachne.Core/Models/SearchEngineType.cs

@@ -5,5 +5,6 @@ public enum SearchEngineType : byte
     Yandere = 0,
     Danbooru = 1,
     Sankaku = 2,
-    Gelbooru = 4
+    Gelbooru = 4,
+    Rule34 = 5
 }

Source/ImoutoRebirth.Arachne/ImoutoRebirth.Arachne.Infrastructure/ImoutoRebirth.Arachne.Infrastructure.csproj

@@ -9,7 +9,7 @@
 
     <ItemGroup>
         <PackageReference Include="AspNetCoreInjection.TypedFactories" Version="1.2.0" />
-        <PackageReference Include="Imouto.BooruParser" Version="3.2.4" />
+        <PackageReference Include="Imouto.BooruParser" Version="3.2.5" />
         <PackageReference Include="Mackiovello.Maybe" Version="1.0.0" />
         <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="7.0.0" />
         <PackageReference Include="Microsoft.Extensions.Http" Version="7.0.0" />

Source/ImoutoRebirth.Arachne/ImoutoRebirth.Arachne.Infrastructure/LoaderFabrics/Rule34LoaderFabric.cs

@@ -0,0 +1,21 @@
+using Flurl.Http.Configuration;
+using Imouto.BooruParser;
+using Imouto.BooruParser.Implementations.Rule34;
+using ImoutoRebirth.Arachne.Core.Models;
+using ImoutoRebirth.Arachne.Infrastructure.Abstract;
+using Microsoft.Extensions.Options;
+
+namespace ImoutoRebirth.Arachne.Infrastructure.LoaderFabrics;
+
+internal class Rule34LoaderFabric : IBooruLoaderFabric
+{
+    private readonly IFlurlClientFactory _flurlClientFactory;
+
+    public SearchEngineType ForType => SearchEngineType.Rule34;
+
+    public Rule34LoaderFabric(IFlurlClientFactory flurlClientFactory) => _flurlClientFactory = flurlClientFactory;
+
+    public IBooruApiLoader Create() => new Rule34ApiLoader(
+        _flurlClientFactory,
+        Options.Create(new Rule34Settings { PauseBetweenRequestsInMs = 1 }));
+}

Source/ImoutoRebirth.Arachne/ImoutoRebirth.Arachne.Infrastructure/ServiceCollectionExtensions.cs

@@ -32,11 +32,13 @@ var policy
         services.AddTransient<DanbooruLoaderFabric>();
         services.AddTransient<SankakuLoaderFabric>();
         services.AddTransient<GelbooruLoaderFabric>();
+        services.AddTransient<Rule34LoaderFabric>();
 
         services.AddTransient<IBooruLoaderFabric>(provider => provider.GetRequiredService<YandereLoaderFabric>());
         services.AddTransient<IBooruLoaderFabric>(provider => provider.GetRequiredService<DanbooruLoaderFabric>());
         services.AddTransient<IBooruLoaderFabric>(provider => provider.GetRequiredService<SankakuLoaderFabric>());
         services.AddTransient<IBooruLoaderFabric>(provider => provider.GetRequiredService<GelbooruLoaderFabric>());
+        services.AddTransient<IBooruLoaderFabric>(provider => provider.GetRequiredService<Rule34LoaderFabric>());
 
 
         services.AddTransient<DanbooruSettings>(x => danbooruSettings);

Source/ImoutoRebirth.Arachne/ImoutoRebirth.Arachne.MessageContracts/Commands/IRule34SearchMetadataCommand.cs

@@ -0,0 +1,5 @@
+namespace ImoutoRebirth.Arachne.MessageContracts.Commands;
+
+public interface IRule34SearchMetadataCommand : ISearchMetadataCommand
+{
+}

Source/ImoutoRebirth.Arachne/ImoutoRebirth.Arachne.MessageContracts/SearchEngineType.cs

@@ -5,5 +5,6 @@ public enum SearchEngineType : byte
     Yandere = 0,
     Danbooru = 1,
     Sankaku = 2,
-    Gelbooru = 4
+    Gelbooru = 4,
+    Rule34 = 5
 }

Source/ImoutoRebirth.Arachne/ImoutoRebirth.Arachne.Service/Consumers/Rule34SearchMetadataCommandConsumer.cs

@@ -0,0 +1,18 @@
+using ImoutoRebirth.Arachne.Core.Models;
+using ImoutoRebirth.Arachne.MessageContracts.Commands;
+using MassTransit;
+
+namespace ImoutoRebirth.Arachne.Service.Consumers;
+
+public class Rule34SearchMetadataCommandConsumer : IConsumer<IRule34SearchMetadataCommand>
+{
+    private readonly ISearchMetadataCommandHandler _searchMetadataCommandHandler;
+
+    public Rule34SearchMetadataCommandConsumer(ISearchMetadataCommandHandler searchMetadataCommandHandler) 
+        => _searchMetadataCommandHandler = searchMetadataCommandHandler;
+
+    public async Task Consume(ConsumeContext<IRule34SearchMetadataCommand> context)
+    {
+        await _searchMetadataCommandHandler.Search(context, SearchEngineType.Rule34);
+    }
+}

Source/ImoutoRebirth.Arachne/ImoutoRebirth.Arachne.Service/Extensions/SearchEngineTypeExtensions.cs

@@ -17,6 +17,8 @@ public static MetadataSource GetMetadataSource(this SearchEngineType searchEngin
                 return MetadataSource.Sankaku;
             case SearchEngineType.Gelbooru:
                 return MetadataSource.Gelbooru;
+            case SearchEngineType.Rule34:
+                return MetadataSource.Rule34;
             default:
                 throw new ArgumentOutOfRangeException(nameof(searchEngineType), searchEngineType, null);
         }
@@ -34,6 +36,8 @@ public static SearchEngineType ToModel(this MessageContracts.SearchEngineType se
                 return SearchEngineType.Sankaku;
             case MessageContracts.SearchEngineType.Gelbooru:
                 return SearchEngineType.Gelbooru;
+            case MessageContracts.SearchEngineType.Rule34:
+                return SearchEngineType.Rule34;
             default:
                 throw new ArgumentOutOfRangeException(nameof(searchEngineType), searchEngineType, null);
         }

Source/ImoutoRebirth.Arachne/ImoutoRebirth.Arachne.Service/Extensions/ServiceCollectionExtensions.cs

@@ -20,6 +20,7 @@ public static IServiceCollection AddArachneServices(this IServiceCollection serv
         services.AddTransient<DanbooruSearchMetadataCommandConsumer>();
         services.AddTransient<SankakuSearchMetadataCommandConsumer>();
         services.AddTransient<GelbooruSearchMetadataCommandConsumer>();
+        services.AddTransient<Rule34SearchMetadataCommandConsumer>();
         services.AddTransient<LoadTagHistoryCommandConsumer>();
         services.AddTransient<LoadNoteHistoryCommandConsumer>();
 
@@ -37,6 +38,7 @@ public static ITrueMassTransitConfigurator AddArachneServicesForRabbit(
             .AddConsumer<DanbooruSearchMetadataCommandConsumer, IDanbooruSearchMetadataCommand>(ArachneReceiverApp.Name)
             .AddConsumer<SankakuSearchMetadataCommandConsumer, ISankakuSearchMetadataCommand>(ArachneReceiverApp.Name)
             .AddConsumer<GelbooruSearchMetadataCommandConsumer, IGelbooruSearchMetadataCommand>(ArachneReceiverApp.Name)
+            .AddConsumer<Rule34SearchMetadataCommandConsumer, IRule34SearchMetadataCommand>(ArachneReceiverApp.Name)
             .AddConsumer<LoadTagHistoryCommandConsumer, ILoadTagHistoryCommand>(
                 ArachneReceiverApp.Name,
                 configurator =>

Source/ImoutoRebirth.Hasami/ImoutoRebirth.Hasami.csproj

@@ -14,7 +14,7 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="Imouto.BooruParser" Version="3.2.4" />
+        <PackageReference Include="Imouto.BooruParser" Version="3.2.5" />
         <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0"/>
         <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0"/>
     </ItemGroup>

Source/ImoutoRebirth.Lilin/ImoutoRebirth.Lilin.Domain/FileInfoAggregate/MetadataSource.cs

@@ -15,5 +15,6 @@ public enum MetadataSource
     Danbooru = 1,
     Sankaku = 2,
     Manual = 3,
-    Gelbooru = 4
+    Gelbooru = 4,
+    Rule34 = 5,
 }

Source/ImoutoRebirth.Lilin/ImoutoRebirth.Lilin.MessageContracts/MetadataSource.cs

@@ -6,5 +6,6 @@ public enum MetadataSource
     Danbooru = 1,
     Sankaku = 2,
     Manual = 3,
-    Gelbooru = 4
+    Gelbooru = 4,
+    Rule34 = 5,
 }

Source/ImoutoRebirth.Lilin/ImoutoRebirth.Lilin.WebApi.Client/Clients.cs

@@ -1875,6 +1875,9 @@ public enum MetadataSource
         [System.Runtime.Serialization.EnumMember(Value = @"Gelbooru")]
         Gelbooru = 4,
 
+        [System.Runtime.Serialization.EnumMember(Value = @"Rule34")]
+        Rule34 = 5,
+
     }
 
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "13.19.0.0 (NJsonSchema v10.9.0.0 (Newtonsoft.Json v13.0.0.0))")]
@@ -2252,4 +2255,4 @@ public WebApiException(string message, int statusCode, string? response, System.
 #pragma warning restore 1591
 #pragma warning restore 8073
 #pragma warning restore 3016
-#pragma warning restore 8603
+#pragma warning restore 8603

Source/ImoutoRebirth.Meido/ImoutoRebirth.Meido.Application/ParsingStatusSlice/Commands/CreateParsingsForNewFileCommand.cs

@@ -34,7 +34,7 @@ public async Task Handle(CreateParsingsForNewFileCommand request, CancellationTo
         var (fileId, md5) = request;
         var now = _clock.GetCurrentInstant();
 
-        // we're ignoring gelbooru since it's unnecessary when danbooru entry is present
+        // we're ignoring gelbooru and rule34 since it's unnecessary when danbooru entry is present
         var allMetadataSources = new[] { MetadataSource.Danbooru, MetadataSource.Yandere, MetadataSource.Sankaku };
 
         foreach (var metadataSource in allMetadataSources)

Source/ImoutoRebirth.Meido/ImoutoRebirth.Meido.Application/ParsingStatusSlice/Commands/SearchCompleteCommand.cs

@@ -42,6 +42,9 @@ public async Task Handle(SaveCompletedSearchCommand request, CancellationToken c
 
         if (source == MetadataSource.Danbooru && searchStatus == SearchStatus.NotFound)
             await CreateGelbooruParsingStatus(fileId);
+
+        if (source == MetadataSource.Gelbooru && searchStatus == SearchStatus.NotFound)
+            await CreateRule34ParsingStatus(fileId);
     }
 
     private async Task SaveSearchResult(
@@ -80,4 +83,25 @@ private async Task CreateGelbooruParsingStatus(Guid fileId)
         await _parsingStatusRepository.Add(result.Result);
         _eventStorage.AddRange(result.EventsCollection);
     }
+
+    private async Task CreateRule34ParsingStatus(Guid fileId)
+    {
+        var now = _clock.GetCurrentInstant();
+
+        // we always have gelbooru status for this file since it's a requirement for rule34
+        var danbooruStatus = await _parsingStatusRepository.Get(fileId, MetadataSource.Gelbooru);
+        var check = await _parsingStatusRepository.Get(fileId, MetadataSource.Rule34);
+        if (check != null || danbooruStatus == null)
+        {
+            _logger.LogWarning(
+                "Can't create a parsing status with duplicate key {FileId}, {Source}",
+                fileId,
+                MetadataSource.Rule34);
+            return;
+        }
+
+        var result = ParsingStatus.Create(fileId, danbooruStatus.Md5, MetadataSource.Rule34, now);
+        await _parsingStatusRepository.Add(result.Result);
+        _eventStorage.AddRange(result.EventsCollection);
+    }
 }

Source/ImoutoRebirth.Meido/ImoutoRebirth.Meido.DataAccess/Migrations/20190531204230_InitSourceActualizingStates.cs

@@ -11,6 +11,7 @@ protected override void Up(MigrationBuilder migrationBuilder)
         InsertInitial(migrationBuilder, MetadataSource.Danbooru);
         InsertInitial(migrationBuilder, MetadataSource.Sankaku);
         InsertInitial(migrationBuilder, MetadataSource.Gelbooru);
+        InsertInitial(migrationBuilder, MetadataSource.Rule34);
     }
 
     protected override void Down(MigrationBuilder migrationBuilder)
@@ -19,6 +20,7 @@ protected override void Down(MigrationBuilder migrationBuilder)
         migrationBuilder.DeleteData("SourceActualizingStates", "Source", 1);
         migrationBuilder.DeleteData("SourceActualizingStates", "Source", 2);
         migrationBuilder.DeleteData("SourceActualizingStates", "Source", 4);
+        migrationBuilder.DeleteData("SourceActualizingStates", "Source", 5);
     }
 
     private static void InsertInitial(MigrationBuilder migrationBuilder, MetadataSource source)

Source/ImoutoRebirth.Meido/ImoutoRebirth.Meido.Domain/MetadataSource.cs

@@ -5,5 +5,6 @@ public enum MetadataSource
     Yandere = 0,
     Danbooru = 1,
     Sankaku = 2,
-    Gelbooru = 4
+    Gelbooru = 4,
+    Rule34 = 5
 }

Source/ImoutoRebirth.Meido/ImoutoRebirth.Meido.Infrastructure/MetadataRequest/Requesters/Rule34MetadataRequester.cs

@@ -0,0 +1,17 @@
+using ImoutoRebirth.Arachne.MessageContracts.Commands;
+using ImoutoRebirth.Meido.Domain;
+using MassTransit;
+
+namespace ImoutoRebirth.Meido.Infrastructure.MetadataRequest.Requesters;
+
+internal class Rule34MetadataRequester : IMetadataRequester
+{
+    private readonly IBus _bus;
+
+    public Rule34MetadataRequester(IBus bus) => _bus = bus;
+
+    public MetadataSource Source => MetadataSource.Rule34;
+
+    public Task SendRequestCommand(Guid fileId, string md5) 
+        => _bus.Send<IRule34SearchMetadataCommand>(new {FileId = fileId, Md5 = md5});
+}

Source/ImoutoRebirth.Meido/ImoutoRebirth.Meido.Infrastructure/ServiceCollectionExtensions.cs

@@ -18,6 +18,7 @@ public static IServiceCollection AddMeidoInfrastructure(this IServiceCollection
         services.AddTransient<IMetadataRequester, DanbooruMetadataRequester>();
         services.AddTransient<IMetadataRequester, SankakuMetadataRequester>();
         services.AddTransient<IMetadataRequester, GelbooruMetadataRequester>();
+        services.AddTransient<IMetadataRequester, Rule34MetadataRequester>();
 
         services.AddDistributedBus();
 

Source/ImoutoRebirth.Meido/ImoutoRebirth.Meido.UI/ServiceCollectionExtensions.cs

@@ -51,6 +51,7 @@ public static ITrueMassTransitConfigurator AddMeidoServicesForRabbit(
             .AddFireAndForget<IYandereSearchMetadataCommand>(ArachneReceiverApp.Name)
             .AddFireAndForget<IDanbooruSearchMetadataCommand>(ArachneReceiverApp.Name)
             .AddFireAndForget<IGelbooruSearchMetadataCommand>(ArachneReceiverApp.Name)
+            .AddFireAndForget<IRule34SearchMetadataCommand>(ArachneReceiverApp.Name)
             .AddFireAndForget<ISankakuSearchMetadataCommand>(ArachneReceiverApp.Name)
             .AddFireAndForget<ILoadTagHistoryCommand>(
                 ArachneReceiverApp.Name,

Source/ImoutoRebirth.Navigator/ImoutoRebirth.Navigator/Services/Tags/AutoMapperProfile.cs

@@ -42,6 +42,7 @@ public AutoMapperProfile()
                     FileTagSource.Sankaku => MetadataSource.Sankaku,
                     FileTagSource.Manual => MetadataSource.Manual,
                     FileTagSource.Gelbooru => MetadataSource.Gelbooru,
+                    FileTagSource.Rule34 => MetadataSource.Rule34,
                     _ => throw new NotImplementedException(x.ToString())
                 });
 
@@ -79,6 +80,7 @@ private void CreateFileTagMaps()
                     MetadataSource.Sankaku => FileTagSource.Sankaku,
                     MetadataSource.Manual => FileTagSource.Manual,
                     MetadataSource.Gelbooru => FileTagSource.Gelbooru,
+                    MetadataSource.Rule34 => FileTagSource.Rule34,
                     _ => throw new NotImplementedException(x.ToString())
                 });
     }

Source/ImoutoRebirth.Navigator/ImoutoRebirth.Navigator/Services/Tags/Model/FileTagSource.cs

@@ -6,5 +6,6 @@ internal enum FileTagSource
     Danbooru = 1,
     Sankaku = 2,
     Manual = 3,
-    Gelbooru = 4
+    Gelbooru = 4,
+    Rule34 = 5
 }