Cache files (#69)

* Remove more json stuff + cache RepositoryAction files and env files for speeding up process.
* Update a shitload of packages
* Replace Moq with FakeItEasy in last test project
* added tests + update log statements

src/RepoM.Api/Git/ProcessExecution/GitCommandException.cs

@@ -2,9 +2,10 @@ namespace RepoM.Api.Git.ProcessExecution;
 
 using System;
 
+[Serializable]
 public class GitCommandException : Exception
 {
-    public GitCommandException() : base() { }
+    public GitCommandException() { } 
 
     public GitCommandException(string message) : base(message) { }
 

src/RepoM.Api/IO/ModuleBasedRepositoryActionProvider/Deserialization/IRepositoryActionDeserializer.cs

@@ -0,0 +1,8 @@
+namespace RepoM.Api.IO.ModuleBasedRepositoryActionProvider.Deserialization;
+
+using RepoM.Api.IO.ModuleBasedRepositoryActionProvider.Data;
+
+public interface IRepositoryActionDeserializer
+{
+    RepositoryActionConfiguration Deserialize(string content);
+}

src/RepoM.Api/IO/ModuleBasedRepositoryActionProvider/Deserialization/JsonDynamicRepositoryActionDeserializer.cs

@@ -10,7 +10,7 @@ namespace RepoM.Api.IO.ModuleBasedRepositoryActionProvider.Deserialization;
 using RepoM.Api.IO.ModuleBasedRepositoryActionProvider.ActionDeserializers;
 using RepoM.Api.IO.ModuleBasedRepositoryActionProvider.Data;
 
-public class JsonDynamicRepositoryActionDeserializer
+internal class JsonDynamicRepositoryActionDeserializer
 {
     private readonly ActionDeserializerComposition _deserializers;
     private static readonly JsonSerializer _jsonSerializer = new()
@@ -29,7 +29,6 @@ public class JsonDynamicRepositoryActionDeserializer
             CommentHandling = CommentHandling.Ignore,
         };
 
-
     public JsonDynamicRepositoryActionDeserializer(ActionDeserializerComposition deserializers)
     {
         _deserializers = deserializers ?? throw new ArgumentNullException(nameof(deserializers));
@@ -99,7 +98,7 @@ private void DeserializeRepositoryActions(JToken? repositoryActionsToken, Reposi
                 continue;
             }
 
-            RepositoryAction? customAction = _deserializers.DeserializeSingleAction(typeValue!, variable, _jsonSerializer);
+            RepositoryAction? customAction = _deserializers.DeserializeSingleAction(typeValue, variable, _jsonSerializer);
             if (customAction == null)
             {
                 continue;
@@ -230,11 +229,9 @@ private static int DeserializeVersion(JToken? versionToken)
             return DEFAULT_VERSION;
         }
 
-        var version = DEFAULT_VERSION;
-
         if (versionToken.Type == JTokenType.Integer)
         {
-            version = versionToken.Value<int>();
+            var version = versionToken.Value<int>();
             return version < 1 ? DEFAULT_VERSION : version;
         }
 
@@ -246,7 +243,7 @@ private static int DeserializeVersion(JToken? versionToken)
                 return DEFAULT_VERSION;
             }
 
-            if (int.TryParse(versionString, out version))
+            if (int.TryParse(versionString, out var version))
             {
                 return version < 1 ? DEFAULT_VERSION : version;
             }

src/RepoM.Api/IO/ModuleBasedRepositoryActionProvider/Deserialization/YamlDynamicRepositoryActionDeserializer.cs

@@ -1,29 +1,29 @@
 namespace RepoM.Api.IO.ModuleBasedRepositoryActionProvider.Deserialization;
 
-using System;
+using RepoM.Api.IO.ModuleBasedRepositoryActionProvider.ActionDeserializers;
 using RepoM.Api.IO.ModuleBasedRepositoryActionProvider.Data;
 using YamlDotNet.Serialization;
 
-public class YamlDynamicRepositoryActionDeserializer
+public class YamlDynamicRepositoryActionDeserializer : IRepositoryActionDeserializer
 {
     private readonly JsonDynamicRepositoryActionDeserializer _jsonDynamicRepositoryActionDeserializer;
     private readonly IDeserializer _deserializer;
     private readonly ISerializer _serializer;
 
-    public YamlDynamicRepositoryActionDeserializer(JsonDynamicRepositoryActionDeserializer jsonDynamicRepositoryActionDeserializer)
+    public YamlDynamicRepositoryActionDeserializer(ActionDeserializerComposition deserializers)
     {
-        _jsonDynamicRepositoryActionDeserializer = jsonDynamicRepositoryActionDeserializer ?? throw new ArgumentNullException(nameof(jsonDynamicRepositoryActionDeserializer));
+        _jsonDynamicRepositoryActionDeserializer = new JsonDynamicRepositoryActionDeserializer(deserializers);
         _deserializer = new DeserializerBuilder().Build();
         _serializer = new SerializerBuilder().JsonCompatible().Build();
     }
 
-    public RepositoryActionConfiguration Deserialize(string rawContent)
+    public RepositoryActionConfiguration Deserialize(string content)
     {
-        var yamlObject = _deserializer.Deserialize(rawContent, typeof(object));
+        var yamlObject = _deserializer.Deserialize(content, typeof(object));
         if (yamlObject == null)
         {
-            // todo, log, throw?, ..
-            return _jsonDynamicRepositoryActionDeserializer.Deserialize(rawContent);
+            // maybe it is json, just give it a try
+            return _jsonDynamicRepositoryActionDeserializer.Deserialize(content);
         }
 
         var json = _serializer.Serialize(yamlObject);

src/RepoM.Api/IO/ModuleBasedRepositoryActionProvider/RepositoryActionsFileStore.cs

@@ -0,0 +1,94 @@
+namespace RepoM.Api.IO.ModuleBasedRepositoryActionProvider;
+
+using System;
+using System.Collections.Generic;
+using System.IO.Abstractions;
+using System.Runtime.Caching;
+using DotNetEnv;
+using RepoM.Api.IO.ModuleBasedRepositoryActionProvider.Data;
+using RepoM.Api.IO.ModuleBasedRepositoryActionProvider.Deserialization;
+
+internal abstract class FileStore<T> where T : class
+{
+    private readonly ObjectCache _cache;
+
+    protected FileStore(ObjectCache cache)
+    {
+        _cache = cache ?? throw new ArgumentNullException(nameof(cache));
+    }
+
+    internal T? Get(string filename) 
+    {
+        if (_cache[filename] is T fileContents)
+        {
+            return fileContents;
+        }
+
+        return null;
+    }
+
+    internal T AddOrGetExisting(string filename, T value)
+    {
+        var policy = new CacheItemPolicy();
+        var filePaths = new List<string>(1) { filename, };
+        policy.ChangeMonitors.Add(new HostFileChangeMonitor(filePaths));
+        var cacheResult = _cache.AddOrGetExisting(filename, value, policy) as T;
+        return cacheResult ?? value;
+    }
+}
+
+internal class RepositoryActionsFileStore : FileStore<RepositoryActionConfiguration>
+{
+    private readonly IFileSystem _fileSystem;
+    private readonly IRepositoryActionDeserializer _repositoryActionsDeserializer;
+
+
+    public RepositoryActionsFileStore(IFileSystem fileSystem, IRepositoryActionDeserializer repositoryActionsDeserializer, ObjectCache cache) : base(cache)
+    {
+        _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
+        _repositoryActionsDeserializer = repositoryActionsDeserializer ?? throw new ArgumentNullException(nameof(repositoryActionsDeserializer));
+    }
+
+    public RepositoryActionConfiguration? TryGet(string filename)
+    {
+        RepositoryActionConfiguration? result = Get(filename);
+
+        if (result != null)
+        {
+            return result;
+        }
+
+        var payload = _fileSystem.File.ReadAllText(filename);
+        RepositoryActionConfiguration? fileContents = _repositoryActionsDeserializer.Deserialize(payload);
+
+        if (fileContents == null)
+        {
+            return null;
+        }
+
+        return AddOrGetExisting(filename, fileContents);
+    }
+}
+
+internal class EnvFileStore : FileStore<Dictionary<string, string>>
+{
+    public EnvFileStore(ObjectCache cache) : base(cache)
+    {
+    }
+
+    public IDictionary<string, string> TryGet(string filename)
+    {
+        Dictionary<string, string>? result = Get(filename);
+
+        if (result != null)
+        {
+            return result;
+        }
+
+        IEnumerable<KeyValuePair<string, string>>? envResult = Env.Load(filename, new LoadOptions(setEnvVars: false));
+
+        Dictionary<string, string>? fileContents = envResult == null ? new Dictionary<string, string>(0) : envResult.ToDictionary();
+
+        return AddOrGetExisting(filename, fileContents);
+    }
+}

src/RepoM.Api/IO/ModuleBasedRepositoryActionProvider/RepositorySpecificConfiguration.cs

@@ -5,7 +5,7 @@ namespace RepoM.Api.IO.ModuleBasedRepositoryActionProvider;
 using System.IO;
 using System.IO.Abstractions;
 using System.Linq;
-using System.Text;
+using System.Runtime.Caching;
 using DotNetEnv;
 using Microsoft.Extensions.Logging;
 using RepoM.Api.Common;
@@ -27,23 +27,26 @@ public class RepositoryConfigurationReader
     public const string FILENAME = "RepositoryActions.yaml";
     private readonly IAppDataPathProvider _appDataPathProvider;
     private readonly IFileSystem _fileSystem;
-    private readonly YamlDynamicRepositoryActionDeserializer _yamlAppSettingsDeserializer;
     private readonly IRepositoryExpressionEvaluator _repoExpressionEvaluator;
     private readonly ILogger _logger;
-
+    private readonly RepositoryActionsFileStore _repositoryActionsFileStore;
+    private readonly EnvFileStore _envFileStore;
 
     public RepositoryConfigurationReader(
         IAppDataPathProvider appDataPathProvider,
         IFileSystem fileSystem,
-        YamlDynamicRepositoryActionDeserializer yamlAppSettingsDeserializer,
+        IRepositoryActionDeserializer repositoryActionsDeserializer,
         IRepositoryExpressionEvaluator repoExpressionEvaluator,
-        ILogger logger)
+        ILogger logger,
+        ObjectCache cache)
     {
         _appDataPathProvider = appDataPathProvider ?? throw new ArgumentNullException(nameof(appDataPathProvider));
         _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
-        _yamlAppSettingsDeserializer = yamlAppSettingsDeserializer ?? throw new ArgumentNullException(nameof(yamlAppSettingsDeserializer));
         _repoExpressionEvaluator = repoExpressionEvaluator ?? throw new ArgumentNullException(nameof(repoExpressionEvaluator));
         _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+
+        _repositoryActionsFileStore = new RepositoryActionsFileStore(_fileSystem, repositoryActionsDeserializer, cache);
+        _envFileStore = new EnvFileStore(cache);
     }
 
     private string GetRepositoryActionsFilename(string basePath)
@@ -107,32 +110,37 @@ private string GetRepositoryActionsFilename(string basePath)
 
         try
         {
-            var content = _fileSystem.File.ReadAllText(filename, Encoding.UTF8);
-            rootFile = Deserialize(_fileSystem.Path.GetExtension(filename), content);
+            rootFile = _repositoryActionsFileStore.TryGet(filename);
         }
         catch (Exception e)
         {
             throw new InvalidConfigurationException(filename, e.Message, e);
         }
-
+
+        if (rootFile == null)
+        {
+            throw new InvalidConfigurationException(filename, "Could not read and deserialize file");
+        }
+
         Redirect? redirect = rootFile.Redirect;
-        if (!string.IsNullOrWhiteSpace(redirect?.Filename))
+        if (!string.IsNullOrWhiteSpace(redirect?.Filename) && IsEnabled(redirect.Enabled, true, null))
         {
-            if (IsEnabled(redirect.Enabled, true, null))
+            filename = EvaluateString(redirect.Filename, null);
+            if (_fileSystem.File.Exists(filename))
             {
-                filename = EvaluateString(redirect.Filename, null);
-                if (_fileSystem.File.Exists(filename))
+                try
                 {
-                    try
-                    {
-                        var content = _fileSystem.File.ReadAllText(filename, Encoding.UTF8);
-                        rootFile = Deserialize(_fileSystem.Path.GetExtension(filename), content);
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogWarning(e, "Could not read and deserialize file '{file}'", filename);
-                        throw new InvalidConfigurationException(filename, e.Message, e);
-                    }
+                    rootFile = _repositoryActionsFileStore.TryGet(filename);
+                }
+                catch (Exception e)
+                {
+                    _logger.LogWarning(e, "Could not read and deserialize file '{file}'", filename);
+                    throw new InvalidConfigurationException(filename, e.Message, e);
+                }
+
+                if (rootFile == null)
+                {
+                    throw new InvalidConfigurationException(filename, "Could not read and deserialize file");
                 }
             }
         }
@@ -177,7 +185,7 @@ List<EvaluatedVariable> EvaluateVariables(IEnumerable<Variable>? vars)
 
                 try
                 {
-                    IEnumerable<KeyValuePair<string, string>>? currentEnvVars = Env.Load(f, new LoadOptions(setEnvVars: false));
+                    IDictionary<string, string> currentEnvVars = _envFileStore.TryGet(f);
                     if (envVars == null || !envVars.Any())
                     {
                         envVars = currentEnvVars.ToDictionary();
@@ -231,8 +239,7 @@ List<EvaluatedVariable> EvaluateVariables(IEnumerable<Variable>? vars)
 
                 try
                 {
-                    var content = _fileSystem.File.ReadAllText(f, Encoding.UTF8);
-                    repoSpecificConfig = Deserialize(_fileSystem.Path.GetExtension(f), content);
+                    repoSpecificConfig = _repositoryActionsFileStore.TryGet(filename);
                 }
                 catch (Exception)
                 {
@@ -284,21 +291,6 @@ private bool IsEnabled(string? booleanExpression, bool defaultWhenNullOrEmpty, I
             ? defaultWhenNullOrEmpty
             : _repoExpressionEvaluator.EvaluateBooleanExpression(booleanExpression!, repository);
     }
-
-    private RepositoryActionConfiguration Deserialize(string extension, string rawContent)
-    {
-        if (extension.StartsWith('.'))
-        {
-            extension = extension[1..];
-        }
-
-        if ("yaml".Equals(extension, StringComparison.CurrentCultureIgnoreCase) || "yml".Equals(extension, StringComparison.CurrentCultureIgnoreCase))
-        {
-            return _yamlAppSettingsDeserializer.Deserialize(rawContent);
-        }
-
-        throw new NotImplementedException("Unknown extension");
-    }
 }
 
 public class RepositoryTagsConfigurationFactory : IRepositoryTagsFactory 

src/RepoM.Api/RepoM.Api.csproj

@@ -7,17 +7,22 @@
     <PackageReference Include="DotNetEnv" Version="2.5.0" />
     <PackageReference Include="ExpressionStringEvaluator" Version="1.0.4" />
     <PackageReference Include="System.Net.Http" Version="4.3.4" />
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="LibGit2Sharp" Version="0.27.2" />
     <PackageReference Include="Mono.Cecil" Version="0.11.5" />
-    <PackageReference Include="System.IO.Abstractions" Version="19.2.29" />
+    <PackageReference Include="System.IO.Abstractions" Version="19.2.51" />
     <PackageReference Include="System.Memory" Version="4.5.5" />
-    <PackageReference Include="YamlDotNet" Version="13.1.0" />
+    <PackageReference Include="System.Runtime.Caching" Version="7.0.0" />
+    <PackageReference Include="YamlDotNet" Version="13.2.0" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\RepoM.Core.Plugin\RepoM.Core.Plugin.csproj" />
   </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Update="JetBrains.Annotations" Version="2023.2.0" />
+  </ItemGroup>
 
 </Project>