Updated serializers to deep compare objects

andresharpe · Aug 7, 2024 · 8780b18 · 8780b18
1 parent 87e9c69
commit 8780b18
Show file tree

Hide file tree

Showing 11 changed files with 214 additions and 65 deletions.
diff --git a/source/Cute.Lib/Cache/HttpResponseFileCache.cs b/source/Cute.Lib/Cache/HttpResponseFileCache.cs
@@ -0,0 +1,35 @@
+using Newtonsoft.Json;
+
+namespace Cute.Lib.Cache;
+
+public class HttpResponseFileCache
+{
+    private readonly string _tempPath = Path.Combine(Path.GetTempPath(), "cute");
+
+    public async Task<T?> Get<T>(string fileBaseName, Func<Task<T>> createObject)
+    {
+        var filename = Path.Combine(_tempPath, $"{fileBaseName}.json");
+
+        Directory.CreateDirectory(_tempPath);
+
+        if (File.Exists(filename))
+        {
+            return JsonConvert.DeserializeObject<T>(await File.ReadAllTextAsync(filename));
+        }
+
+        var newObject = await createObject();
+
+        await File.WriteAllTextAsync(filename, JsonConvert.SerializeObject(newObject));
+
+        return newObject;
+    }
+}
+
+public class HttpResponseCacheEntry
+{
+    public string RequestUri { get; set; } = default!;
+
+    public object? ResponseContent { get; set; } = default!;
+
+    public Dictionary<string, string?> ResponseContentHeaders { get; set; } = [];
+}
diff --git a/source/Cute.Lib/Contentful/BulkActions/BulkActionExecutor.cs b/source/Cute.Lib/Contentful/BulkActions/BulkActionExecutor.cs
@@ -1,4 +1,5 @@
 using Contentful.Core.Models;
+using Cute.Lib.Cache;
 using Cute.Lib.Exceptions;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -21,7 +22,7 @@ public class BulkActionExecutor
     private readonly ContentfulConnection _contentfulConnection;
 
     private readonly HttpClient _httpClient;
-
+    private readonly HttpResponseFileCache _httpResponseCache;
     private string? _contentType;
 
     private ContentType? _contentTypeDefinition;
@@ -32,10 +33,11 @@ public class BulkActionExecutor
 
     private List<Entry<JObject>>? _withNewEntries;
 
-    public BulkActionExecutor(ContentfulConnection contentfulConnection, HttpClient httpClient)
+    public BulkActionExecutor(ContentfulConnection contentfulConnection, HttpClient httpClient, HttpResponseFileCache httpResponseCache)
     {
         _contentfulConnection = contentfulConnection;
         _httpClient = httpClient;
+        _httpResponseCache = httpResponseCache;
     }
 
     public BulkActionExecutor WithContentType(string contentType)
@@ -101,7 +103,7 @@ private async Task ChangePublishRequiredEntries(List<BulkItem> items, BulkAction
             chunkQueue.Enqueue(chunk);
         }
 
-        while (!chunkQueue.IsEmpty)
+        while (!chunkQueue.IsEmpty || bulkActionIds.Count != 0)
         {
             if (!chunkQueue.TryDequeue(out var chunk)) continue;
 

diff --git a/source/Cute.Lib/Contentful/ContentfulEntryEnumerator.cs b/source/Cute.Lib/Contentful/ContentfulEntryEnumerator.cs
@@ -7,20 +7,20 @@ namespace Cute.Lib.Contentful;
 
 public static class ContentfulEntryEnumerator
 {
-    public static async IAsyncEnumerable<(T Entry, ContentfulCollection<T> Entries)> Entries<T>(ContentfulManagementClient client, string contentType,
+    public static async IAsyncEnumerable<(T Entry, ContentfulCollection<T> Entries)> Entries<T>(
+        ContentfulManagementClient client, string contentType,
         string? orderByField = null,
-        int includeLevels = 2, Action<QueryBuilder<T>>? queryConfigurator = null, string? queryString = null)
+        int includeLevels = 2, Action<QueryBuilder<T>>? queryConfigurator = null, string? queryString = null, int pageSize = 1000)
     {
         var skip = 0;
-        var page = 1000;
 
         while (true)
         {
             var queryBuilder = new QueryBuilder<T>()
                 .ContentTypeIs(contentType)
                 .Include(includeLevels)
                 .Skip(skip)
-                .Limit(page);
+                .Limit(pageSize);
 
             if (orderByField != null)
             {
@@ -49,7 +49,7 @@ public static class ContentfulEntryEnumerator
                 yield return (Entry: entry, Entries: entries);
             }
 
-            skip += page;
+            skip += pageSize;
         }
     }
 

diff --git a/source/Cute.Lib/Cute.Lib.csproj b/source/Cute.Lib/Cute.Lib.csproj
@@ -22,6 +22,7 @@
     <PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
     <PackageReference Include="Serilog.Sinks.OpenTelemetry" Version="3.0.0" />
+    <PackageReference Include="System.IO.Hashing" Version="8.0.0" />
     <PackageReference Include="YamlDotNet" Version="16.0.0" />
   </ItemGroup>
 

diff --git a/source/Cute.Lib/GetDataAdapter/HttpDataAdapter/HttpDataAdapter.cs b/source/Cute.Lib/GetDataAdapter/HttpDataAdapter/HttpDataAdapter.cs
@@ -1,5 +1,6 @@
 using Contentful.Core;
 using Contentful.Core.Models;
+using Cute.Lib.Cache;
 using Cute.Lib.Contentful;
 using Cute.Lib.Exceptions;
 using Cute.Lib.Scriban;
@@ -8,20 +9,31 @@
 using Scriban;
 using Scriban.Runtime;
 using Scriban.Syntax;
+using System.IO.Hashing;
+using System.Text;
 
 namespace Cute.Lib.GetDataAdapter;
 
 public class HttpDataAdapter
 {
     private Action<string>? _displayAction;
 
+    private HttpResponseFileCache? _httpResponseFileCache;
+
     public HttpDataAdapter WithDisplayAction(Action<string> displayAction)
     {
         _displayAction = displayAction;
 
         return this;
     }
 
+    public HttpDataAdapter WithHttpResponseFileCache(HttpResponseFileCache httpResponseFileCache)
+    {
+        _httpResponseFileCache = httpResponseFileCache;
+
+        return this;
+    }
+
     public async Task<List<Dictionary<string, string>>?> GetData(HttpDataAdapterConfig adapter,
             IReadOnlyDictionary<string, string?> envSettings,
             ContentfulManagementClient contentfulManagementClient,
@@ -158,16 +170,16 @@ private ScriptObject CreateScripObject(IReadOnlyDictionary<string, string?> cont
 
         var returnValue = new List<Dictionary<string, string>>();
 
-        var cachedResults = new HashSet<string>();
-
-        var requestCount = 1;
+        var requestCount = 0;
 
         var skipTotal = 0;
 
         var baseAddress = uriDict["uri"];
 
         while (true)
         {
+            requestCount++;
+
             var getParameters = string.Empty;
 
             if (adapter.Pagination is not null)
@@ -177,44 +189,20 @@ private ScriptObject CreateScripObject(IReadOnlyDictionary<string, string?> cont
                 skipTotal += adapter.Pagination.LimitMax;
             }
 
-            HttpResponseMessage? endpointResult = null;
+            var requestUri = baseAddress + getParameters;
 
-            string? endpointContent = null;
+            var results = await GetResponseFromEndpointOrCache(adapter, httpClient, formContent, requestUri, requestCount);
 
-            if (adapter.HttpMethod == HttpMethod.Post)
-            {
-                endpointResult = await httpClient.PostAsync(baseAddress + getParameters, formContent);
-            }
-            else if (adapter.HttpMethod == HttpMethod.Get)
-            {
-                endpointResult = await httpClient.GetAsync(baseAddress + getParameters);
-            }
-            else
-            {
-                throw new NotImplementedException($"Unknown method {adapter.HttpMethod}");
-            }
-
-            endpointResult?.EnsureSuccessStatusCode();
-
-            if (endpointResult is not null)
-            {
-                endpointContent = await endpointResult.Content.ReadAsStringAsync();
-            }
-
-            if (endpointContent is null) return [];
-
-            var results = JsonConvert.DeserializeObject(endpointContent);
-
-            if (results is null) return null;
+            if (results is null) return [];
 
             JArray rootArray = [];
 
             if (adapter.ResultsJsonPath is null)
             {
-                rootArray = results as JArray
+                rootArray = results.ResponseContent as JArray
                     ?? throw new CliException("The result of the endpoint call is not a json array.");
             }
-            else if (results is JObject obj)
+            else if (results.ResponseContent is JObject obj)
             {
                 rootArray = obj.SelectToken($"$.{adapter.ResultsJsonPath}") as JArray
                     ?? throw new CliException($"The json path '{adapter.ResultsJsonPath}' does not exist or is not a json array.");
@@ -226,7 +214,7 @@ private ScriptObject CreateScripObject(IReadOnlyDictionary<string, string?> cont
 
             if (_displayAction is not null)
             {
-                _displayAction($"...'{baseAddress + getParameters}' returned {rootArray.Count} entries...");
+                _displayAction($"...'{requestUri}' returned {rootArray.Count} entries...");
             }
 
             var batchValue = MapResultValues(rootArray, compiledTemplates, compiledPreTemplates, scriptObject);
@@ -244,15 +232,7 @@ private ScriptObject CreateScripObject(IReadOnlyDictionary<string, string?> cont
 
             if (adapter.ContinuationTokenHeader is null) break;
 
-            if (cachedResults.Count > 0)
-            {
-                requestCount++;
-                continue;
-            }
-
-            if (endpointResult is null) break;
-
-            if (!endpointResult.Headers.Contains(adapter.ContinuationTokenHeader))
+            if (!results.ResponseContentHeaders.ContainsKey(adapter.ContinuationTokenHeader))
             {
                 break;
             }
@@ -262,16 +242,80 @@ private ScriptObject CreateScripObject(IReadOnlyDictionary<string, string?> cont
                 httpClient.DefaultRequestHeaders.Remove(adapter.ContinuationTokenHeader);
             }
 
-            var token = endpointResult.Headers.GetValues(adapter.ContinuationTokenHeader).First();
+            var token = results.ResponseContentHeaders[adapter.ContinuationTokenHeader];
 
             httpClient.DefaultRequestHeaders.Add(adapter.ContinuationTokenHeader, token);
-
-            requestCount++;
         }
 
         return [.. returnValue.OrderBy(e => e[adapter.ContentKeyField])];
     }
 
+    private async Task<HttpResponseCacheEntry?> GetResponseFromEndpointOrCache(HttpDataAdapterConfig adapter,
+        HttpClient httpClient, FormUrlEncodedContent? formContent, string requestUri, int requestCount)
+    {
+        Task<HttpResponseCacheEntry?> getEntryFunc() => GetResponseFromEndpoint(adapter, httpClient, formContent, requestUri);
+
+        if (_httpResponseFileCache is null) return await getEntryFunc();
+
+        var uri = new Uri(requestUri, UriKind.Absolute);
+
+        var fileBaseName = new StringBuilder();
+
+        fileBaseName.Append(adapter.Id);
+        fileBaseName.Append('_');
+        fileBaseName.Append(uri.Host.Replace('.', '-').Trim('-'));
+        fileBaseName.Append('_');
+        fileBaseName.Append($"{requestCount:D4}");
+
+        if (formContent is not null)
+        {
+            var hashAlgo = new XxHash3();
+            var item = await formContent.ReadAsByteArrayAsync();
+            hashAlgo.Append(item);
+            fileBaseName.Append('_');
+            fileBaseName.Append(hashAlgo.GetCurrentHashAsUInt64());
+        }
+
+        return await _httpResponseFileCache.Get(fileBaseName.ToString(), getEntryFunc);
+    }
+
+    private static async Task<HttpResponseCacheEntry?> GetResponseFromEndpoint(HttpDataAdapterConfig adapter, HttpClient httpClient, FormUrlEncodedContent? formContent, string requestUri)
+    {
+        HttpResponseMessage? endpointResult = null;
+
+        string? endpointContent = null;
+
+        if (adapter.HttpMethod == HttpMethod.Post)
+        {
+            endpointResult = await httpClient.PostAsync(requestUri, formContent);
+        }
+        else if (adapter.HttpMethod == HttpMethod.Get)
+        {
+            endpointResult = await httpClient.GetAsync(requestUri);
+        }
+        else
+        {
+            throw new NotImplementedException($"Unknown method {adapter.HttpMethod}");
+        }
+
+        endpointResult.EnsureSuccessStatusCode();
+
+        endpointContent = await endpointResult.Content.ReadAsStringAsync();
+
+        if (endpointContent is null) return null;
+
+        var results = JsonConvert.DeserializeObject(endpointContent);
+
+        return new HttpResponseCacheEntry
+        {
+            RequestUri = requestUri,
+            ResponseContent = results,
+            ResponseContentHeaders = endpointResult.Headers
+                .AsEnumerable()
+                .ToDictionary(kv => kv.Key, kv => kv.Value.FirstOrDefault()),
+        };
+    }
+
     private static Dictionary<string, string> CompileValuesWithEnvironment(Dictionary<string, string> headers, ScriptObject scriptObject)
     {
         var templates = headers.ToDictionary(m => m.Key, m => Template.Parse(m.Value));

diff --git a/source/Cute.Lib/GetDataAdapter/HttpDataAdapter/Models/HttpDataAdapterConfig.cs b/source/Cute.Lib/GetDataAdapter/HttpDataAdapter/Models/HttpDataAdapterConfig.cs
@@ -2,6 +2,7 @@
 
 public class HttpDataAdapterConfig
 {
+    public string Id { get; set; } = default!;
     public string ContentType { get; set; } = default!;
     public string ContentDisplayField { get; set; } = default!;
     public string ContentKeyField { get; set; } = default!;

diff --git a/source/Cute.Lib/Scriban/CuteFunctions.cs b/source/Cute.Lib/Scriban/CuteFunctions.cs
@@ -3,6 +3,7 @@
 using Cute.Lib.Contentful;
 using Cute.Lib.Utilities;
 using Html2Markdown;
+using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Scriban.Runtime;
 using System.Collections.Concurrent;
@@ -26,6 +27,19 @@ public static string HtmlToMarkdown(string? content)
         return _htmlConverter.Convert(content);
     }
 
+    public static string ToJson(object value)
+    {
+        if (value is JObject jObject)
+        {
+            return jObject.ToString();
+        }
+        else if (value is JArray jArray)
+        {
+            return jArray.ToString();
+        }
+        return JsonConvert.SerializeObject(value);
+    }
+
     public static string UrlLastSegment(string? url)
     {
         if (url == null) return string.Empty;
@@ -94,7 +108,7 @@ private static Dictionary<string, int> GetFromCountCache(string contentType, str
             {
                 if (!_countEntriesCache.TryGetValue(cacheKey, out var _))
                 {
-                    var results = ContentfulEntryEnumerator.Entries<JObject>(ContentfulManagementClient, contentType).ToBlockingEnumerable();
+                    var results = ContentfulEntryEnumerator.Entries<JObject>(ContentfulManagementClient, contentType, includeLevels: 1, pageSize: 500).ToBlockingEnumerable();
 
                     var resultsDictionary = new Dictionary<string, int>();