Beter batch logging

pulibrary · Nov 21, 2024 · 53819e3 · 53819e3
1 parent 1e8e16a
commit 53819e3
Showing 1 changed file with 11 additions and 8 deletions.
diff --git a/lib/dpul_collections/indexing_pipeline/figgy/indexing_consumer.ex b/lib/dpul_collections/indexing_pipeline/figgy/indexing_consumer.ex
@@ -64,11 +64,18 @@ defmodule DpulCollections.IndexingPipeline.Figgy.IndexingConsumer do
   @spec handle_batch(any(), list(Broadway.Message.t()), any(), any()) ::
           list(Broadway.Message.t())
   def handle_batch(_batcher, messages, _batch_info, context) do
-    messages
-    |> Enum.map(&unwrap/1)
-    |> Solr.add(context[:write_collection])
+    unwrapped =
+      messages
+      |> Enum.map(&unwrap/1)
 
-    Logger.warning("indexed batch")
+    filtered =
+      unwrapped
+      |> Enum.filter(fn m -> m != nil end)
+
+    Logger.warning("indexed batch m: #{messages |> Enum.count()} f: #{filtered |> Enum.count()}")
+
+    unwrapped
+    |> Solr.add(context[:write_collection])
 
     messages
   end
@@ -80,10 +87,6 @@ defmodule DpulCollections.IndexingPipeline.Figgy.IndexingConsumer do
   end
 
   defp unwrap(%Broadway.Message{data: %Figgy.TransformationCacheEntry{data: data}}) do
-    if data == nil do
-      Logger.warning("solr record data is nil")
-    end
-
     data
   end
 end