feat: add capability to output Duplicate map from deduplicate command (…

…#103)
DeptSaludPR · Feb 14, 2024 · 24b3495 · 24b3495
1 parent 701678b
commit 24b3495
Show file tree

Hide file tree

Showing 3 changed files with 79 additions and 11 deletions.
diff --git a/src/Biomatch.CLI/Commands/MatchingCommand.Deduplicate.cs b/src/Biomatch.CLI/Commands/MatchingCommand.Deduplicate.cs
@@ -24,7 +24,7 @@ private static Command GetDeduplicateTemplateCommand()
     var outputOption = new Option<FileInfo>(
       name: "--output",
       description: "Output file path",
-      getDefaultValue: () => new FileInfo("Duplicates.csv")
+      getDefaultValue: () => new FileInfo("Deduplicated.csv")
     );
     outputOption.AddAlias("-o");
 
@@ -34,6 +34,16 @@ private static Command GetDeduplicateTemplateCommand()
       getDefaultValue: () => 0.85
     );
 
+    var mapOption = new Option<bool>(name: "--map", description: "Generate duplicate map");
+    mapOption.AddAlias("-m");
+
+    var outputMapOption = new Option<FileInfo>(
+      name: "--output-map",
+      description: "File location for duplicate map",
+      getDefaultValue: () => new FileInfo("DuplicateMap.csv")
+    );
+    outputMapOption.AddAlias("-om");
+
     var dictionaryOptions = GeneralOptions.GetDictionaryOptions();
 
     var command = new Command("deduplicate", "Deduplicate records from a template file")
@@ -44,16 +54,20 @@ private static Command GetDeduplicateTemplateCommand()
       dictionaryOptions.LastNamesDictionaryFilePathOption,
       outputOption,
       scoreOption,
+      mapOption,
+      outputMapOption
     };
 
     command.SetHandler(
-      (
+      async (
         filePathArgumentValue,
         firstNamesDictionaryFilePathOptionValue,
         middleNamesDictionaryFilePathOptionValue,
         lastNamesDictionaryFilePathOptionValue,
         outputOptionValue,
-        scoreOptionValue
+        scoreOptionValue,
+        mapOptionValue,
+        outputMapOptionValue
       ) =>
       {
         var records1FromCsv = PersonRecordTemplate
@@ -67,23 +81,41 @@ private static Command GetDeduplicateTemplateCommand()
             lastNamesDictionaryFilePathOptionValue
           );
 
-        var deduplicatedRecords = Deduplicate.TryDeduplicate(
-          records1FromCsv,
+        var preprocessedRecords = records1FromCsv
+          .PreprocessData(firstNamesDictionary, middleNamesDictionary, lastNamesDictionary)
+          .ToArray();
+
+        var potentialDuplicates = Match.GetPotentialMatchesFromSameDataSet(
+          preprocessedRecords,
+          preprocessedRecords,
           scoreOptionValue,
-          firstNamesDictionary,
-          middleNamesDictionary,
-          lastNamesDictionary,
+          1.0,
           MatchingProgress.GetMatchingProgressReport
         );
 
-        return PersonRecordTemplate.WriteToCsv(deduplicatedRecords, outputOptionValue.FullName);
+        var deduplicatedRecords = Deduplicate.TryDeduplicate(
+          preprocessedRecords,
+          potentialDuplicates
+        );
+
+        if (mapOptionValue)
+        {
+          await PotentialMatchTemplate.WriteToCsv(
+            potentialDuplicates,
+            outputMapOptionValue.FullName
+          );
+        }
+
+        await PersonRecordTemplate.WriteToCsv(deduplicatedRecords, outputOptionValue.FullName);
       },
       filePathArgument,
       dictionaryOptions.FirstNamesDictionaryFilePathOption,
       dictionaryOptions.MiddleNamesDictionaryFilePathOption,
       dictionaryOptions.LastNamesDictionaryFilePathOption,
       outputOption,
-      scoreOption
+      scoreOption,
+      mapOption,
+      outputMapOption
     );
 
     return command;

diff --git a/src/Biomatch.Domain/Biomatch.Domain.csproj b/src/Biomatch.Domain/Biomatch.Domain.csproj
@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="libphonenumber-csharp" Version="8.13.29" />
+    <PackageReference Include="libphonenumber-csharp" Version="8.13.30" />
     <PackageReference Include="Quickenshtein" Version="1.5.1"/>
     <PackageReference Include="SymSpell" Version="6.7.2"/>
     <PackageReference Include="System.Private.Uri" Version="4.3.2"/>

diff --git a/src/Biomatch.Domain/Deduplicate.cs b/src/Biomatch.Domain/Deduplicate.cs
@@ -56,6 +56,42 @@ public static IEnumerable<IPersonRecord> TryDeduplicate(
     }
   }
 
+  public static IEnumerable<IPersonRecord> TryDeduplicate(
+    IEnumerable<PersonRecordForMatch> originalRecords,
+    IEnumerable<PotentialMatch> potentialDuplicates
+  )
+  {
+    var potentialMatchesGroupedByRecord = potentialDuplicates
+      .GroupBy(x => x.Value)
+      .ToDictionary(x => x.Key, x => x.Select(y => y.Match).ToList());
+
+    var duplicates = new Dictionary<string, IPersonRecord>();
+    var uniqueDuplicateRecords = new Dictionary<string, IPersonRecord>();
+    Console.WriteLine("Processing potential matches...");
+    foreach (var potentialMatch in potentialMatchesGroupedByRecord)
+    {
+      if (duplicates.ContainsKey(potentialMatch.Key.RecordId))
+        continue;
+      MarkDuplicates(
+        potentialMatch.Key,
+        potentialMatchesGroupedByRecord,
+        potentialMatch.Value,
+        duplicates
+      );
+      uniqueDuplicateRecords.Add(potentialMatch.Key.RecordId, potentialMatch.Key);
+      yield return potentialMatch.Key;
+    }
+
+    foreach (var record in originalRecords)
+    {
+      if (uniqueDuplicateRecords.ContainsKey(record.RecordId))
+        continue;
+      if (duplicates.ContainsKey(record.RecordId))
+        continue;
+      yield return record;
+    }
+  }
+
   private static void MarkDuplicates(
     IPersonRecord originalRecord,
     Dictionary<IPersonRecord, List<IPersonRecord>> potentialDuplicates,