Adding status priority to cleanDuplicated (ex cleanDoubled)

Cour-de-cassation · Aug 22, 2023 · 7f5035d · 7f5035d
1 parent 161da6b
commit 7f5035d
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 62 deletions.
diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDocuments.ts
@@ -1,7 +1,7 @@
 import { logger } from '../../../utils';
 import { cleanAssignations } from './cleanAssignations';
 import { cleanAssignedDocuments } from './cleanAssignedDocuments';
-import { cleanDoubledDocuments } from './cleanDoubledDocuments';
+import { cleanDuplicatedDocuments } from './cleanDuplicatedDocuments';
 import { cleanFreeDocuments } from './cleanFreeDocuments';
 import { cleanLoadedDocuments } from './cleanLoadedDocuments';
 import { cleanTreatments } from './cleanTreatments';
@@ -11,7 +11,7 @@ export { cleanDocuments };
 async function cleanDocuments() {
   logger.log(`cleanDocuments`);
 
-  await cleanDoubledDocuments();
+  await cleanDuplicatedDocuments();
 
   await cleanAssignedDocuments();
 

diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDoubledDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDoubledDocuments.ts
diff --git a/...anDocuments/cleanDoubledDocuments.spec.ts → ...ocuments/cleanDuplicatedDocuments.spec.ts b/...anDocuments/cleanDoubledDocuments.spec.ts → ...ocuments/cleanDuplicatedDocuments.spec.ts
@@ -1,9 +1,9 @@
 import { documentModule, idModule } from '@label/core';
 import { buildDocumentRepository } from '../../../modules/document';
-import { cleanDoubledDocuments } from './cleanDoubledDocuments';
+import { cleanDuplicatedDocuments } from './cleanDuplicatedDocuments';
 
-describe('cleanDoubledDocuments', () => {
-  it('should clean the doubledDocuments', async () => {
+describe('cleanDuplicatedDocuments', () => {
+  it('should clean the DuplicatedDocuments', async () => {
     const originalDocument = documentModule.generator.generate();
     const otherDocument = documentModule.generator.generate();
     const doubledDocument = documentModule.generator.generate({
@@ -17,7 +17,7 @@ describe('cleanDoubledDocuments', () => {
       doubledDocument,
     ]);
 
-    await cleanDoubledDocuments();
+    await cleanDuplicatedDocuments();
 
     const fetchedDocuments = await documentRepository.findAll();
 

diff --git a/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts b/packages/generic/backend/src/app/scripts/cleanDocuments/cleanDuplicatedDocuments.ts
@@ -0,0 +1,102 @@
+import { documentType } from '@label/core';
+import {
+  buildDocumentRepository,
+  documentService,
+} from '../../../modules/document';
+import { logger } from '../../../utils';
+
+export { cleanDuplicatedDocuments };
+
+/**
+ * Delete all doubled documents (same source, same documentNumber, same text)
+ */
+async function cleanDuplicatedDocuments() {
+  logger.log(`cleanDuplicatedDocuments`);
+
+  const documentRepository = buildDocumentRepository();
+
+  const documents = await documentRepository.findAll();
+  const sortedDocuments = documents.sort(compareDocumentsByStatus)
+
+  logger.log(`${documents.length} documents found. Searching for duplicates...`);
+
+  const documentsToDelete: documentType[] = [];
+
+  for (let index = 0, l = documents.length - 1; index < l; index++) {
+    const currentDocument = sortedDocuments[index];
+    const nextDocument = sortedDocuments[index + 1];
+    if (areDocumentsIdentical(currentDocument, nextDocument)) {
+      documentsToDelete.push(nextDocument);
+    }
+  }
+
+  logger.log(
+    `Found ${documentsToDelete.length} documents to delete. Deleting...`,
+  );
+
+  for (let index = 0, l = documentsToDelete.length; index < l; index++) {
+    await documentService.deleteDocument(documentsToDelete[index]._id);
+  }
+  logger.log('cleanDuplicatedDocuments done!');
+}
+
+function areDocumentsIdentical(
+  document1: documentType,
+  document2: documentType,
+) {
+  return (
+    document1.documentNumber === document2.documentNumber &&
+    document1.source === document2.source &&
+    document1.text === document2.text
+  );
+}
+
+
+function compareDocumentsByStatus(
+  document1: documentType,
+  document2: documentType,
+) {
+
+  const statusToPriorities = {
+    "loaded": 0,
+    "nlpAnnotating": 1,
+    "free": 2,
+    "pending": 3,
+    "saved": 4,
+    "toBePublished": 5,
+    "done": 6,
+    "toBeConfirmed": 7,
+    "locked": 8,
+    "rejected": 9,
+  };
+
+  // comparing source
+  if (document1.source < document2.source) {
+    return -1
+  } else if (document1.source > document2.source) {
+    return 1
+  } else {
+    // comparing documentNumber
+    if (document1.documentNumber < document1.documentNumber) {
+      return -1
+    } else if (document1.documentNumber > document2.documentNumber) {
+      return 1
+    } else {
+      //comparing text
+      if (document1.text < document2.text) {
+        return -1
+      } else if (document1.text > document2.text) {
+        return 1
+      } else {
+        // comparing statuses
+        if (statusToPriorities[document1.status] < statusToPriorities[document2.status]) {
+          return 1 // low priority
+        } else if (statusToPriorities[document1.status] > statusToPriorities[document2.status]) {
+          return -1 // high priority
+        } else {
+          return 0
+        }
+      }
+    }
+  }
+}