Temporary changes

intersystems-ib · Dec 23, 2024 · 3daee81 · 3daee81
1 parent 16c6f1b
commit 3daee81
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 3 deletions.
diff --git a/encoder-ui/src/app/analyzer/analyzer.component.ts b/encoder-ui/src/app/analyzer/analyzer.component.ts
@@ -53,8 +53,11 @@ export class AnalyzerComponent {
     var textHTML = this.textToAnalyze?.value;
     var textOriginal = textHTML;
     var textToProcess = this.textToAnalyze?.value.split(".").filter(Boolean);
-    var piecedTextToProcess: any[] = [textOriginal];
+    var piecedTextToProcess: any[] = [];
 
+    for (var index in textToProcess){
+      piecedTextToProcess = piecedTextToProcess.concat(textToProcess[index].split(","))
+    }
     var forReading = 100/(piecedTextToProcess.length);
     this.totalReceived = 0;
     this.error = false;

diff --git a/src/ENCODER/BP/AnalyzeTextProcess.cls b/src/ENCODER/BP/AnalyzeTextProcess.cls
@@ -5,7 +5,7 @@ Method OnRequest(pInput As ENCODER.Message.AnalysisRequest, Output pOutput As EN
 {
 
         set pOutput = ##class(ENCODER.Message.AnalysisResponse).%New()
-        set result = ..AnalyzeText(pInput.TextToAnalyze, pInput.AnalysisId, pInput.LanguageSelected)
+        set result = ..AnalyzeTextSimplified(pInput.TextToAnalyze, pInput.AnalysisId, pInput.LanguageSelected)
         set pOutput.AnalysisRequestId = pInput.AnalysisId
         return $$$OK
 }
@@ -34,7 +34,7 @@ Method AnalyzeText(text As %String, analysisId As %String, language As %String)
             if phraseToAnalyze != "":
                 embedding = model.encode(phraseToAnalyze, normalize_embeddings=True).tolist()
                 sqlsentence = "INSERT INTO ENCODER_Object.TextMatches (CodeId, Description, Similarity, AnalysisId, RawText) SELECT TOP 50 * FROM (SELECT CodeId, Description, VECTOR_DOT_PRODUCT(VectorDescription, TO_VECTOR('"+str(embedding)+"', DECIMAL)) AS Similarity, '"+analysisId+"', '"+phraseToAnalyze+"' FROM ENCODER_Object.Codes) ORDER BY Similarity DESC"                    
-                stmt = iris.sql.prepare("INSERT INTO ENCODER_Object.TextMatches (CodeId, Description, Similarity, AnalysisId, RawText) SELECT TOP 50 * FROM (SELECT CodeId, Description, VECTOR_DOT_PRODUCT(VectorDescription, TO_VECTOR(?, DECIMAL)) AS Similarity, ?, ? FROM ENCODER_Object.Codes) ORDER BY Similarity DESC")                    
+                stmt = iris.sql.prepare("INSERT INTO ENCODER_Object.TextMatches (CodeId, Description, Similarity, AnalysisId, RawText) SELECT TOP 50 * FROM (SELECT CodeId, Description, VECTOR_DOT_PRODUCT(VectorDescription, TO_VECTOR(?, DECIMAL)) AS Similarity, ?, ? FROM ENCODER_Object.Codes) WHERE Similarity > 0.65 ORDER BY Similarity DESC")                    
                 rs = stmt.execute(str(embedding), analysisId, phraseToAnalyze)        
     except Exception as err:
         iris.cls("Ens.Util.Log").LogInfo("ENCODER.BP.AnalyzeTextProcess", "AnalyzeText", repr(err))
@@ -43,6 +43,35 @@ Method AnalyzeText(text As %String, analysisId As %String, language As %String)
     return "Success"
 }
 
+Method AnalyzeTextSimplified(text As %String, analysisId As %String, language As %String) As %String [ Language = python ]
+{
+    import sentence_transformers
+    import iris
+    import requests
+
+    try:
+        iris.cls("Ens.Util.Log").LogInfo("ENCODER.BP.AnalyzeTextProcess", "AnalyzeText", "Bottle neck 1")
+        model = sentence_transformers.SentenceTransformer('/iris-shared/model/')
+        iris.cls("Ens.Util.Log").LogInfo("ENCODER.BP.AnalyzeTextProcess", "AnalyzeText", "Bottle neck 2")
+        phrases = text.split(".")
+        sqlsentence = ""
+        # iris.cls("Ens.Util.Log").LogInfo("ENCODER.BP.AnalyzeTextProcess", "AnalyzeText", "Starting process")
+        for phraseToAnalyze in phrases :
+            iris.cls("Ens.Util.Log").LogInfo("ENCODER.BP.AnalyzeTextProcess", "AnalyzeText", "Bottle neck 3")
+            if phraseToAnalyze != "":
+                embedding = model.encode(phraseToAnalyze, normalize_embeddings=True).tolist()
+                iris.cls("Ens.Util.Log").LogInfo("ENCODER.BP.AnalyzeTextProcess", "AnalyzeText", "Bottle neck 4")
+                stmt = iris.sql.prepare("INSERT INTO ENCODER_Object.TextMatches (CodeId, Description, Similarity, AnalysisId, RawText) SELECT TOP 50 * FROM (SELECT CodeId, Description, VECTOR_DOT_PRODUCT(VectorDescription, TO_VECTOR(?, DECIMAL)) AS Similarity, ?, ? FROM ENCODER_Object.Codes) WHERE Similarity > 0.65 ORDER BY Similarity DESC")                    
+                iris.cls("Ens.Util.Log").LogInfo("ENCODER.BP.AnalyzeTextProcess", "AnalyzeText", "Bottle neck 5")
+                rs = stmt.execute(str(embedding), analysisId, phraseToAnalyze)        
+                iris.cls("Ens.Util.Log").LogInfo("ENCODER.BP.AnalyzeTextProcess", "AnalyzeText", "Bottle neck 6")
+    except Exception as err:
+        iris.cls("Ens.Util.Log").LogInfo("ENCODER.BP.AnalyzeTextProcess", "AnalyzeText", repr(err))
+        return repr(err)
+
+    return "Success"
+}
+
 Storage Default
 {
 <Type>%Storage.Persistent</Type>