simplify collocation callbacks by using POST requests

python/philologic/runtime/reports/collocation.py

@@ -14,7 +14,7 @@
 from orjson import dumps
 
 
-def collocation_results(request, config):
+def collocation_results(request, config, current_collocates):
     """Fetch collocation results"""
     collocation_object: dict[str, Any] = {"query": dict([i for i in request])}
     db = DB(config.db_path + "/data/")
@@ -81,7 +81,11 @@ def collocation_results(request, config):
         max_time = None
     else:
         max_time = request.max_time or 2
-    all_collocates = {}
+
+    if current_collocates:
+        all_collocates = dict(current_collocates)
+    else:
+        all_collocates = {}
     start_time = timeit.default_timer()
 
     env = lmdb.open(
@@ -129,18 +133,17 @@ def collocation_results(request, config):
                 if collocate is not None:  # in the event lemma is None
                     if collocate_distance is None:
                         if collocate not in all_collocates:
-                            all_collocates[collocate] = {"count": 1}
+                            all_collocates[collocate] = 1
                         else:
-                            all_collocates[collocate]["count"] += 1
+                            all_collocates[collocate] += 1
                     else:
                         if abs(position - q_word_position[0]) <= collocate_distance:  # type: ignore
                             if collocate not in all_collocates:
-                                all_collocates[collocate] = {"count": 1}
+                                all_collocates[collocate] = 1
                             else:
-                                all_collocates[collocate]["count"] += 1
+                                all_collocates[collocate] += 1
 
             hits_done += 1
-
             elapsed = timeit.default_timer() - start_time
             # split the query if more than request.max_time has been spent in the loop
             if max_time is not None:
@@ -149,6 +152,7 @@ def collocation_results(request, config):
     env.close()
     hits.finish()
 
+    all_collocates = sorted(all_collocates.items(), key=lambda item: item[1], reverse=True)
     collocation_object["collocates"] = all_collocates
     collocation_object["results_length"] = len(hits)
     if hits_done < collocation_object["results_length"]:
@@ -158,10 +162,6 @@ def collocation_results(request, config):
         collocation_object["more_results"] = False
         collocation_object["hits_done"] = collocation_object["results_length"]
     collocation_object["distance"] = collocate_distance
-    if len(request.metadata) == 1:  # request.metadata always has philo_type as a key
-        collocation_object["whole_corpus"] = True
-    else:
-        collocation_object["whole_corpus"] = False
 
     return collocation_object
 

www/app/package.json

@@ -53,4 +53,4 @@
         "> 1%",
         "last 2 versions"
     ]
-}
+}

www/app/src/components/Collocation.vue

@@ -9,8 +9,8 @@
                         @click="getFrequency()">
                         <span class="d-none d-sm-none d-md-inline">{{ $t("collocation.collocation") }}</span>
                     </button>
-                    <button type="button" class="btn btn-secondary"
-                        :class="{ active: collocMethod === 'compare', disabled: wholeCorpus }" @click="toggleCompare()">
+                    <button type="button" class="btn btn-secondary" :class="{ active: collocMethod === 'compare' }"
+                        @click="toggleCompare()">
                         <span class="d-none d-sm-none d-md-inline">{{ $t("collocation.compareTo") }}</span>
                     </button>
                 </div>
@@ -155,7 +155,7 @@
                     </div>
                 </div>
                 <button type="button" class="btn btn-secondary" style="width: fit-content"
-                    @click="comparativeCollocations('compare')">{{
+                    @click="getOtherCollocates({}, 0)">{{
             $t('collocation.runComparison') }}
                 </button>
             </div>
@@ -283,6 +283,7 @@ export default {
             biblio: {},
             moreResults: false,
             sortedList: [],
+            collocateCounts: [],
             showFilteredWords: false,
             runningTotal: 0,
             collocCloudWords: [],
@@ -303,10 +304,10 @@ export default {
             otherCollocates: [],
             otherBiblio: {},
             comparedTo: "wholeCorpus",
-            wholeCorpus: true,
             filterMetadataOpen: false,
             compareSearching: false,
-            comparativeSearchStarted: false
+            comparativeSearchStarted: false,
+            otherDone: false,
         };
     },
     created() {
@@ -332,15 +333,14 @@ export default {
     methods: {
         fetchResults() {
             this.localFormData = this.copyObject(this.$store.state.formData);
-            var collocObject = {};
             this.searching = true;
             this.relativeFrequencies = {};
             this.collocMethod = "frequency"
             this.overRepresented = [];
             this.underRepresented = [];
             this.other_corpus_metadata = {};
             this.comparativeSearchStarted = false
-            this.updateCollocation(collocObject, 0);
+            this.updateCollocation({}, 0);
         },
         buildMetadata(metadata) {
             this.metadataDisplay = metadata.display;
@@ -357,55 +357,45 @@ export default {
                 start: start.toString(),
             };
             this.$http
-                .get(`${this.$dbUrl}/reports/collocation.py`, {
-                    params: this.paramsFilter(params),
-                })
+                .post(`${this.$dbUrl}/reports/collocation.py`, {
+                    current_collocates: fullResults,
+                },
+                    {
+                        params: this.paramsFilter(params),
+                    })
                 .then((response) => {
-                    let data = response.data;
-                    this.wholeCorpus = response.data.whole_corpus;
-                    this.resultsLength = data.results_length;
-                    this.moreResults = data.more_results;
-                    this.runningTotal = data.hits_done;
-                    start = data.hits_done;
+                    this.resultsLength = response.data.results_length;
+                    this.moreResults = response.data.more_results;
+                    this.runningTotal = response.data.hits_done;
+                    start = response.data.hits_done;
                     this.searching = false;
                     if (this.resultsLength) {
-                        this.sortAndRenderCollocation(fullResults, data, start);
+                        if (this.moreResults) {
+                            this.sortedList = this.extractSurfaceFromCollocate(response.data.collocates.slice(0, 100));
+                            this.updateCollocation(response.data.collocates, start);
+                        }
+                        else {
+                            this.collocateCounts = response.data.collocates;
+                            this.sortedList = this.extractSurfaceFromCollocate(response.data.collocates.slice(0, 100));
+                            this.done = true
+                        }
                     }
+
                 })
                 .catch((error) => {
                     this.searching = false;
                     this.debug(this, error);
                 });
         },
-        sortAndRenderCollocation(fullResults, data, start) {
-            if (typeof fullResults === "undefined" || Object.keys(fullResults).length === 0) {
-                fullResults = {};
-                this.filterList = data.filter_list;
-            }
-            var collocates = this.mergeResults(fullResults, data.collocates);
-            this.collocatesUnsorted = collocates.unsorted
-            this.sortedList = this.extractSurfaceFromCollocate(collocates.sorted.slice(0, 100));
-            // this.buildWordCloud();
-            if (this.moreResults) {
-                var tempFullResults = collocates.unsorted;
-                var runningQuery = this.$store.state.formData;
-                if (this.report === "collocation" && this.deepEqual(runningQuery, this.localFormData)) {
-                    // make sure we're still running the same query
-                    this.updateCollocation(tempFullResults, start);
-                }
-            } else {
-                this.done = true;
-            }
-        },
         extractSurfaceFromCollocate(words) {
             let newWords = []
-            for (let word of words) {
-                let collocate = `${word.label}`.replace(/lemma:/, "");
+            for (let wordObj of words) {
+                let collocate = `${wordObj[0]}`.replace(/lemma:/, "");
                 if (collocate.search(/\w+:.*/) != -1) {
                     collocate = collocate.replace(/(\p{L}+):.*/u, "$1");
                 }
-                let surfaceForm = word.label;
-                newWords.push({ collocate: collocate, surfaceForm: surfaceForm, count: word.count });
+                let surfaceForm = wordObj[0];
+                newWords.push({ collocate: collocate, surfaceForm: surfaceForm, count: wordObj[1] });
             }
             return newWords
         },
@@ -469,36 +459,72 @@ export default {
                 })
             })
         },
-        comparativeCollocations(method) {
+        getOtherCollocates(fullResults, start) {
+            // Check if this.compareMetadataValues is empty
+            if (Object.keys(this.comparedMetadataValues).length === 0) {
+                this.wholeCorpus = true
+            } else {
+                this.wholeCorpus = false
+            }
+            this.collocMethod = 'compare';
+            let params = {
+                q: this.q,
+                ...this.comparedMetadataValues,
+                start: start.toString(),
+            };
+            this.otherDone = false;
+            this.$http
+                .post(`${this.$dbUrl}/reports/collocation.py`, {
+                    current_collocates: fullResults,
+                },
+                    {
+                        params: this.paramsFilter(params),
+                    })
+                .then((response) => {
+                    let resultsLength = response.data.results_length;
+                    let moreResults = response.data.more_results;
+                    let start = response.data.hits_done;
+                    if (resultsLength) {
+                        if (moreResults) {
+                            this.getOtherCollocates(response.data.collocates, start);
+                        }
+                        else {
+                            this.otherDone = true
+                            this.comparativeCollocations(response.data.collocates)
+                        }
+                    }
+
+                })
+                .catch((error) => {
+                    this.searching = false;
+                    this.debug(this, error);
+                });
+        },
+        comparativeCollocations(otherCollocates) {
             let collapseElement = document.getElementById('other-corpus-metadata')
             if (collapseElement != null) {
                 Collapse.getInstance(collapseElement).hide()
                 this.filterMetadataOpen = false
             }
             this.comparativeSearchStarted = true;
-            this.collocMethod = method;
             this.comparedMetadataValues = this.dateRangeHandler(this.metadataInputStyle, this.dateRange, this.dateType, this.comparedMetadataValues)
             this.otherBiblio = this.buildBiblioCriteria(this.$philoConfig, this.comparedMetadataValues, this.comparedMetadataValues)
             this.compareSearching = true;
             this.overRepresented = [];
             this.underRepresented = [];
             this.otherCollocates = [];
             this.$http.post(`${this.$dbUrl}/scripts/comparative_collocations.py`, {
-                all_collocates: this.collocatesUnsorted,
-                other_corpus_metadata: this.comparedMetadataValues
-            }, {
-                params: {
-                    ...this.$store.state.formData,
-                },
-
+                all_collocates: this.collocateCounts,
+                other_collocates: otherCollocates,
+                whole_corpus: this.wholeCorpus,
             }, {
                 headers: {
                     'Content-Type': 'application/x-www-form-urlencoded'
                 }
             }).then((response) => {
+                this.otherCollocates = this.extractSurfaceFromCollocate(otherCollocates.slice(0, 100));
                 this.overRepresented = this.extractSurfaceFromCollocate(response.data.top);
                 this.underRepresented = this.extractSurfaceFromCollocate(response.data.bottom);
-                this.otherCollocates = this.extractSurfaceFromCollocate(response.data.other_collocates);
                 this.relativeFrequencies = { top: this.overRepresented, bottom: this.underRepresented };
                 this.compareSearching = false;
 

www/app/src/locales/en.json

@@ -63,7 +63,7 @@
         "filterCollocate": "Filter collocates by",
         "collocatesWithin": "Collocates within",
         "words": "words",
-        "collocatesWithinSentence": "Collocates within the same sentence",
+        "collocatesWithinSentence": "Collocates within the same sentence"
     },
     "resultsBiblio": {
         "heading": "Bibliography of results on this page",

www/app/src/locales/fr.json

@@ -62,7 +62,7 @@
         "filterCollocate": "Filtrer les collocations par",
         "collocatesWithin": "Collocations dans une fenêtre de",
         "words": "mots",
-        "collocatesWithinSentence": "Collocations dans la même phrase",
+        "collocatesWithinSentence": "Collocations dans la même phrase"
     },
     "resultsBiblio": {
         "heading": "Bibliographie des résultats sur cette page",

www/reports/collocation.py

@@ -27,9 +27,23 @@
 def collocation(environ, start_response):
     config = WebConfig(os.path.abspath(os.path.dirname(__file__)).replace("reports", ""))
     request = WSGIHandler(environ, config)
+    if environ["REQUEST_METHOD"] == "OPTIONS":
+        # Handle preflight request
+        start_response(
+            "200 OK",
+            [
+                ("Content-Type", "text/plain"),
+                ("Access-Control-Allow-Origin", environ["HTTP_ORIGIN"]),  # Replace with your client domain
+                ("Access-Control-Allow-Methods", "POST, OPTIONS"),
+                ("Access-Control-Allow-Headers", "Content-Type"),  # Adjust if needed for your headers
+            ],
+        )
+        return [b""]  # Empty response body for OPTIONS
     headers = [("Content-type", "application/json; charset=UTF-8"), ("Access-Control-Allow-Origin", "*")]
     start_response("200 OK", headers)
-    collocation_object = collocation_results(request, config)
+    post_data = environ["wsgi.input"].read()
+    current_collocates = orjson.loads(post_data)["current_collocates"]
+    collocation_object = collocation_results(request, config, current_collocates)
     yield orjson.dumps(collocation_object)