Merge pull request #8 from ai-cfia/7-update-finesse-benchmarking-scri…

…pt-to-skip-specific-files

Skip test files when necessary

finesse/FINESSE_USAGE.md

@@ -72,7 +72,7 @@ sequenceDiagram
 ## Example Command
 
 ```cmd
-$locust -f finesse/finesse_test.py --engine azure --path finesse/QnA/good_question --host https://finesse-guidance.ninebasetwo.xyz/api --once
+$locust -f finesse/finesse_test.py --engine azure --path finesse/QnA/sorted-2024-02-22/  --host https://finesse.inspection.alpha.canada.ca/api --once
 Searching with Azure Search...
 
 File: qna_2023-12-08_36.json

finesse/accuracy_functions.py

@@ -19,7 +19,7 @@ def calculate_accuracy(responses_url: list[str], expected_url: str) -> AccuracyR
         if response_number == expected_number:
             position = idx
             score = 1 - (position / total_pages)
-            score= round(score, 2)
+            score = round(score, 2)
             break
 
     return AccuracyResult(position, total_pages, score)
@@ -36,19 +36,20 @@ def save_to_markdown(test_data: dict, engine: str):
         md_file.write("| 📄 File               | 💬 Question                                                                                                                | 📏 Accuracy Score | ⌛ Time     |\n")
         md_file.write("|--------------------|-------------------------------------------------------------------------------------------------------------------------|----------------|----------|\n")
         for key, value in test_data.items():
-            md_file.write(f"| {key} | [{value.get('question')}]({value.get('expected_page').get('url')})' | {value.get('accuracy')*100:.1f}% | {value.get('time')}ms |\n")
+            md_file.write(f"| {key} | [{value.get('question')}]({value.get('expected_page').get('url')}) | {value.get('accuracy')*100:.0f}% | {int(value.get('time'))}ms |\n")
         md_file.write("\n")
         md_file.write(f"Tested on {len(test_data)} files.\n\n")
 
         time_stats, accuracy_stats = calculate_statistical_summary(test_data)
         md_file.write("## Statistical summary\n\n")
         md_file.write("| Statistic             | Time       | Accuracy score|\n")
         md_file.write("|-----------------------|------------|---------|\n")
-        md_file.write(f"|Mean| {time_stats.get('Mean')}ms | {accuracy_stats.get('Mean')*100}% |\n")
-        md_file.write(f"|Median| {time_stats.get('Median')}ms | {accuracy_stats.get('Median')*100}% |\n")
-        md_file.write(f"|Standard Deviation| {time_stats.get('Standard Deviation')}ms | {accuracy_stats.get('Standard Deviation')*100}% |\n")
-        md_file.write(f"|Maximum| {time_stats.get('Maximum')}ms | {accuracy_stats.get('Maximum')*100}% |\n")
-        md_file.write(f"|Minimum| {time_stats.get('Minimum')}ms | {accuracy_stats.get('Minimum')*100}% |\n")
+        md_file.write(f"|Mean| {int(time_stats.get('Mean'))}ms | {int(accuracy_stats.get('Mean')*100)}% |\n")
+        md_file.write(f"|Median| {int(time_stats.get('Median'))}ms | {int(accuracy_stats.get('Median')*100)}% |\n")
+        md_file.write(f"|Standard Deviation| {int(time_stats.get('Standard Deviation'))}ms | {int(accuracy_stats.get('Standard Deviation')*100)}% |\n")
+        md_file.write(f"|Maximum| {int(time_stats.get('Maximum'))}ms | {int(accuracy_stats.get('Maximum')*100)}% |\n")
+        md_file.write(f"|Minimum| {int(time_stats.get('Minimum'))}ms | {int(accuracy_stats.get('Minimum')*100)}% |\n")
+        md_file.write(f"\nThere are a total of {len([result.get('accuracy') for result in test_data.values() if result.get('accuracy') == 0])} null scores\n")
 
 def save_to_csv(test_data: dict, engine: str):
     if not os.path.exists(OUTPUT_FOLDER):
@@ -64,35 +65,35 @@ def save_to_csv(test_data: dict, engine: str):
                 key,
                 value.get("question"),
                 f"{value.get('accuracy')}",
-                f"{value.get('time')}"
+                f"{int(value.get('time'))}"
             ])
         writer.writerow([])
 
         time_stats, accuracy_stats = calculate_statistical_summary(test_data)
         writer.writerow(["Statistic", "Time", "Accuracy Score"])
-        writer.writerow(["Mean", f"{time_stats.get('Mean')}", f"{accuracy_stats.get('Mean')}"])
-        writer.writerow(["Median", f"{time_stats.get('Median')}", f"{accuracy_stats.get('Median')}"])
-        writer.writerow(["Standard Deviation", f"{time_stats.get('Standard Deviation')}", f"{accuracy_stats.get('Standard Deviation')}"])
-        writer.writerow(["Maximum", f"{time_stats.get('Maximum')}", f"{accuracy_stats.get('Maximum')}"])
-        writer.writerow(["Minimum", f"{time_stats.get('Minimum')}", f"{accuracy_stats.get('Minimum')}"])
+        writer.writerow(["Mean", f"{int(time_stats.get('Mean'))}", f"{int(accuracy_stats.get('Mean'))}"])
+        writer.writerow(["Median", f"{int(time_stats.get('Median'))}", f"{int(accuracy_stats.get('Median'))}"])
+        writer.writerow(["Standard Deviation", f"{int(time_stats.get('Standard Deviation'))}", f"{int(accuracy_stats.get('Standard Deviation'))}"])
+        writer.writerow(["Maximum", f"{int(time_stats.get('Maximum'))}", f"{int(accuracy_stats.get('Maximum'))}"])
+        writer.writerow(["Minimum", f"{int(time_stats.get('Minimum'))}", f"{int(accuracy_stats.get('Minimum'))}"])
 
 def log_data(test_data: dict):
     for key, value in test_data.items():
         print("File:", key)
         print("Question:", value.get("question"))
         print("Expected URL:", value.get("expected_page").get("url"))
-        print(f'Accuracy Score: {value.get("accuracy")*100}%')
-        print(f'Time: {value.get("time")}ms')
+        print(f'Accuracy Score: {int(value.get("accuracy")*100)}%')
+        print(f'Time: {int(value.get("time"))}ms')
         print()
     time_stats, accuracy_stats = calculate_statistical_summary(test_data)
     print("---")
     print(f"Tested on {len(test_data)} files.")
     print("Time statistical summary:", end="\n  ")
     for key,value in time_stats.items():
-        print(f"{key}:{value},", end=' ')
+        print(f"{key}:{int(value)},", end=' ')
     print("\nAccuracy statistical summary:", end="\n  ")
     for key,value in accuracy_stats.items():
-        print(f"{key}:{value*100}%,", end=' ')
+        print(f"{key}:{int(value*100)}%,", end=' ')
     print("\n---")
 
 

finesse/finesse_test.py

@@ -29,11 +29,15 @@ class FinesseUser(HttpUser):
     def search_accuracy(self):
         try:
             json_data = next(self.qna_reader)
+            while json_data.get("skip") is True:
+                json_data = next(self.qna_reader)
         except StopIteration:
             if not self.once:
                 # Reset variables
                 self.on_start()
                 json_data = next(self.qna_reader)
+                while json_data.get("skip") is True:
+                    json_data = next(self.qna_reader)
                 print("Restarting the running test")
             else:
                 print("Stopping the running test")

finesse/jsonreader.py

@@ -7,7 +7,7 @@ class JSONReader(Iterator):
 
     def __init__(self, directory):
         self.directory = directory
-        self.file_list = [f for f in os.listdir(directory) if f.endswith('.json')]
+        self.file_list = sorted([f for f in os.listdir(directory) if f.endswith('.json')])
         if not self.file_list:
             raise FileNotFoundError(f"No JSON files found in the directory '{directory}'")
         self.current_file_index = 0