diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..3c3d4c2 --- /dev/null +++ b/.env.template @@ -0,0 +1,2 @@ +BING_SEARCH_KEY = +BING_ENDPOINT = diff --git a/.gitignore b/.gitignore index 12a7043..fba33b9 100644 --- a/.gitignore +++ b/.gitignore @@ -43,5 +43,11 @@ flask_session/ # Ignore local QnA json files QnA -# Ignore output of api-test -output +# Ignore output of api-test and from the scripts +output/ + +# Ignore input of the scripts +input/ + +# Ignore the cache directory +cache/ diff --git a/finesse/FINESSE_USAGE.md b/finesse/FINESSE_USAGE.md index ad0b242..6546ea1 100644 --- a/finesse/FINESSE_USAGE.md +++ b/finesse/FINESSE_USAGE.md @@ -1,8 +1,8 @@ # How to use the Finesse Locust script This tool simplifies the process of comparing different search engines and -assessing their accuracy. It's designed to be straightforward, making it easy -to understand and use. +assessing their accuracy. It's designed to be straightforward, making it easy to +understand and use. ## How it Works @@ -16,8 +16,8 @@ to understand and use. - `static`: Static search engine - `llamaindex`: LlamaIndex search engine - `--path [directory path]`: Point to the directory with files structured - - `--host [API URL]`: Point to the finesse-backend URL - with JSON files with the following properties: + - `--host [API URL]`: Point to the finesse-backend URL with JSON files with + the following properties: - `score`: The score of the page. - `crawl_id`: The unique identifier associated with the crawl table. - `chunk_id`: The unique identifier of the chunk. @@ -43,7 +43,8 @@ to understand and use. - **Round trip time** - Measure round trip time of each request - **Summary statistical value** - - Measure the average, median, standard deviation, minimum and maximal accuracy scores and round trip time + - Measure the average, median, standard deviation, minimum and maximal + accuracy scores and round trip time ## Diagram @@ -100,3 +101,49 @@ Accuracy statistical summary: This example shows how the CLI Output of the tool, analyzing search results from Azure Search and providing an accuracy score for Finesse. + +## Scripts + +### XLSX Converter to JSON πŸ“„ + +This script converts data from an Excel file (.xlsx) into JSON format. It is +used for questions generated created by non-developers. Excel files are easier +to read than JSON files. + +### Usage + +1. **Input Excel File**: Place the Excel file containing the data in the + specified input folder (`--input-folder`). By default, the input folder is + set to `'finesse/scripts/input/'`. + +2. **Output Folder**: Specify the folder where the resulting JSON files will be + saved using the `--output-folder` argument. By default, the output folder is + set to `'finesse/scripts/output/'`. + +3. **Input File Name**: Provide the name of the input Excel file using the + `--file-name` argument.. + +4. **Worksheet Name**: Specify the name of the worksheet containing the data + using the `--sheet-name` argument. By default, it is set to `'To fill'`. + +### Example Command + +```bash +python finesse/scripts/xlsx_converter_json.py --input-folder finesse/scripts/input/ --output-folder finesse/scripts/output/ --file-name Finesse_questions_for_testing.xlsx --sheet-name "To fill" +``` + +Replace `'example.xlsx'` with the actual name of your input Excel file and +`'Sheet1'` with the name of the worksheet containing the data. + +### Output + +The script generates individual JSON files for each row of data in the specified +output folder. Each JSON file contains the following fields: + +- `question`: The question extracted from the Excel file. +- `answer`: The answer extracted from the Excel file. +- `title`: The title(s) extracted from specified columns in the Excel file. +- `url`: The URL(s) extracted from specified columns in the Excel file. + +Upon completion, the script prints "Conversion terminΓ©e !" (Conversion +completed!) to indicate that the conversion process is finished. diff --git a/finesse/accuracy_functions.py b/finesse/accuracy_functions.py index 906f476..5c9bdff 100644 --- a/finesse/accuracy_functions.py +++ b/finesse/accuracy_functions.py @@ -4,23 +4,49 @@ import os from collections import namedtuple import regex as re +from finesse.bing_search import BingSearch +from dotenv import load_dotenv OUTPUT_FOLDER = "./finesse/output" AccuracyResult = namedtuple("AccuracyResult", ["position", "total_pages", "score"]) -def calculate_accuracy(responses_url: list[str], expected_url: str) -> AccuracyResult: +def calculate_accuracy(responses_url: list[str], expected_url: list | str) -> AccuracyResult: + """ + Calculates the accuracy of the responses by comparing the URLs of the responses with the expected URL. + + Args: + responses_url (list[str]): A list of URLs representing the responses. + expected_url (list[str] | str): The expected URL or a list of expected URLs. + + Returns: + AccuracyResult: An object containing the position, total pages, and score of the accuracy calculation. + """ position: int = 0 total_pages: int = len(responses_url) score: float = 0.0 - expected_number = int(re.findall(r'/(\d+)/', expected_url)[0]) + expected_number = [] + + PATTERN = r'/(\d+)/' + if isinstance(expected_url, list): + for url in expected_url: + if url.startswith("https://inspection.canada.ca"): + number = int(re.findall(PATTERN, url)[0]) + expected_number.append(number) + elif isinstance(expected_url, str) and expected_url.startswith("https://inspection.canada.ca"): + number = int(re.findall(PATTERN, expected_url)[0]) + expected_number.append(number) for idx, response_url in enumerate(responses_url): - response_number = int(re.findall(r'/(\d+)/', response_url)[0]) - if response_number == expected_number: - position = idx - score = 1 - (position / total_pages) - score = round(score, 2) - break + if response_url.startswith("https://inspection.canada.ca"): + try: + response_number = int(re.findall(PATTERN, response_url)[0]) + if response_number in expected_number: + position = idx + score = 1 - (position / total_pages) + score= round(score, 2) + break + except IndexError: + pass return AccuracyResult(position, total_pages, score) @@ -33,23 +59,53 @@ def save_to_markdown(test_data: dict, engine: str): with open(output_file, "w") as md_file: md_file.write(f"# Test on the {engine} search engine: {date_string}\n\n") md_file.write("## Test data table\n\n") - md_file.write("| πŸ“„ File | πŸ’¬ Question | πŸ“ Accuracy Score | βŒ› Time |\n") - md_file.write("|--------------------|-------------------------------------------------------------------------------------------------------------------------|----------------|----------|\n") + md_file.write("| πŸ“„ File | πŸ’¬ Question| πŸ”Ž Finesse Accuracy Score | 🌐 Bing Accuracy Score | 🌐 Filtered Bing Accuracy Score |βŒ› Finesse Time | βŒ› Bing Time | βŒ› Filtered Bing Time |\n") + md_file.write("|---|---|---|---|---|---|---|---|\n") for key, value in test_data.items(): - md_file.write(f"| {key} | [{value.get('question')}]({value.get('expected_page').get('url')}) | {value.get('accuracy')*100:.0f}% | {int(value.get('time'))}ms |\n") + question = "" + if isinstance(value.get("expected_page").get("url"), list): + question = f"{value.get('question')} " + for index, url in enumerate(value.get("expected_page").get("url")): + question += f"\| [Link{index+1}]({url}) " + question += "\|" + else: + question = f"[{value.get('question')}]({value.get('expected_page').get('url')})" + md_file.write(f"| {key} | {question} | {int(value.get('accuracy')*100)}% | {int(value.get('bing_accuracy')*100)}% |{int(value.get('bing_filtered_accuracy')*100)}% |{int(value.get('time'))}ms | {int(value.get('bing_time'))}ms | {int(value.get('bing_filtered_time'))}ms |\n") + md_file.write("\n") md_file.write(f"Tested on {len(test_data)} files.\n\n") - time_stats, accuracy_stats = calculate_statistical_summary(test_data) + time_stats, accuracy_stats, bing_accuracy_stats, bing_time_stats, bing_filtered_accuracy_stats, bing_filtered_time_stats = calculate_statistical_summary(test_data) md_file.write("## Statistical summary\n\n") - md_file.write("| Statistic | Time | Accuracy score|\n") - md_file.write("|-----------------------|------------|---------|\n") - md_file.write(f"|Mean| {int(time_stats.get('Mean'))}ms | {int(accuracy_stats.get('Mean')*100)}% |\n") - md_file.write(f"|Median| {int(time_stats.get('Median'))}ms | {int(accuracy_stats.get('Median')*100)}% |\n") - md_file.write(f"|Standard Deviation| {int(time_stats.get('Standard Deviation'))}ms | {int(accuracy_stats.get('Standard Deviation')*100)}% |\n") - md_file.write(f"|Maximum| {int(time_stats.get('Maximum'))}ms | {int(accuracy_stats.get('Maximum')*100)}% |\n") - md_file.write(f"|Minimum| {int(time_stats.get('Minimum'))}ms | {int(accuracy_stats.get('Minimum')*100)}% |\n") - md_file.write(f"\nThere are a total of {len([result.get('accuracy') for result in test_data.values() if result.get('accuracy') == 0])} null scores\n") + md_file.write("| Statistic\Engine | πŸ”Ž Finesse Accuracy score| 🌐 Bing Accuracy Score | 🌐 Filtered Bing Accuracy Score |βŒ› Finesse Time | βŒ› Bing Time | βŒ› Filtered Bing Time |\n") + md_file.write("|---|---|---|---|---|---|---|\n") + for stat in ["Mean", "Median", "Standard Deviation", "Maximum", "Minimum"]: + md_file.write(f"|{stat}| {accuracy_stats.get(stat)}% | {bing_accuracy_stats.get(stat)}% | {bing_filtered_accuracy_stats.get(stat)}% |{time_stats.get(stat)}ms | {bing_time_stats.get(stat)}ms | {bing_filtered_time_stats.get(stat)}ms |\n") + + md_file.write("\n## Count of null and top scores\n\n") + md_file.write("| Score\Engine | πŸ”Ž Finesse Accuracy score| 🌐 Bing Accuracy Score | 🌐 Filtered Bing Accuracy Score |\n") + md_file.write("|---|---|---|---|\n") + finesse_null, finesse_top = count_null_top_scores({key: value.get("accuracy") for key, value in test_data.items()}) + bing_null, bing_top = count_null_top_scores({key: value.get("bing_accuracy") for key, value in test_data.items()}) + bing_filtered_null, bing_filtered_top = count_null_top_scores({key: value.get("bing_filtered_accuracy") for key, value in test_data.items()}) + + md_file.write(f"| Null (0%) | {finesse_null} | {bing_null} |{bing_filtered_null} |\n") + md_file.write(f"| Top (100%)| {finesse_top} | {bing_top} |{bing_filtered_top} |\n") + +def count_null_top_scores(accuracy_scores: dict): + """ + Counts the number of null scores and top scores in the given accuracy_scores dictionary. + + Args: + accuracy_scores (dict): A dictionary containing accuracy scores. + + Returns: + tuple: A tuple containing the count of null scores and top scores, respectively. + """ + null_scores = len([score for score in accuracy_scores.values() if score == 0]) + top_scores = len([score for score in accuracy_scores.values() if score == 1]) + + return null_scores, top_scores def save_to_csv(test_data: dict, engine: str): if not os.path.exists(OUTPUT_FOLDER): @@ -59,59 +115,124 @@ def save_to_csv(test_data: dict, engine: str): output_file = os.path.join(OUTPUT_FOLDER, file_name) with open(output_file, "w", newline="") as csv_file: writer = csv.writer(csv_file) - writer.writerow(["File", "Question", "Accuracy Score", "Time"]) + writer.writerow(["File", "Question", "Finesse Accuracy Score", "Bing Accuracy Score", "Filtered Bing Accuracy Score", "Finesse Time", "Bing Time", "Filtered Bing Time"]) for key, value in test_data.items(): + question = "" + if isinstance(value.get("expected_page").get("url"), list): + question = f"{value.get('question')} " + for index, url in enumerate(value.get("expected_page").get("url")): + question += f"[{index+1}]({url}) " + else: + question = f"[{value.get('question')}]({value.get('expected_page').get('url')})" writer.writerow([ key, - value.get("question"), - f"{value.get('accuracy')}", - f"{int(value.get('time'))}" + question, + f"{int(value.get('accuracy')*100)}%", + f"{int(value.get('bing_accuracy')*100)}%", + f"{int(value.get('bing_filtered_accuracy')*100)}%", + f"{int(value.get('time'))}ms", + f"{int(value.get('bing_time'))}ms", + f"{int(value.get('bing_filtered_time'))}ms" ]) writer.writerow([]) - time_stats, accuracy_stats = calculate_statistical_summary(test_data) - writer.writerow(["Statistic", "Time", "Accuracy Score"]) - writer.writerow(["Mean", f"{int(time_stats.get('Mean'))}", f"{int(accuracy_stats.get('Mean'))}"]) - writer.writerow(["Median", f"{int(time_stats.get('Median'))}", f"{int(accuracy_stats.get('Median'))}"]) - writer.writerow(["Standard Deviation", f"{int(time_stats.get('Standard Deviation'))}", f"{int(accuracy_stats.get('Standard Deviation'))}"]) - writer.writerow(["Maximum", f"{int(time_stats.get('Maximum'))}", f"{int(accuracy_stats.get('Maximum'))}"]) - writer.writerow(["Minimum", f"{int(time_stats.get('Minimum'))}", f"{int(accuracy_stats.get('Minimum'))}"]) - -def log_data(test_data: dict): - for key, value in test_data.items(): - print("File:", key) - print("Question:", value.get("question")) - print("Expected URL:", value.get("expected_page").get("url")) - print(f'Accuracy Score: {int(value.get("accuracy")*100)}%') - print(f'Time: {int(value.get("time"))}ms') - print() - time_stats, accuracy_stats = calculate_statistical_summary(test_data) - print("---") - print(f"Tested on {len(test_data)} files.") - print("Time statistical summary:", end="\n ") - for key,value in time_stats.items(): - print(f"{key}:{int(value)},", end=' ') - print("\nAccuracy statistical summary:", end="\n ") - for key,value in accuracy_stats.items(): - print(f"{key}:{int(value*100)}%,", end=' ') - print("\n---") - - -def calculate_statistical_summary(test_data: dict) -> tuple[dict, dict]: + time_stats, accuracy_stats, bing_accuracy_stats, bing_time_stats, bing_filtered_accuracy_stats, bing_filtered_time_stats = calculate_statistical_summary(test_data) + writer.writerow(["Statistic", "Finesse Accuracy Score", "Bing Accuracy Score", "Filtered Bing Accuracy Score", "Finesse Time", "Bing Time", "Filtered Bing Time"]) + writer.writerow(["Mean", f"{accuracy_stats.get('Mean')}%", f"{bing_accuracy_stats.get('Mean')}%", f"{bing_filtered_accuracy_stats.get('Mean')}%", f"{time_stats.get('Mean')}ms", f"{bing_time_stats.get('Mean')}ms", f"{bing_filtered_time_stats.get('Mean')}ms"]) + writer.writerow(["Median", f"{accuracy_stats.get('Median')}%", f"{bing_accuracy_stats.get('Median')}%", f"{bing_filtered_accuracy_stats.get('Median')}%", f"{time_stats.get('Median')}ms", f"{bing_time_stats.get('Median')}ms", f"{bing_filtered_time_stats.get('Median')}ms"]) + writer.writerow(["Standard Deviation", f"{accuracy_stats.get('Standard Deviation')}%", f"{bing_accuracy_stats.get('Standard Deviation')}%", f"{bing_filtered_accuracy_stats.get('Standard Deviation')}%", f"{time_stats.get('Standard Deviation')}ms", f"{bing_time_stats.get('Standard Deviation')}ms", f"{bing_filtered_time_stats.get('Standard Deviation')}ms"]) + writer.writerow(["Maximum", f"{accuracy_stats.get('Maximum')}%", f"{bing_accuracy_stats.get('Maximum')}%", f"{bing_filtered_accuracy_stats.get('Maximum')}%", f"{time_stats.get('Maximum')}ms", f"{bing_time_stats.get('Maximum')}ms", f"{bing_filtered_time_stats.get('Maximum')}ms"]) + writer.writerow(["Minimum", f"{accuracy_stats.get('Minimum')}%", f"{bing_accuracy_stats.get('Minimum')}%", f"{bing_filtered_accuracy_stats.get('Minimum')}%", f"{time_stats.get('Minimum')}ms", f"{bing_time_stats.get('Minimum')}ms", f"{bing_filtered_time_stats.get('Minimum')}ms"]) + +def calculate_statistical_summary(test_data: dict) -> tuple[dict, dict, dict, dict, dict, dict]: + """ + Calculate the statistical summary of the test data. + + Args: + test_data (dict): A dictionary containing the test data. + + Returns: + tuple[dict, dict, dict, dict, dict, dict]: A tuple containing the statistical summary for different metrics. + The tuple contains the following dictionaries: + - time_stats: Statistical summary for the 'time' metric. + - accuracy_stats: Statistical summary for the 'accuracy' metric. + - bing_accuracy_stats: Statistical summary for the 'bing_accuracy' metric. + - bing_times_stats: Statistical summary for the 'bing_times' metric. + - bing_filtered_accuracy_stats: Statistical summary for the 'bing_filtered_accuracy' metric. + - bing_filtered_times_stats: Statistical summary for the 'bing_filtered_times' metric. + """ + def calculate_stats(data: list) -> dict: + stats = { + "Mean": statistics.mean(data), + "Median": statistics.median(data), + "Standard Deviation": statistics.stdev(data), + "Maximum": max(data), + "Minimum": min(data), + } + return stats + + def round_values(stats: dict) -> dict: + return {key: int(round(value, 3)) for key, value in stats.items()} + + def convert_to_percentage(stats: dict) -> dict: + return {key: int(round(value * 100, 2)) for key, value in stats.items()} + times = [result.get("time") for result in test_data.values()] accuracies = [result.get("accuracy") for result in test_data.values()] - time_stats = { - "Mean": round(statistics.mean(times), 3), - "Median": round(statistics.median(times), 3), - "Standard Deviation": round(statistics.stdev(times), 3), - "Maximum": round(max(times), 3), - "Minimum": round(min(times), 3), - } - accuracy_stats = { - "Mean": round(statistics.mean(accuracies), 2), - "Median": round(statistics.median(accuracies), 2), - "Standard Deviation": round(statistics.stdev(accuracies), 2), - "Maximum": round(max(accuracies), 2), - "Minimum": round(min(accuracies), 2), - } - return time_stats, accuracy_stats + bing_accuracies = [result.get("bing_accuracy") for result in test_data.values()] + bing_times = [result.get("bing_time") for result in test_data.values()] + bing_filtered_accuracies = [result.get("bing_filtered_accuracy") for result in test_data.values()] + bing_filtered_times = [result.get("bing_filtered_time") for result in test_data.values()] + + time_stats = calculate_stats(times) + accuracy_stats = calculate_stats(accuracies) + bing_accuracy_stats = calculate_stats(bing_accuracies) + bing_times_stats = calculate_stats(bing_times) + bing_filtered_accuracy_stats = calculate_stats(bing_filtered_accuracies) + bing_filtered_times_stats = calculate_stats(bing_filtered_times) + + time_stats = round_values(time_stats) + bing_times_stats = round_values(bing_times_stats) + bing_filtered_times_stats = round_values(bing_filtered_times_stats) + bing_accuracy_stats = convert_to_percentage(bing_accuracy_stats) + accuracy_stats = convert_to_percentage(accuracy_stats) + bing_filtered_accuracy_stats = convert_to_percentage(bing_filtered_accuracy_stats) + + return time_stats, accuracy_stats, bing_accuracy_stats, bing_times_stats, bing_filtered_accuracy_stats, bing_filtered_times_stats + +def update_dict_bing_data(test_data: dict): + """ + Updates the given test_data dictionary with the bing accuracy results. + + Args: + test_data (dict): The dictionary containing the test data. + """ + copy_data = test_data.copy() + load_dotenv() + endpoint = os.getenv("BING_ENDPOINT") + subscription_key = os.getenv("BING_SEARCH_KEY") + search_engine = BingSearch(endpoint, subscription_key) + count = 1 + for key, value in copy_data.items(): + question = value.get("question") + expected_url = value.get("expected_page").get("url") + top = value.get("top") + response_url, time_elapsed = search_engine.search_urls(question, top) + accuracy_result = calculate_accuracy(response_url, expected_url) + test_data[key]["bing_accuracy"] = accuracy_result.score + test_data[key]["bing_time"] = time_elapsed + print(f"{count} files are done") + count += 1 + + print("Second Bing Search Test") + count = 1 + for key, value in copy_data.items(): + question = f"site:inspection.canada.ca {value.get('question')}" + expected_url = value.get("expected_page").get("url") + top = value.get("top") + response_url, time_elapsed = search_engine.search_urls(question, top) + accuracy_result = calculate_accuracy(response_url, expected_url) + test_data[key]["bing_filtered_accuracy"] = accuracy_result.score + test_data[key]["bing_filtered_time"] = time_elapsed + print(f"{count} files are done") + count += 1 diff --git a/finesse/bing_search.py b/finesse/bing_search.py new file mode 100644 index 0000000..a9c5016 --- /dev/null +++ b/finesse/bing_search.py @@ -0,0 +1,42 @@ +from azure.cognitiveservices.search.websearch import WebSearchClient +from msrest.authentication import CognitiveServicesCredentials +import time +import statistics +class BingSearch(): + """ + A class for performing web searches using the Bing Search API. + """ + + def __init__(self, endpoint, subscription_key): + self.endpoint = endpoint + self.subscription_key = subscription_key + self.client = WebSearchClient(endpoint=self.endpoint, credentials=CognitiveServicesCredentials(self.subscription_key)) + self.client.config.base_url = '{Endpoint}/v7.0' # Temporary change to fix the error. Issue opened https://github.com/Azure/azure-sdk-for-python/issues/34917 + + def search_urls(self, query: str, num_results: int = 100) -> tuple[list[str], float]: + """ + Search for URLs using the Bing Search API. + + Args: + query (str): The search query. + num_results (int, optional): The number of results to retrieve. Defaults to 100. + + Returns: + tuple[list[str], float]: A tuple containing a list of URLs and the average elapsed time for the search. + """ + urls = [] + elapsed_time = [] + offset = 0 + # Limit of 50 results per query and Bing Search return less than 50 web results + while len(urls) < num_results: + start_time = time.time() + web_data = self.client.web.search(query=query, market="en-ca", count=50, response_filter=["Webpages"], offset=offset) + elapsed_time.append(time.time() - start_time) + if hasattr(web_data, 'web_pages') and web_data.web_pages is not None: + urls.extend([item.url for item in web_data.web_pages.value]) + try: + offset += len([item.url for item in web_data.web_pages.value]) + except AttributeError: + break + urls = urls[:num_results] + return urls, statistics.mean(elapsed_time) * 1000 diff --git a/finesse/finesse_test.py b/finesse/finesse_test.py index d36fb99..8ed9aee 100644 --- a/finesse/finesse_test.py +++ b/finesse/finesse_test.py @@ -2,9 +2,11 @@ from jsonreader import JSONReader import os import json -from accuracy_functions import save_to_markdown, save_to_csv, log_data, calculate_accuracy +from accuracy_functions import save_to_markdown, save_to_csv, calculate_accuracy, update_dict_bing_data from host import is_host_up +global_test_data = dict() +settings = dict() class NoTestDataError(Exception): """Raised when all requests have failed and there is no test data""" @@ -58,12 +60,11 @@ def search_accuracy(self): for page in response_pages: response_url.append(page.get("url")) accuracy_result = calculate_accuracy(response_url, expected_url) - time_taken = round(response.elapsed.microseconds/1000,3) - + time_taken = round(response.elapsed.total_seconds()*1000,3) expected_page = json_data.copy() del expected_page['question'] del expected_page['answer'] - self.qna_results[file_name] = { + global_test_data[file_name] = { "question": question, "expected_page": expected_page, "response_pages": response_pages, @@ -71,22 +72,16 @@ def search_accuracy(self): "total_pages": accuracy_result.total_pages, "accuracy": accuracy_result.score, "time": time_taken, + "top": self.top, } def on_start(self): self.qna_reader = JSONReader(self.path) - self.qna_results = dict() def on_stop(self): - if not self.qna_results: + if not global_test_data: raise NoTestDataError - log_data(self.qna_results) - if self.format == "md": - save_to_markdown(self.qna_results, self.engine) - elif self.format == "csv": - save_to_csv(self.qna_results, self.engine) - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.path = self.environment.parsed_options.path @@ -94,3 +89,19 @@ def __init__(self, *args, **kwargs): self.format = self.environment.parsed_options.format self.once = self.environment.parsed_options.once self.top = self.environment.parsed_options.top + settings["engine"] = self.engine + settings["format"] = self.format + settings["once"] = self.once + settings["top"] = self.top + settings["path"] = self.path + + +@events.quit.add_listener +def quit(**_kwargs): + print("Search accuracy test completed") + print("Starting bing search test") + update_dict_bing_data(global_test_data) + if settings.get("format") == "md": + save_to_markdown(global_test_data, "azure") + elif settings.get("format") == "csv": + save_to_csv(global_test_data, settings.get("engine")) diff --git a/finesse/jsonreader.py b/finesse/jsonreader.py index 8f1a8ef..224e964 100644 --- a/finesse/jsonreader.py +++ b/finesse/jsonreader.py @@ -1,13 +1,14 @@ import json from typing import Iterator import os - +from natsort import natsorted class JSONReader(Iterator): "Read test data from JSON files using an iterator" def __init__(self, directory): self.directory = directory - self.file_list = sorted([f for f in os.listdir(directory) if f.endswith('.json')]) + self.file_list = natsorted([f for f in os.listdir(directory) if f.endswith('.json')]) + if not self.file_list: raise FileNotFoundError(f"No JSON files found in the directory '{directory}'") self.current_file_index = 0 diff --git a/finesse/scripts/xlsx_converter_json.py b/finesse/scripts/xlsx_converter_json.py new file mode 100644 index 0000000..e455c66 --- /dev/null +++ b/finesse/scripts/xlsx_converter_json.py @@ -0,0 +1,56 @@ +import openpyxl +import os +import json +import argparse + +parser = argparse.ArgumentParser(description='XLSX Converter to JSON') +parser.add_argument('--input-folder', dest='input_folder', default='finesse/scripts/input/', help='Path to the input folder') +parser.add_argument('--output-folder', dest='output_folder', default='finesse/scripts/output/', help='Path to the output folder') +parser.add_argument('--file-name', dest='file_name', help='Name of the input file') +parser.add_argument('--sheet-name', dest='sheet_name', default='To fill', help='Name of the worksheet') + +args = parser.parse_args() + +INPUT_FOLDER = args.input_folder +OUTPUT_FOLDER = args.output_folder +FILE_NAME = args.file_name +SHEET_NAME = args.sheet_name +FILE_PATH = os.path.join(INPUT_FOLDER, FILE_NAME) + +workbook = openpyxl.load_workbook(FILE_PATH) +worksheet = workbook.active +count = 1 + +for row in range(5, worksheet.max_row + 1): + question = worksheet.cell(row=row, column=2).value + if question is None: + continue + + answer = worksheet.cell(row=row, column=3).value + + titles = [] + links = [] + for col in range(5, 10): + title = worksheet.cell(row=row, column=col).value + link = worksheet.cell(row=row, column=col).hyperlink + if title: + titles.append(title) + if link: + links.append(link.target) + + data = { + 'question': question or "", + 'answer': answer or "", + 'title': titles[0] if len(titles) == 1 else titles or "", + 'url': links[0] if len(links) == 1 else links or "" + } + + # Enregistrement du fichier JSON + output_file = os.path.join(OUTPUT_FOLDER, f'question_{count}.json') + if not os.path.exists(OUTPUT_FOLDER): + os.makedirs(OUTPUT_FOLDER) + with open(output_file, 'w', encoding='utf-8') as json_file: + json.dump(data, json_file, ensure_ascii=False, indent=4) + count += 1 + +print("Conversion completed successfully!") diff --git a/requirements.txt b/requirements.txt index ff525a0..90effa8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,7 @@ locust regex +python-dotenv +azure-cognitiveservices-search-websearch +msrest +openpyxl +natsort diff --git a/tests/test_accuracy_functions.py b/tests/test_accuracy_functions.py index f50e357..b11fe57 100644 --- a/tests/test_accuracy_functions.py +++ b/tests/test_accuracy_functions.py @@ -16,5 +16,23 @@ def test_calculate_accuracy(self): self.assertEqual(result.total_pages, 4) self.assertEqual(result.score, 0.75) + def test_calculate_accuracy_multiple_expected_urls(self): + responses_url = [ + "https://inspection.canada.ca/exporting-food-plants-or-animals/food-exports/food-specific-export-requirements/meat/crfpcp/eng/1434119937443/1434120400252", + "https://inspection.canada.ca/protection-des-vegetaux/especes-envahissantes/directives/date/d-08-04/fra/1323752901318/1323753612811", + "https://inspection.canada.ca/varietes-vegetales/vegetaux-a-caracteres-nouveaux/demandeurs/directive-94-08/documents-sur-la-biologie/lens-culinaris-medikus-lentille-/fra/1330978380871/1330978449837", + "https://inspection.canada.ca/protection-des-vegetaux/especes-envahissantes/directives/date/d-96-15/fra/1323854808025/1323854941807" + ] + expected_urls = [ + "https://inspection.canada.ca/animal-health/terrestrial-animals/exports/pets/brunei-darussalam/eng/1475849543824/1475849672294", + "https://inspection.canada.ca/animal-health/terrestrial-animals/exports/pets/eu-commercial-/instructions/eng/1447782811647/1447782887583", + "https://inspection.canada.ca/protection-des-vegetaux/especes-envahissantes/directives/date/d-96-15/fra/1323854808025/1323854941807", + "https://inspection.canada.ca/varietes-vegetales/vegetaux-a-caracteres-nouveaux/demandeurs/directive-94-08/documents-sur-la-biologie/lens-culinaris-medikus-lentille-/fra/1330978380871/1330978449837" + ] + result = calculate_accuracy(responses_url, expected_urls) + self.assertEqual(result.position, 2) + self.assertEqual(result.total_pages, 4) + self.assertEqual(result.score, 0.5) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_bing_search.py b/tests/test_bing_search.py new file mode 100644 index 0000000..4c73a7a --- /dev/null +++ b/tests/test_bing_search.py @@ -0,0 +1,22 @@ +import unittest +from finesse.bing_search import BingSearch +from dotenv import load_dotenv +import os +class TestBingSearch(unittest.TestCase): + def test_search_urls(self): + load_dotenv() + endpoint = os.getenv("BING_ENDPOINT") + subscription_key = os.getenv("BING_SEARCH_KEY") + bing_search = BingSearch(endpoint, subscription_key) + + query = "Canadian Food Inspection Agency" + num_results = 100 + + urls, elapsed_time = bing_search.search_urls(query, num_results) + + self.assertEqual(len(urls), num_results) + self.assertTrue(all(url.startswith("http") for url in urls)) + self.assertIsInstance(elapsed_time, float) + +if __name__ == "__main__": + unittest.main()