From 6e4dd81b76a23b5c98e1d2ff15eeb63336aa59ef Mon Sep 17 00:00:00 2001
From: cvermand <33010418+bidoubiwa@users.noreply.github.com>
Date: Wed, 3 Jun 2020 15:10:51 +0200
Subject: [PATCH] Immutable settings (#22)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Make setting immutable between each scrape

* Validate that custom settings is a dictionary

* Change value of custom settings in tests

* lint

* Removed unecessary try and catch

* Update scraper/src/meilisearch_helper.py

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>
---
 scraper/src/config/config_validator.py      | 5 +++++
 scraper/src/index.py                        | 1 +
 scraper/src/meilisearch_helper.py           | 5 +++--
 scraper/src/tests/config_loader/abstract.py | 2 +-
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/scraper/src/config/config_validator.py b/scraper/src/config/config_validator.py
index f432d0d..70d2291 100644
--- a/scraper/src/config/config_validator.py
+++ b/scraper/src/config/config_validator.py
@@ -25,6 +25,11 @@ def validate(self):
                                                     list):
             raise Exception('stop_urls should be list')
 
+        # Custom settings must be a dict
+        if self.config.custom_settings and not isinstance(self.config.custom_settings,
+                                                    dict):
+            raise Exception('custom_settings must be a dictionary')
+
         if self.config.js_render and not isinstance(self.config.js_render,
                                                     bool):
             raise Exception('js_render should be boolean')
diff --git a/scraper/src/index.py b/scraper/src/index.py
index 2b06928..7472544 100644
--- a/scraper/src/index.py
+++ b/scraper/src/index.py
@@ -40,6 +40,7 @@ def run_config(config):
         config.app_id,
         config.api_key,
         config.index_uid,
+        config.custom_settings
     )
 
     root_module = 'src.' if __name__ == '__main__' else 'scraper.src.'
diff --git a/scraper/src/meilisearch_helper.py b/scraper/src/meilisearch_helper.py
index 1e40e06..3383f90 100644
--- a/scraper/src/meilisearch_helper.py
+++ b/scraper/src/meilisearch_helper.py
@@ -101,11 +101,12 @@ class MeiliSearchHelper:
         'acceptNewFields': False
     }
 
-    def __init__(self, host_url, api_key, index_uid):
+    def __init__(self, host_url, api_key, index_uid, custom_settings):
         self.meilisearch_client = meilisearch.Client(host_url, api_key)
         self.__delete_and_create_index(index_uid)
         self.meilisearch_index = self.__delete_and_create_index(index_uid)
-        self.meilisearch_index.update_settings(MeiliSearchHelper.SETTINGS)
+        settings = {**MeiliSearchHelper.SETTINGS, **custom_settings}
+        self.meilisearch_index.update_settings(settings)
 
     def add_records(self, records, url, from_sitemap):
         """Add new records to the index"""
diff --git a/scraper/src/tests/config_loader/abstract.py b/scraper/src/tests/config_loader/abstract.py
index 198cf6e..c9f230f 100644
--- a/scraper/src/tests/config_loader/abstract.py
+++ b/scraper/src/tests/config_loader/abstract.py
@@ -7,7 +7,7 @@ def config(additional_config={}):
         'allowed_domains': 'allowed_domains',
         'api_key': 'api_key',
         'app_id': 'app_id',
-        'custom_settings': 'custom_settings',
+        'custom_settings': {},
         'hash_strategy': 'hash_strategy',
         'index_uid': 'index_uid',
         'selectors': [],