Correções para ajustar o Opensearch

okfn-brasil · Dec 1, 2023 · 18924b1 · 18924b1
1 parent fad2c68
commit 18924b1
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 23 deletions.
diff --git a/Makefile b/Makefile
@@ -117,7 +117,7 @@ shell:
 		bash
 
 .PHONY: run
-run: create-pod database rerun
+run: create-pod opensearch database rerun
 
 .PHONY:load-data
 load-data:

diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
@@ -24,10 +24,10 @@ Já leu? Então vamos às informações específicas deste repositório:
 | Serviço   | [`themed_excerpts`](/themed_excerpts)              | Consultas ao índices de busca textual temáticos do QD.                                             | index                               |
 | Módulo    | [`database`](/database)                            | Classe de interação com bancos de dados Postgres.                                                  | Postgres                            |
 | Módulo    | [`config`](/config)                                | Configuração de variáveis de ambiente.                                                             |                                     |
-| Módulo    | [`index`](/index)                                  | Classe de interação com índices Elasticsearch.                                                     | Elasticsearch                       |
+| Módulo    | [`index`](/index)                                  | Classe de interação com índices Opensearch.                                                        | Opensearch                          |
 | Recurso   | Postgres                                           | Banco de dados de CNPJ. Contém informações sobre empresas e sócios cadastrados na Receita Federal. |                                     |
 | Recurso   | Banco de dados do [Censo](https://censo.ok.org.br) | Banco de dados de municípios. Contém metadados municipais.                                         |                                     |
-| Recurso   | Elasticsearch                                      | Índices de busca textual.                                                                          |                                     |
+| Recurso   | Opensearch                                         | Índices de busca textual.                                                                          |                                     |
 | Recurso   | Mailjet                                            | Serviço de envio de email.                                                                         |                                     |
 
 ## Como configurar o ambiente de desenvolvimento
@@ -69,4 +69,4 @@ make coverage
 ```
 
 # Mantendo
-As pessoas mantenedoras devem seguir as diretrizes do [Guia para Mantenedoras](https://github.com/okfn-brasil/querido-diario-comunidade/blob/main/.github/CONTRIBUTING.md#mantendo) do Querido Diário.
+As pessoas mantenedoras devem seguir as diretrizes do [Guia para Mantenedoras](https://github.com/okfn-brasil/querido-diario-comunidade/blob/main/.github/CONTRIBUTING.md#mantendo) do Querido Diário.
diff --git a/gazettes/gazette_access.py b/gazettes/gazette_access.py
@@ -395,7 +395,6 @@ def create_gazettes_data_gateway(
         raise Exception(
             "Query builder should implement the QueryBuilderInterface interface"
         )
-
     return GazetteSearchEngineGateway(search_engine, query_builder, index)
 
 

diff --git a/index/opensearch.py b/index/opensearch.py
@@ -3,7 +3,7 @@
 from datetime import date
 from enum import Enum, unique
 from typing import Dict, List, Union
-
+import os
 import opensearchpy
 
 

diff --git a/scripts/load_fake_gazettes.py b/scripts/load_fake_gazettes.py
@@ -10,54 +10,54 @@
 INDEX = "gazettes"
 
 
-def delete_index(es):
+def delete_index(search_engine):
     for attempt in range(3):
         try:
-            es.indices.delete(index=INDEX, ignore_unavailable=True, timeout="30s")
-            es.indices.refresh()
+            search_engine.indices.delete(index=INDEX, ignore_unavailable=True, timeout="30s")
+            search_engine.indices.refresh()
             print("Index deleted")
             return
         except Exception as e:
             print("Index deletion failed")
             time.sleep(10)
 
 
-def create_index(es):
+def create_index(search_engine):
     for attempt in range(3):
         try:
-            es.indices.create(
+            search_engine.indices.create(
                 index=INDEX,
                 body={"mappings": {"properties": {"date": {"type": "date"}}}},
-                timeout="30s",
+                timeout=30,
             )
-            es.indices.refresh()
+            search_engine.indices.refresh()
             print(f"Index {INDEX} created")
             return
         except Exception as e:
             print(f"Index creation failed: {e}")
             time.sleep(10)
 
 
-def recreate_index(es):
-    # delete_index(es)
-    create_index(es)
+def recreate_index(search_engine):
+    # delete_index(search_engine)
+    create_index(search_engine)
 
 
-def try_push_data_to_index(es, bulk_data):
+def try_push_data_to_index(search_engine, bulk_data):
     for attempt in range(3):
         try:
-            es.bulk(bulk_data, index=INDEX, refresh=True, timeout="30s")
+            search_engine.bulk(bulk_data, index=INDEX, refresh=True, timeout="30s")
             return
         except Exception as e:
             time.sleep(10)
 
 
-def add_data_on_index(data, es):
+def add_data_on_index(data, search_engine):
     bulk_data = []
     for gazette in data:
         bulk_data.append({"index": {"_index": INDEX, "_id": gazette["file_checksum"]}})
         bulk_data.append(gazette)
-    try_push_data_to_index(es, bulk_data)
+    try_push_data_to_index(search_engine, bulk_data)
     print("Index populated")
 
 

diff --git a/tests/test_opensearch.py b/tests/test_opensearch.py
@@ -592,7 +592,7 @@ def delete_index(self):
         for attempt in range(3):
             try:
                 self.search_engine.indices.delete(
-                    index=self.INDEX, ignore_unavailable=True, timeout="30s"
+                    index=self.INDEX, ignore_unavailable=True, timeout=30
                 )
                 self.search_engine.indices.refresh()
                 return
@@ -605,7 +605,7 @@ def create_index(self):
                 self.search_engine.indices.create(
                     index=self.INDEX,
                     body={"mappings": {"properties": {"date": {"type": "date"}}}},
-                    timeout="30s",
+                    timeout=30,
                 )
                 self.search_engine.indices.refresh()
                 return
@@ -619,7 +619,7 @@ def recreate_index(self):
     def try_push_data_to_index(self, bulk_data):
         for attempt in range(3):
             try:
-                self.search_engine.bulk(bulk_data, index=self.INDEX, refresh=True, timeout="30s")
+                self.search_engine.bulk(bulk_data, index=self.INDEX, refresh=True, timeout=30)
                 return
             except Exception as e:
                 time.sleep(10)