Merge pull request #111 from geonetwork/elasticssearch-8-upgrade

Elasticsearch 8 migration.
geonetwork · Mar 14, 2024 · b84f16c · b84f16c
2 parents 4dff66a + 31efe6f
commit b84f16c
Show file tree

Hide file tree

Showing 28 changed files with 71 additions and 80 deletions.
diff --git a/.env b/.env
@@ -1,6 +1,6 @@
 # docker-compose default environment variable values
 COMPOSE_PROJECT_NAME=gncloud
-TAG=4.2.8-0
+TAG=4.4.3-0
 EUREKA_SERVER_URL=http://discovery:8761/eureka
 
 # gn metadta postgres environment variables

diff --git a/modules/library/common-error/pom.xml b/modules/library/common-error/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>gn-cloud-lib</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/modules/library/common-index-model/pom.xml b/modules/library/common-index-model/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>gn-cloud-lib</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/modules/library/common-persistence/pom.xml b/modules/library/common-persistence/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>gn-cloud-lib</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/modules/library/common-search/pom.xml b/modules/library/common-search/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>gn-cloud-lib</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/modules/library/common-search/src/main/resources/application.yml b/modules/library/common-search/src/main/resources/application.yml
@@ -3,7 +3,7 @@ gn:
     # Full text on all fields
     # 'queryBase': '${any}',
     # Full text but more boost on title match
-    queryFilter: '+isTemplate:n AND -indexingError:true'
+    queryFilter: '+isTemplate:n'
     queryBase: '(any.\*:(${any}) OR resourceTitleObject.\*:(${any})^2)'
     trackTotalHits: true
     scoreConfig: >

diff --git a/modules/library/common-security/pom.xml b/modules/library/common-security/pom.xml
@@ -11,7 +11,7 @@
   <parent>
     <artifactId>gn-cloud-lib</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-cloud-common-security</artifactId>

diff --git a/modules/library/common-standards/pom.xml b/modules/library/common-standards/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>gn-cloud-lib</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/modules/library/common-utility/pom.xml b/modules/library/common-utility/pom.xml
@@ -11,7 +11,7 @@
   <parent>
     <artifactId>gn-cloud-lib</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-cloud-common-utility</artifactId>

diff --git a/modules/library/common-view/pom.xml b/modules/library/common-view/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>gn-cloud-lib</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/modules/library/pom.xml b/modules/library/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>gn-cloud-parent</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <artifactId>gn-cloud-lib</artifactId>
   <packaging>pom</packaging>

diff --git a/modules/pom.xml b/modules/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>gn-cloud-microservices</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <artifactId>gn-cloud-parent</artifactId>
   <packaging>pom</packaging>

diff --git a/modules/services/authorizing/pom.xml b/modules/services/authorizing/pom.xml
@@ -10,7 +10,7 @@
   <parent>
     <artifactId>gn-cloud-services</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

diff --git a/modules/services/gateway/pom.xml b/modules/services/gateway/pom.xml
@@ -11,7 +11,7 @@
   <parent>
     <artifactId>gn-cloud-services</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-cloud-gateway</artifactId>

diff --git a/modules/services/indexing/pom.xml b/modules/services/indexing/pom.xml
@@ -11,7 +11,7 @@
   <parent>
     <artifactId>gn-cloud-services</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-cloud-indexing</artifactId>
@@ -97,12 +97,8 @@
       <artifactId>camel-micrometer-starter</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-high-level-client</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>org.elasticsearch</groupId>
-      <artifactId>elasticsearch</artifactId>
+      <groupId>co.elastic.clients</groupId>
+      <artifactId>elasticsearch-java</artifactId>
     </dependency>
 
     <!--  test dependencies  -->

diff --git a/...s/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java b/...s/services/indexing/src/main/java/org/fao/geonet/indexing/controller/IndexController.java
@@ -45,7 +45,7 @@ public ResponseEntity<?> indexAll(
   @PostMapping("/event/{bucket}/{uuid}")
   public void sendEvent(
       @PathVariable("bucket")String bucket, 
-      @PathVariable("uuid") String uuid) throws Exception {
+      @PathVariable("uuid") String uuid) {
     eventStreamService.produceEvent(bucket, uuid);
   }
 }
diff --git a/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java b/modules/services/indexing/src/main/java/org/fao/geonet/indexing/service/IndexingService.java
@@ -6,9 +6,15 @@
 
 package org.fao.geonet.indexing.service;
 
-import static org.elasticsearch.rest.RestStatus.CREATED;
-import static org.elasticsearch.rest.RestStatus.OK;
-
+import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.AcknowledgedResponse;
+import co.elastic.clients.elasticsearch._types.ElasticsearchException;
+import co.elastic.clients.elasticsearch._types.Refresh;
+import co.elastic.clients.elasticsearch.core.BulkRequest;
+import co.elastic.clients.elasticsearch.core.BulkResponse;
+import co.elastic.clients.elasticsearch.core.IndexRequest;
+import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
+import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.File;
@@ -28,15 +34,7 @@
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.camel.Exchange;
-import org.elasticsearch.ElasticsearchStatusException;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.support.master.AcknowledgedResponse;
 import org.elasticsearch.client.RequestOptions;
-import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.fao.geonet.common.xml.XsltUtil;
 import org.fao.geonet.domain.AbstractMetadata;
 import org.fao.geonet.domain.Metadata;
@@ -63,16 +61,19 @@ public class IndexingService {
   MetadataRepository metadataRepository;
 
   @Autowired
-  RestHighLevelClient client;
+  ElasticsearchClient client;
 
   /**
    * Delete index.
    */
   public void deleteIndex(Exchange e) {
     try {
+      DeleteIndexRequest deleteIndexRequest = DeleteIndexRequest.of(
+          b -> b.index(index)
+      );
       AcknowledgedResponse deleteIndexResponse = client.indices()
-          .delete(new DeleteIndexRequest(index), RequestOptions.DEFAULT);
-      if (deleteIndexResponse.isAcknowledged()) {
+          .delete(deleteIndexRequest);
+      if (deleteIndexResponse.acknowledged()) {
         log.info(String.format(
             "Index %s removed.",
             index));
@@ -202,37 +203,43 @@ protected static String collectDbProperties(AbstractMetadata r) {
 
 
   private BulkRequest buildBulkRequest(IndexRecords indexRecords) {
-    BulkRequest bulkRequest = new BulkRequest(index);
+    BulkRequest.Builder requestBuilder = new BulkRequest.Builder()
+        .index(index)
+        .refresh(Refresh.True);
     ObjectMapper mapper = new ObjectMapper();
 
     indexRecords.getIndexRecord().forEach(r -> {
       try {
-        IndexRequest indexRequest = new IndexRequest();
-        indexRequest.id(r.getId());
-        indexRequest.source(mapper.writeValueAsString(r), XContentType.JSON);
-        bulkRequest.add(indexRequest);
+        String json = mapper.writeValueAsString(r);
+        requestBuilder.operations(op -> op
+            .index(idx -> idx
+                .index(index)
+                .id(r.getId())
+                .document(json)
+            )
+        );
       } catch (JsonProcessingException jsonProcessingException) {
         jsonProcessingException.printStackTrace();
       }
     });
-    return bulkRequest;
+    return requestBuilder.build();
   }
 
   private void sendToIndex(IndexRecords indexRecords,
       IndexingReport report) {
     BulkRequest bulkRequest = buildBulkRequest(indexRecords);
     try {
       // TODO: Asynchronous?
-      BulkResponse bulkItemResponses = client.bulk(bulkRequest, RequestOptions.DEFAULT);
+      BulkResponse bulkItemResponses = client.bulk(bulkRequest);
       log.info(String.format(
           "Indexing operation took %d.",
-          bulkItemResponses.getIngestTookInMillis()
+          bulkItemResponses.took()
       ));
-      if (bulkItemResponses.hasFailures()) {
+      if (bulkItemResponses.errors()) {
         AtomicInteger failureCount = new AtomicInteger();
-        Arrays.stream(bulkItemResponses.getItems()).forEach(item -> {
-          if (item.status() != OK
-              && item.status() != CREATED) {
+        bulkItemResponses.items().forEach(item -> {
+          if (item.status() != 200
+              && item.status() != 201) {
             failureCount.getAndIncrement();
             // TODO: Index error document
           }
@@ -243,16 +250,10 @@ private void sendToIndex(IndexRecords indexRecords,
             failureCount
         ));
       }
-    } catch (ElasticsearchStatusException indexException) {
-      report.setNumberOfRecordsWithIndexingErrors(indexRecords.getIndexRecord().size());
-      log.error(String.format(
-          "Error while saving records %d in index. Error is: %s.",
-          indexException.getMessage()
-      ));
-    } catch (IOException ioException) {
+    } catch (ElasticsearchException | IOException esException) {
       log.error(String.format(
           "Error while sending records to index. Error is: %s.",
-          ioException.getMessage()
+              esException.getMessage()
       ));
     }
   }

diff --git a/modules/services/ogc-api-records/README.md b/modules/services/ogc-api-records/README.md
@@ -76,7 +76,7 @@ docker run -it -p8080:8080 \
   -e "SPRING_PROFILES_ACTIVE=standalone" \
   -e "SPRING_CONFIG_LOCATION=/ogcapiconfig/" \
   -e "JAVA_OPTS=-Dfile.encoding=UTF-8" \
-  gn-cloud-ogc-api-records-service:4.2.8-0
+  gn-cloud-ogc-api-records-service:4.4.3-0
 ```
 
 or use a published release (create your configuration first like above):

diff --git a/modules/services/ogc-api-records/pom.xml b/modules/services/ogc-api-records/pom.xml
@@ -10,7 +10,7 @@
   <parent>
     <groupId>org.geonetwork-opensource.cloud</groupId>
     <artifactId>gn-cloud-services</artifactId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-cloud-ogc-api-records-service</artifactId>

diff --git a/...records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java b/...records/src/test/java/org/fao/geonet/ogcapi/records/controller/ItemApiControllerTest.java
@@ -19,6 +19,7 @@
 import org.fao.geonet.ogcapi.records.MvcConfigurer;
 import org.fao.geonet.repository.LanguageRepository;
 import org.fao.geonet.repository.MetadataRepository;
+import org.fao.geonet.repository.SettingRepository;
 import org.fao.geonet.repository.SourceRepository;
 import org.fao.geonet.repository.UiSettingsRepository;
 import org.fao.geonet.view.ViewUtility;
@@ -64,6 +65,9 @@ public class ItemApiControllerTest {
     @MockBean
     private MetadataRepository mockMetadataRepository;
 
+    @MockBean
+    private SettingRepository mockSettingRepository;
+
     @Autowired
     private MockMvc mockMvc;
 

diff --git a/modules/services/pom.xml b/modules/services/pom.xml
@@ -13,7 +13,7 @@
   <parent>
     <artifactId>gn-cloud-parent</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <artifactId>gn-cloud-services</artifactId>
   <packaging>pom</packaging>

diff --git a/modules/services/searching/pom.xml b/modules/services/searching/pom.xml
@@ -11,7 +11,7 @@
   <parent>
     <groupId>org.geonetwork-opensource.cloud</groupId>
     <artifactId>gn-cloud-services</artifactId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-cloud-searching</artifactId>
@@ -93,11 +93,6 @@
       <artifactId>lombok</artifactId>
     </dependency>
 
-    <!--<dependency>
-      <groupId>org.elasticsearch.client</groupId>
-      <artifactId>elasticsearch-rest-high-level-client</artifactId>
-    </dependency>-->
-
     <dependency>
       <groupId>org.springdoc</groupId>
       <artifactId>springdoc-openapi-common</artifactId>

diff --git a/modules/standards/iso19139/pom.xml b/modules/standards/iso19139/pom.xml
@@ -3,7 +3,7 @@
   <parent>
     <artifactId>gn-cloud-standards</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-cloud-standard-iso19139</artifactId>

diff --git a/modules/standards/pom.xml b/modules/standards/pom.xml
@@ -4,7 +4,7 @@
   <parent>
     <artifactId>gn-cloud-parent</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <artifactId>gn-cloud-standards</artifactId>
   <packaging>pom</packaging>

diff --git a/modules/support-services/configuring/pom.xml b/modules/support-services/configuring/pom.xml
@@ -10,7 +10,7 @@
   <parent>
     <artifactId>gn-cloud-support-services</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-cloud-configuring</artifactId>

diff --git a/modules/support-services/discovery/pom.xml b/modules/support-services/discovery/pom.xml
@@ -10,7 +10,7 @@
   <parent>
     <artifactId>gn-cloud-support-services</artifactId>
     <groupId>org.geonetwork-opensource.cloud</groupId>
-    <version>4.2.8-0</version>
+    <version>4.4.3-0</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>gn-cloud-discovery</artifactId>