From 8d07508ae4457dc99ed49185875c1a8b53d9ff19 Mon Sep 17 00:00:00 2001 From: Nico Koprowski Date: Thu, 9 Feb 2023 14:17:56 +0100 Subject: [PATCH] fix(search): fix bug when deep paginating over legal entites and address business partners * fixes bug leading to 500 due to opensearch not supporting deep paginations * logic now distinguishes between pagination with search parameters and without * with search parameters put a limit to page number due to opensearch limitations * without search parameters just paginate in database directly without opensearch --- .../impl/service/SearchServiceImpl.kt | 55 +++++++++++++++++-- .../pool/config/OpenSearchConfigProperties.kt | 3 +- .../bpdm/pool/controller/AddressController.kt | 3 +- .../BusinessPartnerLegacyController.kt | 3 +- .../pool/controller/LegalEntityController.kt | 3 +- .../request/AddressPartnerSearchRequest.kt | 6 +- .../request/AddressPropertiesSearchRequest.kt | 6 +- .../request/BusinessPartnerSearchRequest.kt | 10 +++- .../LegalEntityPropertiesSearchRequest.kt | 6 +- .../request/SitePropertiesSearchRequest.kt | 6 +- .../exception/BpdmOpenSearchUserException.kt | 27 +++++++++ .../service/BusinessPartnerFetchService.kt | 2 + .../src/main/resources/application.properties | 1 + 13 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/exception/BpdmOpenSearchUserException.kt diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/opensearch/impl/service/SearchServiceImpl.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/opensearch/impl/service/SearchServiceImpl.kt index 9562d2d3a..eef9d6ad2 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/opensearch/impl/service/SearchServiceImpl.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/opensearch/impl/service/SearchServiceImpl.kt @@ -29,6 +29,7 @@ import org.eclipse.tractusx.bpdm.pool.component.opensearch.impl.doc.TextDoc import org.eclipse.tractusx.bpdm.pool.component.opensearch.impl.repository.AddressDocSearchRepository import org.eclipse.tractusx.bpdm.pool.component.opensearch.impl.repository.LegalEntityDocSearchRepository import org.eclipse.tractusx.bpdm.pool.component.opensearch.impl.repository.TextDocSearchRepository +import org.eclipse.tractusx.bpdm.pool.config.OpenSearchConfigProperties import org.eclipse.tractusx.bpdm.pool.dto.request.AddressPartnerSearchRequest import org.eclipse.tractusx.bpdm.pool.dto.request.BusinessPartnerSearchRequest import org.eclipse.tractusx.bpdm.pool.dto.request.PaginationRequest @@ -38,12 +39,10 @@ import org.eclipse.tractusx.bpdm.pool.dto.response.LegalEntityMatchResponse import org.eclipse.tractusx.bpdm.pool.dto.response.SuggestionResponse import org.eclipse.tractusx.bpdm.pool.entity.AddressPartner import org.eclipse.tractusx.bpdm.pool.entity.LegalEntity +import org.eclipse.tractusx.bpdm.pool.exception.BpdmOpenSearchUserException import org.eclipse.tractusx.bpdm.pool.repository.AddressPartnerRepository import org.eclipse.tractusx.bpdm.pool.repository.LegalEntityRepository -import org.eclipse.tractusx.bpdm.pool.service.AddressService -import org.eclipse.tractusx.bpdm.pool.service.BusinessPartnerFetchService -import org.eclipse.tractusx.bpdm.pool.service.toBusinessPartnerMatchDto -import org.eclipse.tractusx.bpdm.pool.service.toMatchDto +import org.eclipse.tractusx.bpdm.pool.service.* import org.springframework.context.annotation.Primary import org.springframework.data.domain.PageRequest import org.springframework.stereotype.Service @@ -62,7 +61,8 @@ class SearchServiceImpl( val addressService: AddressService, val textDocSearchRepository: TextDocSearchRepository, val businessPartnerFetchService: BusinessPartnerFetchService, - val objectMapper: ObjectMapper + val objectMapper: ObjectMapper, + val openSearchConfigProperties: OpenSearchConfigProperties ) : SearchService { private val logger = KotlinLogging.logger { } @@ -159,9 +159,49 @@ class SearchServiceImpl( private fun searchAndPreparePage( searchRequest: BusinessPartnerSearchRequest, paginationRequest: PaginationRequest + ): PageResponse> { + return if (searchRequest == BusinessPartnerSearchRequest.EmptySearchRequest) { + paginateLegalEntities(paginationRequest) + } else { + searchIndex(searchRequest, paginationRequest) + } + } + + private fun searchAndPreparePage( + searchRequest: AddressPartnerSearchRequest, + paginationRequest: PaginationRequest + ): PageResponse> { + + return if (searchRequest == AddressPartnerSearchRequest.EmptySearchRequest) { + paginateAddressPartner(paginationRequest) + } else { + searchIndex(searchRequest, paginationRequest) + } + } + + private fun paginateLegalEntities(paginationRequest: PaginationRequest): PageResponse> { + logger.debug { "Paginate database for legal entities" } + val legalEntityPage = legalEntityRepository.findAll(PageRequest.of(paginationRequest.page, paginationRequest.size)) + + return legalEntityPage.toDto(legalEntityPage.content.map { Pair(0f, it) }) // assign 0 score as no search has been conducted + } + + private fun paginateAddressPartner(paginationRequest: PaginationRequest): PageResponse> { + logger.debug { "Paginate database for address partners" } + val addressPage = addressPartnerRepository.findAll(PageRequest.of(paginationRequest.page, paginationRequest.size)) + + return addressPage.toDto(addressPage.content.map { Pair(0f, it) }) // assign 0 score as no search has been conducted + } + + private fun searchIndex( + searchRequest: BusinessPartnerSearchRequest, + paginationRequest: PaginationRequest ): PageResponse> { logger.debug { "Search index for legal entities" } + if (paginationRequest.page > openSearchConfigProperties.maxPage) + throw BpdmOpenSearchUserException("When using search parameters page can't exceed ${openSearchConfigProperties.maxPage} but was ${paginationRequest.page} instead") + val searchResult = legalEntityDocSearchRepository.findBySearchRequest( searchRequest, PageRequest.of(paginationRequest.page, paginationRequest.size) @@ -184,12 +224,15 @@ class SearchServiceImpl( return PageResponse(totalHits, totalPages, paginationRequest.page, legalEntities.size, scoreLegalEntityPairs) } - private fun searchAndPreparePage( + private fun searchIndex( searchRequest: AddressPartnerSearchRequest, paginationRequest: PaginationRequest ): PageResponse> { logger.debug { "Search index for addresses" } + if (paginationRequest.page > openSearchConfigProperties.maxPage) + throw BpdmOpenSearchUserException("When using search parameters page can't exceed ${openSearchConfigProperties.maxPage} but was ${paginationRequest.page} instead") + val searchResult = addressDocSearchRepository.findBySearchRequest( searchRequest, PageRequest.of(paginationRequest.page, paginationRequest.size) diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/config/OpenSearchConfigProperties.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/config/OpenSearchConfigProperties.kt index d1785d998..35a3aebc1 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/config/OpenSearchConfigProperties.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/config/OpenSearchConfigProperties.kt @@ -30,5 +30,6 @@ class OpenSearchConfigProperties( val scheme: String = "http", val exportPageSize: Int = 100, val exportSchedulerCronExpr: String = "-", - val refreshOnWrite: Boolean = false + val refreshOnWrite: Boolean = false, + val maxPage: Int = 20 ) \ No newline at end of file diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/AddressController.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/AddressController.kt index 08a3dcb9c..f897fa827 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/AddressController.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/AddressController.kt @@ -51,7 +51,8 @@ class AddressController( summary = "Get page of addresses matching the search criteria", description = "This endpoint tries to find matches among all existing business partners of type address, " + "filtering out partners which entirely do not match and ranking the remaining partners according to the accuracy of the match. " + - "The match of a partner is better the higher its relevancy score." + "The match of a partner is better the higher its relevancy score. " + + "Note that when using search parameters the max page is \${bpdm.opensearch.max-page}." ) @ApiResponses( value = [ diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/BusinessPartnerLegacyController.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/BusinessPartnerLegacyController.kt index c1ea9cab6..6c806b176 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/BusinessPartnerLegacyController.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/BusinessPartnerLegacyController.kt @@ -49,7 +49,8 @@ class BusinessPartnerLegacyController( summary = "Get page of business partners matching the search criteria", description = "This endpoint tries to find matches among all existing business partners, " + "filtering out partners which entirely do not match and ranking the remaining partners according to the accuracy of the match. " + - "The match of a partner is better the higher its relevancy score.", + "The match of a partner is better the higher its relevancy score." + + "Note that when using search parameters the max page is \${bpdm.opensearch.max-page}.", deprecated = true ) @ApiResponses( diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/LegalEntityController.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/LegalEntityController.kt index b41b7439b..53891dd72 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/LegalEntityController.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/controller/LegalEntityController.kt @@ -55,7 +55,8 @@ class LegalEntityController( summary = "Get page of legal entity business partners matching the search criteria", description = "This endpoint tries to find matches among all existing business partners of type legal entity, " + "filtering out partners which entirely do not match and ranking the remaining partners according to the accuracy of the match. " + - "The match of a partner is better the higher its relevancy score." + "The match of a partner is better the higher its relevancy score. " + + "Note that when using search parameters the max page is \${bpdm.opensearch.max-page}." ) @ApiResponses( value = [ diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/AddressPartnerSearchRequest.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/AddressPartnerSearchRequest.kt index 58de9eb62..42aa92738 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/AddressPartnerSearchRequest.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/AddressPartnerSearchRequest.kt @@ -40,4 +40,8 @@ data class AddressPartnerSearchRequest constructor( var postalDeliveryPoint: String? = null, @field:Parameter(description = "Filter business partners by ISO 3166-1 alpha-2 country code") var countryCode: CountryCode? = null -) +) { + companion object { + val EmptySearchRequest = AddressPartnerSearchRequest() + } +} diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/AddressPropertiesSearchRequest.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/AddressPropertiesSearchRequest.kt index d3dfaac5e..d8dfe9c3d 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/AddressPropertiesSearchRequest.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/AddressPropertiesSearchRequest.kt @@ -37,4 +37,8 @@ data class AddressPropertiesSearchRequest constructor( var premise: String? = null, @field:Parameter(description = "Filter business partners by postal delivery point full denotation") var postalDeliveryPoint: String? = null -) +) { + companion object { + val EmptySearchRequest = AddressPropertiesSearchRequest() + } +} diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/BusinessPartnerSearchRequest.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/BusinessPartnerSearchRequest.kt index ff131e0bc..55814981c 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/BusinessPartnerSearchRequest.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/BusinessPartnerSearchRequest.kt @@ -23,4 +23,12 @@ data class BusinessPartnerSearchRequest( val partnerProperties: LegalEntityPropertiesSearchRequest, val addressProperties: AddressPropertiesSearchRequest, val siteProperties: SitePropertiesSearchRequest -) \ No newline at end of file +) { + companion object { + val EmptySearchRequest = BusinessPartnerSearchRequest( + LegalEntityPropertiesSearchRequest.EmptySearchRequest, + AddressPropertiesSearchRequest.EmptySearchRequest, + SitePropertiesSearchRequest.EmptySearchRequest + ) + } +} \ No newline at end of file diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/LegalEntityPropertiesSearchRequest.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/LegalEntityPropertiesSearchRequest.kt index c6bc67504..260f689e2 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/LegalEntityPropertiesSearchRequest.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/LegalEntityPropertiesSearchRequest.kt @@ -33,5 +33,9 @@ data class LegalEntityPropertiesSearchRequest constructor( val status: String?, @field:Parameter(description = "Filter legal entities by classification denotation") val classification: String?, -) +) { + companion object { + val EmptySearchRequest = LegalEntityPropertiesSearchRequest(null, null, null, null) + } +} diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/SitePropertiesSearchRequest.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/SitePropertiesSearchRequest.kt index 11032fd6b..879778a00 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/SitePropertiesSearchRequest.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/dto/request/SitePropertiesSearchRequest.kt @@ -27,4 +27,8 @@ import io.swagger.v3.oas.annotations.media.Schema data class SitePropertiesSearchRequest constructor( @field:Parameter(description = "Filter sites by name") val siteName: String? -) +) { + companion object { + val EmptySearchRequest = SitePropertiesSearchRequest(null) + } +} diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/exception/BpdmOpenSearchUserException.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/exception/BpdmOpenSearchUserException.kt new file mode 100644 index 000000000..207a9b6cf --- /dev/null +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/exception/BpdmOpenSearchUserException.kt @@ -0,0 +1,27 @@ +/******************************************************************************* + * Copyright (c) 2021,2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ******************************************************************************/ + +package org.eclipse.tractusx.bpdm.pool.exception + +import org.springframework.http.HttpStatus +import org.springframework.web.bind.annotation.ResponseStatus + +@ResponseStatus(HttpStatus.BAD_REQUEST) +class BpdmOpenSearchUserException(message: String) : RuntimeException(message) { +} \ No newline at end of file diff --git a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/service/BusinessPartnerFetchService.kt b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/service/BusinessPartnerFetchService.kt index 51470db24..e52fb263f 100644 --- a/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/service/BusinessPartnerFetchService.kt +++ b/bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/service/BusinessPartnerFetchService.kt @@ -54,6 +54,7 @@ class BusinessPartnerFetchService( return findOrThrow(bpn).toBusinessPartnerDto() } + /** * Fetch a business partner by [identifierValue] (ignoring case) of [identifierType] and return as [LegalEntityPartnerResponse] */ @@ -62,6 +63,7 @@ class BusinessPartnerFetchService( return findOrThrow(identifierType, identifierValue).toPoolDto() } + @Transactional fun findBusinessPartnerIgnoreCase(identifierType: String, identifierValue: String): BusinessPartnerResponse { return findOrThrow(identifierType, identifierValue).toBusinessPartnerDto() diff --git a/bpdm-pool/src/main/resources/application.properties b/bpdm-pool/src/main/resources/application.properties index b73c8394e..b709eeea5 100644 --- a/bpdm-pool/src/main/resources/application.properties +++ b/bpdm-pool/src/main/resources/application.properties @@ -54,6 +54,7 @@ bpdm.opensearch.host=localhost bpdm.opensearch.port=9200 bpdm.opensearch.scheme=http bpdm.opensearch.export-page-size=100 +bpdm.opensearch.max-page=20 # Special value "-" disables scheduling. See javadoc of org.springframework.scheduling.support.CronExpression.parse for format. bpdm.opensearch.export-scheduler-cron-expr=- bpdm.opensearch.refresh-on-write=false