Merge pull request #338 from charvolant/master

Release 1.5

.travis.yml

@@ -7,6 +7,7 @@ branches:
   only:
   - master
   - develop
+  - hotfix
   - 1.4.2.1
 before_cache:
   - rm -f  $HOME/.gradle/caches/modules-2/modules-2.lock

build.gradle

@@ -11,7 +11,7 @@ buildscript {
     }
 }
 
-version "1.4.13"
+version "1.5"
 group "au.org.ala"
 
 apply plugin:"eclipse"
@@ -60,15 +60,12 @@ dependencies {
     }
 
     compile 'net.sf.opencsv:opencsv:2.3'
-    compile "org.apache.solr:solr-solrj:6.6.5"
+    compile "org.apache.solr:solr-solrj:8.1.0"
     compile("org.gbif:dwca-io:1.24") {
         exclude group: 'com.google.guava', module: 'guava'
     }
     compile "com.google.guava:guava:19.0"
-    compile("au.org.ala:ala-name-matching:3.4") {
-        exclude group: 'org.slf4j', module: 'slf4j-log4j12'
-        exclude group: 'org.apache.bval', module: 'org.apache.bval.bundle'
-    }
+    compile group: "au.org.ala", name: "ala-name-matching-model", version:"4.0"
     compile "org.jsoup:jsoup:1.8.3"
     compile 'org.grails.plugins:external-config:1.1.1'
 

grails-app/conf/application.yml

@@ -205,16 +205,19 @@ solr:
       connection: http://localhost:8983/solr/bie
       queueSize: 10
       threadCount: 4
+      timeout: 120000
     updatingLive:
       type: UPDATE
       connection: http://localhost:8983/solr/bie
       queueSize: 10
       threadCount: 4
+      timeout: 300000
     offline:
       type: UPDATE
       connection: http://localhost:8983/solr/bie-offline
       queueSize: 10
       threadCount: 4
+      timeout: 300000
     search:
       qf:
         - exact_text

grails-app/conf/spring/resources.groovy

@@ -7,19 +7,22 @@ beans = {
         application.config.solr.live.type,
         application.config.solr.live.connection,
         application.config.solr.live.queueSize as Integer,
-        application.config.solr.live.threadCount as Integer
+        application.config.solr.live.threadCount as Integer,
+        application.config.solr.live.timeout as Integer
     )
     offlineSolrClient(SolrClientBean,
         application.config.solr.offline.type,
         application.config.solr.offline.connection,
         application.config.solr.offline.queueSize as Integer,
-        application.config.solr.offline.threadCount as Integer
+        application.config.solr.offline.threadCount as Integer,
+        application.config.solr.offline.timeout as Integer
     )
     updatingLiveSolrClient(SolrClientBean,
         application.config.solr.updatingLive.type,
         application.config.solr.updatingLive.connection,
         application.config.solr.updatingLive.queueSize as Integer,
-        application.config.solr.updatingLive.threadCount as Integer
+        application.config.solr.updatingLive.threadCount as Integer,
+        application.config.solr.updatingLive.timeout as Integer
     )
     conservationListsSource(ConservationListsSource, application.config.conservationListsUrl)
 }

grails-app/controllers/au/org/ala/bie/SearchController.groovy

@@ -188,13 +188,13 @@ class SearchController implements GrailsConfigurationAware {
         if (!req) {
             response.sendError(400, "Body could not be parsed or was empty")
         }
-        boolean includeVernacular = req.optBoolean('vernacular')
+        boolean includeVernacular = req.optBoolean('vernacular', true)
         List<String> names = req['names']
         List result = []
 
         names.eachWithIndex { name, idx ->
             log.debug "$idx. Looking up name: ${name}"
-            result.add(searchService.getLongProfileForName(name))
+            result.add(searchService.getLongProfileForName(name, includeVernacular))
         }
 
         render result as JSON

grails-app/init/bie/index/Application.groovy

@@ -2,7 +2,10 @@ package bie.index
 
 import grails.boot.GrailsApp
 import grails.boot.config.GrailsAutoConfiguration
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration
+import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration
 
+@EnableAutoConfiguration(exclude = [SolrAutoConfiguration])
 class Application extends GrailsAutoConfiguration {
     static void main(String[] args) {
         GrailsApp.run(Application, args)

grails-app/services/au/org/ala/bie/ImportService.groovy

@@ -48,6 +48,7 @@ import java.text.SimpleDateFormat
 import java.util.concurrent.ConcurrentLinkedQueue
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
+import java.util.regex.Pattern
 import java.util.zip.GZIPInputStream
 
 /**
@@ -106,7 +107,8 @@ class ImportService implements GrailsConfigurationAware {
     static ACCEPTED_STATUS = TaxonomicType.values().findAll({ it.accepted }).collect({ "taxonomicStatus:${it.term}" }).join(' OR ')
     // Synonym status
     static SYNONYM_STATUS = TaxonomicType.values().findAll({ it.synonym }).collect({ "taxonomicStatus:${it.term}" }).join(' OR ')
-
+    // A pattern indicating that we have a URL embedded in an anchor. Yuk
+    static SOURCE_IN_ANCHOR = Pattern.compile(/<[Aa] [^>]*[Hh][Rr][Ee][Ff]\s*=\s*"([^"]+)"[^>]*>.*<\/[Aa]>/)
 
     def indexService, searchService, biocacheService
     def listService, layerService, collectoryService, wordpressService, knowledgeBaseService
@@ -947,7 +949,11 @@ class ImportService implements GrailsConfigurationAware {
 
     private boolean addVernacularName(String taxonID, String name, String kingdom, String vernacularName, String nameId, Object status, String language, String source, String datasetID, String taxonRemarks, String provenance, Map additional, List buffer, Object defaultStatus) {
         def taxonDoc = null
-
+        if (source) { // Extract URL from anchor if needed
+            def sia = SOURCE_IN_ANCHOR.matcher(source)
+            if (sia.matches())
+                source = sia.group(1)
+        }
         if (taxonID)
             taxonDoc = searchService.lookupTaxon(taxonID, true)
         if (!taxonDoc && name)
@@ -2129,17 +2135,16 @@ class ImportService implements GrailsConfigurationAware {
         def variants = searchService.lookupVariant(guid, !online)
         if (variants) {
             priority = variants.collect({ (double) it.priority ?: weightNorm }).max()
-            double boost = Math.min(weightMax, Math.max(weightMin, priority))
             def names = (variants.collect { it.scientificName }) as Set
             names.addAll(variants.collect { it.nameComplete })
             names.remove(null)
             names.remove(scientificName)
             names.remove(nameComplete)
             if (names)
-                update["nameVariant"] = [boost: boost, set: names]
-            update["scientificName"] = [boost: boost, set: scientificName]
+                update["nameVariant"] = [set: names]
+            update["scientificName"] = [set: scientificName]
             if (nameComplete)
-                update["nameComplete"] = [boost: boost, set: nameComplete]
+                update["nameComplete"] = [set: nameComplete]
         }
         update['priority'] = [set: (int) Math.round(priority)]
         def commonNames = searchService.lookupVernacular(guid, !online)

grails-app/services/au/org/ala/bie/IndexService.groovy

@@ -20,7 +20,6 @@ import org.apache.solr.common.params.SolrParams
  * The interface to SOLR based logic.
  */
 class IndexService implements GrailsConfigurationAware {
-    def grailsApplication
     def liveSolrClient
     def offlineSolrClient
     def updatingLiveSolrClient
@@ -30,18 +29,21 @@ class IndexService implements GrailsConfigurationAware {
     SolrQuery suggestTemplate
     List<String> searchFq
     String searchDefType
+    String searchBoost
     List<String> searchQf
 
     @Override
     void setConfiguration(Config config) {
         def search = config.solr.search
+        searchBoost = search.boost
+        searchDefType = search.defType
         searchTemplate = new SolrQuery()
         search.fq.each { searchTemplate.addFilterQuery(it) }
-        searchTemplate.set('defType', search.defType)
+        searchTemplate.set('defType', searchDefType)
         searchTemplate.set('qf', search.qf.join(' '))
         search.bq.each { searchTemplate.add('bq', it) }
         searchTemplate.set('q.alt', search.qAlt)
-        searchTemplate.set('boost', search.boost)
+        searchTemplate.set('boost', searchBoost)
         if (search.hl.hl as Boolean) {
             searchTemplate.highlightFields = (search.hl.fl as String).split(',')
             searchTemplate.highlightSimplePre = search.hl.simple.pre
@@ -84,17 +86,12 @@ class IndexService implements GrailsConfigurationAware {
         docsToIndex.each { map ->
             def solrDoc = new SolrInputDocument()
             map.each{ fieldName, fieldValue ->
-                def boost = 1.0f
-                if (fieldValue && Map.class.isAssignableFrom(fieldValue.getClass()) && fieldValue["boost"]) {
-                    boost = fieldValue.boost
-                    fieldValue.remove("boost")
-                }
                 if(isList(fieldValue)){
                     fieldValue.each {
-                        solrDoc.addField(fieldName, it, (float) boost)
+                        solrDoc.addField(fieldName, it)
                     }
                 } else {
-                    solrDoc.addField(fieldName, fieldValue, (float) boost)
+                    solrDoc.addField(fieldName, fieldValue)
                 }
             }
             buffer << solrDoc
@@ -169,25 +166,38 @@ class IndexService implements GrailsConfigurationAware {
      * @param sort sort field
      * @param dir Sort direction
      * @param cursor Cursor mark if paginating
+     * @param useBoost Use the search boost functionality
+     * @param useDefType Use the default defType
      *
      * @return The query result
      */
-    QueryResponse query(boolean online, String q, List fq = [], Integer rows = 10, Integer start = 0, String context = null, String sort = null, String dir = 'asc', String cursor = null) {
+    QueryResponse query(boolean online, String q, List fq = [], Integer rows = 10, Integer start = 0, String context = null, String sort = null, String dir = 'asc', String cursor = null, boolean useBoost = false, boolean useDefType = false) {
         def query = new SolrQuery(q)
 
-        if (context)
+        if (context) {
             query.add(context(context))
-        if (fq)
+        }
+        if (fq) {
             fq.each { query.addFilterQuery(it) }
-        if (rows)
+        }
+        if (rows) {
             query.setRows(rows)
-        if (start)
+        }
+        if (start) {
             query.setStart(start)
+        }
         if (sort) {
             query.sort = new SolrQuery.SortClause(sort, dir == 'desc' ? SolrQuery.ORDER.desc : SolrQuery.ORDER.asc)
         }
-        if (cursor)
+        if (cursor) {
             query.set(CursorMarkParams.CURSOR_MARK_PARAM, cursor)
+        }
+        if (useBoost) {
+            query.set('boost', searchBoost)
+        }
+        if (useDefType) {
+            query.set("defType", searchDefType)
+        }
         return this.query(query, online)
     }
 

grails-app/services/au/org/ala/bie/SearchService.groovy

@@ -284,34 +284,40 @@ class SearchService {
         }
     }
 
+    private queryChildConcepts(q, fqs, queryString, children) {
+        def response = indexService.query(true, q, fqs, 1000, 0, queryString)
+        def taxa = response.results
+        taxa.each { taxon ->
+            children << [
+                    guid         : taxon.guid,
+                    parentGuid   : taxon.parentGuid,
+                    name         : taxon.scientificName,
+                    nameComplete : taxon.nameComplete ?: taxon.scientificName,
+                    nameFormatted: taxon.nameFormatted,
+                    author       : taxon.scientificNameAuthorship,
+                    rank         : taxon.rank,
+                    rankID       : taxon.rankID
+            ]
+        }
+    }
 
     def getChildConcepts(taxonID, queryString, within, unranked){
         def baseTaxon = lookupTaxon(taxonID)
         def baseRankID = baseTaxon?.rankID ?: -1
         def baseFq = "idxtype:${IndexDocType.TAXON.name()}"
         def fqs = [ baseFq ]
-        if (baseRankID > 0 && within) {
-            fqs << "rankID:[${unranked ? -1 : baseRankID + 1} TO ${baseRankID + within}]"
-        }
         def q = "parentGuid:\"${ Encoder.escapeSolr(taxonID) }\""
-        def response = indexService.query(true, q, fqs, 1000, 0, queryString)
-        if (response.results.numFound == 0) {
-            response = indexService.query(true, q, [ baseFq ], 1000, 0, queryString)
-        }
         def children = []
-        def taxa = response.results
-        taxa.each { taxon ->
-            children << [
-                    guid:taxon.guid,
-                    parentGuid: taxon.parentGuid,
-                    name: taxon.scientificName,
-                    nameComplete: taxon.nameComplete ?: taxon.scientificName,
-                    nameFormatted: taxon.nameFormatted,
-                    author: taxon.scientificNameAuthorship,
-                    rank: taxon.rank,
-                    rankID:taxon.rankID
-            ]
+
+        if (baseRankID > 0 && within) {
+             fqs << "rankID:[${unranked ? -1 : baseRankID + 1} TO ${baseRankID + within}]"
         }
+
+        this.queryChildConcepts(q, fqs, queryString, children)
+        if (unranked && baseRankID > 0 && within)  // Long running bug in SOLR unable to handle (- +) subqueries
+            this.queryChildConcepts(q, [ baseFq, "-rankID:*"], queryString, children)
+        if (children.isEmpty() && fqs.size() > 1)
+            this.queryChildConcepts(q, [ baseFq ], queryString, children)
         children.sort { c1, c2 ->
             def r1 = c1.rankID
             def r2 = c2.rankID
@@ -353,10 +359,22 @@ class SearchService {
      */
     def lookupTaxonByName(String taxonName, String kingdom, Boolean useOfflineIndex = false){
         taxonName = Encoder.escapeSolr(taxonName)
-        def q = "+commonNameExact:\"${taxonName}\" OR +scientificName:\"${taxonName}\" OR +nameComplete:\"${taxonName} OR +exact_text:\"${taxonName}\""
+        def q = "scientificName:\"${taxonName}\" OR nameComplete:\"${taxonName}\" OR commonName:\"${taxonName}\""
         if (kingdom)
             q = "(${q}) AND rk_kingdom:\"${ Encoder.escapeSolr(kingdom) }\""
-        def response = indexService.search(!useOfflineIndex, q, [ "idxtype:${ IndexDocType.TAXON.name() }" ], [], 0, 1)
+        def response = indexService.query(
+                !useOfflineIndex,
+                q,
+                [ "idxtype:${IndexDocType.TAXON.name()}" ],
+                1,
+                0,
+                null,
+                null,
+                null,
+                null,
+                true,
+                true
+        )
         if (response.results.isEmpty()) {
             q = "+scientificName:\"${taxonName}\" OR +nameComplete:\"${taxonName}\""
             response = indexService.query(!useOfflineIndex, q, [ "idxtype:${ IndexDocType.TAXONVARIANT.name() }" ], 1, 0)
@@ -463,7 +481,20 @@ class SearchService {
      */
     def getProfileForName(String name){
         name = Encoder.escapeSolr(name)
-        def response = indexService.search(true, '"' + name + '"', [ "idxtype:${IndexDocType.TAXON.name()}" ])
+        def query = "scientificName:\"${name}\" OR nameComplete:\"${name}\" OR commonName:\"${name}\""
+        def response = indexService.query(
+                true,
+                query,
+                [ "idxtype:${IndexDocType.TAXON.name()}" ],
+                  10,
+                0,
+                null,
+                null,
+                null,
+                null,
+                true,
+                true
+        )
         def model = []
 
         if (response.results.numFound > 0) {
@@ -480,9 +511,24 @@ class SearchService {
         model
     }
 
-    Map getLongProfileForName(String name){
+    Map getLongProfileForName(String name, boolean includeVernacular){
         name = Encoder.escapeSolr(name)
-        def response = indexService.search(true, '"' + name + '"', [ "idxtype:${IndexDocType.TAXON.name()}" ])
+        def query = "scientificName:\"${name}\" OR nameComplete:\"${name}\""
+        if (includeVernacular)
+            query = query + " OR commonName:\"${name}\""
+        def response = indexService.query(
+                true,
+                query,
+                [ "idxtype:${IndexDocType.TAXON.name()}" ],
+                1,
+                0,
+                null,
+                null,
+                null,
+                null,
+                true,
+                true
+        )
         def model = [:]
         if (response.results.numFound > 0) {
             def result = response.results.get(0)
@@ -517,7 +563,6 @@ class SearchService {
                 ]
 
         }
-
         model
     }