diff --git a/README.md b/README.md index d195c08..ecd2549 100644 --- a/README.md +++ b/README.md @@ -240,13 +240,18 @@ Property | Description ### Record handler -Each search handler may also provide a result handler to finely tune how you wish to display or filter the results. At its most simplest, the record handler simply expects an array to be returned for each record that contains 4 properties: +Each search handler may also provide a result handler to finely tune how you wish to display or filter the results. At its most simplest, the record handler simply expects an array to be returned for each record that contains 3 properties: - `title`: The title of the result. -- `image`: The path to a corresponding image for the result. - `description`: Additional context for the result. - `url`: The URL that the result will point to. +It may also optionally provide the following properties to provide more context: + +- `group`: The group that this result belongs to, when using grouped results. +- `label`: The label of the result, which may provide more context or grouping for results. +- `image`: The path to a corresponding image for the result. + You may, of course, define additional properties in your array. The record handler can be configured in a number of different ways. diff --git a/behaviors/Searchable.php b/behaviors/Searchable.php index 2dfcd5d..c2902e3 100644 --- a/behaviors/Searchable.php +++ b/behaviors/Searchable.php @@ -10,7 +10,7 @@ use Winter\Storm\Support\Facades\Config; use Winter\Storm\Database\Traits\SoftDelete; use Illuminate\Support\Collection as BaseCollection; -use Laravel\Scout\Builder; +use Winter\Search\Classes\Builder; use Laravel\Scout\Scout; use Laravel\Scout\Searchable as BaseSearchable; @@ -133,7 +133,7 @@ public function queueRemoveFromSearch($models) * * @param string $query * @param \Closure $callback - * @return \Laravel\Scout\Builder + * @return \Winter\Search\Classes\Builder */ public static function search($query = '', $callback = null) { @@ -154,7 +154,7 @@ public static function search($query = '', $callback = null) * * @param string $query * @param \Closure $callback - * @return \Laravel\Scout\Builder + * @return \Winter\Search\Classes\Builder */ public function doSearch($query = '', $callback = null) { @@ -227,7 +227,7 @@ public static function removeAllFromSearch() /** * Get the requested models from an array of object IDs. * - * @param \Laravel\Scout\Builder $builder + * @param \Winter\Search\Classes\Builder $builder * @param array $ids * @return mixed */ @@ -239,7 +239,7 @@ public function getScoutModelsByIds(Builder $builder, array $ids) /** * Get a query builder for retrieving the requested models from an array of object IDs. * - * @param \Laravel\Scout\Builder $builder + * @param \Winter\Search\Classes\Builder $builder * @param array $ids * @return mixed */ diff --git a/behaviors/halcyon/Searchable.php b/behaviors/halcyon/Searchable.php index ed7e44f..f73ae8d 100644 --- a/behaviors/halcyon/Searchable.php +++ b/behaviors/halcyon/Searchable.php @@ -3,7 +3,7 @@ namespace Winter\Search\Behaviors\Halcyon; use Cms\Classes\Theme; -use Laravel\Scout\Builder; +use Winter\Search\Classes\Builder; use Illuminate\Support\Collection as BaseCollection; use Winter\Search\Behaviors\Searchable as BaseSearchable; use Winter\Search\Classes\HalcyonModelObserver; @@ -70,7 +70,7 @@ public static function makeAllSearchable($chunk = null) /** * Get the requested models from an array of object IDs. * - * @param \Laravel\Scout\Builder $builder + * @param \Winter\Search\Classes\Builder $builder * @param array $ids * @return mixed */ @@ -82,7 +82,7 @@ public function getScoutModelsByIds(Builder $builder, array $ids) /** * Get a query builder for retrieving the requested models from an array of object IDs. * - * @param \Laravel\Scout\Builder $builder + * @param \Winter\Search\Classes\Builder $builder * @param array $ids * @return mixed */ diff --git a/classes/Builder.php b/classes/Builder.php new file mode 100644 index 0000000..03529cd --- /dev/null +++ b/classes/Builder.php @@ -0,0 +1,108 @@ +engine()->get($this); + + $relevanceCalculator = $relevanceCalculator ?? \Closure::fromCallable([$this, 'relevanceCalculator']); + + return $collection->map(function ($model) use ($relevanceCalculator) { + $model->relevance = $relevanceCalculator($model, $this->wordifyQuery($this->query)); + return $model; + })->sortByDesc('relevance'); + } + + public function firstRelevant(?callable $relevanceCalculator = null) + { + $collection = $this->engine()->get($this); + + $relevanceCalculator = $relevanceCalculator ?? \Closure::fromCallable([$this, 'relevanceCalculator']); + + return $collection->map(function ($model) use ($relevanceCalculator) { + $model->relevance = $relevanceCalculator($model, $this->wordifyQuery($this->query)); + return $model; + })->sortByDesc('relevance')->first(); + } + + /** + * Calculates the relevance of a model to a query. + * + * @param \Winter\Storm\Database\Model|\Winter\Storm\Halcyon\Model $model + * @param string $query + * @return float|int + */ + public function relevanceCalculator($model, array $queryWords) + { + // Get ranking map + $rankingMap = $this->getRankingMap($model); + + $relevance = 0; + $multiplier = 2; + + // Go through and find each word in the searchable fields, with the first word being the most important, and + // each word thereafter being less important + foreach ($rankingMap as $field => $rank) { + foreach ($queryWords as $query) { + $multiplier /= 2; + + if (stripos($model->{$field}, $query) !== false) { + // Count matches and multiply by rank + $relevance += ( + (substr_count(strtolower($model->{$field}), strtolower($query)) * $rank) + * $multiplier + ); + } + } + } + + return $relevance; + } + + /** + * Gets a ranking map of the searchable fields. + * + * Searchable fields are ordered by descending importance, with the most important field first. It applies ranking + * based on a double sequence. + * + * If no searchable fields are provided, this will return `false`. + * + * @return int[]|false + */ + protected function getRankingMap($model) + { + if (!$model->propertyExists('searchable')) { + return false; + } + + $searchable = array_reverse($model->searchable); + $rankingMap = []; + $rank = 1; + + foreach ($searchable as $field) { + $rankingMap[$field] = $rank; + $rank *= 2; + } + + return array_reverse($rankingMap, true); + } + + /** + * Convert a query string into an array of applicable words. + * + * This will strip all stop words and punctuation from the query string, then split each word into an array. + */ + protected function wordifyQuery($query): array + { + $query = preg_replace('/[% ]+/', ' ', strtolower($query)); + + return array_map(function ($word) { + return trim($word, ' .,'); + }, preg_split('/ +/', $query)); + } +} diff --git a/components/Search.php b/components/Search.php index d116369..8f33cfe 100644 --- a/components/Search.php +++ b/components/Search.php @@ -2,15 +2,17 @@ namespace Winter\Search\Components; -use Lang; -use Winter\Search\Plugin; use Cms\Classes\ComponentBase; use Illuminate\Database\Eloquent\Model as DbModel; +use Illuminate\Pagination\LengthAwarePaginator; +use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Lang; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Request; use System\Classes\PluginManager; +use TeamTNT\TNTSearch\Stemmer\PorterStemmer; +use Winter\Search\Plugin; use Winter\Storm\Exception\ApplicationException; -use Winter\Storm\Support\Arr; use Winter\Storm\Support\Facades\Validator; use Winter\Storm\Halcyon\Model as HalcyonModel; @@ -47,6 +49,24 @@ public function defineProperties() 'required' => true, 'placeholder' => Lang::get(Plugin::LANG . 'components.search.handler.placeholder'), ], + 'fuzzySearch' => [ + 'title' => Plugin::LANG . 'components.search.fuzzySearch.title', + 'description' => Plugin::LANG . 'components.search.fuzzySearch.description', + 'type' => 'checkbox', + 'default' => false, + ], + 'orderByRelevance' => [ + 'title' => Plugin::LANG . 'components.search.orderByRelevance.title', + 'description' => Plugin::LANG . 'components.search.orderByRelevance.description', + 'type' => 'checkbox', + 'default' => false, + ], + 'showExcerpts' => [ + 'title' => Plugin::LANG . 'components.search.showExcerpts.title', + 'description' => Plugin::LANG . 'components.search.showExcerpts.description', + 'type' => 'checkbox', + 'default' => true, + ], 'limit' => [ 'title' => Plugin::LANG . 'components.search.limit.title', 'description' => Plugin::LANG . 'components.search.limit.description', @@ -64,7 +84,23 @@ public function defineProperties() 'validationPattern' => '^[0-9]+$', 'validationMessage' => Plugin::LANG . 'components.search.limit.validationMessage', 'group' => Plugin::LANG . 'components.search.groups.pagination', - ] + ], + 'grouping' => [ + 'title' => Plugin::LANG . 'components.search.grouping.title', + 'description' => Plugin::LANG . 'components.search.grouping.description', + 'type' => 'checkbox', + 'default' => false, + 'group' => Plugin::LANG . 'components.search.groups.grouping', + ], + 'perGroup' => [ + 'title' => Plugin::LANG . 'components.search.perGroup.title', + 'description' => Plugin::LANG . 'components.search.perGroup.description', + 'type' => 'string', + 'default' => 5, + 'validationPattern' => '^[0-9]+$', + 'validationMessage' => Plugin::LANG . 'components.search.perGroup.validationMessage', + 'group' => Plugin::LANG . 'components.search.groups.grouping', + ], ]; } @@ -179,6 +215,19 @@ public function onSearch() $totalCount = 0; $totalTotal = 0; + if ($this->property('fuzzySearch', false)) { + $processedQuery = $this->processQuery($query); + if (empty($processedQuery)) { + return [ + '#' . $this->getId('results') => $this->renderPartial('@no-query'), + 'results' => [], + 'count' => 0, + ]; + } + } else { + $processedQuery = $query; + } + foreach ($handlers as $id => $handler) { $class = $handler['model']; if (is_string($class)) { @@ -189,10 +238,17 @@ public function onSearch() sprintf('Model for handler "%s" must be a database or Halcyon model, or a callback', $id) ); } + if (is_callable($class)) { - $results = $class()->doSearch($query)->paginate($this->property('perPage', 20), 'page', ($handlerPage === $id) ? $page : 1); + $search = $class()->doSearch($processedQuery); + } else { + $search = $class->doSearch($processedQuery); + } + + if ($this->property('orderByRelevance', false)) { + $results = $search->getWithRelevance(); } else { - $results = $class->doSearch($query)->paginate($this->property('perPage', 20), 'page', ($handlerPage === $id) ? $page : 1); + $results = $search->get(); } if ($results->count() === 0) { @@ -209,6 +265,12 @@ public function onSearch() continue; } + if ($handlerPage !== $id) { + $page = 1; + } + + $results = $this->paginateResults($results, $page); + foreach ($results as $result) { $processed = $this->processRecord($result, $query, $handler['record']); @@ -233,6 +295,10 @@ public function onSearch() $handlerResults[$id]['results'][] = $processed; } + + if ($this->property('grouping', false)) { + $handlerResults[$id]['results'] = $this->applyGrouping($handlerResults[$id]['results']); + } } return [ @@ -253,19 +319,45 @@ public function onSearch() ]; } + /** + * Processes each record returned by the search handler. + * + * This method calls the search handler and allows the search handler to manage and format how the result appears + * in the results list. Each result should be an array with at least the following information: + * + * - `title`: The title of the result. + * - `description`: A brief description of the result. + * - `url`: The URL to the result. + * + * In addition, you may provide these optional attributes: + * + * - `group`: The group to which the result belongs. + * - `label`: A label to display for the result. + * - `image`: An image to display next to the result. + */ protected function processRecord($record, string $query, array|callable $handler) { $requiredAttributes = ['title', 'description', 'url']; + $optionalAttributes = ['group', 'label', 'image']; if (is_callable($handler)) { $processed = $handler($record, $query); + if (!is_array($processed)) { + return false; + } + foreach ($requiredAttributes as $attr) { if (!array_key_exists($attr, $processed)) { return false; } } + // Remove processed values that are not required or optional + $processed = array_filter($processed, function ($key) use ($requiredAttributes, $optionalAttributes) { + return in_array($key, array_merge($requiredAttributes, $optionalAttributes)); + }, ARRAY_FILTER_USE_KEY); + return $processed; } else { $processed = []; @@ -274,12 +366,10 @@ protected function processRecord($record, string $query, array|callable $handler if (!isset($handler[$attr])) { return false; } - - $processed[$attr] = Arr::get($record, $handler[$attr], null); } foreach ($handler as $attr => $value) { - if (in_array($attr, $requiredAttributes)) { + if (in_array($attr, array_merge($requiredAttributes, $optionalAttributes))) { continue; } @@ -300,4 +390,226 @@ public function getId(string $id): string { return $this->alias . '-' . $id; } + + /** + * Determines if results are being grouped. + */ + public function isGrouped(): bool + { + return $this->property('grouping', false); + } + + /** + * Determines if excerpts should be shown. + */ + public function showExcerpts(): bool + { + return $this->property('showExcerpts', true); + } + + /** + * Creates a paginator for search results. + */ + protected function paginateResults(Collection $results, int $page = 1) + { + return new LengthAwarePaginator( + $results, + $results->count(), + $this->property('perPage', 20), + $page, + ); + } + + /** + * Applies grouping to results, if required. + */ + protected function applyGrouping(array $results) + { + $grouped = []; + + foreach ($results as $result) { + $group = $result['group'] ?? 'Other results'; + + if (!isset($grouped[$group])) { + $grouped[$group] = []; + } + + if (count($grouped[$group]) >= $this->property('perGroup', 5)) { + continue; + } + + $grouped[$group][] = $result; + } + + return $grouped; + } + + /** + * Processes the search query. + * + * This applies some processing of the search query to get better search results. It does the following: + * + * - Removes any percentage signs from the query, in order to not get full wildcard searches. + * - Strips punctuation from the query. + * - Removes any stop words. + * - Stems each word in the query, so words with the incorrect inflection or pluralisation can still be found. + * - Replaces spacing with percentages, in order to allow words that aren't next to each other to be found. + * - Trims the query. + */ + protected function processQuery(string $query): string + { + $query = str_replace('%', '', strtolower($query)); + $words = preg_split('/[ -]+/', $query, -1, PREG_SPLIT_NO_EMPTY); + + // Strip punctuation + $words = array_map(function ($word) { + return preg_replace('/[^a-z0-9]/', '', $word); + }, $words); + + // Remove stop words + $words = array_filter($words, function ($word) { + return $this->isStopWord($word); + }); + + // Stem words + $words = array_map(function ($word) { + return PorterStemmer::stem($word); + }, $words); + + // Replace spaces with wildcards to match partial words and trim query + return trim(implode('% %', $words)); + } + + protected function isStopWord(string $word): bool + { + return !in_array(strtolower($word), [ + 'i', + 'me', + 'my', + 'myself', + 'we', + 'our', + 'ours', + 'ourselves', + 'you', + 'your', + 'yours', + 'yourself', + 'yourselves', + 'he', + 'him', + 'his', + 'himself', + 'she', + 'her', + 'hers', + 'herself', + 'it', + 'its', + 'itself', + 'they', + 'them', + 'their', + 'theirs', + 'themselves', + 'what', + 'which', + 'who', + 'whom', + 'this', + 'that', + 'these', + 'those', + 'am', + 'is', + 'are', + 'was', + 'were', + 'be', + 'been', + 'being', + 'have', + 'has', + 'had', + 'having', + 'do', + 'does', + 'did', + 'doing', + 'a', + 'an', + 'the', + 'and', + 'but', + 'if', + 'or', + 'because', + 'as', + 'until', + 'while', + 'of', + 'at', + 'by', + 'for', + 'with', + 'about', + 'against', + 'between', + 'into', + 'through', + 'during', + 'before', + 'after', + 'above', + 'below', + 'to', + 'from', + 'up', + 'down', + 'in', + 'out', + 'on', + 'off', + 'over', + 'under', + 'again', + 'further', + 'then', + 'once', + 'here', + 'there', + 'when', + 'where', + 'why', + 'how', + 'all', + 'any', + 'both', + 'each', + 'few', + 'more', + 'most', + 'other', + 'some', + 'such', + 'no', + 'nor', + 'not', + 'only', + 'own', + 'same', + 'so', + 'than', + 'too', + 'very', + 's', + 't', + 'can', + 'will', + 'just', + 'don', + 'should', + 'now', + ]); + } } diff --git a/components/search/results.htm b/components/search/results.htm index 20b34c8..0563f9c 100644 --- a/components/search/results.htm +++ b/components/search/results.htm @@ -14,12 +14,44 @@ diff --git a/composer.json b/composer.json index f8b9b3f..96ddae9 100644 --- a/composer.json +++ b/composer.json @@ -25,7 +25,8 @@ "require": { "php": "^8.0", "composer/installers": "~1.0", - "laravel/scout": "^9.4.5" + "laravel/scout": "^9.4.5", + "teamtnt/tntsearch": "^4.0" }, "suggest": { "algolia/algoliasearch-client-php": "Required to use the Algolia engine (^3.2).", diff --git a/database/factories/SearchableModelFactory.php b/database/factories/SearchableModelFactory.php new file mode 100644 index 0000000..031e5ce --- /dev/null +++ b/database/factories/SearchableModelFactory.php @@ -0,0 +1,24 @@ +addProvider(\Faker\Provider\en_US\Text::class); + + return [ + 'title' => $faker->sentence(), + 'description' => $faker->sentence(30), + 'content' => $faker->paragraph(16), + 'keywords' => $faker->words(8, true), + ]; + } +} diff --git a/engines/CollectionEngine.php b/engines/CollectionEngine.php index 10db83d..5949e5e 100644 --- a/engines/CollectionEngine.php +++ b/engines/CollectionEngine.php @@ -2,16 +2,16 @@ namespace Winter\Search\Engines; -use Arr; use Laravel\Scout\Engines\CollectionEngine as BaseCollectionEngine; use Winter\Storm\Database\Traits\SoftDelete; +use Winter\Storm\Support\Arr; class CollectionEngine extends BaseCollectionEngine { /** * Ensure that soft delete handling is properly applied to the query. * - * @param \Laravel\Scout\Builder $builder + * @param \Winter\Search\Classes\Builder $builder * @param \Illuminate\Database\Query\Builder $query * @return \Illuminate\Database\Query\Builder */ diff --git a/engines/DatabaseEngine.php b/engines/DatabaseEngine.php index ad1aed9..fee8236 100644 --- a/engines/DatabaseEngine.php +++ b/engines/DatabaseEngine.php @@ -2,21 +2,21 @@ namespace Winter\Search\Engines; -use Arr; use Laravel\Scout\Builder; use Laravel\Scout\Engines\DatabaseEngine as BaseDatabaseEngine; use Winter\Storm\Database\Traits\SoftDelete; +use Winter\Storm\Support\Arr; class DatabaseEngine extends BaseDatabaseEngine { /** * Ensure that soft delete handling is properly applied to the query. * - * @param \Laravel\Scout\Builder $builder + * @param \Winter\Search\Classes\Builder $builder * @param \Illuminate\Database\Query\Builder $query * @return \Illuminate\Database\Query\Builder */ - protected function ensureSoftDeletesAreHandled($builder, $query) + protected function constrainForSoftDeletes($builder, $query) { if (Arr::get($builder->wheres, '__soft_deleted') === 0) { return $query->withoutTrashed(); @@ -36,7 +36,7 @@ protected function ensureSoftDeletesAreHandled($builder, $query) * Since Winter adds Scout capabilities through behaviours, we have no way to support the * attributes method of defining columns. * - * @param \Laravel\Scout\Builder $builder + * @param \Winter\Search\Classes\Builder $builder * @param string $attributeClass * @return array */ @@ -51,7 +51,7 @@ protected function getAttributeColumns(Builder $builder, $attributeClass) * Since Winter adds Scout capabilities through behaviours, we have no way to support the * attributes method of defining columns. * - * @param \Laravel\Scout\Builder $builder + * @param \Winter\Search\Classes\Builder $builder * @return array */ protected function getFullTextOptions(Builder $builder) diff --git a/lang/en/lang.php b/lang/en/lang.php index 535a598..7644c78 100644 --- a/lang/en/lang.php +++ b/lang/en/lang.php @@ -8,6 +8,7 @@ 'otherPlugins' => [ 'cmsPages' => 'CMS Pages', 'staticPages' => 'Static Pages', + 'winterBlog' => 'Winter Blog', ], 'components' => [ 'search' => [ @@ -16,12 +17,25 @@ 'groups' => [ 'pagination' => 'Pagination', 'display' => 'Display', + 'grouping' => 'Result grouping', ], 'handler' => [ 'title' => 'Search handlers', 'description' => 'Select search handlers that have been registered through a plugin\'s "registerSearchHandlers" method. You may select more than one.', 'placeholder' => 'Select one or more', ], + 'fuzzySearch' => [ + 'title' => 'Fuzzy search?', + 'description' => 'Allows the search query to match records more loosely. Some index providers may already provide fuzzy searching, so only enable this if necessary.', + ], + 'orderByRelevance' => [ + 'title' => 'Order by relevance?', + 'description' => 'Runs a custom relevance algorithm on results and orders based on relevancy. This is recommended only for database or collection search indexes, as other providers have their own relevance algorithms.', + ], + 'showExcerpts' => [ + 'title' => 'Show excerpts?', + 'description' => 'If checked, excerpts from the result content will be displayed in search results.', + ], 'limit' => [ 'title' => 'Results limit', 'description' => 'Define the total amount of results you wish to retrieve. Set to 0 to have no limit.', @@ -32,20 +46,14 @@ 'description' => 'Define the amount of results you wish to retrieve per page. Set to 0 to have no pagination.', 'validationMessage' => 'Results per page must be a number', ], - 'combineResults' => [ - 'title' => 'Combine results', - 'description' => 'If multiple search handlers are included, ticking this will combine the results into one result array. Otherwise, the results array will be grouped by the search handler name.', - ], - 'displayImages' => [ - 'title' => 'Show images', - ], - 'displayHandlerName' => [ - 'title' => 'Show search handler name', - 'description' => 'Useful if you have combined results and wish to show the handler that provided the result', + 'grouping' => [ + 'title' => 'Enable grouping?', + 'description' => 'If enabled, results will be grouped by logical groupings, such as categories or pages.', ], - 'displayPluginName' => [ - 'title' => 'Show plugin name', - 'description' => 'Useful if you have combined results and wish to show the plugin that provided the result', + 'perGroup' => [ + 'title' => 'Results per group', + 'description' => 'Define the upper limit of results you wish to retrieve per group. Set to 0 to have no limit.', + 'validationMessage' => 'Results per group must be a number', ], ], ], diff --git a/tests/cases/models/SearchableModelTest.php b/tests/cases/models/SearchableModelTest.php new file mode 100644 index 0000000..ed240d1 --- /dev/null +++ b/tests/cases/models/SearchableModelTest.php @@ -0,0 +1,41 @@ +count(50)->create(); + + // Second most relevant + $records[0]->update(['title' => 'TestQuery TestQuery TestQuery']); + // Third most relevant + $records[1]->update(['title' => 'TestQuery TestQuery']); + // Fourth most relevant + $records[2]->update(['description' => 'TestQuery TestQuery TestQuery']); + // Fifth most relevant + $records[3]->update(['description' => 'TestQuery TestQuery']); + // Most relevant + $records[4]->update(['title' => 'TestQuery TestQuery TestQuery', 'description' => 'TestQuery TestQuery TestQuery']); + + $results = SearchableModel::search('TestQuery')->getWithRelevance(); + + $recordIds = $records->slice(0, 5)->pluck('id')->toArray(); + $resultIds = $results->slice(0, 5)->pluck('id')->toArray(); + + $this->assertEquals($recordIds[4], $resultIds[0]); + $this->assertEquals($recordIds[0], $resultIds[1]); + $this->assertEquals($recordIds[1], $resultIds[2]); + $this->assertEquals($recordIds[2], $resultIds[3]); + $this->assertEquals($recordIds[3], $resultIds[4]); + + $result = SearchableModel::search('TestQuery')->firstRelevant(); + + $this->assertEquals($recordIds[4], $result->id); + } +} diff --git a/tests/fixtures/SearchableModel.php b/tests/fixtures/SearchableModel.php new file mode 100644 index 0000000..220ae8a --- /dev/null +++ b/tests/fixtures/SearchableModel.php @@ -0,0 +1,45 @@ + 'string', + 'description' => 'string', + 'content' => 'text', + 'keywords' => 'string', + ]; + + protected static function newFactory() + { + return SearchableModelFactory::new(); + } +}