From 085c9a2197efe6d1b4732e41462a01de2a8e3d3b Mon Sep 17 00:00:00 2001 From: Ethan Hathaway Date: Thu, 2 Apr 2020 18:25:55 -0700 Subject: [PATCH] Feat: add terms filter --- dev/app/context.ts | 15 +- package.json | 2 +- src/filters/index.ts | 1 + src/filters/multi_select_filter.ts | 6 +- src/filters/terms_filter.ts | 516 +++++++++++++++++++++++++++++ src/manager.ts | 6 +- src/types.ts | 24 ++ 7 files changed, 564 insertions(+), 6 deletions(-) create mode 100644 src/filters/terms_filter.ts diff --git a/dev/app/context.ts b/dev/app/context.ts index c812a93..467ba59 100644 --- a/dev/app/context.ts +++ b/dev/app/context.ts @@ -14,7 +14,8 @@ import { PrefixSuggestion, RangeFilter, History, - localStorageHistoryPersister + localStorageHistoryPersister, + TermsFilter } from '../../src'; import {IRangeConfig} from '../../src/filters/range_filter'; import {reaction} from 'mobx'; @@ -67,6 +68,15 @@ const customPrefixSuggestion = new PrefixSuggestion({ fieldNameModifierAggs: (fieldName: string) => `${fieldName}.keyword` }); +const customTermsFilter = new TermsFilter({ + defaultFilterKind: 'must', + defaultFilterInclusion: 'include', + getCount: false, + aggsEnabled: false, + fieldNameModifierQuery: (fieldName: string) => `${fieldName}.keyword`, + fieldNameModifierAggs: (fieldName: string) => `${fieldName}.keyword` +}); + const defaultRangeFilterConfig: IRangeConfig = { field: '', aggsEnabled: false, @@ -106,7 +116,8 @@ const creatorCRM = new Manager(client, { // 'user_profile.children.year_born' // ], filters: { - range: customRangeFilter + range: customRangeFilter, + terms: customTermsFilter }, suggestions: { prefix: customPrefixSuggestion diff --git a/package.json b/package.json index 165fbea..7909946 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@social-native/snpkg-client-elasticsearch", - "version": "4.0.0", + "version": "4.1.0", "description": "", "main": "dist/index.cjs.js", "module": "dist/index.es.js", diff --git a/src/filters/index.ts b/src/filters/index.ts index 23e0331..ff18120 100644 --- a/src/filters/index.ts +++ b/src/filters/index.ts @@ -5,6 +5,7 @@ export {default as ExistsFilter} from './exists_filter'; export {default as MultiSelectFilter} from './multi_select_filter'; export {default as DateRangeFilter} from './date_range_filter'; export {default as GeoFilter} from './geo_filter'; +export {default as TermsFilter} from './terms_filter'; export {default as filterUtils} from './utils'; diff --git a/src/filters/multi_select_filter.ts b/src/filters/multi_select_filter.ts index 2817e45..be10563 100644 --- a/src/filters/multi_select_filter.ts +++ b/src/filters/multi_select_filter.ts @@ -373,8 +373,12 @@ class MultiSelectFilter extends BaseFilter< return objKeys(filter as MultiSelectFieldFilter).reduce( (newQuery, selectedValue) => { const selectedValueFilter = filter[selectedValue]; + + const inclusion = + selectedValueFilter.inclusion || config.defaultFilterInclusion; + const newFilter = - selectedValueFilter.inclusion === 'include' + inclusion === 'include' ? {match: {[fieldNameModifier(name)]: selectedValue}} : { bool: { diff --git a/src/filters/terms_filter.ts b/src/filters/terms_filter.ts new file mode 100644 index 0000000..5da4383 --- /dev/null +++ b/src/filters/terms_filter.ts @@ -0,0 +1,516 @@ +import {runInAction, decorate, observable, set, reaction} from 'mobx'; +import {objKeys} from '../utils'; +import { + ESRequest, + ESResponse, + FilterKind, + BaseFilterConfig, + IBaseOptions, + ESMappingType, + TermsSubFieldFilterValue, + TermsFieldFilter, + RawTermsAggs, + FieldFilters, + FieldNameModifier, + FieldKinds +} from '../types'; +import BaseFilter from './base'; +import utils from './utils'; + +/** + * Config + */ +const CONFIG_DEFAULT = { + defaultFilterKind: 'should', + defaultFilterInclusion: 'include', + getCount: true, + aggsEnabled: false, + fieldNameModifierQuery: (fieldName: string) => fieldName, + fieldNameModifierAggs: (fieldName: string) => fieldName +}; + +export interface IConfig extends BaseFilterConfig { + field: string; + defaultFilterKind?: 'should' | 'must'; + defaultFilterInclusion?: 'include' | 'exclude'; + getCount?: boolean; + aggsEnabled?: boolean; + fieldNameModifierQuery?: FieldNameModifier; + fieldNameModifierAggs?: FieldNameModifier; +} + +export type IConfigs = { + [esFieldName in Fields]: IConfig; +}; + +/** + * Results + */ + +export type TermsCountResult = { + [selectedValue: string]: number; +}; + +export type CountResults = { + [esFieldName in Fields]: TermsCountResult; +}; + +// use with all fields b/c exists can check any field data value for existence +export const shouldUseField = (_fieldName: string, fieldType: ESMappingType) => + fieldType === 'keyword' || fieldType === 'text'; + +class TermsFilter extends BaseFilter { + public filteredCount: CountResults; + public unfilteredCount: CountResults; + + constructor( + defaultConfig?: Omit, 'field'>, + specificConfigs?: IConfigs, + options?: IBaseOptions + ) { + super( + 'terms', + defaultConfig || (CONFIG_DEFAULT as Omit, 'field'>), + specificConfigs as IConfigs + ); + runInAction(() => { + this._shouldUseField = (options && options.shouldUseField) || shouldUseField; + this.filteredCount = {} as CountResults; + this.unfilteredCount = {} as CountResults; + }); + + reaction( + () => { + const filteredCountFieldNames = objKeys(this.filteredCount); + + const fieldsMissingUnfilteredCounts = filteredCountFieldNames.reduce( + (acc, fieldName) => { + const filteredSubFieldNameValues = Object.keys( + this.filteredCount[fieldName] || {} + ); + const unfilteredSubFieldNameObj = this.unfilteredCount[fieldName] || {}; + + const fieldIsMissingUnfilteredCounts = filteredSubFieldNameValues.reduce( + (missingUnfilteredCounts, name) => { + if (unfilteredSubFieldNameObj[name] === undefined) { + return true; + } else { + return missingUnfilteredCounts; + } + }, + false + ); + + if (fieldIsMissingUnfilteredCounts) { + return [...acc, fieldName]; + } else { + return acc; + } + }, + [] as string[] + ); + + return fieldsMissingUnfilteredCounts; + }, + fieldsMissingUnfilteredCounts => { + if (fieldsMissingUnfilteredCounts && fieldsMissingUnfilteredCounts.length > 0) { + fieldsMissingUnfilteredCounts.forEach(field => { + this._shouldUpdateUnfilteredAggsSubscribers.forEach(s => + s(this.filterKind, field as Fields) + ); + }); + } + } + ); + } + + public userState(): { + fieldKinds?: FieldKinds; + fieldFilters?: FieldFilters; + } | void { + const kinds = Object.keys(this.fieldFilters).reduce((fieldKinds, fieldName) => { + return { + ...fieldKinds, + [fieldName]: this.kindForField(fieldName as Fields) + }; + }, {} as FieldKinds); + + const fieldFilters = Object.keys(this.fieldFilters).reduce((fieldFilterAcc, fieldName) => { + const filter = this.fieldFilters[fieldName as Fields] as TermsFieldFilter; + if (filter && Object.keys(filter).length > 0) { + return { + ...fieldFilterAcc, + [fieldName]: filter + }; + } else { + return fieldFilterAcc; + } + }, {} as FieldFilters); + + if (Object.keys(kinds).length > 0 && Object.keys(fieldFilters).length > 0) { + return { + fieldKinds: kinds, + fieldFilters + }; + } else if (Object.keys(kinds).length > 0) { + return { + fieldKinds: kinds + }; + } else if (Object.keys(fieldFilters).length > 0) { + return { + fieldFilters + }; + } else { + return; + } + } + + /** + * Alias to getter b/c computed getters can't be inherited + */ + + public get fields() { + return this._fields; + } + /** + * Alias to getter b/c computed getters can't be inherited + */ + public get activeFields() { + return this._activeFields; + } + + /** + * Clears all field filters for this filter. + * Clears all state related to aggregations. + */ + public clearAllFieldFilters = () => { + runInAction(() => { + this.fieldFilters = {} as FieldFilters; + this.filteredCount = {} as CountResults; + this.unfilteredCount = {} as CountResults; + }); + }; + + /** + * Sets a sub filter for a field. + */ + public addToFilter( + field: Fields, + subFilterName: string, + subFilterValue: TermsSubFieldFilterValue + ): void { + runInAction(() => { + const subFilters = this.fieldFilters[field]; + const newSubFilters = { + ...subFilters, + [subFilterName]: subFilterValue + }; + set(this.fieldFilters, { + [field]: newSubFilters + }); + }); + } + + /** + * Deletes a sub filter for a field. + */ + public removeFromFilter(field: Fields, subFilterName: string): void { + runInAction(() => { + const subFilters = this.fieldFilters[field]; + if (!subFilters) { + return; + } + + delete subFilters[subFilterName]; + + set(this.fieldFilters, { + [field]: subFilters + }); + }); + } + + /** + * State that should cause a global ES query request using all filters + * + * Changes to this state is tracked by the manager so that it knows when to run a new filter query + */ + public get _shouldRunFilteredQueryAndAggs(): object { + const fieldFilters = objKeys(this.fieldFilters).reduce((acc, fieldName) => { + const subFields = this.fieldFilters[fieldName] as TermsFieldFilter; + if (!subFields) { + return {...acc}; + } + // access sub field filters so those changes are tracked too + const subFieldFilters = Object.keys(subFields).reduce((accc, subFieldName) => { + return { + ...accc, + [`_$_${fieldName}-${subFieldName}`]: subFields[subFieldName] + }; + }, {} as TermsFieldFilter); + return {...acc, ...subFieldFilters}; + }, {}); + return {filters: {...fieldFilters}, kinds: {...this.fieldKinds}}; + } + + /** + * *************************************************************************** + * REQUEST BUILDERS + * *************************************************************************** + */ + + /** + * Transforms the request obj. + * + * Adds aggs to the request, but no query. + */ + public _addUnfilteredQueryAndAggsToRequest = (request: ESRequest): ESRequest => { + return [this._addCountAggsToEsRequest].reduce((newRequest, fn) => fn(newRequest), request); + }; + + /** + * Transforms the request obj. + * + * Adds aggs to the request, but no query. + */ + public _addUnfilteredAggsToRequest = ( + request: ESRequest, + fieldToFilterOn: string + ): ESRequest => { + return [this._addCountAggsToEsRequest].reduce( + (newRequest, fn) => fn(newRequest, fieldToFilterOn), + request + ); + }; + + /** + * Transforms the request obj. + * + * Adds aggs to the request, but no query. + */ + public _addFilteredAggsToRequest = (request: ESRequest, fieldToFilterOn: string): ESRequest => { + return [this._addQueriesToESRequest, this._addCountAggsToEsRequest].reduce( + (newRequest, fn) => fn(newRequest, fieldToFilterOn), + request + ); + }; + + /** + * Transforms the request obj. + * + * Adds query and aggs to the request. + */ + public _addFilteredQueryAndAggsToRequest = (request: ESRequest): ESRequest => { + return [this._addQueriesToESRequest, this._addCountAggsToEsRequest].reduce( + (newRequest, fn) => fn(newRequest), + request + ); + }; + + /** + * Transforms the request obj. + * + * Adds query to the request, but no aggs. + */ + public _addFilteredQueryToRequest = (request: ESRequest): ESRequest => { + return [this._addQueriesToESRequest].reduce((newRequest, fn) => fn(newRequest), request); + }; + + /** + * *************************************************************************** + * RESPONSE PARSERS + * *************************************************************************** + */ + + /** + * Extracts unfiltered agg stats from a response obj. + */ + public _extractUnfilteredAggsStateFromResponse = (response: ESResponse): void => { + [this._parseCountFromResponse].forEach(fn => fn(true, response)); + }; + + /** + * Extracts filtered agg stats from a response obj. + */ + public _extractFilteredAggsStateFromResponse = (response: ESResponse): void => { + [this._parseCountFromResponse].forEach(fn => fn(false, response)); + }; + + /** + * *************************************************************************** + * CUSTOM TO TEMPLATE + * *************************************************************************** + */ + + public _addQueriesToESRequest = (request: ESRequest): ESRequest => { + if (!this.fieldFilters) { + return request; + } + // tslint:disable-next-line + return objKeys(this.fieldConfigs).reduce((acc, fieldName) => { + if (!this.fieldFilters) { + return acc; + } + const config = this.fieldConfigs[fieldName]; + const name = config.field; + + const filter = this.fieldFilters[fieldName]; + if (!filter) { + return acc; + } + + const kind = this.kindForField(fieldName); + if (!kind) { + throw new Error(`kind is not set for terms filter type ${fieldName}`); + } + + const fieldNameModifier = config.fieldNameModifierQuery; + + if (filter) { + return objKeys(filter as TermsFieldFilter).reduce((newQuery, selectedValue) => { + const selectedValueFilter = filter[selectedValue]; + const terms = selectedValueFilter.terms; + + const inclusion = + selectedValueFilter.inclusion || config.defaultFilterInclusion; + + const newFilter = + inclusion === 'include' + ? {terms: {[fieldNameModifier(name)]: terms}} + : { + bool: { + must_not: { + terms: {[fieldNameModifier(name)]: terms} + } + } + }; + const kindForSelectedValue = selectedValueFilter.kind || kind; + const existingFiltersForKind = + newQuery.query.bool[kindForSelectedValue as FilterKind] || []; + + return { + ...newQuery, + query: { + ...newQuery.query, + bool: { + ...newQuery.query.bool, + [kindForSelectedValue as FilterKind]: [ + ...existingFiltersForKind, + newFilter + ] + } + } + }; + }, acc); + } else { + return acc; + } + }, request); + }; + + public _addCountAggsToEsRequest = (request: ESRequest, fieldToFilterOn?: string): ESRequest => { + // tslint:disable-next-line + return objKeys(this.fieldConfigs || {}).reduce((acc, fieldName) => { + if (fieldToFilterOn && fieldName !== fieldToFilterOn) { + return acc; + } + const config = this.fieldConfigs[fieldName]; + + const name = config.field; + if (!config || !config.aggsEnabled) { + return acc; + } + + const fieldNameModifier = config.fieldNameModifierAggs; + + const filter = this.fieldFilters[fieldName]; + if (!filter) { + return acc; + } + const valuesToFilterOn = objKeys(filter as TermsFieldFilter); + + const aggsToAdd = valuesToFilterOn.reduce((aggFilters, value) => { + return { + ...aggFilters, + [value]: { + match: { + [fieldNameModifier(name)]: value + } + } + }; + }, {}); + + if (config.getCount) { + return { + ...acc, + aggs: { + ...acc.aggs, + [`${name}__terms_count`]: { + filters: { + filters: aggsToAdd + } + } + } + }; + } else { + return acc; + } + }, request); + }; + + public _parseCountFromResponse = (isUnfilteredQuery: boolean, response: ESResponse): void => { + if (!this.fieldFilters) { + return; + } + const existingCount = isUnfilteredQuery ? this.unfilteredCount : this.filteredCount; + const count = objKeys(this.fieldConfigs).reduce( + // tslint:disable-next-line + (acc, termsFieldName) => { + const config = this.fieldConfigs[termsFieldName]; + const name = config.field; + if (config.getCount && response.aggregations) { + const allCounts = response.aggregations[`${name}__terms_count`] as RawTermsAggs; + if (allCounts && allCounts.buckets) { + const countedSelections = Object.keys(allCounts.buckets); + const countsForSelections = countedSelections.reduce( + (newState, selection) => { + return { + ...newState, + [selection]: allCounts.buckets[selection as any].doc_count + }; + }, + {} + ); + + return { + ...acc, + [termsFieldName]: countsForSelections + }; + } else { + return acc; + } + } else { + return acc; + } + }, + {...existingCount} as CountResults + ); + if (isUnfilteredQuery) { + runInAction(() => { + this.unfilteredCount = count; + }); + } else { + runInAction(() => { + this.filteredCount = count; + }); + } + }; +} + +decorate(TermsFilter, { + filteredCount: observable, + unfilteredCount: observable +}); + +utils.decorateFilter(TermsFilter); + +export default TermsFilter; diff --git a/src/manager.ts b/src/manager.ts index 04bdc57..3b93a9d 100644 --- a/src/manager.ts +++ b/src/manager.ts @@ -5,7 +5,8 @@ import { ExistsFilter, MultiSelectFilter, GeoFilter, - DateRangeFilter + DateRangeFilter, + TermsFilter } from './filters'; import { ESRequest, @@ -78,7 +79,8 @@ const DEFAULT_MANAGER_OPTIONS: Omit< boolean: new BooleanFilter(), range: new RangeFilter(), dateRange: new DateRangeFilter(), - geo: new GeoFilter() + geo: new GeoFilter(), + terms: new TermsFilter() }, suggestions: { fuzzy: new FuzzySuggestion(), diff --git a/src/types.ts b/src/types.ts index 2c6bea4..5d8824c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -9,6 +9,7 @@ import { DateRangeFilter } from './filters'; import {FuzzySuggestion, BaseSuggestion} from './suggestions'; +import TermsFilter from 'filters/terms_filter'; /** * *********************************** * ES Request @@ -43,6 +44,7 @@ export type ESRequest = { export type AggregationResults = | RawMultiSelectAggs + | RawTermsAggs | RawExistsAggs | RawRangeBoundAggs | RawRangeDistributionAggs @@ -265,6 +267,7 @@ export interface IFiltersOptions { boolean?: BooleanFilter; exists?: ExistsFilter; geo?: GeoFilter; + terms?: TermsFilter; [customFilter: string]: BaseFilter | undefined; } @@ -281,6 +284,7 @@ export interface IFilters { boolean: BooleanFilter; exists: ExistsFilter; geo: GeoFilter; + terms: TermsFilter; [customFilter: string]: BaseFilter; } @@ -320,6 +324,26 @@ export type RawMultiSelectAggs = { }>; }; +/** + * Terms Filter + */ + +export type TermsSubFieldFilterValue = { + inclusion: 'include' | 'exclude'; + kind?: 'should' | 'must'; + terms: string[]; +}; + +export type TermsFieldFilter = { + [selectedValue: string]: TermsSubFieldFilterValue; +}; + +export type RawTermsAggs = { + buckets: Array<{ + doc_count: number; + }>; +}; + /** * Geo Filter */