diff --git a/.eslintrc.yml b/.eslintrc.yml index 3d5e3ba9..9c2670b7 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -9,7 +9,8 @@ rules: "@kbn/eslint/require-license-header": off "prettier/prettier": off "import/no-extraneous-dependencies": off - "template-curly-spacing" : 0 + "template-curly-spacing": 0 + "no-setter-return": off indent: - off - 2 diff --git a/public/agg_table/agg_table.js b/public/agg_table/agg_table.js index cfb7b1df..7e51525e 100644 --- a/public/agg_table/agg_table.js +++ b/public/agg_table/agg_table.js @@ -6,7 +6,7 @@ import { CSV_SEPARATOR_SETTING, CSV_QUOTE_VALUES_SETTING } from '../../../../src import aggTableTemplate from './agg_table.html'; import { getFormatService } from '../services'; import { fieldFormatter } from '../field_formatter';import { computeColumnTotal } from '../column_total_computer'; -import { handleCourierRequest } from '../data_load/kibana_cloned_code/courier'; +import { handleRequest } from '../data_load/kibana_cloned_code/request_handler'; import { createTable } from '../data_load/document-table-response-handler'; import { streamSaver } from './stream_saver'; @@ -54,11 +54,11 @@ function KbnEnhancedAggTableController($scope, tableConfig){ const table = $scope.table; if ($scope.csvFullExport && self.csv.totalHits === undefined) { - self.csv.totalHits = _.get(table.request.searchSource, 'finalResponse.hits.total', -1); + self.csv.totalHits = table.totalHits; } if ($scope.csvFullExport && self.csv.totalHits > table.rows.length) { - self.exportFullAsCsv(formatted, table.request); + self.exportFullAsCsv(formatted, table.request, table.hits); } else { const csvContent = self.toCsv(table, formatted, true); @@ -67,11 +67,10 @@ function KbnEnhancedAggTableController($scope, tableConfig){ } }; - self.exportFullAsCsv = async function (formatted, request) { + self.exportFullAsCsv = async function (formatted, request, initialHits) { // store initial table last sort value if (self.csv.lastSortValue === undefined) { - const initialHits = _.get(request.searchSource, 'finalResponse.hits.hits', []); self.csv.lastSortValue = initialHits[initialHits.length - 1].sort; } @@ -91,11 +90,10 @@ function KbnEnhancedAggTableController($scope, tableConfig){ let searchAfter = self.csv.lastSortValue; do { const hitsSize = Math.min(remainingSize, self.csv.maxHitsSize); - request.searchSource.setField('size', hitsSize); - request.searchSource.setField('search_after', searchAfter); - const response = await handleCourierRequest(request); + request.searchSourceFields.size = hitsSize; + request.searchSourceFields.search_after = searchAfter; + const response = await handleRequest(request).toPromise(); response.aggs = request.aggs; - response.hits = _.get(request.searchSource, 'finalResponse.hits.hits', []); response.fieldColumns = $scope.fieldColumns; const table = createTable(response); csvBuffer = self.toCsv(table, formatted, false); diff --git a/public/data_load/document-table-response-handler.js b/public/data_load/document-table-response-handler.js index 3e6e2524..33f18d4f 100644 --- a/public/data_load/document-table-response-handler.js +++ b/public/data_load/document-table-response-handler.js @@ -55,7 +55,7 @@ function createRow(hit, columns) { } export function createTable(response) { - const table = { columns: [], rows: [], request: response.request }; + const table = { columns: [], rows: [], request: response.request, hits: response.hits, totalHits: response.totalHits }; const aggConfigs = response.aggs; aggConfigs.aggs = []; diff --git a/public/data_load/enhanced-table-request-handler.js b/public/data_load/enhanced-table-request-handler.js index 2c5e3f6f..5ac9d5fe 100644 --- a/public/data_load/enhanced-table-request-handler.js +++ b/public/data_load/enhanced-table-request-handler.js @@ -1,37 +1,34 @@ -import _ from 'lodash'; import { getSearchService } from '../services'; -import { handleCourierRequest } from './kibana_cloned_code/courier'; +import { handleRequest } from './kibana_cloned_code/request_handler'; export async function enhancedTableRequestHandler ({ - partialRows, - metricsAtAllLevels, - visParams, - timeRange, - query, + abortSignal, + aggs, filters, + indexPattern, inspectorAdapters, - forceFetch, - aggs, - queryFilter, - searchSessionId + partialRows, + query, + searchSessionId, + timeFields, + timeRange, + executionContext, + visParams, }) { - const filterManager = queryFilter; const MAX_HITS_SIZE = 10000; - // create search source with query parameters - const searchService = getSearchService(); - const searchSource = await searchService.searchSource.create(); - searchSource.setField('index', aggs.indexPattern); + // create search source fields + const searchSourceFields = {}; let hitsSize = (visParams.hitsSize !== undefined ? Math.min(visParams.hitsSize, MAX_HITS_SIZE) : 0); - searchSource.setField('size', hitsSize); + searchSourceFields.size = hitsSize; // specific request params for "field columns" if (visParams.fieldColumns !== undefined) { if (!visParams.fieldColumns.some(fieldColumn => fieldColumn.field.name === '_source')) { - searchSource.setField('_source', visParams.fieldColumns.map(fieldColumn => fieldColumn.field.name)); + searchSourceFields._source = visParams.fieldColumns.map(fieldColumn => fieldColumn.field.name); } - searchSource.setField('docvalue_fields', visParams.fieldColumns.filter(fieldColumn => fieldColumn.field.readFromDocValues).map(fieldColumn => fieldColumn.field.name)); + searchSourceFields.docvalue_fields = visParams.fieldColumns.filter(fieldColumn => fieldColumn.field.readFromDocValues).map(fieldColumn => fieldColumn.field.name); const scriptFields = {}; visParams.fieldColumns.filter(fieldColumn => fieldColumn.field.scripted).forEach(fieldColumn => { scriptFields[fieldColumn.field.name] = { @@ -40,18 +37,18 @@ export async function enhancedTableRequestHandler ({ } }; }); - searchSource.setField('script_fields', scriptFields); + searchSourceFields.script_fields = scriptFields; } // set search sort if (visParams.sortField !== undefined) { - searchSource.setField('sort', [{ + searchSourceFields.sort = [{ [visParams.sortField.name]: { order: visParams.sortOrder } - }]); + }]; if ((visParams.hitsSize !== undefined && visParams.hitsSize > MAX_HITS_SIZE) || visParams.csvFullExport) { - searchSource.getField('sort').push({'_id': {'order': 'asc','unmapped_type': 'keyword'}}); + searchSourceFields.sort.push({'_id': {'order': 'asc','unmapped_type': 'keyword'}}); } } @@ -68,20 +65,21 @@ export async function enhancedTableRequestHandler ({ // execute elasticsearch query const request = { - searchSource: searchSource, - aggs: aggs, - indexPattern: aggs.indexPattern, - timeRange: timeRange, - query: query, - filters: filters, - forceFetch: forceFetch, - metricsAtAllLevels: metricsAtAllLevels, - partialRows: partialRows, - inspectorAdapters: inspectorAdapters, - filterManager: filterManager, - searchSessionId: searchSessionId + abortSignal, + aggs, + filters, + indexPattern, + inspectorAdapters, + partialRows, + query, + searchSessionId, + searchSourceService: getSearchService().searchSource, + timeFields, + timeRange, + executionContext, + searchSourceFields, }; - const response = await handleCourierRequest(request); + const response = await handleRequest(request).toPromise(); // set 'split tables' direction const splitAggs = aggs.bySchemaName('split'); @@ -89,8 +87,7 @@ export async function enhancedTableRequestHandler ({ splitAggs[0].params.row = visParams.row; } - // enrich response: total & aggs - response.totalHits = _.get(searchSource, 'finalResponse.hits.total', -1); + // enrich response: aggs response.aggs = aggs; // enrich columns: aggConfig @@ -101,7 +98,6 @@ export async function enhancedTableRequestHandler ({ // enrich response: hits if (visParams.fieldColumns !== undefined) { response.fieldColumns = visParams.fieldColumns; - response.hits = _.get(searchSource, 'finalResponse.hits.hits', []); // continue requests until expected hits size is reached if (visParams.hitsSize !== undefined && visParams.hitsSize > MAX_HITS_SIZE && response.totalHits > MAX_HITS_SIZE) { @@ -110,12 +106,11 @@ export async function enhancedTableRequestHandler ({ remainingSize -= hitsSize; const searchAfter = response.hits[response.hits.length - 1].sort; hitsSize = Math.min(remainingSize, MAX_HITS_SIZE); - searchSource.setField('size', hitsSize); - searchSource.setField('search_after', searchAfter); - await handleCourierRequest(request); - const nextResponseHits = _.get(searchSource, 'finalResponse.hits.hits', []); - for (let i = 0; i < nextResponseHits.length; i++) { - response.hits.push(nextResponseHits[i]); + searchSourceFields.size = hitsSize; + searchSourceFields.search_after = searchAfter; + const nextResponse = await handleRequest(request).toPromise(); + for (let i = 0; i < nextResponse.hits.length; i++) { + response.hits.push(nextResponse.hits[i]); } } while (remainingSize > hitsSize); } diff --git a/public/data_load/kibana_cloned_code/courier.ts b/public/data_load/kibana_cloned_code/courier.ts deleted file mode 100644 index b408a014..00000000 --- a/public/data_load/kibana_cloned_code/courier.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { i18n } from '@kbn/i18n'; - -import { Adapters } from '../../../../../src/plugins/inspector/common'; - -import { IAggConfigs } from '../../../../../src/plugins/data/common/search/aggs'; -import { ISearchSource } from '../../../../../src/plugins/data/common/search/search_source'; -import { - calculateBounds, - Filter, - IndexPattern, - Query, - tabifyAggResponse, - TimeRange, -} from '../../../../../src/plugins/data/common'; - -/** - * Clone of: ../../../../../src/plugins/data/common/search/expressions/esaggs/request_handler.ts - * Components: RequestHandlerParams and handleCourierRequest - */ -interface RequestHandlerParams { - abortSignal?: AbortSignal; - aggs: IAggConfigs; - filters?: Filter[]; - indexPattern?: IndexPattern; - inspectorAdapters: Adapters; - metricsAtAllLevels?: boolean; - partialRows?: boolean; - query?: Query; - searchSessionId?: string; - searchSource: ISearchSource; - timeFields?: string[]; - timeRange?: TimeRange; - getNow?: () => Date; -} - -export const handleCourierRequest = async ({ - abortSignal, - aggs, - filters, - indexPattern, - inspectorAdapters, - metricsAtAllLevels, - partialRows, - query, - searchSessionId, - searchSource, - timeFields, - timeRange, - getNow, -}: RequestHandlerParams) => { - const forceNow = getNow?.(); - - // Create a new search source that inherits the original search source - // but has the appropriate timeRange applied via a filter. - // This is a temporary solution until we properly pass down all required - // information for the request to the request handler (https://github.com/elastic/kibana/issues/16641). - // Using callParentStartHandlers: true we make sure, that the parent searchSource - // onSearchRequestStart will be called properly even though we use an inherited - // search source. - const timeFilterSearchSource = searchSource.createChild({ callParentStartHandlers: true }); - const requestSearchSource = timeFilterSearchSource.createChild({ callParentStartHandlers: true }); - - aggs.setTimeRange(timeRange as TimeRange); - - // For now we need to mirror the history of the passed search source, since - // the request inspector wouldn't work otherwise. - Object.defineProperty(requestSearchSource, 'history', { - get() { - return searchSource.history; - }, - set(history) { - searchSource.history = history; - }, - }); - - requestSearchSource.setField('aggs', aggs); - - requestSearchSource.onRequestStart((paramSearchSource, options) => { - return aggs.onSearchRequestStart(paramSearchSource, options); - }); - - // If timeFields have been specified, use the specified ones, otherwise use primary time field of index - // pattern if it's available. - const defaultTimeField = indexPattern?.getTimeField?.(); - const defaultTimeFields = defaultTimeField ? [defaultTimeField.name] : []; - const allTimeFields = timeFields && timeFields.length > 0 ? timeFields : defaultTimeFields; - - aggs.setForceNow(forceNow); - aggs.setTimeFields(allTimeFields); - - // If a timeRange has been specified and we had at least one timeField available, create range - // filters for that those time fields - if (timeRange && allTimeFields.length > 0) { - timeFilterSearchSource.setField('filter', () => { - return aggs.getSearchSourceTimeFilter(forceNow); - }); - } - - requestSearchSource.setField('filter', filters); - requestSearchSource.setField('query', query); - - inspectorAdapters.requests?.reset(); - - const { rawResponse: response } = await requestSearchSource - .fetch$({ - abortSignal, - sessionId: searchSessionId, - inspector: { - adapter: inspectorAdapters.requests, - title: i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { - defaultMessage: 'Data', - }), - description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { - defaultMessage: - 'This request queries Elasticsearch to fetch the data for the visualization.', - }), - }, - }) - .toPromise(); - - const parsedTimeRange = timeRange ? calculateBounds(timeRange, { forceNow }) : null; - const tabifyParams = { - metricsAtAllLevels, - partialRows, - timeRange: parsedTimeRange - ? { from: parsedTimeRange.min, to: parsedTimeRange.max, timeFields: allTimeFields } - : undefined, - }; - - // Need this so the enhancedTableRequestHandler can recover the hits for the document table - (searchSource as any).finalResponse = response; - - const tabifiedResponse = tabifyAggResponse(aggs, response, tabifyParams); - - return tabifiedResponse; -}; diff --git a/public/data_load/kibana_cloned_code/request_handler.ts b/public/data_load/kibana_cloned_code/request_handler.ts new file mode 100644 index 00000000..db36cd09 --- /dev/null +++ b/public/data_load/kibana_cloned_code/request_handler.ts @@ -0,0 +1,162 @@ +import { i18n } from '@kbn/i18n'; +import { defer } from 'rxjs'; +import { map, switchMap } from 'rxjs/operators'; +import { get } from 'lodash'; + + +import type { KibanaExecutionContext } from '../../../../../src/core/public'; +import { Adapters } from '../../../../../src/plugins/inspector/common'; + +import { + calculateBounds, + Filter, + IndexPattern, + Query, + TimeRange, + IAggConfigs, + ISearchStartSearchSource, + tabifyAggResponse, +} from '../../../../../src/plugins/data/common'; + +/** + * Clone of: ../../../../../src/plugins/data/common/search/expressions/esaggs/request_handler.ts + * Customizations: + * - searchSourceFields param + * - response.totalHits + * - response.hits + */ + +/** @internal */ +export interface RequestHandlerParams { + abortSignal?: AbortSignal; + aggs: IAggConfigs; + filters?: Filter[]; + indexPattern?: IndexPattern; + inspectorAdapters: Adapters; + metricsAtAllLevels?: boolean; + partialRows?: boolean; + query?: Query; + searchSessionId?: string; + searchSourceService: ISearchStartSearchSource; + timeFields?: string[]; + timeRange?: TimeRange; + getNow?: () => Date; + executionContext?: KibanaExecutionContext; + searchSourceFields: {[key: string]: any}; +} + +export const handleRequest = ({ + abortSignal, + aggs, + filters, + indexPattern, + inspectorAdapters, + partialRows, + query, + searchSessionId, + searchSourceService, + timeFields, + timeRange, + getNow, + executionContext, + searchSourceFields, +}: RequestHandlerParams) => { + return defer(async () => { + const forceNow = getNow?.(); + const searchSource = await searchSourceService.create(); + + searchSource.setField('index', indexPattern); + Object.keys(searchSourceFields).forEach(fieldName => { + searchSource.setField(fieldName as any, searchSourceFields[fieldName]); + }); + + // Create a new search source that inherits the original search source + // but has the appropriate timeRange applied via a filter. + // This is a temporary solution until we properly pass down all required + // information for the request to the request handler (https://github.com/elastic/kibana/issues/16641). + // Using callParentStartHandlers: true we make sure, that the parent searchSource + // onSearchRequestStart will be called properly even though we use an inherited + // search source. + const timeFilterSearchSource = searchSource.createChild({ callParentStartHandlers: true }); + const requestSearchSource = timeFilterSearchSource.createChild({ + callParentStartHandlers: true, + }); + + // If timeFields have been specified, use the specified ones, otherwise use primary time field of index + // pattern if it's available. + const defaultTimeField = indexPattern?.getTimeField?.(); + const defaultTimeFields = defaultTimeField ? [defaultTimeField.name] : []; + const allTimeFields = timeFields?.length ? timeFields : defaultTimeFields; + + aggs.setTimeRange(timeRange as TimeRange); + aggs.setForceNow(forceNow); + aggs.setTimeFields(allTimeFields); + + // For now we need to mirror the history of the passed search source, since + // the request inspector wouldn't work otherwise. + Object.defineProperty(requestSearchSource, 'history', { + get() { + return searchSource.history; + }, + set(history) { + return (searchSource.history = history); + }, + }); + + requestSearchSource.setField('aggs', aggs); + + requestSearchSource.onRequestStart((paramSearchSource, options) => { + return aggs.onSearchRequestStart(paramSearchSource, options); + }); + + // If a timeRange has been specified and we had at least one timeField available, create range + // filters for that those time fields + if (timeRange && allTimeFields.length > 0) { + timeFilterSearchSource.setField('filter', () => { + return aggs.getSearchSourceTimeFilter(forceNow); + }); + } + + requestSearchSource.setField('filter', filters); + requestSearchSource.setField('query', query); + + return { allTimeFields, forceNow, requestSearchSource }; + }).pipe( + switchMap(({ allTimeFields, forceNow, requestSearchSource }) => + requestSearchSource + .fetch$({ + abortSignal, + sessionId: searchSessionId, + inspector: { + adapter: inspectorAdapters.requests, + title: i18n.translate('data.functions.esaggs.inspector.dataRequest.title', { + defaultMessage: 'Data', + }), + description: i18n.translate('data.functions.esaggs.inspector.dataRequest.description', { + defaultMessage: + 'This request queries Elasticsearch to fetch the data for the visualization.', + }), + }, + executionContext, + }) + .pipe( + map(({ rawResponse: response }) => { + const parsedTimeRange = timeRange ? calculateBounds(timeRange, { forceNow }) : null; + const tabifyParams = { + metricsAtAllLevels: aggs.hierarchical, + partialRows, + timeRange: parsedTimeRange + ? { from: parsedTimeRange.min, to: parsedTimeRange.max, timeFields: allTimeFields } + : undefined, + }; + + return { + ...tabifyAggResponse(aggs, response, tabifyParams), + totalHits: get(response, 'hits.total', -1), + hits: get(response, 'hits.hits', []), + }; + }) + ) + ) + ); +}; diff --git a/public/data_load/kibana_cloned_code/utils.ts b/public/data_load/kibana_cloned_code/utils.ts index 8b28fccf..b04e82be 100644 --- a/public/data_load/kibana_cloned_code/utils.ts +++ b/public/data_load/kibana_cloned_code/utils.ts @@ -1,8 +1,6 @@ -import { IFieldFormat } from '../../../../../src/plugins/data/common'; - /** - * Clone of: '../../../../../src/plugins/data/public/search/expressions/utils/serialize_agg_config.ts' - * Component: serializeAggConfig + * Inspired from: '../../../../../src/plugins/data/common/search/tabify/response_writer.ts' + * Function: response */ export function serializeAggConfig(aggConfig) { const sourceParams = { @@ -18,9 +16,3 @@ export function serializeAggConfig(aggConfig) { sourceParams }; } - -/** - * Clone of: '../../../../../src/plugins/data/common/field_formats/utils.ts' - * Component: FormatFactory -*/ -export type FormatFactory = (mapping?) => IFieldFormat; diff --git a/public/data_load/visualization_fn.ts b/public/data_load/visualization_fn.ts index a3919319..a6bbe8a4 100644 --- a/public/data_load/visualization_fn.ts +++ b/public/data_load/visualization_fn.ts @@ -1,7 +1,7 @@ import { get } from 'lodash'; import { i18n } from '@kbn/i18n'; import { ExpressionFunctionDefinition, Render } from '../../../../src/plugins/expressions/public'; -import { getIndexPatterns, getFilterManager, getSearchService, getVisualization } from '../services'; +import { getIndexPatterns, getSearchService, getVisualization } from '../services'; import { enhancedTableRequestHandler } from './enhanced-table-request-handler'; import { enhancedTableResponseHandler } from './enhanced-table-response-handler'; import { documentTableResponseHandler } from './document-table-response-handler'; @@ -15,6 +15,7 @@ interface Arguments { visConfig?: string; uiState?: string; aggConfigs?: string; + timeFields?: string[]; } export interface CommonVisRenderValue { @@ -87,7 +88,7 @@ const expressionFunction = (visName: VisName, responseHandler: ResponseHandler): help: 'Aggregation configurations', }, }, - async fn(input, args, { inspectorAdapters, getSearchSessionId }) { + async fn(input, args, { inspectorAdapters, abortSignal, getSearchSessionId, getExecutionContext }) { const visConfigParams = args.visConfig ? JSON.parse(args.visConfig) : {}; const schemas = args.schemas ? JSON.parse(args.schemas) : {}; const indexPattern = args.index ? await getIndexPatterns().get(args.index) : null; @@ -96,20 +97,22 @@ const expressionFunction = (visName: VisName, responseHandler: ResponseHandler): const aggs = indexPattern ? getSearchService().aggs.createAggConfigs(indexPattern, aggConfigsState) : undefined; + aggs.hierarchical = args.metricsAtAllLevels; const visType = getVisualization().get(visName); input = await enhancedTableRequestHandler({ - partialRows: args.partialRows, - metricsAtAllLevels: args.metricsAtAllLevels, - visParams: visConfigParams, - timeRange: get(input, 'timeRange', null), - query: get(input, 'query', null), + abortSignal, + aggs, filters: get(input, 'filters', null), + indexPattern, inspectorAdapters, - queryFilter: getFilterManager(), - aggs, - forceFetch: true, + partialRows: args.partialRows, + query: get(input, 'query', null), searchSessionId: getSearchSessionId(), + timeFields: args.timeFields, + timeRange: get(input, 'timeRange', null), + executionContext: getExecutionContext(), + visParams: visConfigParams, });