From e40ac3116d46105565e249fd2ee18b06183e65d2 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Thu, 11 Apr 2024 15:11:25 -0300 Subject: [PATCH 01/13] implements the query method with DataFrame transformation --- .../src/datasource.ts | 171 ++++++++++++++++-- bitmovin-analytics-datasource/src/types.ts | 9 + .../src/utils/dataUtils.ts | 45 +++++ .../src/utils/intervalUtils.ts | 23 +++ 4 files changed, 232 insertions(+), 16 deletions(-) create mode 100644 bitmovin-analytics-datasource/src/utils/dataUtils.ts create mode 100644 bitmovin-analytics-datasource/src/utils/intervalUtils.ts diff --git a/bitmovin-analytics-datasource/src/datasource.ts b/bitmovin-analytics-datasource/src/datasource.ts index 08d2d86..ff6d7df 100644 --- a/bitmovin-analytics-datasource/src/datasource.ts +++ b/bitmovin-analytics-datasource/src/datasource.ts @@ -1,15 +1,29 @@ import { + createDataFrame, DataQueryRequest, DataQueryResponse, DataSourceApi, DataSourceInstanceSettings, - MutableDataFrame, + Field, FieldType, } from '@grafana/data'; import { getBackendSrv } from '@grafana/runtime'; -import { catchError, lastValueFrom, map, of, Observable } from 'rxjs'; +import { catchError, lastValueFrom, map, Observable, of } from 'rxjs'; -import { MyQuery, MyDataSourceOptions } from './types'; +import { MyDataSourceOptions, MyQuery } from './types'; +import { zip } from 'lodash'; +import { padAndSortTimeSeries } from './utils/dataUtils'; + +type AnalyticsQuery = { + filters: { name: string; operator: string; value: number }[]; + groupBy: string[]; + orderBy: { name: string; order: string }[]; + dimension: string; + start: Date; + end: Date; + licenseKey: string; + interval?: string; +}; export class DataSource extends DataSourceApi { baseUrl: string; @@ -26,31 +40,156 @@ export class DataSource extends DataSourceApi { this.baseUrl = instanceSettings.url!; } + /** + * The Bitmovin API Response follows this rules: + * - if the interval property is given on the request query, then time series data is returned and the first value of a row is a timestamp in milliseconds + * - if the groupBy property array is not empty on the request query, then depending on whether the interval property is set all values + * - between the first one (if interval is set) and the last one (not included) can be considered string values + * - up to the last one (not included) can be considered string values + * - the last value of a row will always be a number + * */ async query(options: DataQueryRequest): Promise { const { range } = options; - const from = range!.from.valueOf(); - const to = range!.to.valueOf(); - - // Return a constant for each query. - const data = options.targets.map((target) => { - return new MutableDataFrame({ - refId: target.refId, - fields: [ - { name: 'Time', values: [from, to], type: FieldType.time }, - { name: 'Value', values: [target.constant, target.constant], type: FieldType.number }, - ], + const from = new Date(range!.from.toDate().setSeconds(0, 0)); + const to = range!.to.toDate(); + + const query: AnalyticsQuery = { + filters: [ + { + name: 'VIDEO_STARTUPTIME', + operator: 'GT', + value: 0, + }, + ], + groupBy: [], + orderBy: [ + { + name: 'MINUTE', + order: 'DESC', + }, + ], + dimension: 'IMPRESSION_ID', + start: from, + end: to, + licenseKey: '45adcf9b-8f7c-4e28-91c5-50ba3d442cd4', + interval: 'MINUTE', + }; + + const promises = options.targets.map(async (target) => { + const response = await lastValueFrom(this.request(this.getRequestUrl(), 'POST', query)); + + //TODOMY implement error handling + + const dataRows: Array> = response.data.data.result.rows; + const columnLabels: Array<{ key: string; label: string }> = response.data.data.result.columnLabels; + + const fields: Array> = []; + + //one row will look like this [timestamp, groupBy1, ... groupByN, value] + if (query.interval && query.groupBy.length > 0) { + //group the result by the groupBy values to be able to display it as multiple time series in one graph + const groupedTimeSeriesMap = new Map>>(); + dataRows.forEach((row) => { + const groupKey = row.slice(1, row.length - 1).toString(); + if (!groupedTimeSeriesMap.has(groupKey)) { + groupedTimeSeriesMap.set(groupKey, []); + } + groupedTimeSeriesMap.get(groupKey)?.push(row as []); + }); + + //pad grouped data as there can only be one time field for a graph with multiple time series + const paddedTimeSeries: Array>> = []; + groupedTimeSeriesMap.forEach((data) => { + paddedTimeSeries.push(padAndSortTimeSeries(data, from.getTime(), to.getTime(), query.interval!)); + }); + + //TODOMY we could probably also just use the range fucntion to save the timestamps, not sure whats better here? + //extract timestamps + const transposedFirstGroupData = zip(...paddedTimeSeries[0]); + const timestamps = transposedFirstGroupData[0]; + + fields.push({ name: 'Time', values: timestamps, type: FieldType.time }); + + // extract and save the values for every grouped time series + paddedTimeSeries.forEach((data, key) => { + //Field name that consists of the groupBy values of the current timeSeries + const name = data[0].slice(1, data[0].length - 1).join(', '); + + //extract values + const columns = zip(...data); + const valueColumn = columns.slice(columns.length - 1); + + fields.push({ + name: name, + values: valueColumn[0] as number[], + type: FieldType.number, + }); + }); + } else { + //data is a time series data so padding is needed and time data needs to be extracted + if (query.interval) { + const paddedData = padAndSortTimeSeries(dataRows, from.getTime(), to.getTime(), 'MINUTE'); + const columns = zip(...paddedData); + fields.push({ name: 'Time', values: columns[0] as number[], type: FieldType.time }); + fields.push({ + name: columnLabels[columnLabels.length - 1].label, + values: columns[columns.length - 1] as number[], + type: FieldType.number, + }); + } else { + //data is no timeseries, no padding required + const columns = zip(...dataRows); + + if (query.groupBy.length > 0) { + const start = query.interval !== undefined ? 1 : 0; + const end = columns.length - 1; + + const groupByColumns = columns.slice(start, end); + + groupByColumns.forEach((column, index) => { + fields.push({ + name: columnLabels[index].label, + values: column as string[], + type: FieldType.string, + }); + }); + } + + fields.push({ + name: columnLabels[columnLabels.length - 1].label, + values: columns[columns.length - 1] as number[], + type: FieldType.number, + }); + } + } + + return createDataFrame({ + fields: fields, }); }); - return { data }; + //TODOMY show error message beside the Dimension if aggregation was called on a non numeric dimension + + //TODOMY toggle for timeseries or table + return Promise.all(promises).then((data) => ({ data })); } - async request(url: string, method: string) { + getRequestUrl(): string { + if (this.adAnalytics === true) { + return '/analytics/ads/queries'; + } + + return '/analytics/queries/count'; + } + + request(url: string, method: string, payload?: any): Observable> { const options = { url: this.baseUrl + url, headers: { 'X-Api-Key': this.apiKey }, method: method, + data: payload, }; + return getBackendSrv().fetch(options); } diff --git a/bitmovin-analytics-datasource/src/types.ts b/bitmovin-analytics-datasource/src/types.ts index 811a529..1a30cc2 100644 --- a/bitmovin-analytics-datasource/src/types.ts +++ b/bitmovin-analytics-datasource/src/types.ts @@ -18,3 +18,12 @@ export interface MyDataSourceOptions extends DataSourceJsonData { tenantOrgId?: string; adAnalytics?: boolean; } + +export enum QUERY_INTERVAL { + SECOND = 'SECOND', + MINUTE = 'MINUTE', + HOUR = 'HOUR', + DAY = 'DAY', + MONTH = 'MONTH', + AUTO = 'AUTO', +} diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.ts new file mode 100644 index 0000000..b27f7e2 --- /dev/null +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.ts @@ -0,0 +1,45 @@ +import { differenceWith, sortBy } from 'lodash'; +import { intervalToMilliseconds } from './intervalUtils'; + +/** + * Adds padding to a given time series to fill in any missing timestamps for a given interval. + * @param {Array>} data The time series data to be padded. + * @param {number} startTimestamp The start timestamp for the padding interval. + * @param {number} endTimestamp The end timestamp for the padding interval. + * @param {String} interval The interval used for the query, e.g. SECOND, MINUTE, HOUR, ... . + * @returns {Array>} The padded and sorted time series data. + */ +export function padAndSortTimeSeries( + data: Array>, + startTimestamp: number, + endTimestamp: number, + interval: string +): Array> { + //TODOMY error handling for when this method returns -1 and the data is empty + const intervalInMs = intervalToMilliseconds(interval); + + let rowData: (string | number)[] = [0]; + const zeroValueTimeSeries: Array> = []; + + // Preserve groupBys in the data if present + if (data[0].length > 2) { + rowData = [...data[0].slice(1, data[0].length - 1), 0]; + } + + // Create zero value time series data for the entire interval + for (let timestamp = startTimestamp; timestamp <= endTimestamp; timestamp += intervalInMs) { + const row = [timestamp, ...rowData]; + zeroValueTimeSeries.push(row); + } + + // Find the missing time series data + const missingTimestampRows = differenceWith(zeroValueTimeSeries, data, (first, second) => first[0] === second[0]); + + // Pad data with the zero value data + const paddedData = data.concat(missingTimestampRows); + + // Sort data by timestamp + const sortedData = sortBy(paddedData, (row) => row[0]); + + return sortedData; +} diff --git a/bitmovin-analytics-datasource/src/utils/intervalUtils.ts b/bitmovin-analytics-datasource/src/utils/intervalUtils.ts new file mode 100644 index 0000000..858d59a --- /dev/null +++ b/bitmovin-analytics-datasource/src/utils/intervalUtils.ts @@ -0,0 +1,23 @@ +import { QUERY_INTERVAL } from '../types'; + +/** + * Get corresponding interval in milliseconds. + * @param {String} interval The interval + * @returns {number} Interval in milliseconds or -1 if unknown. + */ +export const intervalToMilliseconds = (interval: string): number => { + switch (interval) { + case QUERY_INTERVAL.SECOND: + return 1000; + case QUERY_INTERVAL.MINUTE: + return 1000 * 60; + case QUERY_INTERVAL.HOUR: + return 1000 * 60 * 60; + case QUERY_INTERVAL.DAY: + return 1000 * 60 * 60 * 24; + case QUERY_INTERVAL.MONTH: + return 1000 * 60 * 60 * 24 * 30; + default: + return -1; + } +}; From dc0650fb6c14305a7efc8df205b8139dab89537f Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Thu, 11 Apr 2024 15:32:34 -0300 Subject: [PATCH 02/13] extracts data transforming into own functions and improves code readability --- .../src/datasource.ts | 41 +----------- .../src/utils/dataUtils.ts | 66 +++++++++++++++++-- 2 files changed, 64 insertions(+), 43 deletions(-) diff --git a/bitmovin-analytics-datasource/src/datasource.ts b/bitmovin-analytics-datasource/src/datasource.ts index ff6d7df..fb9196c 100644 --- a/bitmovin-analytics-datasource/src/datasource.ts +++ b/bitmovin-analytics-datasource/src/datasource.ts @@ -12,7 +12,7 @@ import { catchError, lastValueFrom, map, Observable, of } from 'rxjs'; import { MyDataSourceOptions, MyQuery } from './types'; import { zip } from 'lodash'; -import { padAndSortTimeSeries } from './utils/dataUtils'; +import { padAndSortTimeSeries, transformGroupedTimeSeriesData } from './utils/dataUtils'; type AnalyticsQuery = { filters: { name: string; operator: string; value: number }[]; @@ -87,44 +87,7 @@ export class DataSource extends DataSourceApi { //one row will look like this [timestamp, groupBy1, ... groupByN, value] if (query.interval && query.groupBy.length > 0) { - //group the result by the groupBy values to be able to display it as multiple time series in one graph - const groupedTimeSeriesMap = new Map>>(); - dataRows.forEach((row) => { - const groupKey = row.slice(1, row.length - 1).toString(); - if (!groupedTimeSeriesMap.has(groupKey)) { - groupedTimeSeriesMap.set(groupKey, []); - } - groupedTimeSeriesMap.get(groupKey)?.push(row as []); - }); - - //pad grouped data as there can only be one time field for a graph with multiple time series - const paddedTimeSeries: Array>> = []; - groupedTimeSeriesMap.forEach((data) => { - paddedTimeSeries.push(padAndSortTimeSeries(data, from.getTime(), to.getTime(), query.interval!)); - }); - - //TODOMY we could probably also just use the range fucntion to save the timestamps, not sure whats better here? - //extract timestamps - const transposedFirstGroupData = zip(...paddedTimeSeries[0]); - const timestamps = transposedFirstGroupData[0]; - - fields.push({ name: 'Time', values: timestamps, type: FieldType.time }); - - // extract and save the values for every grouped time series - paddedTimeSeries.forEach((data, key) => { - //Field name that consists of the groupBy values of the current timeSeries - const name = data[0].slice(1, data[0].length - 1).join(', '); - - //extract values - const columns = zip(...data); - const valueColumn = columns.slice(columns.length - 1); - - fields.push({ - name: name, - values: valueColumn[0] as number[], - type: FieldType.number, - }); - }); + fields.push(...transformGroupedTimeSeriesData(dataRows, from.getTime(), to.getTime(), query.interval)); } else { //data is a time series data so padding is needed and time data needs to be extracted if (query.interval) { diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.ts index b27f7e2..d97a733 100644 --- a/bitmovin-analytics-datasource/src/utils/dataUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.ts @@ -1,5 +1,6 @@ -import { differenceWith, sortBy } from 'lodash'; +import { differenceWith, sortBy, zip } from 'lodash'; import { intervalToMilliseconds } from './intervalUtils'; +import { Field, FieldType } from '@grafana/data'; /** * Adds padding to a given time series to fill in any missing timestamps for a given interval. @@ -18,17 +19,17 @@ export function padAndSortTimeSeries( //TODOMY error handling for when this method returns -1 and the data is empty const intervalInMs = intervalToMilliseconds(interval); - let rowData: (string | number)[] = [0]; + let dataRows: (string | number)[] = [0]; const zeroValueTimeSeries: Array> = []; // Preserve groupBys in the data if present if (data[0].length > 2) { - rowData = [...data[0].slice(1, data[0].length - 1), 0]; + dataRows = [...data[0].slice(1, -1), 0]; } // Create zero value time series data for the entire interval for (let timestamp = startTimestamp; timestamp <= endTimestamp; timestamp += intervalInMs) { - const row = [timestamp, ...rowData]; + const row = [timestamp, ...dataRows]; zeroValueTimeSeries.push(row); } @@ -43,3 +44,60 @@ export function padAndSortTimeSeries( return sortedData; } + +/** + * Transforms grouped time series data into the Data Frame format. + * + * @param {Array>} dataRows The grouped time series data to be transformed. + * @param {number} startTimestamp The start timestamp for the time series data. + * @param {number} endTimestamp The end timestamp for the time series data. + * @param {string} interval The interval used for the time series data. + * @returns {Array>} The transformed time series data. + */ +export function transformGroupedTimeSeriesData( + dataRows: Array>, + startTimestamp: number, + endTimestamp: number, + interval: string +): Array> { + const fields: Array> = []; + + // Group the data by the groupBy values to display multiple time series in one graph + const groupedTimeSeriesMap = new Map>>(); + dataRows.forEach((row) => { + const groupKey = row.slice(1, -1).toString(); + if (!groupedTimeSeriesMap.has(groupKey)) { + groupedTimeSeriesMap.set(groupKey, []); + } + groupedTimeSeriesMap.get(groupKey)?.push(row as []); + }); + + // Pad grouped data as there can only be one time field for a graph with multiple time series + const paddedTimeSeries: Array>> = []; + groupedTimeSeriesMap.forEach((data) => { + paddedTimeSeries.push(padAndSortTimeSeries(data, startTimestamp, endTimestamp, interval!)); + }); + + // Extract and save timestamps from the first group data + const transposedFirstGroupData = zip(...paddedTimeSeries[0]); + const timestamps = transposedFirstGroupData[0]; + fields.push({ name: 'Time', values: timestamps, type: FieldType.time }); + + // Extract and save the values for every grouped time series + paddedTimeSeries.forEach((data) => { + // Field name consisting of the groupBy values of the current time series + const name = data[0].slice(1, -1).join(', '); + + //extract values + const columns = zip(...data); + const valueColumn = columns.slice(-1); + + fields.push({ + name: name, + values: valueColumn[0] as number[], + type: FieldType.number, + }); + }); + + return fields; +} From d23965777eb74fa75f2b24d52bab250b6602ef48 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Fri, 12 Apr 2024 07:40:49 -0300 Subject: [PATCH 03/13] extracts data transformation of table data and simple time series data into own functions --- .../src/datasource.ts | 51 +++++--------- .../src/utils/dataUtils.ts | 67 +++++++++++++++++++ 2 files changed, 82 insertions(+), 36 deletions(-) diff --git a/bitmovin-analytics-datasource/src/datasource.ts b/bitmovin-analytics-datasource/src/datasource.ts index fb9196c..943ba29 100644 --- a/bitmovin-analytics-datasource/src/datasource.ts +++ b/bitmovin-analytics-datasource/src/datasource.ts @@ -5,14 +5,12 @@ import { DataSourceApi, DataSourceInstanceSettings, Field, - FieldType, } from '@grafana/data'; import { getBackendSrv } from '@grafana/runtime'; import { catchError, lastValueFrom, map, Observable, of } from 'rxjs'; import { MyDataSourceOptions, MyQuery } from './types'; -import { zip } from 'lodash'; -import { padAndSortTimeSeries, transformGroupedTimeSeriesData } from './utils/dataUtils'; +import { transformGroupedTimeSeriesData, transformSimpleTimeSeries, transformTableData } from './utils/dataUtils'; type AnalyticsQuery = { filters: { name: string; operator: string; value: number }[]; @@ -85,44 +83,25 @@ export class DataSource extends DataSourceApi { const fields: Array> = []; - //one row will look like this [timestamp, groupBy1, ... groupByN, value] + // Determine the appropriate transformation based on query parameters if (query.interval && query.groupBy.length > 0) { + // If the query has an interval and group by columns, transform the data as grouped time series fields.push(...transformGroupedTimeSeriesData(dataRows, from.getTime(), to.getTime(), query.interval)); } else { - //data is a time series data so padding is needed and time data needs to be extracted if (query.interval) { - const paddedData = padAndSortTimeSeries(dataRows, from.getTime(), to.getTime(), 'MINUTE'); - const columns = zip(...paddedData); - fields.push({ name: 'Time', values: columns[0] as number[], type: FieldType.time }); - fields.push({ - name: columnLabels[columnLabels.length - 1].label, - values: columns[columns.length - 1] as number[], - type: FieldType.number, - }); + // If the query has an interval but no group by columns, transform the data as simple time series + fields.push( + ...transformSimpleTimeSeries( + dataRows as number[][], + columnLabels[columnLabels.length - 1].label, + from.getTime(), + to.getTime(), + query.interval + ) + ); } else { - //data is no timeseries, no padding required - const columns = zip(...dataRows); - - if (query.groupBy.length > 0) { - const start = query.interval !== undefined ? 1 : 0; - const end = columns.length - 1; - - const groupByColumns = columns.slice(start, end); - - groupByColumns.forEach((column, index) => { - fields.push({ - name: columnLabels[index].label, - values: column as string[], - type: FieldType.string, - }); - }); - } - - fields.push({ - name: columnLabels[columnLabels.length - 1].label, - values: columns[columns.length - 1] as number[], - type: FieldType.number, - }); + // If no interval is specified, transform the data as table data + fields.push(...transformTableData(dataRows, columnLabels)); } } diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.ts index d97a733..f08e1f4 100644 --- a/bitmovin-analytics-datasource/src/utils/dataUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.ts @@ -101,3 +101,70 @@ export function transformGroupedTimeSeriesData( return fields; } + +/** + * Transforms simple time series data into the Data Frame format. + * + * @param {Array>} dataRows The time series data to be transformed. Each data row must have the following structure: [timestamp: number, value: number] + * @param {string} columnName The name for the value column in the time series data. + * @param {number} startTimestamp The start timestamp in milliseconds for the time series data. + * @param {number} endTimestamp The end timestamp in milliseconds for the time series data. + * @param {string} interval The interval used for the time series data. + * @returns {Array>} The transformed time series data. + */ +export function transformSimpleTimeSeries( + dataRows: Array>, + columnName: string, + startTimestamp: number, + endTimestamp: number, + interval: string +): Array> { + const fields: Array> = []; + const paddedData = padAndSortTimeSeries(dataRows, startTimestamp, endTimestamp, interval); + const columns = zip(...paddedData); + + fields.push({ name: 'Time', values: columns[0] as number[], type: FieldType.time }); + fields.push({ + name: columnName, + values: columns[columns.length - 1] as number[], + type: FieldType.number, + }); + + return fields; +} + +/** + * Transforms table data into the Data Frame format. + * + * @param {Array>} dataRows The table data to be transformed. Each data row must have the following structure: [groupBy1: string, groupBy2: string, ... ,groupByN: string, value: number] + * @param {Array<{ key: string; label: string }>} columnLabels The labels for each column in the table data. + * @returns {Array>} The transformed table data. + */ +export function transformTableData( + dataRows: Array>, + columnLabels: Array<{ key: string; label: string }> +): Array> { + const fields: Array> = []; + const columns = zip(...dataRows); + + if (dataRows[0].length > 1) { + const groupByColumns = columns.slice(0, -1); + + groupByColumns.forEach((column, index) => { + fields.push({ + name: columnLabels[index].label, + values: column as string[], + type: FieldType.string, + }); + }); + } + + // Add the last column as a number field + fields.push({ + name: columnLabels[columnLabels.length - 1].label, + values: columns[columns.length - 1] as number[], + type: FieldType.number, + }); + + return fields; +} From dcd4e6fd07d2ef81dbcfc36fe12fea8ebeab12dc Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Fri, 12 Apr 2024 07:41:09 -0300 Subject: [PATCH 04/13] improves java doc and comments --- bitmovin-analytics-datasource/src/datasource.ts | 13 +++++++------ .../src/utils/dataUtils.ts | 15 ++++++++------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/bitmovin-analytics-datasource/src/datasource.ts b/bitmovin-analytics-datasource/src/datasource.ts index 943ba29..b504967 100644 --- a/bitmovin-analytics-datasource/src/datasource.ts +++ b/bitmovin-analytics-datasource/src/datasource.ts @@ -39,12 +39,13 @@ export class DataSource extends DataSourceApi { } /** - * The Bitmovin API Response follows this rules: - * - if the interval property is given on the request query, then time series data is returned and the first value of a row is a timestamp in milliseconds - * - if the groupBy property array is not empty on the request query, then depending on whether the interval property is set all values - * - between the first one (if interval is set) and the last one (not included) can be considered string values - * - up to the last one (not included) can be considered string values - * - the last value of a row will always be a number + * The Bitmovin API Response follows these rules: + * - If the interval property is provided in the request query, time series data is returned and the first value of each row is a timestamp in milliseconds. + * - If the groupBy property array is not empty in the request query: + * - Depending on whether the interval property is set: + * - Interval is set: All values between the first one (timestamp) and the last one (not included) can be considered string values. + * - Interval is not set: All values up to the last one (not included) can be considered string values + * - The last value of each row is always be a number. * */ async query(options: DataQueryRequest): Promise { const { range } = options; diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.ts index f08e1f4..f0a1b0f 100644 --- a/bitmovin-analytics-datasource/src/utils/dataUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.ts @@ -4,10 +4,11 @@ import { Field, FieldType } from '@grafana/data'; /** * Adds padding to a given time series to fill in any missing timestamps for a given interval. - * @param {Array>} data The time series data to be padded. - * @param {number} startTimestamp The start timestamp for the padding interval. - * @param {number} endTimestamp The end timestamp for the padding interval. - * @param {String} interval The interval used for the query, e.g. SECOND, MINUTE, HOUR, ... . + * + * @param {Array>} data The time series data to be padded. Each data row must have the following structure: [timestamp: number, groupBy1?: string, ... ,groupByN?: string, value: number] + * @param {number} startTimestamp The start timestamp in milliseconds for the padding interval. + * @param {number} endTimestamp The end timestamp in milliseconds for the padding interval. + * @param {String} interval The interval used for the query, e.g. MINUTE, HOUR, ... . * @returns {Array>} The padded and sorted time series data. */ export function padAndSortTimeSeries( @@ -48,9 +49,9 @@ export function padAndSortTimeSeries( /** * Transforms grouped time series data into the Data Frame format. * - * @param {Array>} dataRows The grouped time series data to be transformed. - * @param {number} startTimestamp The start timestamp for the time series data. - * @param {number} endTimestamp The end timestamp for the time series data. + * @param {Array>} dataRows The grouped time series data to be transformed. Each data row must have the following structure: [timestamp: number, groupBy1: string, groupBy2: string, ... ,groupByN: string, value: number] + * @param {number} startTimestamp The start timestamp in milliseconds for the time series data. + * @param {number} endTimestamp The end timestamp in milliseconds for the time series data. * @param {string} interval The interval used for the time series data. * @returns {Array>} The transformed time series data. */ From b8a3e1e7d5b3173c82824ba27dc89babbf6eb155 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Mon, 15 Apr 2024 11:15:59 -0300 Subject: [PATCH 05/13] adds unit tests for dataUtils and adds error and edge case handling --- .../src/datasource.ts | 9 +- .../src/utils/dataUtils.test.ts | 269 ++++++++++++++++++ .../src/utils/dataUtils.ts | 35 ++- .../src/utils/intervalUtils.ts | 1 + 4 files changed, 303 insertions(+), 11 deletions(-) create mode 100644 bitmovin-analytics-datasource/src/utils/dataUtils.test.ts diff --git a/bitmovin-analytics-datasource/src/datasource.ts b/bitmovin-analytics-datasource/src/datasource.ts index b504967..cef8e1e 100644 --- a/bitmovin-analytics-datasource/src/datasource.ts +++ b/bitmovin-analytics-datasource/src/datasource.ts @@ -77,15 +77,13 @@ export class DataSource extends DataSourceApi { const promises = options.targets.map(async (target) => { const response = await lastValueFrom(this.request(this.getRequestUrl(), 'POST', query)); - //TODOMY implement error handling - const dataRows: Array> = response.data.data.result.rows; const columnLabels: Array<{ key: string; label: string }> = response.data.data.result.columnLabels; const fields: Array> = []; // Determine the appropriate transformation based on query parameters - if (query.interval && query.groupBy.length > 0) { + if (query.interval && query.groupBy?.length > 0) { // If the query has an interval and group by columns, transform the data as grouped time series fields.push(...transformGroupedTimeSeriesData(dataRows, from.getTime(), to.getTime(), query.interval)); } else { @@ -94,7 +92,7 @@ export class DataSource extends DataSourceApi { fields.push( ...transformSimpleTimeSeries( dataRows as number[][], - columnLabels[columnLabels.length - 1].label, + columnLabels.length > 0 ? columnLabels[columnLabels.length - 1].label : 'Column 1', from.getTime(), to.getTime(), query.interval @@ -111,9 +109,6 @@ export class DataSource extends DataSourceApi { }); }); - //TODOMY show error message beside the Dimension if aggregation was called on a non numeric dimension - - //TODOMY toggle for timeseries or table return Promise.all(promises).then((data) => ({ data })); } diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.test.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.test.ts new file mode 100644 index 0000000..71fdc3f --- /dev/null +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.test.ts @@ -0,0 +1,269 @@ +import { + padAndSortTimeSeries, + transformGroupedTimeSeriesData, + transformSimpleTimeSeries, + transformTableData, +} from './dataUtils'; +import { FieldType } from '@grafana/data'; + +describe('padAndSortTimeSeries', () => { + it('should return sorted and padded data for simple time series data', () => { + //arrange + const data = [ + [1712919600000, 2], //Friday, 12 April 2024 11:00:00 + [1712919720000, 5], //Friday, 12 April 2024 11:02:00 + ]; + + const startTimestamp = 1712919540000; //Friday, 12 April 2024 10:59:00 + const endTimestamp = 1712919780000; //Friday, 12 April 2024 11:03:00 + + //act + const result = padAndSortTimeSeries(data, startTimestamp, endTimestamp, 'MINUTE'); + + //assert + expect(result).toEqual([ + [1712919540000, 0], + [1712919600000, 2], + [1712919660000, 0], + [1712919720000, 5], + [1712919780000, 0], + ]); + }); + + it('should return sorted and padded data for grouped time series data', () => { + //arrange + const data = [ + [1712919600000, 'BROWSER', 'DEVICE_TYPE', 2], //Friday, 12 April 2024 11:00:00 + [1712919720000, 'BROWSER', 'DEVICE_TYPE', 5], //Friday, 12 April 2024 11:02:00 + ]; + + const startTimestamp = 1712919540000; //Friday, 12 April 2024 10:59:00 + const endTimestamp = 1712919780000; //Friday, 12 April 2024 11:03:00 + + //act + const result = padAndSortTimeSeries(data, startTimestamp, endTimestamp, 'MINUTE'); + + //assert + expect(result).toEqual([ + [1712919540000, 'BROWSER', 'DEVICE_TYPE', 0], + [1712919600000, 'BROWSER', 'DEVICE_TYPE', 2], + [1712919660000, 'BROWSER', 'DEVICE_TYPE', 0], + [1712919720000, 'BROWSER', 'DEVICE_TYPE', 5], + [1712919780000, 'BROWSER', 'DEVICE_TYPE', 0], + ]); + }); + + it('should throw error when interval is not valid', () => { + //arrange + const data = [[0, 0]]; + + //act && assert + expect(() => padAndSortTimeSeries(data, 0, 0, 'INVALID INTERVAL')).toThrow( + new Error('Query interval INVALID INTERVAL is not a valid interval.') + ); + }); + + it('should return empty array when provided data is empty', () => { + // arrange && act + const result = padAndSortTimeSeries([], 0, 0, 'MINUTE'); + + //assert + expect(result).toEqual([]); + }); +}); + +describe('transformGroupedTimeSeriesData', () => { + it('should return correctly grouped Data Frame time series data', () => { + //arrange + const data = [ + [1712919540000, 'Firefox', 'Mac', 3], + [1712919540000, 'Safari', 'Mac', 3], + [1712919540000, 'Safari', 'iPhone', 6], + [1712919600000, 'Safari', 'Mac', 10], + [1712919660000, 'Firefox', 'Mac', 9], + [1712919720000, 'Safari', 'iPhone', 10], + [1712919780000, 'Chrome Mobile', 'iPhone', 1], + ]; + + const startTimestamp = 1712919540000; //Friday, 12 April 2024 10:59:00 + const endTimestamp = 1712919780000; //Friday, 12 April 2024 11:03:00 + + //act + const result = transformGroupedTimeSeriesData(data, startTimestamp, endTimestamp, 'MINUTE'); + + //assert + expect(result).toEqual([ + { + name: 'Time', + values: [1712919540000, 1712919600000, 1712919660000, 1712919720000, 1712919780000], + type: FieldType.time, + }, + { name: 'Firefox, Mac', values: [3, 0, 9, 0, 0], type: FieldType.number }, + { name: 'Safari, Mac', values: [3, 10, 0, 0, 0], type: FieldType.number }, + { name: 'Safari, iPhone', values: [6, 0, 0, 10, 0], type: FieldType.number }, + { name: 'Chrome Mobile, iPhone', values: [0, 0, 0, 0, 1], type: FieldType.number }, + ]); + }); + + it('should return empty array for empty dataRows', () => { + //arrange + const startTimestamp = 1712919540000; //Friday, 12 April 2024 10:59:00 + const endTimestamp = 1712919780000; //Friday, 12 April 2024 11:03:00 + + //act + const result = transformGroupedTimeSeriesData([], startTimestamp, endTimestamp, 'MINUTE'); + + //assert + expect(result).toEqual([]); + }); + + it('should throw error if interval is not valid', () => { + //arrange + const data = [ + [1712919540000, 'Firefox', 'Mac', 3], + [1712919540000, 'Safari', 'Mac', 3], + [1712919540000, 'Safari', 'iPhone', 6], + [1712919600000, 'Safari', 'Mac', 10], + [1712919660000, 'Firefox', 'Mac', 9], + [1712919720000, 'Safari', 'iPhone', 10], + [1712919780000, 'Chrome Mobile', 'iPhone', 1], + ]; + + const startTimestamp = 1712919540000; //Friday, 12 April 2024 10:59:00 + const endTimestamp = 1712919780000; //Friday, 12 April 2024 11:03:00 + + //act && assert + expect(() => transformGroupedTimeSeriesData(data, startTimestamp, endTimestamp, 'INVALID INTERVAL')).toThrow( + new Error('Query interval INVALID INTERVAL is not a valid interval.') + ); + }); +}); + +describe('transformSimpleTimeSeries', () => { + it('should return correct Data Frame time series data', () => { + //arrange + const data = [ + [1712919540000, 3], + [1712919600000, 10], + [1712919780000, 1], + ]; + + const startTimestamp = 1712919540000; //Friday, 12 April 2024 10:59:00 + const endTimestamp = 1712919780000; //Friday, 12 April 2024 11:03:00 + + //act + const result = transformSimpleTimeSeries(data, 'Impression Id', startTimestamp, endTimestamp, 'MINUTE'); + + //assert + expect(result).toEqual([ + { + name: 'Time', + values: [1712919540000, 1712919600000, 1712919660000, 1712919720000, 1712919780000], + type: FieldType.time, + }, + { name: 'Impression Id', values: [3, 10, 0, 0, 1], type: FieldType.number }, + ]); + }); + + it('should return empty array for empty dataRows', () => { + //arrange + const startTimestamp = 1712919540000; //Friday, 12 April 2024 10:59:00 + const endTimestamp = 1712919780000; //Friday, 12 April 2024 11:03:00 + + //act + const result = transformSimpleTimeSeries([], 'Impression Id', startTimestamp, endTimestamp, 'MINUTE'); + + //assert + expect(result).toEqual([]); + }); + + it('should throw error when interval is not valid', () => { + //arrange + const data = [ + [1712919540000, 3], + [1712919600000, 10], + [1712919780000, 1], + ]; + + const startTimestamp = 1712919540000; //Friday, 12 April 2024 10:59:00 + const endTimestamp = 1712919780000; //Friday, 12 April 2024 11:03:00 + + //act && assert + expect(() => + transformSimpleTimeSeries(data, 'Impression Id', startTimestamp, endTimestamp, 'INVALID INTERVAL') + ).toThrow(new Error('Query interval INVALID INTERVAL is not a valid interval.')); + }); +}); + +describe('transformTableData', () => { + it('should return correct Data Frame table data', () => { + //arrange + const data = [ + ['Firefox', 'Mac', 3], + ['Opera', 'iPhone', 10], + ['Chrome', 'iPhone', 1], + ['Chrome', 'Pixel 7', 4], + ]; + + const columnLabels = [ + { key: 'BROWSER', label: 'Browser Name' }, + { key: 'DEVICE_TYPE', label: 'Device Type' }, + { key: 'IMPRESSION_ID', label: 'Impression Id' }, + ]; + + //act + const result = transformTableData(data, columnLabels); + + //assert + expect(result).toEqual([ + { + name: 'Browser Name', + values: ['Firefox', 'Opera', 'Chrome', 'Chrome'], + type: FieldType.string, + }, + { + name: 'Device Type', + values: ['Mac', 'iPhone', 'iPhone', 'Pixel 7'], + type: FieldType.string, + }, + { name: 'Impression Id', values: [3, 10, 1, 4], type: FieldType.number }, + ]); + }); + + it('should return empty array for empty data rows', () => { + //arrange && act + + const result = transformTableData([], []); + + //assert + expect(result).toEqual([]); + }); + + it('should return array with auto generated labels for empty labels array', () => { + //arrange + const data = [ + ['Firefox', 'Mac', 3], + ['Opera', 'iPhone', 10], + ['Chrome', 'iPhone', 1], + ['Chrome', 'Pixel 7', 4], + ]; + + // act + const result = transformTableData(data, []); + + //assert + expect(result).toEqual([ + { + name: 'Column 1', + values: ['Firefox', 'Opera', 'Chrome', 'Chrome'], + type: FieldType.string, + }, + { + name: 'Column 2', + values: ['Mac', 'iPhone', 'iPhone', 'Pixel 7'], + type: FieldType.string, + }, + { name: 'Column 3', values: [3, 10, 1, 4], type: FieldType.number }, + ]); + }); +}); diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.ts index f0a1b0f..714f431 100644 --- a/bitmovin-analytics-datasource/src/utils/dataUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.ts @@ -17,8 +17,14 @@ export function padAndSortTimeSeries( endTimestamp: number, interval: string ): Array> { - //TODOMY error handling for when this method returns -1 and the data is empty + if (data.length === 0) { + return []; + } + const intervalInMs = intervalToMilliseconds(interval); + if (intervalInMs < 0) { + throw new Error(`Query interval ${interval} is not a valid interval.`); + } let dataRows: (string | number)[] = [0]; const zeroValueTimeSeries: Array> = []; @@ -61,6 +67,10 @@ export function transformGroupedTimeSeriesData( endTimestamp: number, interval: string ): Array> { + if (dataRows.length === 0) { + return []; + } + const fields: Array> = []; // Group the data by the groupBy values to display multiple time series in one graph @@ -120,6 +130,10 @@ export function transformSimpleTimeSeries( endTimestamp: number, interval: string ): Array> { + if (dataRows.length === 0) { + return []; + } + const fields: Array> = []; const paddedData = padAndSortTimeSeries(dataRows, startTimestamp, endTimestamp, interval); const columns = zip(...paddedData); @@ -137,7 +151,7 @@ export function transformSimpleTimeSeries( /** * Transforms table data into the Data Frame format. * - * @param {Array>} dataRows The table data to be transformed. Each data row must have the following structure: [groupBy1: string, groupBy2: string, ... ,groupByN: string, value: number] + * @param {Array>} dataRows The table data to be transformed. Each data row must have the following structure: [groupBy1: string, groupBy2: string, ... , groupByN: string, value: number] * @param {Array<{ key: string; label: string }>} columnLabels The labels for each column in the table data. * @returns {Array>} The transformed table data. */ @@ -145,15 +159,28 @@ export function transformTableData( dataRows: Array>, columnLabels: Array<{ key: string; label: string }> ): Array> { + if (dataRows.length === 0) { + return []; + } + const fields: Array> = []; const columns = zip(...dataRows); + let columnNames: string[] = []; + if (columnLabels.length === 0) { + for (let i = 0; i < columns.length; i++) { + columnNames.push(`Column ${i + 1}`); + } + } else { + columnNames.push(...columnLabels.map((label) => label.label)); + } + if (dataRows[0].length > 1) { const groupByColumns = columns.slice(0, -1); groupByColumns.forEach((column, index) => { fields.push({ - name: columnLabels[index].label, + name: columnNames[index], values: column as string[], type: FieldType.string, }); @@ -162,7 +189,7 @@ export function transformTableData( // Add the last column as a number field fields.push({ - name: columnLabels[columnLabels.length - 1].label, + name: columnNames[columnNames.length - 1], values: columns[columns.length - 1] as number[], type: FieldType.number, }); diff --git a/bitmovin-analytics-datasource/src/utils/intervalUtils.ts b/bitmovin-analytics-datasource/src/utils/intervalUtils.ts index 858d59a..e8f0f22 100644 --- a/bitmovin-analytics-datasource/src/utils/intervalUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/intervalUtils.ts @@ -2,6 +2,7 @@ import { QUERY_INTERVAL } from '../types'; /** * Get corresponding interval in milliseconds. + * * @param {String} interval The interval * @returns {number} Interval in milliseconds or -1 if unknown. */ From 2d6e88597c5af6fe97dac63284782360d78d7a91 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Mon, 15 Apr 2024 11:22:42 -0300 Subject: [PATCH 06/13] changes query parameter --- bitmovin-analytics-datasource/src/datasource.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitmovin-analytics-datasource/src/datasource.ts b/bitmovin-analytics-datasource/src/datasource.ts index cef8e1e..da7b822 100644 --- a/bitmovin-analytics-datasource/src/datasource.ts +++ b/bitmovin-analytics-datasource/src/datasource.ts @@ -70,7 +70,7 @@ export class DataSource extends DataSourceApi { dimension: 'IMPRESSION_ID', start: from, end: to, - licenseKey: '45adcf9b-8f7c-4e28-91c5-50ba3d442cd4', + licenseKey: '', interval: 'MINUTE', }; From c03c5b2053a9ea88cdfc4f90a9b1699ebe06bdf0 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Mon, 15 Apr 2024 11:59:23 -0300 Subject: [PATCH 07/13] trigger tests on every branch push --- bitmovin-analytics-datasource/.github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bitmovin-analytics-datasource/.github/workflows/ci.yml b/bitmovin-analytics-datasource/.github/workflows/ci.yml index 2bd2344..37f124f 100644 --- a/bitmovin-analytics-datasource/.github/workflows/ci.yml +++ b/bitmovin-analytics-datasource/.github/workflows/ci.yml @@ -3,12 +3,10 @@ name: CI on: push: branches: - - master - - main + - '**' pull_request: branches: - - master - - main + - '**' permissions: read-all From c1897c57da3ecb60bd35016a2134b8e09d2916b3 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Mon, 15 Apr 2024 12:03:01 -0300 Subject: [PATCH 08/13] Revert "trigger tests on every branch push" This reverts commit c03c5b2053a9ea88cdfc4f90a9b1699ebe06bdf0. --- bitmovin-analytics-datasource/.github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bitmovin-analytics-datasource/.github/workflows/ci.yml b/bitmovin-analytics-datasource/.github/workflows/ci.yml index 37f124f..2bd2344 100644 --- a/bitmovin-analytics-datasource/.github/workflows/ci.yml +++ b/bitmovin-analytics-datasource/.github/workflows/ci.yml @@ -3,10 +3,12 @@ name: CI on: push: branches: - - '**' + - master + - main pull_request: branches: - - '**' + - master + - main permissions: read-all From 14265e306ca8d8cc245493592be75ed47bd4ebb0 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Tue, 16 Apr 2024 07:16:22 -0300 Subject: [PATCH 09/13] implements PR feedback --- .../src/datasource.ts | 6 +-- bitmovin-analytics-datasource/src/types.ts | 6 +++ .../src/utils/dataUtils.ts | 47 ++++++++++--------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/bitmovin-analytics-datasource/src/datasource.ts b/bitmovin-analytics-datasource/src/datasource.ts index da7b822..23aea5d 100644 --- a/bitmovin-analytics-datasource/src/datasource.ts +++ b/bitmovin-analytics-datasource/src/datasource.ts @@ -9,7 +9,7 @@ import { import { getBackendSrv } from '@grafana/runtime'; import { catchError, lastValueFrom, map, Observable, of } from 'rxjs'; -import { MyDataSourceOptions, MyQuery } from './types'; +import { MixedDataRowList, MyDataSourceOptions, MyQuery, NumberDataRowList } from './types'; import { transformGroupedTimeSeriesData, transformSimpleTimeSeries, transformTableData } from './utils/dataUtils'; type AnalyticsQuery = { @@ -77,7 +77,7 @@ export class DataSource extends DataSourceApi { const promises = options.targets.map(async (target) => { const response = await lastValueFrom(this.request(this.getRequestUrl(), 'POST', query)); - const dataRows: Array> = response.data.data.result.rows; + const dataRows: MixedDataRowList = response.data.data.result.rows; const columnLabels: Array<{ key: string; label: string }> = response.data.data.result.columnLabels; const fields: Array> = []; @@ -91,7 +91,7 @@ export class DataSource extends DataSourceApi { // If the query has an interval but no group by columns, transform the data as simple time series fields.push( ...transformSimpleTimeSeries( - dataRows as number[][], + dataRows as NumberDataRowList, columnLabels.length > 0 ? columnLabels[columnLabels.length - 1].label : 'Column 1', from.getTime(), to.getTime(), diff --git a/bitmovin-analytics-datasource/src/types.ts b/bitmovin-analytics-datasource/src/types.ts index 1a30cc2..c0fe27b 100644 --- a/bitmovin-analytics-datasource/src/types.ts +++ b/bitmovin-analytics-datasource/src/types.ts @@ -27,3 +27,9 @@ export enum QUERY_INTERVAL { MONTH = 'MONTH', AUTO = 'AUTO', } + +export type MixedDataRow = Array; +export type MixedDataRowList = Array; + +export type NumberDataRow = Array; +export type NumberDataRowList = Array; diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.ts index 714f431..2c7ba04 100644 --- a/bitmovin-analytics-datasource/src/utils/dataUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.ts @@ -1,22 +1,23 @@ import { differenceWith, sortBy, zip } from 'lodash'; import { intervalToMilliseconds } from './intervalUtils'; import { Field, FieldType } from '@grafana/data'; +import { MixedDataRowList, NumberDataRow, NumberDataRowList } from '../types'; /** * Adds padding to a given time series to fill in any missing timestamps for a given interval. * - * @param {Array>} data The time series data to be padded. Each data row must have the following structure: [timestamp: number, groupBy1?: string, ... ,groupByN?: string, value: number] + * @param {MixedDataRowList} data The time series data to be padded. Each data row must have the following structure: [timestamp: number, groupBy1?: string, ... , groupByN?: string, value: number] where each row has the same groupByValue. If the groupByValues differ from row to row, only the groupByValues of the first row are considered. * @param {number} startTimestamp The start timestamp in milliseconds for the padding interval. * @param {number} endTimestamp The end timestamp in milliseconds for the padding interval. * @param {String} interval The interval used for the query, e.g. MINUTE, HOUR, ... . - * @returns {Array>} The padded and sorted time series data. + * @returns {MixedDataRowList} The padded and sorted time series data. */ export function padAndSortTimeSeries( - data: Array>, + data: MixedDataRowList, startTimestamp: number, endTimestamp: number, interval: string -): Array> { +): MixedDataRowList { if (data.length === 0) { return []; } @@ -27,7 +28,7 @@ export function padAndSortTimeSeries( } let dataRows: (string | number)[] = [0]; - const zeroValueTimeSeries: Array> = []; + const zeroValueTimeSeries: MixedDataRowList = []; // Preserve groupBys in the data if present if (data[0].length > 2) { @@ -55,14 +56,14 @@ export function padAndSortTimeSeries( /** * Transforms grouped time series data into the Data Frame format. * - * @param {Array>} dataRows The grouped time series data to be transformed. Each data row must have the following structure: [timestamp: number, groupBy1: string, groupBy2: string, ... ,groupByN: string, value: number] + * @param {MixedDataRowList} dataRows The grouped time series data to be transformed. Each data row must have the following structure: [timestamp: number, groupBy1: string, groupBy2: string, ... ,groupByN: string, value: number] * @param {number} startTimestamp The start timestamp in milliseconds for the time series data. * @param {number} endTimestamp The end timestamp in milliseconds for the time series data. * @param {string} interval The interval used for the time series data. * @returns {Array>} The transformed time series data. */ export function transformGroupedTimeSeriesData( - dataRows: Array>, + dataRows: MixedDataRowList, startTimestamp: number, endTimestamp: number, interval: string @@ -74,7 +75,7 @@ export function transformGroupedTimeSeriesData( const fields: Array> = []; // Group the data by the groupBy values to display multiple time series in one graph - const groupedTimeSeriesMap = new Map>>(); + const groupedTimeSeriesMap = new Map(); dataRows.forEach((row) => { const groupKey = row.slice(1, -1).toString(); if (!groupedTimeSeriesMap.has(groupKey)) { @@ -84,17 +85,16 @@ export function transformGroupedTimeSeriesData( }); // Pad grouped data as there can only be one time field for a graph with multiple time series - const paddedTimeSeries: Array>> = []; + const paddedTimeSeries: Array = []; groupedTimeSeriesMap.forEach((data) => { - paddedTimeSeries.push(padAndSortTimeSeries(data, startTimestamp, endTimestamp, interval!)); + paddedTimeSeries.push(padAndSortTimeSeries(data, startTimestamp, endTimestamp, interval)); }); // Extract and save timestamps from the first group data - const transposedFirstGroupData = zip(...paddedTimeSeries[0]); - const timestamps = transposedFirstGroupData[0]; - fields.push({ name: 'Time', values: timestamps, type: FieldType.time }); + const timestamps = paddedTimeSeries[0].map((row) => row[0]); + fields.push({ name: 'Time', values: timestamps as NumberDataRow, type: FieldType.time }); - // Extract and save the values for every grouped time series + // Extract time series values per group paddedTimeSeries.forEach((data) => { // Field name consisting of the groupBy values of the current time series const name = data[0].slice(1, -1).join(', '); @@ -105,7 +105,7 @@ export function transformGroupedTimeSeriesData( fields.push({ name: name, - values: valueColumn[0] as number[], + values: valueColumn[0] as NumberDataRow, type: FieldType.number, }); }); @@ -116,7 +116,7 @@ export function transformGroupedTimeSeriesData( /** * Transforms simple time series data into the Data Frame format. * - * @param {Array>} dataRows The time series data to be transformed. Each data row must have the following structure: [timestamp: number, value: number] + * @param {NumberDataRowList} dataRows The time series data to be transformed. Each data row must have the following structure: [timestamp: number, value: number] * @param {string} columnName The name for the value column in the time series data. * @param {number} startTimestamp The start timestamp in milliseconds for the time series data. * @param {number} endTimestamp The end timestamp in milliseconds for the time series data. @@ -124,7 +124,7 @@ export function transformGroupedTimeSeriesData( * @returns {Array>} The transformed time series data. */ export function transformSimpleTimeSeries( - dataRows: Array>, + dataRows: NumberDataRowList, columnName: string, startTimestamp: number, endTimestamp: number, @@ -138,10 +138,10 @@ export function transformSimpleTimeSeries( const paddedData = padAndSortTimeSeries(dataRows, startTimestamp, endTimestamp, interval); const columns = zip(...paddedData); - fields.push({ name: 'Time', values: columns[0] as number[], type: FieldType.time }); + fields.push({ name: 'Time', values: columns[0] as NumberDataRow, type: FieldType.time }); fields.push({ name: columnName, - values: columns[columns.length - 1] as number[], + values: columns[columns.length - 1] as NumberDataRow, type: FieldType.number, }); @@ -151,12 +151,12 @@ export function transformSimpleTimeSeries( /** * Transforms table data into the Data Frame format. * - * @param {Array>} dataRows The table data to be transformed. Each data row must have the following structure: [groupBy1: string, groupBy2: string, ... , groupByN: string, value: number] + * @param {MixedDataRowList} dataRows The table data to be transformed. Each data row must have the following structure: [groupBy1: string, groupBy2: string, ... , groupByN: string, value: number] * @param {Array<{ key: string; label: string }>} columnLabels The labels for each column in the table data. * @returns {Array>} The transformed table data. */ export function transformTableData( - dataRows: Array>, + dataRows: MixedDataRowList, columnLabels: Array<{ key: string; label: string }> ): Array> { if (dataRows.length === 0) { @@ -175,7 +175,8 @@ export function transformTableData( columnNames.push(...columnLabels.map((label) => label.label)); } - if (dataRows[0].length > 1) { + const containsGroupByValues = dataRows[0].length > 1; + if (containsGroupByValues) { const groupByColumns = columns.slice(0, -1); groupByColumns.forEach((column, index) => { @@ -190,7 +191,7 @@ export function transformTableData( // Add the last column as a number field fields.push({ name: columnNames[columnNames.length - 1], - values: columns[columns.length - 1] as number[], + values: columns[columns.length - 1] as NumberDataRow, type: FieldType.number, }); From e700f32aa80d9aef387ea7ed503419f591bd0448 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Tue, 16 Apr 2024 09:55:55 -0300 Subject: [PATCH 10/13] lint --- bitmovin-analytics-datasource/src/datasource.ts | 8 +++++--- bitmovin-analytics-datasource/src/types.ts | 6 +++--- bitmovin-analytics-datasource/src/utils/dataUtils.ts | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/bitmovin-analytics-datasource/src/datasource.ts b/bitmovin-analytics-datasource/src/datasource.ts index 23aea5d..fb926af 100644 --- a/bitmovin-analytics-datasource/src/datasource.ts +++ b/bitmovin-analytics-datasource/src/datasource.ts @@ -13,9 +13,9 @@ import { MixedDataRowList, MyDataSourceOptions, MyQuery, NumberDataRowList } fro import { transformGroupedTimeSeriesData, transformSimpleTimeSeries, transformTableData } from './utils/dataUtils'; type AnalyticsQuery = { - filters: { name: string; operator: string; value: number }[]; + filters: Array<{ name: string; operator: string; value: number }>; groupBy: string[]; - orderBy: { name: string; order: string }[]; + orderBy: Array<{ name: string; order: string }>; dimension: string; start: Date; end: Date; @@ -142,7 +142,9 @@ export class DataSource extends DataSourceApi { }), catchError((err) => { let message = 'Bitmovin: '; - if (err.status) message += err.status + ' '; + if (err.status) { + message += err.status + ' '; + } if (err.statusText) { message += err.statusText; } else { diff --git a/bitmovin-analytics-datasource/src/types.ts b/bitmovin-analytics-datasource/src/types.ts index c0fe27b..6a87f3c 100644 --- a/bitmovin-analytics-datasource/src/types.ts +++ b/bitmovin-analytics-datasource/src/types.ts @@ -29,7 +29,7 @@ export enum QUERY_INTERVAL { } export type MixedDataRow = Array; -export type MixedDataRowList = Array; +export type MixedDataRowList = MixedDataRow[]; -export type NumberDataRow = Array; -export type NumberDataRowList = Array; +export type NumberDataRow = number[]; +export type NumberDataRowList = NumberDataRow[]; diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.ts index 2c7ba04..64b4009 100644 --- a/bitmovin-analytics-datasource/src/utils/dataUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.ts @@ -1,7 +1,7 @@ import { differenceWith, sortBy, zip } from 'lodash'; import { intervalToMilliseconds } from './intervalUtils'; import { Field, FieldType } from '@grafana/data'; -import { MixedDataRowList, NumberDataRow, NumberDataRowList } from '../types'; +import { MixedDataRow, MixedDataRowList, NumberDataRow, NumberDataRowList } from '../types'; /** * Adds padding to a given time series to fill in any missing timestamps for a given interval. @@ -27,7 +27,7 @@ export function padAndSortTimeSeries( throw new Error(`Query interval ${interval} is not a valid interval.`); } - let dataRows: (string | number)[] = [0]; + let dataRows: MixedDataRow = [0]; const zeroValueTimeSeries: MixedDataRowList = []; // Preserve groupBys in the data if present @@ -85,7 +85,7 @@ export function transformGroupedTimeSeriesData( }); // Pad grouped data as there can only be one time field for a graph with multiple time series - const paddedTimeSeries: Array = []; + const paddedTimeSeries: MixedDataRowList[] = []; groupedTimeSeriesMap.forEach((data) => { paddedTimeSeries.push(padAndSortTimeSeries(data, startTimestamp, endTimestamp, interval)); }); From e29277647a43eba998b5a749b6717e58148fb64c Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Tue, 16 Apr 2024 09:59:45 -0300 Subject: [PATCH 11/13] extract groupedTimeSeriesTimestamps with zip instead of map to align code --- bitmovin-analytics-datasource/src/utils/dataUtils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.ts index 64b4009..6a80944 100644 --- a/bitmovin-analytics-datasource/src/utils/dataUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.ts @@ -91,7 +91,8 @@ export function transformGroupedTimeSeriesData( }); // Extract and save timestamps from the first group data - const timestamps = paddedTimeSeries[0].map((row) => row[0]); + const transposedFirstGroupTimeSeriesData = zip(...paddedTimeSeries[0]); + const timestamps = transposedFirstGroupTimeSeriesData[0]; fields.push({ name: 'Time', values: timestamps as NumberDataRow, type: FieldType.time }); // Extract time series values per group From bd2d2dbf6a38b7dfcfc40158b4f26816dcca0f87 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Tue, 16 Apr 2024 15:19:32 -0300 Subject: [PATCH 12/13] simplify QueryInterval type and move it to intervalUtils file --- bitmovin-analytics-datasource/src/datasource.ts | 3 ++- bitmovin-analytics-datasource/src/types.ts | 9 --------- .../src/utils/dataUtils.ts | 8 ++++---- .../src/utils/intervalUtils.ts | 16 ++++++++-------- 4 files changed, 14 insertions(+), 22 deletions(-) diff --git a/bitmovin-analytics-datasource/src/datasource.ts b/bitmovin-analytics-datasource/src/datasource.ts index fb926af..fc7eb92 100644 --- a/bitmovin-analytics-datasource/src/datasource.ts +++ b/bitmovin-analytics-datasource/src/datasource.ts @@ -11,6 +11,7 @@ import { catchError, lastValueFrom, map, Observable, of } from 'rxjs'; import { MixedDataRowList, MyDataSourceOptions, MyQuery, NumberDataRowList } from './types'; import { transformGroupedTimeSeriesData, transformSimpleTimeSeries, transformTableData } from './utils/dataUtils'; +import { QueryInterval } from './utils/intervalUtils'; type AnalyticsQuery = { filters: Array<{ name: string; operator: string; value: number }>; @@ -20,7 +21,7 @@ type AnalyticsQuery = { start: Date; end: Date; licenseKey: string; - interval?: string; + interval?: QueryInterval; }; export class DataSource extends DataSourceApi { diff --git a/bitmovin-analytics-datasource/src/types.ts b/bitmovin-analytics-datasource/src/types.ts index 6a87f3c..bed9c31 100644 --- a/bitmovin-analytics-datasource/src/types.ts +++ b/bitmovin-analytics-datasource/src/types.ts @@ -19,15 +19,6 @@ export interface MyDataSourceOptions extends DataSourceJsonData { adAnalytics?: boolean; } -export enum QUERY_INTERVAL { - SECOND = 'SECOND', - MINUTE = 'MINUTE', - HOUR = 'HOUR', - DAY = 'DAY', - MONTH = 'MONTH', - AUTO = 'AUTO', -} - export type MixedDataRow = Array; export type MixedDataRowList = MixedDataRow[]; diff --git a/bitmovin-analytics-datasource/src/utils/dataUtils.ts b/bitmovin-analytics-datasource/src/utils/dataUtils.ts index 6a80944..1467836 100644 --- a/bitmovin-analytics-datasource/src/utils/dataUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/dataUtils.ts @@ -1,5 +1,5 @@ import { differenceWith, sortBy, zip } from 'lodash'; -import { intervalToMilliseconds } from './intervalUtils'; +import { intervalToMilliseconds, QueryInterval } from './intervalUtils'; import { Field, FieldType } from '@grafana/data'; import { MixedDataRow, MixedDataRowList, NumberDataRow, NumberDataRowList } from '../types'; @@ -16,7 +16,7 @@ export function padAndSortTimeSeries( data: MixedDataRowList, startTimestamp: number, endTimestamp: number, - interval: string + interval: QueryInterval ): MixedDataRowList { if (data.length === 0) { return []; @@ -66,7 +66,7 @@ export function transformGroupedTimeSeriesData( dataRows: MixedDataRowList, startTimestamp: number, endTimestamp: number, - interval: string + interval: QueryInterval ): Array> { if (dataRows.length === 0) { return []; @@ -129,7 +129,7 @@ export function transformSimpleTimeSeries( columnName: string, startTimestamp: number, endTimestamp: number, - interval: string + interval: QueryInterval ): Array> { if (dataRows.length === 0) { return []; diff --git a/bitmovin-analytics-datasource/src/utils/intervalUtils.ts b/bitmovin-analytics-datasource/src/utils/intervalUtils.ts index e8f0f22..7fb18bb 100644 --- a/bitmovin-analytics-datasource/src/utils/intervalUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/intervalUtils.ts @@ -1,22 +1,22 @@ -import { QUERY_INTERVAL } from '../types'; +export type QueryInterval = 'SECOND' | 'MINUTE' | 'HOUR' | 'DAY' | 'MONTH' | 'AUTO'; /** * Get corresponding interval in milliseconds. * - * @param {String} interval The interval + * @param {QueryInterval} interval The interval * @returns {number} Interval in milliseconds or -1 if unknown. */ -export const intervalToMilliseconds = (interval: string): number => { +export const intervalToMilliseconds = (interval: QueryInterval): number => { switch (interval) { - case QUERY_INTERVAL.SECOND: + case 'SECOND': return 1000; - case QUERY_INTERVAL.MINUTE: + case 'MINUTE': return 1000 * 60; - case QUERY_INTERVAL.HOUR: + case 'HOUR': return 1000 * 60 * 60; - case QUERY_INTERVAL.DAY: + case 'DAY': return 1000 * 60 * 60 * 24; - case QUERY_INTERVAL.MONTH: + case 'MONTH': return 1000 * 60 * 60 * 24 * 30; default: return -1; From 9c76de00ae0794c1b1f59946a9354c83554ea8b5 Mon Sep 17 00:00:00 2001 From: MGJamJam Date: Wed, 17 Apr 2024 06:43:43 -0300 Subject: [PATCH 13/13] delete unused QueriIntervals --- bitmovin-analytics-datasource/src/utils/intervalUtils.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bitmovin-analytics-datasource/src/utils/intervalUtils.ts b/bitmovin-analytics-datasource/src/utils/intervalUtils.ts index 7fb18bb..1cf9787 100644 --- a/bitmovin-analytics-datasource/src/utils/intervalUtils.ts +++ b/bitmovin-analytics-datasource/src/utils/intervalUtils.ts @@ -1,4 +1,4 @@ -export type QueryInterval = 'SECOND' | 'MINUTE' | 'HOUR' | 'DAY' | 'MONTH' | 'AUTO'; +export type QueryInterval = 'MINUTE' | 'HOUR' | 'DAY'; /** * Get corresponding interval in milliseconds. @@ -8,16 +8,12 @@ export type QueryInterval = 'SECOND' | 'MINUTE' | 'HOUR' | 'DAY' | 'MONTH' | 'AU */ export const intervalToMilliseconds = (interval: QueryInterval): number => { switch (interval) { - case 'SECOND': - return 1000; case 'MINUTE': return 1000 * 60; case 'HOUR': return 1000 * 60 * 60; case 'DAY': return 1000 * 60 * 60 * 24; - case 'MONTH': - return 1000 * 60 * 60 * 24 * 30; default: return -1; }