From aa5e8b034955cef225bd3a76d647c41385beb447 Mon Sep 17 00:00:00 2001 From: Pranciskus Ambrazas Date: Mon, 5 Aug 2024 12:47:54 +0300 Subject: [PATCH] fixing stats endpoint --- services/events.service.ts | 119 ++++++++++++++++++++++++++++++++++++- services/stats.service.ts | 83 -------------------------- 2 files changed, 117 insertions(+), 85 deletions(-) delete mode 100644 services/stats.service.ts diff --git a/services/events.service.ts b/services/events.service.ts index 5c10919..4d0f916 100644 --- a/services/events.service.ts +++ b/services/events.service.ts @@ -1,7 +1,7 @@ 'use strict'; import moleculer, { Context } from 'moleculer'; -import { Method, Service } from 'moleculer-decorators'; +import { Action, Method, Service } from 'moleculer-decorators'; import PostgisMixin, { intersectsQuery } from 'moleculer-postgis'; import DbConnection from '../mixins/database.mixin'; import { @@ -15,10 +15,12 @@ import { UserAuthMeta, QueryObject, } from '../types'; -import { App } from './apps.service'; +import { App, APP_TYPE } from './apps.service'; import { LKS_SRID, parseToJsonIfNeeded } from '../utils'; import { Subscription } from './subscriptions.service'; import { Tag } from './tags.service'; +import { Knex } from 'knex'; +import _ from 'lodash'; interface Fields extends CommonFields { app: number; @@ -181,6 +183,109 @@ export function applyEventsQueryBySubscriptions(query: QueryObject, subscription }, }) export default class EventsService extends moleculer.Service { + @Action({ + rest: { + method: 'GET', + path: '/', + basePath: '/stats', + }, + auth: EndpointType.PUBLIC, + }) + async stats(ctx: Context<{ query: any }>) { + const adapter = await this.getAdapter(ctx); + const table = adapter.getTable(); + const knex: Knex = adapter.client; + + const query = await this.getComputedQuery(ctx); + const eventsQuery = adapter.computeQuery(table, query); + const tagsById: { [key: string]: Tag } = await ctx.call('tags.find', { mapping: 'id' }); + + const appTypeCaseWhenClause = Object.keys(APP_TYPE).reduce((acc: string[], key: string) => { + if (key && APP_TYPE[key]) { + acc.push(`WHEN apps.key = '${key}' THEN '${APP_TYPE[key]}'`); + } + return acc; + }, []); + + const appTypeCaseClause = `CASE ${appTypeCaseWhenClause.join(' ')} END AS app_type`; + + const eventsCountByAppType = await knex + .select('ecat.appType') + .count('ecat.id') + .from( + knex + .select('events.id', knex.raw(appTypeCaseClause)) + .from(eventsQuery.as('events')) + .leftJoin('apps', 'events.appId', 'apps.id') + .as('ecat'), + ) + .groupBy('ecat.appType'); + + const eventsCountByTagId = await knex + .select(knex.raw('jsonb_array_elements(events.tags)::numeric as tag_id')) + .count('events.id') + .from(eventsQuery.as('events')) + .groupBy('tagId'); + + const eventsCountByTagData = await knex + .select(knex.raw('td.tag_id::numeric'), 'td.tagName') + .sum(knex.raw('td.tag_value::numeric')) + .from( + knex + .select( + knex.raw(`jsonb_array_elements(events.tags_data)->>'id' as tag_id`), + knex.raw(`jsonb_array_elements(events.tags_data)->>'name' as tag_name`), + knex.raw(`jsonb_array_elements(events.tags_data)->>'value' as tag_value`), + ) + .from(eventsQuery.as('events')) + .whereNotNull('events.tagsData') + .as('td'), + ) + .groupBy(['tagId', 'tagName']); + + const stats: { + count: number; + + byApp: { + [key: App['key']]: { + count: number; + byTag?: { [key: Tag['name']]: { count: number; [key: string]: number } }; + }; + }; + } = { + byApp: {}, + count: 0, + }; + + eventsCountByAppType?.forEach((item) => { + const count = Number(item.count); + const path = `byApp.${item.appType}.count`; + const existingCount = _.get(stats, path, 0); + _.set(stats, path, existingCount + count); + stats.count += count; + }); + + eventsCountByTagId?.forEach((item) => { + const tag = tagsById[item.tagId]; + const count = Number(item.count); + + const path = `byApp.${tag.appType}.byTag.${tag.name}.count`; + const existingCount = _.get(stats, path, 0); + _.set(stats, path, existingCount + count); + }); + + eventsCountByTagData?.forEach((item) => { + const tag = tagsById[item.tagId]; + const count = Number(item.sum); + + const path = `byApp.${tag.appType}.byTag.${tag.name}.${item.tagName}`; + const existingCount = _.get(stats, path, 0); + _.set(stats, path, existingCount + count); + }); + + return stats; + } + @Method async applyFilters(ctx: Context) { ctx.params.query = parseToJsonIfNeeded(ctx.params.query) || {}; @@ -196,4 +301,14 @@ export default class EventsService extends moleculer.Service { return ctx; } + + @Method + async getComputedQuery(ctx: Context<{ query: any }>) { + let { params } = ctx; + params = this.sanitizeParams(params); + params = await this._applyScopes(params, ctx); + params = this.paramsFieldNameConversion(params); + + return parseToJsonIfNeeded(params.query) || {}; + } } diff --git a/services/stats.service.ts b/services/stats.service.ts deleted file mode 100644 index 69eafa9..0000000 --- a/services/stats.service.ts +++ /dev/null @@ -1,83 +0,0 @@ -'use strict'; - -import moleculer, { Context } from 'moleculer'; -import { Action, Service } from 'moleculer-decorators'; -import { Tag } from './tags.service'; -import { Event } from './events.service'; -import { App, APP_TYPE } from './apps.service'; -import { EndpointType } from '../types'; - -@Service({ - name: 'stats', -}) -export default class StatsService extends moleculer.Service { - @Action({ - rest: 'GET /', - auth: EndpointType.PUBLIC, - timeout: 0, - cache: { - keys: ['query'], - }, - }) - async all(ctx: Context<{ query?: any }>) { - const tagsById: { [key: string]: Tag } = await ctx.call('tags.find', { mapping: 'id' }); - - const appsById: { [key: string]: App } = await ctx.call('apps.find', { - mapping: 'id', - }); - - const events: Event[] = await ctx.call( - 'events.find', - { - query: ctx.params.query, - fields: ['id', 'app', 'tags', 'tagsData'], - }, - { - timeout: 0, - }, - ); - - const stats: { - count: number; - - byApp: { - [key: App['key']]: { - count: number; - byTag?: { [key: Tag['name']]: { count: number; [key: string]: number } }; - }; - }; - } = { - byApp: {}, - count: 0, - }; - - stats.count = events.length; - - events?.forEach((e) => { - const appKey = appsById[e.app]?.key; - const appType = APP_TYPE[appKey]; - const appStats = stats.byApp[appType] || { count: 0 }; - appStats.count++; - - e.tags?.forEach((t) => { - const tagKey = `${tagsById[t].name}`; - if (!tagKey) return; - appStats.byTag = appStats.byTag || {}; - appStats.byTag[tagKey] = appStats.byTag[tagKey] || { count: 0 }; - appStats.byTag[tagKey].count++; - - if (e.tagsData?.length) { - const matchingTags = e.tagsData.filter((i) => i.id === t); - matchingTags?.forEach((tag) => { - appStats.byTag[tagKey][tag.name] = appStats.byTag[tagKey][tag.name] || 0; - appStats.byTag[tagKey][tag.name] += tag.value; - }); - } - }); - - stats.byApp[appType] = appStats; - }); - - return stats; - } -}