From a95c18655688110295d50735078070a634c44868 Mon Sep 17 00:00:00 2001 From: Felix Brucker Date: Mon, 26 Feb 2024 09:01:33 +0700 Subject: [PATCH] Improve performance --- src/analyze-chia-log.ts | 10 +++++---- src/analyzer/corrupt-plot.ts | 5 +++-- src/analyzer/duplicate-plot.ts | 5 +++-- src/analyzer/plot-nfts.ts | 5 +++-- src/analyzer/re-org.ts | 5 +++-- src/analyzer/slow-block-validation.ts | 5 +++-- src/analyzer/slow-plots.ts | 5 +++-- src/analyzer/startup-info.ts | 5 +++-- src/extensions/array-extensions.ts | 17 ++++++++++++++ src/extensions/map-extensions.ts | 32 +++++++++++++++++++++++++++ src/log-line-mapper.ts | 8 +++---- src/util/grouping.ts | 15 +++++++++++++ 12 files changed, 95 insertions(+), 22 deletions(-) create mode 100644 src/extensions/array-extensions.ts create mode 100644 src/extensions/map-extensions.ts create mode 100644 src/util/grouping.ts diff --git a/src/analyze-chia-log.ts b/src/analyze-chia-log.ts index cc6c84f..c59c488 100644 --- a/src/analyze-chia-log.ts +++ b/src/analyze-chia-log.ts @@ -11,6 +11,7 @@ import {detectPlotNfts, PlotNft} from './analyzer/plot-nfts.js' import {detectReOrgs, ReOrg} from './analyzer/re-org.js' import {detectSpSpam, SpSpam} from './analyzer/sp-spam.js' import {detectUnsupportedDb} from './analyzer/unsupported-db.js' +import {grouping} from './util/grouping.js' export interface LogAnalyzationResult { groupedCriticalLines: GroupedLines @@ -36,11 +37,12 @@ export interface AnalyzeOptions { export function analyzeChiaLog(logFileContent: string, options?: AnalyzeOptions): LogAnalyzationResult { const logLines = mapLogFileContentsToLogLines(logFileContent) + const logLinesByLogLevel = grouping(logLines, logLine => logLine.logLevel) - const criticalLogLines = logLines.filter(logLine => logLine.logLevel === LogLevel.critical) - const errorLogLines = logLines.filter(logLine => logLine.logLevel === LogLevel.error) - const warningLogLines = logLines.filter(logLine => logLine.logLevel === LogLevel.warning) - const infoLogLines = logLines.filter(logLine => logLine.logLevel === LogLevel.info) + const criticalLogLines = logLinesByLogLevel.get(LogLevel.critical) ?? [] + const errorLogLines = logLinesByLogLevel.get(LogLevel.error) ?? [] + const warningLogLines = logLinesByLogLevel.get(LogLevel.warning) ?? [] + const infoLogLines = logLinesByLogLevel.get(LogLevel.info) ?? [] const reversedInfoLogLines = infoLogLines .slice() .reverse() diff --git a/src/analyzer/corrupt-plot.ts b/src/analyzer/corrupt-plot.ts index 772b8ce..649d169 100644 --- a/src/analyzer/corrupt-plot.ts +++ b/src/analyzer/corrupt-plot.ts @@ -1,3 +1,5 @@ +import '../extensions/array-extensions.js' + import {LogLine} from '../types/log-line.js' const plotRegex = /^File: (.+\.plot) Plot ID.*$/ @@ -9,7 +11,7 @@ export interface CorruptPlot { export function detectCorruptPlots(errorLogLines: LogLine[]): CorruptPlot[] { const plotsWithErrors = errorLogLines - .map(logLine => { + .mapAndFilter(logLine => { const matches = logLine.message.match(plotRegex) if (matches === null || matches.length !== 2) { return @@ -17,7 +19,6 @@ export function detectCorruptPlots(errorLogLines: LogLine[]): CorruptPlot[] { return matches[1] }) - .filter((plotPath): plotPath is string => plotPath !== undefined) const numberOfErrorsByPlotPath = plotsWithErrors.reduce((agg, plotPath) => { const errorCount = agg.get(plotPath) ?? 0 diff --git a/src/analyzer/duplicate-plot.ts b/src/analyzer/duplicate-plot.ts index b7d64bf..84654b8 100644 --- a/src/analyzer/duplicate-plot.ts +++ b/src/analyzer/duplicate-plot.ts @@ -1,3 +1,5 @@ +import '../extensions/array-extensions.js' + import {LogLine} from '../types/log-line.js' const duplicatePlotRegex = /^Have multiple copies of the plot (.+\.plot) in \[(.*)].*$/ @@ -9,7 +11,7 @@ export interface DuplicatePlot { export function detectDuplicatePlots(warningLogLines: LogLine[]): DuplicatePlot[] { return warningLogLines - .map((logLine): DuplicatePlot|undefined => { + .mapAndFilter((logLine): DuplicatePlot|undefined => { if (logLine.file !== 'chia.plotting.manager') { return } @@ -23,5 +25,4 @@ export function detectDuplicatePlots(warningLogLines: LogLine[]): DuplicatePlot[ plotPaths: matches[2].split(', ').map(plotPath => plotPath.trim().replaceAll(`'`, '').replaceAll('\\\\', '\\')), } }) - .filter((duplicatePlot): duplicatePlot is DuplicatePlot => duplicatePlot !== undefined) } diff --git a/src/analyzer/plot-nfts.ts b/src/analyzer/plot-nfts.ts index 11377a2..d1d813d 100644 --- a/src/analyzer/plot-nfts.ts +++ b/src/analyzer/plot-nfts.ts @@ -1,3 +1,5 @@ +import '../extensions/array-extensions.js' + import {LogLine} from '../types/log-line.js' import {convertPuzzleHashToAddress} from '../util/puzzle-hash.js' @@ -12,7 +14,7 @@ export interface PlotNft { export function detectPlotNfts(infoLogLines: LogLine[]): PlotNft[] { const plotNfts = infoLogLines - .map((logLine): PlotNft|undefined => { + .mapAndFilter((logLine): PlotNft|undefined => { if (logLine.file !== 'chia.farmer.farmer') { return } @@ -38,7 +40,6 @@ export function detectPlotNfts(infoLogLines: LogLine[]): PlotNft[] { payoutAddress, } }) - .filter((plotNft): plotNft is PlotNft => plotNft !== undefined) .reduce((agg, plotNft) => { agg.set(plotNft.launcherId, plotNft) diff --git a/src/analyzer/re-org.ts b/src/analyzer/re-org.ts index d4fb924..483d91e 100644 --- a/src/analyzer/re-org.ts +++ b/src/analyzer/re-org.ts @@ -1,3 +1,5 @@ +import '../extensions/array-extensions.js' + import {LogLine} from '../types/log-line.js' import {toChiaLogDate} from '../util/to-chia-log-date.js' @@ -12,7 +14,7 @@ export interface ReOrg { export function detectReOrgs(infoLogLines: LogLine[]): ReOrg[] { return infoLogLines - .map((logLine): ReOrg|undefined => { + .mapAndFilter((logLine): ReOrg|undefined => { if (logLine.file !== 'chia.full_node.full_node') { return } @@ -32,5 +34,4 @@ export function detectReOrgs(infoLogLines: LogLine[]): ReOrg[] { } } }) - .filter((reOrg): reOrg is ReOrg => reOrg !== undefined) } diff --git a/src/analyzer/slow-block-validation.ts b/src/analyzer/slow-block-validation.ts index 642b471..b164671 100644 --- a/src/analyzer/slow-block-validation.ts +++ b/src/analyzer/slow-block-validation.ts @@ -1,3 +1,5 @@ +import '../extensions/array-extensions.js' + import {LogLine} from '../types/log-line.js' import {Dayjs} from 'dayjs' @@ -29,7 +31,7 @@ const slowBlockValidationRegex = /^Block validation time: ([0-9]+\.[0-9]+) secon export function analyzeForSlowBlockValidation(warningLogLines: LogLine[]): SlowBlockValidationResult { const blockValidationWarnings = warningLogLines - .map((logLine): BlockValidationWarning|undefined => { + .mapAndFilter((logLine): BlockValidationWarning|undefined => { const matches = logLine.message.match(slowBlockValidationRegex) if (matches === null || (matches.length !== 5 && matches.length !== 3)) { return @@ -43,7 +45,6 @@ export function analyzeForSlowBlockValidation(warningLogLines: LogLine[]): SlowB percentFull: matches.length === 5 ? parseFloat(matches[4]) : undefined, } }) - .filter((warning): warning is BlockValidationWarning => warning !== undefined) const slowBlockValidationWarnings = blockValidationWarnings.filter(warning => warning.blockValidationTimeInSeconds >= 4) if (slowBlockValidationWarnings.length === 0) { diff --git a/src/analyzer/slow-plots.ts b/src/analyzer/slow-plots.ts index dba2a77..656c244 100644 --- a/src/analyzer/slow-plots.ts +++ b/src/analyzer/slow-plots.ts @@ -1,3 +1,5 @@ +import '../extensions/array-extensions.js' + import {LogLine} from '../types/log-line.js' import {Dayjs} from 'dayjs' @@ -10,7 +12,7 @@ export interface SlowPlotScan { export function detectSlowPlotScans(infoLogLines: LogLine[]): SlowPlotScan[] { return infoLogLines - .map((logLine): SlowPlotScan|undefined => { + .mapAndFilter((logLine): SlowPlotScan|undefined => { const matches = logLine.message.match(plotScanRegex) if (matches === null || matches.length !== 3) { return @@ -21,5 +23,4 @@ export function detectSlowPlotScans(infoLogLines: LogLine[]): SlowPlotScan[] { durationInSeconds: parseFloat(matches[2]), } }) - .filter((slowPlotScan): slowPlotScan is SlowPlotScan => slowPlotScan !== undefined && slowPlotScan.durationInSeconds > 20) } diff --git a/src/analyzer/startup-info.ts b/src/analyzer/startup-info.ts index 5aa7c5f..e348137 100644 --- a/src/analyzer/startup-info.ts +++ b/src/analyzer/startup-info.ts @@ -1,3 +1,5 @@ +import '../extensions/array-extensions.js' + import {LogLine} from '../types/log-line.js' import {Dayjs} from 'dayjs' import {mapFind} from '../util/map-find.js' @@ -141,7 +143,7 @@ export function detectStartupInfo(infoLogLines: LogLine[], reversedInfoLogLines: } const startedServices = infoLogLines - .map(logLine => { + .mapAndFilter(logLine => { const matches = logLine.message.match(startingServiceRegex) if (matches === null || matches.length !== 2) { return @@ -149,7 +151,6 @@ export function detectStartupInfo(infoLogLines: LogLine[], reversedInfoLogLines: return matches[1] }) - .filter((serviceName): serviceName is string => serviceName !== undefined) const databaseInfo = mapFind( infoLogLinesSinceStartup, diff --git a/src/extensions/array-extensions.ts b/src/extensions/array-extensions.ts new file mode 100644 index 0000000..02d1222 --- /dev/null +++ b/src/extensions/array-extensions.ts @@ -0,0 +1,17 @@ +declare global { + export interface Array { + mapAndFilter(mapper: (value: T) => V|undefined): V[] + } +} + +Array.prototype.mapAndFilter = function(mapper: (value: T) => V|undefined): V[] { + const result: V[] = [] + this.forEach((element: T) => { + const mapped = mapper(element) + if (mapped !== undefined) { + result.push(mapped) + } + }) + + return result +} diff --git a/src/extensions/map-extensions.ts b/src/extensions/map-extensions.ts new file mode 100644 index 0000000..877714b --- /dev/null +++ b/src/extensions/map-extensions.ts @@ -0,0 +1,32 @@ +declare global { + export interface Map { + getWithDefault(key: K, defaultValue: V): V + } + export interface MapConstructor { + fromObject(obj: Record): Map + fromArray(array: Array, mapToKey: (value: V) => string): Map + } +} + +Map.prototype.getWithDefault = function(key: K, defaultValue: V): V { + if (this.has(key)) { + return this.get(key) + } + + this.set(key, defaultValue) + + return defaultValue +} + +Map.fromObject = function(obj: Record): Map { + return new Map(Object.entries(obj)) +} + +Map.fromArray = function(array: Array, mapToKey: (value: V) => string): Map { + const map = new Map() + for (const value of array) { + map.set(mapToKey(value), value) + } + + return map +} diff --git a/src/log-line-mapper.ts b/src/log-line-mapper.ts index 7f582a8..56a1e3e 100644 --- a/src/log-line-mapper.ts +++ b/src/log-line-mapper.ts @@ -1,3 +1,5 @@ +import './extensions/array-extensions.js' + import dayjs from 'dayjs' import {LogLevel, LogLine} from './types/log-line.js' @@ -10,7 +12,7 @@ export function mapLogFileContentsToLogLines(logFileContents: string): LogLine[] const logLines: string[] = [] for (const line of lines) { if (line.match(logLineTimestampRegex) === null) { - if (logLines.length >= 0) { + if (logLines.length > 0) { logLines[logLines.length -1] = `${logLines[logLines.length - 1]}\n${line}` continue @@ -19,9 +21,7 @@ export function mapLogFileContentsToLogLines(logFileContents: string): LogLine[] logLines.push(line) } - return logLines - .map(mapToLogLine) - .filter((logLine): logLine is LogLine => logLine !== undefined) + return logLines.mapAndFilter(mapToLogLine) } diff --git a/src/util/grouping.ts b/src/util/grouping.ts new file mode 100644 index 0000000..701a192 --- /dev/null +++ b/src/util/grouping.ts @@ -0,0 +1,15 @@ +import '../extensions/map-extensions.js' + +export function grouping( + elements: ElementType[], + mapper: (value: ElementType) => string, +): Map { + return elements.reduce((result, element) => { + const key = mapper(element) + result + .getWithDefault(key, []) + .push(element) + + return result + }, new Map) +}