From fc51ead9f2ef5311a548073b85432716c622a50c Mon Sep 17 00:00:00 2001 From: Ivan Kerin Date: Fri, 30 Dec 2022 13:24:30 +0200 Subject: [PATCH] Implement logging for debugging / diagnostics --- package.json | 2 +- src/parser.ts | 50 +- src/types.ts | 8 + .../keppel/__snapshots__/grammer.spec.ts.snap | 548 ++++++++++++++++++ test/keppel/grammer.spec.ts | 17 + 5 files changed, 617 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 524b64e..774637f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ikerin/rd-parse", - "version": "4.0.1", + "version": "4.1.0", "main": "dist/index.js", "types": "dist/index.d.ts", "description": "Recursive descent parser generator. Define your grammar in pure Javascript.", diff --git a/src/parser.ts b/src/parser.ts index 3f9528c..158c6ce 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1,5 +1,5 @@ import { ParserError } from './ParserError'; -import { Context, Stack, FunctionRule, Rule, Token } from './types'; +import { Context, Stack, FunctionRule, Rule, Token, Trace } from './types'; function locAt(text: string, newPos: number, { pos, line, column }: Context): Context { while (pos < newPos) { @@ -18,9 +18,37 @@ function markSeen($: Stack): void { } } +function logRule($: Stack, rule: string | RegExp): void { + if ($.log) { + const label = String(rule); + $.log.ruleCalls[label] = label in $.log.ruleCalls ? $.log.ruleCalls[label] + 1 : 1; + } +} + +function logNode($: Stack): [number, number]; +function logNode($: Stack, start: [number, number], node: Token): void; +function logNode($: Stack, start?: [number, number], node?: Token): void | [number, number] { + if ($.log) { + const time = process.hrtime(start); + if (start) { + const name = $.log.nodeTrace(node); + if (name) { + $.log.nodeExecutionTime[name] = + name in $.log.nodeExecutionTime + ? [$.log.nodeExecutionTime[name][0] + time[0], $.log.nodeExecutionTime[name][1] + time[1]] + : time; + $.log.nodeCalls[name] = name in $.log.nodeCalls ? $.log.nodeCalls[name] + 1 : 1; + } + } else { + return time; + } + } +} + export function RegexToken(pattern: RegExp): FunctionRule { return ($) => { markSeen($); + logRule($, pattern); const match = pattern.exec($.text.substring($.pos)); if (!match) return $; @@ -39,6 +67,7 @@ export function RegexToken(pattern: RegExp): FunctionRule { export function StringToken(pattern: string): FunctionRule { return ($) => { markSeen($); + logRule($, pattern); return $.text.startsWith(pattern, $.pos) ? { ...$, pos: $.pos + pattern.length } : $; }; } @@ -187,14 +216,19 @@ export function Node const functionRule = Use(rule); return ($) => { + const time = logNode($); const $next = functionRule($); + if ($next === $) return $; // We have a match const node = reducer($.stack.slice($.sp, $next.sp) as Capture, $, $next); - $next.sp = $.sp; - if (node !== null) $.stack[$next.sp++] = node; + $next.sp = $.sp; + if (node !== null) { + $.stack[$next.sp++] = node; + logNode($, time, node); + } return $next; }; } @@ -239,7 +273,7 @@ export function Y(proc: (x: FunctionRule) => FunctionRule): FunctionRule { return ((x) => proc((y) => x(x)(y)))((x: any) => proc((y) => x(x)(y))); } -const START = (text: string, pos = 0): Stack => ({ +const START = (text: string, pos = 0, log?: (node: Token) => string): Stack => ({ text, ignore: [], stack: [], @@ -247,15 +281,17 @@ const START = (text: string, pos = 0): Stack => ({ sp: 0, lastSeen: locAt(text, pos, { pos: 0, line: 1, column: 1 }), pos, + ...(log ? { log: { nodeTrace: log, nodeExecutionTime: {}, ruleCalls: {}, nodeCalls: {} } } : {}), }); export function Parser( Grammar: FunctionRule, pos = 0, partial = false, + log?: (node: Token) => string, ) { - return (text: string): { ast: TAstToken; comments: TCommentToken[] } => { - const $ = START(text, pos); + return (text: string): { ast: TAstToken; comments: TCommentToken[]; log?: Trace } => { + const $ = START(text, pos, log); const $next = Grammar($); if ($ === $next || (!partial && $next.pos < text.length)) { @@ -263,6 +299,6 @@ export function Parser string; + ruleCalls: Record; + nodeCalls: Record; + nodeExecutionTime: Record; +} + export interface Stack { /** * Current position within 'text' @@ -35,6 +42,7 @@ export interface Stack { */ stack: Token[]; comments: Token[]; + log?: Trace; } export type FunctionRule = ($: Stack) => Stack; diff --git a/test/keppel/__snapshots__/grammer.spec.ts.snap b/test/keppel/__snapshots__/grammer.spec.ts.snap index 74be114..b553396 100644 --- a/test/keppel/__snapshots__/grammer.spec.ts.snap +++ b/test/keppel/__snapshots__/grammer.spec.ts.snap @@ -64,6 +64,168 @@ body [ // I am a comment " `; +exports[`Keppel Keppel grammar parser error for partial: correct 1`] = ` +Object { + "ast": Array [ + Object { + "body": Array [ + Object { + "body": Array [ + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "href", + "value": "#", + }, + Object { + "name": "test", + "value": "My test", + }, + ], + "body": Array [ + Object { + "end": 156, + "start": 148, + "type": "free text", + "value": "Awesome", + }, + ], + "classes": Array [ + "brand", + ], + "end": 157, + "name": "a", + "start": 113, + "type": "tag", + }, + Object { + "body": Array [ + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "href", + "value": "#", + }, + ], + "body": Array [ + Object { + "end": 206, + "start": 199, + "type": "free text", + "value": "Item 1", + }, + ], + "end": 207, + "name": "a", + "start": 186, + "type": "tag", + }, + ], + "end": 209, + "name": "li", + "start": 182, + "type": "tag", + }, + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "href", + "value": "#", + }, + ], + "body": Array [ + Object { + "end": 243, + "start": 236, + "type": "free text", + "value": "Item 2", + }, + ], + "end": 244, + "name": "a", + "start": 223, + "type": "tag", + }, + ], + "end": 246, + "name": "li", + "start": 219, + "type": "tag", + }, + ], + "classes": Array [ + "nav", + ], + "end": 254, + "name": "ul", + "start": 165, + "type": "tag", + }, + ], + "classes": Array [ + "navbar-inner", + ], + "end": 260, + "name": "div", + "start": 88, + "type": "tag", + }, + ], + "classes": Array [ + "navbar", + "navbar-inverse", + "navbar-fixed-top", + ], + "end": 264, + "id": "navbar", + "name": "div", + "start": 30, + "type": "tag", + }, + ], + "end": 266, + "name": "body", + "start": 0, + "type": "tag", + }, + ], + "comments": Array [ + Object { + "end": 27, + "start": 10, + "type": "comment", + "value": " I am a comment", + }, + Object { + "end": 305, + "start": 268, + "type": "comment", + "value": " I am a comment in the end of input", + }, + ], +} +`; + +exports[`Keppel Keppel grammar parser error for partial: wrong 1`] = ` +Object { + "ast": Array [ + Object { + "end": 3, + "name": "body", + "start": 0, + "type": "tag", + }, + ], + "comments": Array [], +} +`; + exports[`Keppel Keppel grammar parser generation 1 1`] = ` Object { "ast": Array [ @@ -572,6 +734,392 @@ Object { } `; +exports[`Keppel Keppel grammar parser with log 1`] = ` +Object { + "ast": Array [ + Object { + "end": 16, + "start": 0, + "type": "free text", + "value": "", + }, + Object { + "attributes": Array [ + Object { + "name": "lang", + "value": "en", + }, + ], + "body": Array [ + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "charset", + "value": "utf-8", + }, + ], + "end": 69, + "name": "meta", + "start": 49, + "type": "tag", + }, + Object { + "body": Array [ + Object { + "end": 106, + "start": 82, + "type": "free text", + "value": "Awesome nav bar example", + }, + ], + "end": 107, + "name": "title", + "start": 75, + "type": "tag", + }, + Object { + "attributes": Array [ + Object { + "name": "rel", + "value": "stylesheet", + }, + Object { + "name": "href", + "value": "http://netdna.bootstrapcdn.com/twitter-bootstrap/2.3.1/css/bootstrap-combined.no-icons.min.css", + }, + ], + "end": 237, + "name": "link", + "start": 113, + "type": "tag", + }, + Object { + "attributes": Array [ + Object { + "name": "rel", + "value": "stylesheet", + }, + Object { + "name": "href", + "value": "http://netdna.bootstrapcdn.com/font-awesome/3.1.1/css/font-awesome.min.css", + }, + ], + "end": 347, + "name": "link", + "start": 243, + "type": "tag", + }, + ], + "end": 351, + "name": "head", + "start": 38, + "type": "tag", + }, + Object { + "body": Array [ + Object { + "body": Array [ + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "href", + "value": "#", + }, + ], + "body": Array [ + Object { + "end": 480, + "start": 472, + "type": "free text", + "value": "Awesome", + }, + ], + "classes": Array [ + "brand", + ], + "end": 481, + "name": "a", + "start": 453, + "type": "tag", + }, + Object { + "body": Array [ + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "href", + "value": "#", + }, + ], + "body": Array [ + Object { + "end": 534, + "start": 527, + "type": "free text", + "value": "Item 1", + }, + ], + "end": 535, + "name": "a", + "start": 514, + "type": "tag", + }, + ], + "end": 537, + "name": "li", + "start": 510, + "type": "tag", + }, + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "href", + "value": "#", + }, + ], + "body": Array [ + Object { + "end": 573, + "start": 566, + "type": "free text", + "value": "Item 2", + }, + ], + "end": 574, + "name": "a", + "start": 553, + "type": "tag", + }, + ], + "end": 576, + "name": "li", + "start": 549, + "type": "tag", + }, + ], + "classes": Array [ + "nav", + ], + "end": 586, + "name": "ul", + "start": 491, + "type": "tag", + }, + Object { + "body": Array [ + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "href", + "value": "#", + }, + ], + "body": Array [ + Object { + "body": Array [ + Object { + "end": 746, + "start": 720, + "type": "free text", + "value": " Logged in as: Mr. Right ", + }, + ], + "classes": Array [ + "icon-user", + "icon-white", + ], + "end": 747, + "name": "i", + "start": 696, + "type": "tag", + }, + Object { + "classes": Array [ + "caret", + ], + "end": 769, + "name": "b", + "start": 763, + "type": "tag", + }, + ], + "classes": Array [ + "dropdown-toggle", + ], + "end": 783, + "name": "a", + "start": 652, + "type": "tag", + }, + Object { + "body": Array [ + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "href", + "value": "#", + }, + ], + "body": Array [ + Object { + "end": 855, + "start": 847, + "type": "free text", + "value": "Profile", + }, + ], + "end": 856, + "name": "a", + "start": 834, + "type": "tag", + }, + ], + "end": 858, + "name": "li", + "start": 830, + "type": "tag", + }, + Object { + "classes": Array [ + "divider", + ], + "end": 883, + "name": "li", + "start": 874, + "type": "tag", + }, + Object { + "body": Array [ + Object { + "attributes": Array [ + Object { + "name": "href", + "value": "#", + }, + ], + "body": Array [ + Object { + "end": 925, + "start": 916, + "type": "free text", + "value": "Sign out", + }, + ], + "end": 926, + "name": "a", + "start": 903, + "type": "tag", + }, + ], + "end": 928, + "name": "li", + "start": 899, + "type": "tag", + }, + ], + "classes": Array [ + "dropdown-menu", + ], + "end": 942, + "name": "ul", + "start": 797, + "type": "tag", + }, + ], + "classes": Array [ + "dropdown", + ], + "end": 954, + "name": "li", + "start": 626, + "type": "tag", + }, + ], + "classes": Array [ + "nav", + "pull-right", + ], + "end": 964, + "name": "ul", + "start": 596, + "type": "tag", + }, + ], + "classes": Array [ + "navbar-inner", + ], + "end": 972, + "name": "div", + "start": 426, + "type": "tag", + }, + ], + "classes": Array [ + "navbar", + "navbar-inverse", + "navbar-fixed-top", + ], + "end": 978, + "id": "navbar", + "name": "div", + "start": 366, + "type": "tag", + }, + ], + "end": 982, + "name": "body", + "start": 355, + "type": "tag", + }, + ], + "end": 984, + "name": "html", + "start": 18, + "type": "tag", + }, + ], + "comments": Array [], + "log": Object { + "nodeCalls": Object { + "free text": 16, + "tag": 40, + }, + "nodeExecutionTime": Object { + "free text": Any, + "tag": Any, + }, + "nodeTrace": [Function], + "ruleCalls": Object { + "#": 26, + "(": 26, + ")": 10, + ",": 12, + ".": 41, + "/^\\"([^\\"]*)\\"/": 34, + "/^'([^']*)'/": 42, + "/^([a-zA-Z][a-zA-Z0-9_-]*)/": 84, + "/^\\\\/\\\\/([^\\\\r\\\\n]*)\\\\n/": 329, + "/^\\\\s+/": 454, + "=": 12, + "[": 26, + "]": 21, + }, + }, +} +`; + exports[`Keppel markTextError should display correctly 1`] = ` "Test message ^ diff --git a/test/keppel/grammer.spec.ts b/test/keppel/grammer.spec.ts index ef23899..e6bc629 100644 --- a/test/keppel/grammer.spec.ts +++ b/test/keppel/grammer.spec.ts @@ -4,6 +4,8 @@ import { Parser, ParserError, markTextError } from '../../src'; import { Keppel } from '../../examples/keppel'; const keppelParser = Parser(Keppel); +const keppelParserLog = Parser(Keppel, 0, false, (node) => node.type ?? node?.[0]?.type); +const keppelPartialParser = Parser(Keppel, 0, true); describe('Keppel', () => { test('Keppel grammar parser generation 1', () => { @@ -34,6 +36,14 @@ describe('Keppel', () => { } }); + test('Keppel grammar parser error for partial', () => { + const wrongKeppel = readFileSync(join(__dirname, '/wrong.keppel'), 'utf8'); + expect(keppelPartialParser(wrongKeppel)).toMatchSnapshot('wrong'); + + const correctKeppel = readFileSync(join(__dirname, '/test1.keppel'), 'utf8'); + expect(keppelParser(correctKeppel)).toMatchSnapshot('correct'); + }); + test.each([ [`Test message`, 'Error1', -3], [`Test message`, 'Error1', 2], @@ -41,4 +51,11 @@ describe('Keppel', () => { ])('markTextError should display correctly', (text, message, pos) => { expect(markTextError(text, message, pos)).toMatchSnapshot(); }); + + test('Keppel grammar parser with log', () => { + const keppel = readFileSync(join(__dirname, '/test2.keppel'), 'utf8'); + expect(keppelParserLog(keppel)).toMatchSnapshot({ + log: { nodeExecutionTime: { 'free text': expect.any(Array), tag: expect.any(Array) } }, + }); + }); });