BrsFile ast toString()

package.json

@@ -145,7 +145,7 @@
         "require-relative": "^0.8.7",
         "roku-deploy": "^3.9.2",
         "serialize-error": "^7.0.1",
-        "source-map": "^0.7.3",
+        "source-map": "^0.7.4",
         "vscode-languageserver": "7.0.0",
         "vscode-languageserver-protocol": "3.16.0",
         "vscode-languageserver-textdocument": "^1.0.1",

src/astUtils/creators.ts

@@ -89,7 +89,8 @@ export function createToken<T extends TokenKind>(kind: T, text?: string, range =
         text: text ?? tokenDefaults[kind as string] ?? kind.toString().toLowerCase(),
         isReserved: !text || text === kind.toString(),
         range: range,
-        leadingWhitespace: ''
+        leadingWhitespace: '',
+        leadingTrivia: []
     };
 }
 
@@ -99,7 +100,8 @@ export function createIdentifier(name: string, range?: Range): Identifier {
         text: name,
         isReserved: false,
         range: range,
-        leadingWhitespace: ''
+        leadingWhitespace: '',
+        leadingTrivia: []
     };
 }
 
@@ -169,7 +171,8 @@ export function createCall(callee: Expression, args?: Expression[]) {
         callee,
         createToken(TokenKind.LeftParen, '('),
         createToken(TokenKind.RightParen, ')'),
-        args || []
+        args || [],
+        []
     );
 }
 

src/astUtils/reflection.spec.ts

@@ -47,8 +47,8 @@ describe('reflection', () => {
         const fors = new ForStatement(token, assignment, token, expr, block, token, token, expr);
         const foreach = new ForEachStatement({ forEach: token, in: token, endFor: token }, token, expr, block);
         const whiles = new WhileStatement({ while: token, endWhile: token }, expr, block);
-        const dottedSet = new DottedSetStatement(expr, ident, expr);
-        const indexedSet = new IndexedSetStatement(expr, expr, expr, token, token);
+        const dottedSet = new DottedSetStatement(expr, ident, expr, token, token);
+        const indexedSet = new IndexedSetStatement(expr, expr, expr, token, token, token);
         const library = new LibraryStatement({ library: token, filePath: token });
         const namespace = new NamespaceStatement(token, new NamespacedVariableNameExpression(createVariableExpression('a', range)), body, token);
         const cls = new ClassStatement(token, ident, [], token);
@@ -193,28 +193,29 @@ describe('reflection', () => {
             range: range,
             isReserved: false,
             charCode: 0,
-            leadingWhitespace: ''
+            leadingWhitespace: '',
+            leadingTrivia: []
         };
         const nsVar = new NamespacedVariableNameExpression(createVariableExpression('a', range));
         const binary = new BinaryExpression(expr, token, expr);
-        const call = new CallExpression(expr, token, token, []);
+        const call = new CallExpression(expr, token, token, [], []);
         const fun = new FunctionExpression([], block, token, token, token, token);
         const dottedGet = new DottedGetExpression(expr, ident, token);
         const xmlAttrGet = new XmlAttributeGetExpression(expr, ident, token);
         const indexedGet = new IndexedGetExpression(expr, expr, token, token);
         const grouping = new GroupingExpression({ left: token, right: token }, expr);
         const literal = createStringLiteral('test');
         const escapedCarCode = new EscapedCharCodeLiteralExpression(charCode);
-        const arrayLit = new ArrayLiteralExpression([], token, token);
+        const arrayLit = new ArrayLiteralExpression([], token, token, []);
         const aaLit = new AALiteralExpression([], token, token);
         const unary = new UnaryExpression(token, expr);
         const variable = new VariableExpression(ident);
         const sourceLit = new SourceLiteralExpression(token);
         const newx = new NewExpression(token, call);
-        const callfunc = new CallfuncExpression(expr, token, ident, token, [], token);
+        const callfunc = new CallfuncExpression(expr, token, ident, token, [], token, []);
         const tplQuasi = new TemplateStringQuasiExpression([expr]);
-        const tplString = new TemplateStringExpression(token, [tplQuasi], [], token);
-        const taggedTpl = new TaggedTemplateStringExpression(ident, token, [tplQuasi], [], token);
+        const tplString = new TemplateStringExpression(token, [tplQuasi], [], token, [], []);
+        const taggedTpl = new TaggedTemplateStringExpression(ident, token, [tplQuasi], [], token, [], []);
         const annotation = new AnnotationExpression(token, token);
 
         it('isExpression', () => {

src/astUtils/reflection.ts

@@ -20,6 +20,7 @@ import { DynamicType } from '../types/DynamicType';
 import type { InterfaceType } from '../types/InterfaceType';
 import type { ObjectType } from '../types/ObjectType';
 import type { AstNode, Expression, Statement } from '../parser/AstNode';
+import type { Token } from '../lexer/Token';
 
 // File reflection
 
@@ -183,9 +184,9 @@ export function isThrowStatement(element: AstNode | undefined): element is Throw
  * this will work for StringLiteralExpression -> Expression,
  * but will not work CustomStringLiteralExpression -> StringLiteralExpression -> Expression
  */
-export function isExpression(element: AstNode | undefined): element is Expression {
+export function isExpression(element: AstNode | Token | undefined): element is Expression {
     // eslint-disable-next-line no-bitwise
-    return !!(element && element.visitMode & InternalWalkMode.visitExpressions);
+    return !!(element && (element as any).visitMode & InternalWalkMode.visitExpressions);
 }
 
 export function isBinaryExpression(element: AstNode | undefined): element is BinaryExpression {

src/lexer/Lexer.spec.ts

@@ -1,8 +1,8 @@
 /* eslint no-template-curly-in-string: 0 */
 import { expect } from '../chai-config.spec';
-
 import { TokenKind } from './TokenKind';
-import { Lexer } from './Lexer';
+import { Lexer, triviaKinds } from './Lexer';
+import type { Token } from './Token';
 import { isToken } from './Token';
 import { rangeToArray } from '../parser/Parser.spec';
 import { Range } from 'vscode-languageserver';
@@ -1377,6 +1377,63 @@ describe('lexer', () => {
             TokenKind.Eof
         ]);
     });
+
+    describe('trivia', () => {
+        function stringify(tokens: Token[]) {
+            return tokens
+                //exclude the explicit triva tokens since they'll be included in the leading/trailing arrays
+                .filter(x => !triviaKinds.includes(x.kind))
+                .flatMap(x => [...x.leadingTrivia, x])
+                .map(x => x.text)
+                .join('');
+        }
+
+        it('combining token text and trivia can reproduce full input', () => {
+            const input = `
+                function   test(  )
+                    'comment
+                    print   alpha   '  blabla
+                end function 'trailing
+                'trailing2
+            `;
+            expect(
+                stringify(
+                    Lexer.scan(input).tokens
+                )
+            ).to.eql(input);
+        });
+
+        function expectTrivia(text: string, expected: Array<{ text: string; leadingTrivia?: string[]; trailingTrivia?: string[] }>) {
+            const tokens = Lexer.scan(text).tokens.filter(x => !triviaKinds.includes(x.kind));
+            expect(
+                tokens.map(x => {
+                    return {
+                        text: x.text,
+                        leadingTrivia: x.leadingTrivia.map(x => x.text)
+                    };
+                })
+            ).to.eql(
+                expected.map(x => ({
+                    leadingTrivia: [],
+                    ...x
+                }))
+            );
+        }
+
+        it('associates trailing items on same line with the preceeding token', () => {
+            expectTrivia(
+                `'leading\n` +
+                `alpha = true 'trueComment\n` +
+                `'eof`
+                , [
+                    { leadingTrivia: [`'leading`, `\n`], text: `alpha` },
+                    { leadingTrivia: [` `], text: `=` },
+                    { leadingTrivia: [` `], text: `true` },
+                    //EOF
+                    { leadingTrivia: [` `, `'trueComment`, `\n`, `'eof`], text: `` }
+                ]);
+        });
+    });
 });
 
 function expectKinds(text: string, tokenKinds: TokenKind[]) {

src/lexer/Lexer.ts

@@ -6,6 +6,13 @@ import type { Range, Diagnostic } from 'vscode-languageserver';
 import { DiagnosticMessages } from '../DiagnosticMessages';
 import util from '../util';
 
+export const triviaKinds: ReadonlyArray<TokenKind> = [
+    TokenKind.Newline,
+    TokenKind.Whitespace,
+    TokenKind.Comment,
+    TokenKind.Colon
+];
+
 export class Lexer {
     /**
      * The zero-indexed position at which the token under consideration begins.
@@ -103,12 +110,22 @@ export class Lexer {
             isReserved: false,
             text: '',
             range: util.createRange(this.lineBegin, this.columnBegin, this.lineEnd, this.columnEnd + 1),
-            leadingWhitespace: this.leadingWhitespace
+            leadingWhitespace: this.leadingWhitespace,
+            leadingTrivia: this.leadingTrivia
         });
         this.leadingWhitespace = '';
         return this;
     }
 
+    private leadingTrivia: Token[] = [];
+
+    /**
+     * Pushes a token into the leadingTrivia list
+     */
+    private pushTrivia(token: Token) {
+        this.leadingTrivia.push(token);
+    }
+
     /**
      * Fill in missing/invalid options with defaults
      */
@@ -1031,6 +1048,13 @@ export class Lexer {
         return false;
     }
 
+    /**
+     * Determine if this token is a trivia token
+     */
+    private isTrivia(token: Token) {
+        return triviaKinds.includes(token.kind);
+    }
+
     /**
      * Creates a `Token` and adds it to the `tokens` array.
      * @param kind the type of token to produce.
@@ -1042,8 +1066,15 @@ export class Lexer {
             text: text,
             isReserved: ReservedWords.has(text.toLowerCase()),
             range: this.rangeOf(),
-            leadingWhitespace: this.leadingWhitespace
+            leadingWhitespace: this.leadingWhitespace,
+            leadingTrivia: []
         };
+        if (this.isTrivia(token)) {
+            this.pushTrivia(token);
+        } else {
+            token.leadingTrivia.push(...this.leadingTrivia);
+            this.leadingTrivia = [];
+        }
         this.leadingWhitespace = '';
         this.tokens.push(token);
         this.sync();

src/lexer/Token.ts

@@ -17,6 +17,10 @@ export interface Token {
      * Any leading whitespace found prior to this token. Excludes newline characters.
      */
     leadingWhitespace?: string;
+    /**
+     * Any tokens starting on the next line of the previous token, up to the start of this token
+     */
+    leadingTrivia: Token[];
 }
 
 /**

src/parser/AstNode.spec.ts

@@ -6,6 +6,7 @@ import { expect } from '../chai-config.spec';
 import type { DottedGetExpression } from './Expression';
 import { expectZeroDiagnostics } from '../testHelpers.spec';
 import { tempDir, rootDir, stagingDir } from '../testHelpers.spec';
+import { Parser } from './Parser';
 
 describe('Program', () => {
     let program: Program;
@@ -42,4 +43,45 @@ describe('Program', () => {
             });
         });
     });
+
+    describe('toString', () => {
+        function testToString(text: string) {
+            expect(
+                Parser.parse(text).ast.toString()
+            ).to.eql(
+                text
+            );
+        }
+        it('retains full fidelity', () => {
+            testToString(`
+                thing = true
+
+                if true
+                    thing = true
+                end if
+
+                if true
+                    thing = true
+                else
+                    thing = true
+                end if
+
+                if true
+                    thing = true
+                else if true
+                    thing = true
+                else
+                    thing = true
+                end if
+
+                for i = 0 to 10 step 1
+                    print true,false;3
+                end for
+
+                for each item in thing
+                    print 1
+                end for
+            `);
+        });
+    });
 });

src/parser/AstNode.ts

@@ -8,6 +8,8 @@ import type { BrsTranspileState } from './BrsTranspileState';
 import type { TranspileResult } from '../interfaces';
 import type { AnnotationExpression } from './Expression';
 import util from '../util';
+import type { SourceNode } from 'source-map';
+import { TranspileState } from './TranspileState';
 
 /**
  * A BrightScript AST node
@@ -104,6 +106,20 @@ export abstract class AstNode {
             walkMode: WalkMode.visitAllRecursive
         });
     }
+
+    /**
+     * Return the string value of this AstNode
+     */
+    public toString() {
+        return this
+            .toSourceNode(new TranspileState('', {}))
+            .toString();
+    }
+
+    /**
+     * Generate a SourceNode that represents the stringified value of this node (used to generate sourcemaps and transpile the code
+     */
+    public abstract toSourceNode(state: TranspileState): SourceNode;
 }
 
 export abstract class Statement extends AstNode {