Enhance lexer to support long numeric literals with type designators (#…

…1351)
rokucommunity · Nov 21, 2024 · f6dedf7 · f6dedf7
1 parent 4fd325b
commit f6dedf7
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 4 deletions.
diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts
@@ -851,6 +851,22 @@ describe('lexer', () => {
             expect(f.text).to.eql('2.5e3');
         });
 
+        it('supports very long numbers with !', () => {
+            function doTest(number: string) {
+                let f = Lexer.scan(number).tokens[0];
+                expect(f.kind).to.equal(TokenKind.FloatLiteral);
+                expect(f.text).to.eql(number);
+            }
+            doTest('0!');
+            doTest('0!');
+            doTest('147483648!');
+            doTest('2147483648!');
+            doTest('2147483648111!');
+            doTest('2.4e-38!');
+            doTest('2.4e-32342342342342342342342342348!');
+            doTest('2.4e+32342342342342342342342342348!');
+        });
+
         it('supports larger-than-supported-precision floats to be defined with exponents', () => {
             let f = Lexer.scan('2.3659475627512424e-38').tokens[0];
             expect(f.kind).to.equal(TokenKind.FloatLiteral);

diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts
@@ -6,6 +6,11 @@ import type { Range, Diagnostic } from 'vscode-languageserver';
 import { DiagnosticMessages } from '../DiagnosticMessages';
 import util from '../util';
 
+/**
+ * Numeric type designators can only be one of these characters
+ */
+const numericTypeDesignatorCharsRegexp = /[#d!e&%]/;
+
 export class Lexer {
     /**
      * The zero-indexed position at which the token under consideration begins.
@@ -675,8 +680,12 @@ export class Lexer {
         let asString = this.source.slice(this.start, this.current);
         let numberOfDigits = containsDecimal ? asString.length - 1 : asString.length;
         let designator = this.peek().toLowerCase();
+        //set to undefined if it's not one of the supported designator chars
+        if (!numericTypeDesignatorCharsRegexp.test(designator)) {
+            designator = undefined;
+        }
 
-        if (numberOfDigits >= 10 && designator !== '&' && designator !== 'e') {
+        if (numberOfDigits >= 10 && !designator) {
             // numeric literals over 10 digits with no type designator are implicitly Doubles
             this.addToken(TokenKind.DoubleLiteral);
         } else if (designator === '#') {
@@ -707,9 +716,9 @@ export class Lexer {
             this.advance();
             this.addToken(TokenKind.FloatLiteral);
         } else if (designator === 'e') {
-            // literals that use "E" as the exponent are also automatic Floats
+            // literals that use "e" as the exponent are also automatic Floats
 
-            // consume the "E"
+            // consume the "e"
             this.advance();
 
             // exponents are optionally signed
@@ -722,6 +731,11 @@ export class Lexer {
                 this.advance();
             }
 
+            //optionally consume a trailing type designator
+            if (numericTypeDesignatorCharsRegexp.test(this.peek())) {
+                this.advance();
+            }
+
             this.addToken(TokenKind.FloatLiteral);
         } else if (containsDecimal) {
             // anything with a decimal but without matching Double rules is a Float
@@ -737,7 +751,6 @@ export class Lexer {
         } else {
             // otherwise, it's a regular integer
             this.addToken(TokenKind.IntegerLiteral);
-
         }
     }