Add support for npm: loaded library grammars

peggyjs · Dec 21, 2023 · e158774 · e158774
1 parent 8a5e05d
commit e158774
Show file tree

Hide file tree

Showing 17 changed files with 236 additions and 67 deletions.
diff --git a/bin/peggy-cli.js b/bin/peggy-cli.js
@@ -115,7 +115,7 @@ class PeggyCLI extends Command {
 
     this
       .version(peggy.VERSION, "-v, --version")
-      .argument("[input_file...]", 'Grammar file(s) to read.  Use "-" to read stdin.  If multiple files are given, they are combined in the given order to produce a single output.', ["-"])
+      .argument("[input_file...]", 'Grammar file(s) to read.  Use "-" to read stdin.  If multiple files are given, they are combined in the given order to produce a single output.  Use npm:"<pacakgeName>/file.peggy" to import from an npm dependency.', ["-"])
       .allowExcessArguments(false)
       .addOption(
         new Option(
@@ -298,10 +298,17 @@ class PeggyCLI extends Command {
 
         if (!this.outputFile) {
           if (this.inputFiles.indexOf("-") === -1) {
-            this.outputJS = this.inputFiles[0].slice(
+            let inFile = this.inputFiles[0];
+            // You might just want to run a fragment grammar as-is,
+            // particularyly with a specified start rule.
+            const m = inFile.match(/^npm:.*\/([^/]+)$/);
+            if (m) {
+              inFile = m[1];
+            }
+            this.outputJS = inFile.slice(
               0,
-              this.inputFiles[0].length
-                - path.extname(this.inputFiles[0]).length
+              inFile.length
+                - path.extname(inFile).length
             ) + ".js";
 
             this.outputFile = ((typeof this.progOptions.test !== "string")
@@ -636,6 +643,7 @@ class PeggyCLI extends Command {
 
     let exitCode = 1;
     let errorText = "";
+    let prevSource = process.cwd() + "/";
     try {
       for (const source of this.inputFiles) {
         const input = { source, text: null };
@@ -644,7 +652,17 @@ class PeggyCLI extends Command {
           input.source = "stdin";
           this.std.in.resume();
           input.text = await readStream(this.std.in);
+        } else if (source.startsWith("npm:")) {
+          const req = (
+            // In node 12+, createRequire is documented.
+            // In node 10, createRequireFromPath is the least-undocumented approach.
+            Module.createRequire || Module.createRequireFromPath
+          )(prevSource);
+          prevSource = req.resolve(source.slice(4));
+          input.source = prevSource;
+          input.text = await fs.promises.readFile(prevSource, "utf8");
         } else {
+          prevSource = path.resolve(source);
           input.text = await fs.promises.readFile(source, "utf8");
         }
         sources.push(input);

diff --git a/docs/js/benchmark-bundle.min.js b/docs/js/benchmark-bundle.min.js
diff --git a/docs/js/test-bundle.min.js b/docs/js/test-bundle.min.js
diff --git a/docs/vendor/peggy/peggy.min.js b/docs/vendor/peggy/peggy.min.js
diff --git a/lib/compiler/asts.js b/lib/compiler/asts.js
@@ -14,7 +14,11 @@ const visitor = require("./visitor");
 function combinePossibleArrays(a, b) {
   const aa = Array.isArray(a) ? a : [a];
   const bb = Array.isArray(b) ? b : [b];
-  return aa.concat(bb);
+
+  // Put library code above code that uses it, so that variables
+  // will be in scope.
+  const res = bb.filter(x => x).concat(aa.filter(x => x)); // Remove nulls.
+  return res;
 }
 
 // AST utilities.

diff --git a/lib/parser.js b/lib/parser.js
@@ -998,6 +998,9 @@ function peg$parse(input, options) {
             if (s3 !== peg$FAILED) {
               s4 = peg$parse__();
               s5 = peg$parseNameSpaceImport();
+              if (s5 === peg$FAILED) {
+                s5 = peg$parseNamedImports();
+              }
               if (s5 !== peg$FAILED) {
                 s1 = [s1, s2, s3, s4, s5];
                 s0 = s1;
@@ -1013,37 +1016,6 @@ function peg$parse(input, options) {
             peg$currPos = s0;
             s0 = peg$FAILED;
           }
-          if (s0 === peg$FAILED) {
-            s0 = peg$currPos;
-            s1 = peg$parseBindingIdentifier();
-            if (s1 !== peg$FAILED) {
-              s2 = peg$parse__();
-              if (input.charCodeAt(peg$currPos) === 44) {
-                s3 = peg$c4;
-                peg$currPos++;
-              } else {
-                s3 = peg$FAILED;
-                if (peg$silentFails === 0) { peg$fail(peg$e4); }
-              }
-              if (s3 !== peg$FAILED) {
-                s4 = peg$parse__();
-                s5 = peg$parseNamedImports();
-                if (s5 !== peg$FAILED) {
-                  s1 = [s1, s2, s3, s4, s5];
-                  s0 = s1;
-                } else {
-                  peg$currPos = s0;
-                  s0 = peg$FAILED;
-                }
-              } else {
-                peg$currPos = s0;
-                s0 = peg$FAILED;
-              }
-            } else {
-              peg$currPos = s0;
-              s0 = peg$FAILED;
-            }
-          }
         }
       }
     }
@@ -1256,29 +1228,23 @@ function peg$parse(input, options) {
   function peg$parseImportSpecifier() {
     var s0, s1, s2, s3, s4, s5;
 
-    s0 = peg$parseBindingIdentifier();
-    if (s0 === peg$FAILED) {
-      s0 = peg$currPos;
-      s1 = peg$parseModuleExportName();
-      if (s1 !== peg$FAILED) {
-        s2 = peg$parse__();
-        if (input.substr(peg$currPos, 2) === peg$c6) {
-          s3 = peg$c6;
-          peg$currPos += 2;
-        } else {
-          s3 = peg$FAILED;
-          if (peg$silentFails === 0) { peg$fail(peg$e6); }
-        }
-        if (s3 !== peg$FAILED) {
-          s4 = peg$parse__();
-          s5 = peg$parseBindingIdentifier();
-          if (s5 !== peg$FAILED) {
-            s1 = [s1, s2, s3, s4, s5];
-            s0 = s1;
-          } else {
-            peg$currPos = s0;
-            s0 = peg$FAILED;
-          }
+    s0 = peg$currPos;
+    s1 = peg$parseModuleExportName();
+    if (s1 !== peg$FAILED) {
+      s2 = peg$parse__();
+      if (input.substr(peg$currPos, 2) === peg$c6) {
+        s3 = peg$c6;
+        peg$currPos += 2;
+      } else {
+        s3 = peg$FAILED;
+        if (peg$silentFails === 0) { peg$fail(peg$e6); }
+      }
+      if (s3 !== peg$FAILED) {
+        s4 = peg$parse__();
+        s5 = peg$parseBindingIdentifier();
+        if (s5 !== peg$FAILED) {
+          s1 = [s1, s2, s3, s4, s5];
+          s0 = s1;
         } else {
           peg$currPos = s0;
           s0 = peg$FAILED;
@@ -1287,6 +1253,12 @@ function peg$parse(input, options) {
         peg$currPos = s0;
         s0 = peg$FAILED;
       }
+    } else {
+      peg$currPos = s0;
+      s0 = peg$FAILED;
+    }
+    if (s0 === peg$FAILED) {
+      s0 = peg$parseBindingIdentifier();
     }
 
     return s0;

diff --git a/src/parser.pegjs b/src/parser.pegjs
@@ -92,8 +92,7 @@ ImportClause
   = ImportedDefaultBinding
   / NameSpaceImport
   / NamedImports
-  / ImportedDefaultBinding __ "," __ NameSpaceImport
-  / ImportedDefaultBinding __ "," __ NamedImports
+  / ImportedDefaultBinding __ "," __ (NameSpaceImport / NamedImports)
 
 ImportedDefaultBinding = ImportedBinding
 NameSpaceImport = "*" __ "as" __ ImportedBinding
@@ -103,8 +102,8 @@ NamedImports
 FromClause = "from" __ ModuleSpecifier
 ImportsList = ImportSpecifier|1.., __ "," __|
 ImportSpecifier
-  = ImportedBinding
-  / ModuleExportName __ "as" __ ImportedBinding
+  = ModuleExportName __ "as" __ ImportedBinding
+  / ImportedBinding
 ModuleSpecifier = StringLiteral
 ImportedBinding
   = BindingIdentifier

diff --git a/test/cli/fixtures/frags/.gitignore b/test/cli/fixtures/frags/.gitignore
@@ -0,0 +1,2 @@
+*.js
+node_modules/
diff --git a/test/cli/fixtures/frags/package.json b/test/cli/fixtures/frags/package.json
@@ -0,0 +1,8 @@
+{
+  "name": "frags",
+  "version": "0.0.0",
+  "description": "Fragments of peggy grammars, in an npm package",
+  "private": true,
+  "author": "Joe Hildebrand <joe-github@cursive.net>",
+  "license": "MIT"
+}
diff --git a/test/cli/fixtures/frags/path.peggy b/test/cli/fixtures/frags/path.peggy
@@ -0,0 +1,12 @@
+{{
+  // From path.peggy
+  import path from "path";
+
+  const PATH = "path";
+}}
+
+{
+  const PATH2 = "path2";
+}
+
+pth = p:$.* &{ return p === path.resolve(p) }
diff --git a/test/cli/fixtures/frags/unicode.peggy b/test/cli/fixtures/frags/unicode.peggy
@@ -0,0 +1,65 @@
+{{
+  import path from 'node:path';
+}}
+
+{
+  const c = 'too';
+}
+IdentifierName
+  = $(IdentifierStart IdentifierPart*)
+
+IdentifierStart
+  = UnicodeIDStart
+  / "$"
+  / "_"
+  / "\\" UnicodeEscapeSequence
+
+IdentifierPart
+  = UnicodeIDContinue
+  / "$"
+  / "\\" UnicodeEscapeSequence
+  / "\u200C"
+  / "\u200D"
+
+UnicodeEscapeSequence
+  = "u" Hex4Digits
+  / "u{" CodePoint "}"
+
+CodePoint
+   = HexDigits ?{ return parseInt(n, 16) <= 0x10FFFF }
+
+Hex4Digits
+  = HexDigit HexDigit HexDigit HexDigit
+
+HexDigits
+  = HexDigit+
+
+HexDigit = [0-9a-f]i
+
+
+UnicodeIDStart = ID_Start
+
+UnicodeIDContinue = ID_Continue
+
+// Separator, Space
+Zs = c:SourceCharacter &{ return /\p{Zs}/u.test(c) }
+
+ID_Start
+  = c:SourceCharacter &{ return /\p{ID_Start}/u.test(c) }
+
+ID_Continue
+  = c:SourceCharacter &{ return /\p{ID_Continue}/u.test(c) }
+
+SourceCharacter
+  = SourceCharacterLow
+  / SourceCharacterHigh
+
+// Not surrogates
+SourceCharacterLow
+  = [\u0000-\uD7FF\uE000-\uFFFF]
+
+// Can be properly-matched surrogates or lone surrogates.
+SourceCharacterHigh
+  = $([\uD800-\uDBFF][\uDC00-\uDFFF]) // Surrogate pair
+  / [\uD800-\uDBFF] // Lone first surrogate
+  / [\uDC00-\uDFFF] // Lone second surrogate
diff --git a/test/cli/fixtures/useFrags/.gitignore b/test/cli/fixtures/useFrags/.gitignore
@@ -0,0 +1,2 @@
+*.js
+node_modules/
diff --git a/test/cli/fixtures/useFrags/fs.peggy b/test/cli/fixtures/useFrags/fs.peggy
@@ -0,0 +1,16 @@
+{{
+  // From fs.peggy
+  import {readFileSync as rfs1} from "fs";
+
+  const FS = "fs";
+}}
+
+{
+  // From fs.peggy
+  const pk = JSON.parse(rfs1('package.json', 'utf8'));
+  console.log({name: pk.name})
+}
+
+pkg
+  = p:$.* &{ return p === pk.name }
+  / pth
diff --git a/test/cli/fixtures/useFrags/identifier.peggy b/test/cli/fixtures/useFrags/identifier.peggy
@@ -0,0 +1 @@
+id = IdentifierName
diff --git a/test/cli/fixtures/useFrags/package-lock.json b/test/cli/fixtures/useFrags/package-lock.json
diff --git a/test/cli/fixtures/useFrags/package.json b/test/cli/fixtures/useFrags/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "usefrags",
+  "version": "0.0.0",
+  "description": "Use fragments from another npm package",
+  "type": "module",
+  "private": "true",
+  "scripts": {
+    "build": "../../../../bin/peggy.js identifier.peggy npm:frags/unicode.peggy && ../../../../bin/peggy.js fs.peggy npm:frags/path.peggy"
+  },
+  "author": "Joe Hildebrand <joe-github@cursive.net>",
+  "license": "MIT",
+  "dependencies": {
+    "frags": "file:../frags"
+  }
+}
diff --git a/test/cli/run.spec.ts b/test/cli/run.spec.ts
@@ -329,7 +329,9 @@ Arguments:
   input_file                       Grammar file(s) to read.  Use "-" to read
                                    stdin.  If multiple files are given, they
                                    are combined in the given order to produce a
-                                   single output. (default: ["-"])
+                                   single output.  Use
+                                   npm:"<pacakgeName>/file.peggy" to import
+                                   from an npm dependency. (default: ["-"])
 
 Options:
   -v, --version                    output the version number
@@ -850,6 +852,35 @@ Options:
     fs.unlinkSync(out);
   });
 
+  it("handles npm: sources", async() => {
+    let input1 = path.join(__dirname, "fixtures", "useFrags", "identifier.peggy");
+    let out = path.join(__dirname, "fixtures", "useFrags", "identifier.js");
+    await expect(exec({
+      args: [input1, "npm:frags/unicode.peggy"],
+      exitCode: 0,
+    })).resolves.toBe("");
+
+    fs.unlinkSync(out);
+
+    input1 = path.join(__dirname, "fixtures", "useFrags", "fs.peggy");
+    out = path.join(__dirname, "fixtures", "useFrags", "fs.js");
+    await expect(exec({
+      args: [input1, "npm:frags/path.peggy"],
+      exitCode: 0,
+    })).resolves.toBe("");
+
+    fs.unlinkSync(out);
+
+    out = path.join(__dirname, "..", "..", "path.js");
+    await expect(exec({
+      args: ["npm:frags/path.peggy"],
+
+      exitCode: 0,
+    })).resolves.toBe("");
+
+    fs.unlinkSync(out);
+  });
+
   describe("handles source map", () => {
     describe("with default name without --output", () => {
       const sourceMap = path.resolve(__dirname, "..", "..", "source.map");