diff --git a/bin/fromMem.js b/bin/fromMem.js new file mode 100644 index 00000000..ddbeb01c --- /dev/null +++ b/bin/fromMem.js @@ -0,0 +1,188 @@ +"use strict"; + +// Import or require module text from memory, rather than disk. Runs +// in a node vm, very similar to how node loads modules. +// +// Ideas taken from the "module-from-string" and "eval" modules, neither of +// which were situated correctly to be used as-is. + +const vm = require("vm"); +const { Module } = require("module"); +const path = require("path"); +const url = require("url"); + +// These already exist in a new, blank VM. Date, JSON, NaN, etc. +// Things from the core language. +const vmGlobals = new vm + .Script("Object.getOwnPropertyNames(globalThis)") + .runInNewContext() + .sort(); +vmGlobals.push("global", "globalThis", "sys"); + +// These are the things that are normally in the environment, that vm doesn't +// make available. This that you expect to be available in a node environment +// that aren't in the laguage itself. +const neededKeys = Object + .getOwnPropertyNames(global) + .filter(k => !vmGlobals.includes(k)) + .sort(); +const globalContext = Object.fromEntries( + neededKeys.map(k => [k, global[k]]) +); + +// In node <15, console is in vmGlobals. +globalContext.console = console; + +/** + * Options for how to process code. + * + * @typedef {object} FromMemOptions + * @property {"amd"|"bare"|"commonjs"|"es"|"globals"|"umd"} [format="commonjs"] + * What format does the code have? Throws an error if the format is not + * "commonjs", "es", "umd", or "bare". + * @property {string} [filename=__filename] What is the fully-qualified synthetic + * filename for the code? Most important is the directory, which is used to + * find modules that the code import's or require's. + * @property {object} [context={}] Variables to make availble in the global + * scope while code is being evaluated. + * @property {boolean} [includeGlobals=true] Include the typical global + * properties that node gives to all modules. (e.g. Buffer, process). + * @property {string} [globalExport=null] For type "globals", what name is + * exported from the module? + */ + +/** + * Treat the given code as a node module as if require() had been called + * on a file containing the code. + * + * @param {string} code Source code in commonjs format. + * @param {string} dirname Used for __dirname. + * @param {FromMemOptions} options + * @returns {object} The module exports from code + */ +function requireString(code, dirname, options) { + const m = new Module(options.filename, module); // Current module is parent. + // This is the function that will be called by `require()` in the parser. + m.require = Module.createRequire(options.filename); + const script = new vm.Script(code, { filename: options.filename }); + return script.runInNewContext({ + module: m, + exports: m.exports, + require: m.require, + __dirname: dirname, + __filename: options.filename, + ...options.context, + }); +} + +/** + * If the given specifier starts with a ".", path.resolve it to the given + * directory. Otherwise, it's a fully-qualified path, a node internal + * module name, an npm-provided module name, or a URL. + * + * @param {string} dirname Owning directory + * @param {string} specifier String from the rightmost side of an import statement + * @returns {string} Resolved path name or original string + */ +function resolveIfNeeded(dirname, specifier) { + if (specifier.startsWith(".")) { + specifier = path.resolve(dirname, specifier); + } + return specifier; +} + +/** + * Treat the given code as a node module as if import had been called + * on a file containing the code. + * + * @param {string} code Source code in es6 format. + * @param {string} dirname Where the synthetic file would have lived. + * @param {FromMemOptions} options + * @returns {object} The module exports from code + */ +async function importString(code, dirname, options) { + if (!vm.SourceTextModule) { + throw new Error("Start node with --experimental-vm-modules for this to work"); + } + + const [maj, min] = process.version + .match(/^v(\d+)\.(\d+)\.(\d+)/) + .slice(1) + .map(x => parseInt(x, 10)); + if ((maj < 20) || ((maj === 20) && (min < 8))) { + throw new Error("Requires node.js 20.8+ or 21."); + } + + const mod = new vm.SourceTextModule(code, { + identifier: options.filename, + context: vm.createContext(options.context), + initializeImportMeta(meta) { + meta.url = String(url.pathToFileURL(options.filename)); + }, + importModuleDynamically(specifier) { + return import(resolveIfNeeded(dirname, specifier)); + }, + }); + + await mod.link(async(specifier, referencingModule) => { + const resolvedSpecifier = resolveIfNeeded(dirname, specifier); + const targetModule = await import(resolvedSpecifier); + const exports = Object.keys(targetModule); + + // DO NOT change function to () =>, or `this` will be wrong. + return new vm.SyntheticModule(exports, function() { + for (const e of exports) { + this.setExport(e, targetModule[e]); + } + }, { + context: referencingModule.context, + }); + }); + await mod.evaluate(); + return mod.namespace; +} + +/** + * Import or require the given code from memory. Knows about the different + * Peggy output formats. Returns the exports of the module. + * + * @param {string} code Code to import + * @param {FromMemOptions} [options] Options. Most important is filename. + * @returns {Promise} The evaluated code. + */ +// eslint-disable-next-line require-await -- Always want to return a Promise +module.exports = async function fromMem(code, options) { + options = { + format: "commonjs", + filename: `${__filename}-string`, + context: {}, + includeGlobals: true, + globalExport: null, + ...options, + }; + + if (options.includeGlobals) { + options.context = { + ...globalContext, + ...options.context, + }; + } + options.context.global = options.context; + options.context.globalThis = options.context; + + options.filename = path.resolve(options.filename); + const dirname = path.dirname(options.filename); + + switch (options.format) { + case "bare": + case "commonjs": + case "umd": + return requireString(code, dirname, options); + case "es": + // Returns promise + return importString(code, dirname, options); + // I don't care enough about amd and globals to figure out how to load them. + default: + throw new Error(`Unsupported output format: "${options.format}"`); + } +}; diff --git a/bin/peggy-cli.js b/bin/peggy-cli.js index dd91f765..50ba8ab6 100644 --- a/bin/peggy-cli.js +++ b/bin/peggy-cli.js @@ -3,12 +3,11 @@ const { Command, CommanderError, InvalidArgumentError, Option, } = require("commander"); -const { Module } = require("module"); +const Module = require("module"); const fs = require("fs"); const path = require("path"); const peggy = require("../lib/peg.js"); const util = require("util"); -const vm = require("vm"); exports.CommanderError = CommanderError; exports.InvalidArgumentError = InvalidArgumentError; @@ -589,39 +588,12 @@ class PeggyCLI extends Command { const filename = this.outputJS ? path.resolve(this.outputJS) : path.join(process.cwd(), "stdout.js"); // Synthetic - const dirname = path.dirname(filename); - const m = new Module(filename, module); - // This is the function that will be called by `require()` in the parser. - m.require = Module.createRequire(filename); - const script = new vm.Script(source, { filename }); - const exec = script.runInNewContext({ - // Anything that is normally in the global scope that we think - // might be needed. Limit to what is available in lowest-supported - // engine version. - - // See: https://github.com/nodejs/node/blob/master/lib/internal/bootstrap/node.js - // for more things to add. - module: m, - exports: m.exports, - require: m.require, - __dirname: dirname, - __filename: filename, - - Buffer, - TextDecoder: (typeof TextDecoder === "undefined") ? undefined : TextDecoder, - TextEncoder: (typeof TextEncoder === "undefined") ? undefined : TextEncoder, - URL, - URLSearchParams, - atob: Buffer.atob, - btoa: Buffer.btoa, - clearImmediate, - clearInterval, - clearTimeout, - console, - process, - setImmediate, - setInterval, - setTimeout, + + const fromMem = require("./fromMem.js"); + const exec = await fromMem(source, { + filename, + format: this.argv.format, + globalExport: this.argv.exportVar, }); const opts = { diff --git a/bin/peggy.js b/bin/peggy.js index 9162688b..e42c0fcb 100755 --- a/bin/peggy.js +++ b/bin/peggy.js @@ -1,4 +1,4 @@ -#!/usr/bin/env node +#!/usr/bin/env -S node --experimental-vm-modules --no-warnings "use strict"; diff --git a/package.json b/package.json index 31d34a67..decc3f0f 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "lint": "eslint . --ext js,ts,mjs", "ts": "tsc --build tsconfig.json", "docs": "cd docs && npm run build", - "test": "jest", + "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js", "test:web": "cd web-test && npm test", "test:all": "npm run test && npm run test:web", "benchmark": "node ./benchmark/run_bench.js", diff --git a/test/cli/fixtures/imp.peggy b/test/cli/fixtures/imp.peggy new file mode 100644 index 00000000..855c0c0e --- /dev/null +++ b/test/cli/fixtures/imp.peggy @@ -0,0 +1,12 @@ +{{ +import opts from "./options.mjs"; +// Cause importModuleDynamically to fire +const opts2 = await import("./options.mjs"); +}} + +foo='1' { return [ + opts.cli_test.words, + opts2.default.cli_test.words, + // Needs to use import.meta to cause initializeImportMeta to fire. + import.meta.url.length > 0 +]; } diff --git a/test/cli/fixtures/options.mjs b/test/cli/fixtures/options.mjs new file mode 100644 index 00000000..6e824292 --- /dev/null +++ b/test/cli/fixtures/options.mjs @@ -0,0 +1,9 @@ +export default { + cli_test: { + words: ["zazzy"], + }, + dependencies: { + j: "jest", + commander: "commander", + }, +}; diff --git a/test/cli/run.spec.ts b/test/cli/run.spec.ts index 9babfc0b..a41a611d 100644 --- a/test/cli/run.spec.ts +++ b/test/cli/run.spec.ts @@ -1173,6 +1173,35 @@ Error: Expected "1" but end of input found. }); }); + it("handles tests that import other modules", async() => { + if ((await import("vm")).SourceTextModule) { + const grammar = path.join(__dirname, "fixtures", "imp.peggy"); + try { + await exec({ + args: ["--format", "es", "-t", "1", grammar], + expected: "[ [ 'zazzy' ], [ 'zazzy' ], true ]\n", + }); + } catch (e) { + expect((e as Error).message).toMatch("Requires node.js 20.8+ or 21"); + } + await exec({ + args: ["--format", "amd", "-t", "1", grammar], + error: /Unsupported output format/, + }); + await exec({ + args: ["--format", "globals", "-t", "1", grammar], + error: /Unsupported output format/, + }); + await exec({ + args: ["--format", "bare", "-t", "1"], + stdin: "foo = '1'\n", + expected: "'1'\n", + }); + } else { + throw new Error("Use --experimental-vm-modules"); + } + }); + it("handles grammar errors", async() => { await exec({ stdin: "foo=unknownRule",