From 759470b2e83c8f480610e08c555fb98d3b29b69c Mon Sep 17 00:00:00 2001 From: Joe Hildebrand Date: Fri, 28 Apr 2023 16:44:43 -0400 Subject: [PATCH] Allow multiple files on the command line, all of which get combined into one output. Not ready. Still needs change to source-map generation, for topLevelInitializer to be an array, and to make moving imports to the top of all topLevelInitializers (keeping track of source info) a JS-specific pass. --- bin/peggy-cli.js | 82 +++++++++++++------------- bin/watcher.js | 120 +++++++++++++++++++++++++-------------- lib/compiler/asts.js | 44 ++++++++++++++ lib/peg.js | 20 +++++-- test/cli/run.spec.ts | 19 +++---- test/cli/watcher.spec.ts | 37 ++++++++++-- 6 files changed, 220 insertions(+), 102 deletions(-) diff --git a/bin/peggy-cli.js b/bin/peggy-cli.js index 7267953f..46ffcb2d 100644 --- a/bin/peggy-cli.js +++ b/bin/peggy-cli.js @@ -94,8 +94,8 @@ class PeggyCLI extends Command { /** @type {peggy.BuildOptionsBase} */ this.argv = {}; - /** @type {string?} */ - this.inputFile = null; + /** @type {string[]} */ + this.inputFiles = []; /** @type {string?} */ this.outputFile = null; /** @type {object} */ @@ -115,7 +115,7 @@ class PeggyCLI extends Command { this .version(peggy.VERSION, "-v, --version") - .argument("[input_file]", 'Grammar file to read. Use "-" to read stdin.', "-") + .argument("[input_file...]", 'Grammar file(s) to read. Use "-" to read stdin. If multiple files are given, they are combined in the given order to produce a single output.', ["-"]) .allowExcessArguments(false) .addOption( new Option( @@ -214,8 +214,8 @@ class PeggyCLI extends Command { .hideHelp() .default(false) ) - .action((inputFile, opts) => { // On parse() - this.inputFile = inputFile; + .action((inputFiles, opts) => { // On parse() + this.inputFiles = inputFiles; this.argv = opts; if ((typeof this.argv.startRule === "string") @@ -284,21 +284,24 @@ class PeggyCLI extends Command { this.argv.output = "source"; if ((this.args.length === 0) && this.progOptions.input) { // Allow command line to override config file. - this.inputFile = this.progOptions.input; + this.inputFiles = Array.isArray(this.progOptions.input) + ? this.progOptions.input + : [this.progOptions.input]; } this.outputFile = this.progOptions.output; this.outputJS = this.progOptions.output; - if ((this.inputFile === "-") && this.argv.watch) { + if ((this.inputFiles.indexOf("-") !== -1) && this.argv.watch) { this.argv.watch = false; // Make error throw. this.error("Can't watch stdin"); } if (!this.outputFile) { - if (this.inputFile !== "-") { - this.outputJS = this.inputFile.slice( + if (this.inputFiles.indexOf("-") === -1) { + this.outputJS = this.inputFiles[0].slice( 0, - this.inputFile.length - path.extname(this.inputFile).length + this.inputFiles[0].length + - path.extname(this.inputFiles[0]).length ) + ".js"; this.outputFile = ((typeof this.progOptions.test !== "string") @@ -345,7 +348,7 @@ class PeggyCLI extends Command { } this.verbose("PARSER OPTIONS:", this.argv); this.verbose("PROGRAM OPTIONS:", this.progOptions); - this.verbose('INPUT: "%s"', this.inputFile); + this.verbose('INPUT: "%s"', this.inputFiles); this.verbose('OUTPUT: "%s"', this.outputFile); if (this.progOptions.verbose) { this.argv.info = (pass, msg) => PeggyCLI.print(this.std.err, `INFO(${pass}): ${msg}`); @@ -561,7 +564,7 @@ class PeggyCLI extends Command { if (this.testFile === "-") { this.testText = await readStream(this.std.in); } else { - this.testText = fs.readFileSync(this.testFile, "utf8"); + this.testText = await fs.promises.readFile(this.testFile, "utf8"); } } if (typeof this.testText === "string") { @@ -629,26 +632,29 @@ class PeggyCLI extends Command { * @returns {Promise} */ async run() { - let inputStream = undefined; - - if (this.inputFile === "-") { - this.std.in.resume(); - inputStream = this.std.in; - this.argv.grammarSource = "stdin"; - } else { - this.argv.grammarSource = this.inputFile; - inputStream = fs.createReadStream(this.inputFile); - } + const sources = []; let exitCode = 1; let errorText = ""; - let input = ""; try { - this.verbose("CLI", errorText = "reading input stream"); - input = await readStream(inputStream); + for (const source of this.inputFiles) { + const input = { source, text: null }; + this.verbose("CLI", errorText = `reading input "${source}"`); + if (source === "-") { + input.source = "stdin"; + this.std.in.resume(); + input.text = await readStream(this.std.in); + } else { + input.text = await fs.promises.readFile(source, "utf8"); + } + sources.push(input); + } + + // This is wrong. It's a hack in place until source generation is fixed. + this.argv.grammarSource = sources[0].source; this.verbose("CLI", errorText = "parsing grammar"); - const source = peggy.generate(input, this.argv); // All of the real work. + const source = peggy.generate(sources, this.argv); // All of the real work. this.verbose("CLI", errorText = "open output stream"); const outputStream = await this.openOutputStream(); @@ -669,10 +675,6 @@ class PeggyCLI extends Command { await this.test(mappedSource); } } catch (error) { - const sources = [{ - source: this.argv.grammarSource, - text: input, - }]; if (this.testGrammarSource) { sources.push({ source: this.testGrammarSource, @@ -711,19 +713,21 @@ class PeggyCLI extends Command { if (this.argv.watch) { const Watcher = require("./watcher.js"); // Lazy: usually not needed. const hasTest = this.progOptions.test || this.progOptions.testFile; + const watchFiles = [...this.inputFiles]; + if (this.progOptions.testFile) { + watchFiles.push(this.progOptions.testFile); + } + this.watcher = new Watcher(...watchFiles); - this.watcher = new Watcher(this.inputFile); - - const that = this; - this.watcher.on("change", async() => { - PeggyCLI.print(this.std.err, `"${that.inputFile}" changed...`); + this.watcher.on("change", async fn => { + PeggyCLI.print(this.std.err, `"${fn}" changed...`); this.lastError = null; - await that.run(); + await this.run(); - if (that.lastError) { - PeggyCLI.print(this.std.err, that.lastError); + if (this.lastError) { + PeggyCLI.print(this.std.err, this.lastError); } else if (!hasTest) { - PeggyCLI.print(this.std.err, `Wrote: "${that.outputFile}"`); + PeggyCLI.print(this.std.err, `Wrote: "${this.outputFile}"`); } }); diff --git a/bin/watcher.js b/bin/watcher.js index 9e0dfcfc..965ec0da 100644 --- a/bin/watcher.js +++ b/bin/watcher.js @@ -6,6 +6,8 @@ const { EventEmitter } = require("events"); // This may have to be tweaked based on experience. const DEBOUNCE_MS = 100; +const CLOSING = Symbol("CLOSING"); +const ERROR = Symbol("ERROR"); /** * Relatively feature-free file watcher that deals with some of the @@ -18,51 +20,70 @@ const DEBOUNCE_MS = 100; */ class Watcher extends EventEmitter { /** - * Creates an instance of Watcher. + * Creates an instance of Watcher. This only works for files in a small + * number of directories. * - * @param {string} filename The file to watch. Should be a plain file, - * not a directory, pipe, etc. + * @param {string[]} filenames The files to watch. Should be one or more + * strings, each of which is the name of a plain file, not a directory, + * pipe, etc. */ - constructor(filename) { + constructor(...filenames) { super(); - const rfile = path.resolve(filename); - const { dir, base } = path.parse(rfile); - let timeout = null; + const resolved = new Set(filenames.map(fn => path.resolve(fn))); + const dirs = new Set([...resolved].map(fn => path.dirname(fn))); - // eslint-disable-next-line func-style -- Needs this. - const changed = (typ, fn) => { - if (fn === base) { - if (!timeout) { - fs.stat(rfile, (er, stats) => { - if (!er && stats.isFile()) { - this.emit("change", stats); - } - }); - } else { - clearTimeout(timeout); + this.timeout = null; + this.watchers = []; + + for (const dir of dirs) { + // eslint-disable-next-line func-style -- Needs "this" + const changed = (_typ, fn) => { + if (typeof this.timeout === "symbol") { + return; } + const filename = path.join(dir, fn); + // Might be a different fil changing in one of the target dirs + if (resolved.has(filename)) { + if (!this.timeout) { + fs.stat(filename, (er, stats) => { + if (!er && stats.isFile()) { + this.emit("change", filename, stats); + } + }); + } else { + clearTimeout(this.timeout); + } - // De-bounce - timeout = setTimeout(() => { - timeout = null; - }, Watcher.interval); - } - }; - const closed = () => this.emit("close"); + // De-bounce, across all files + this.timeout = setTimeout(() => { + this.timeout = null; + }, Watcher.interval); + } + }; - this.watcher = fs.watch(dir); - this.watcher.on("error", er => { - this.watcher.off("close", closed); - this.watcher.once("close", () => this.emit("error", er)); - this.watcher.close(); - this.watcher = null; - }); - this.watcher.on("close", closed); - this.watcher.on("change", changed); + const w = fs.watch(dir); + w.on("error", er => { + const t = this.timeout; + this.timeout = ERROR; + if (t && (typeof t !== "symbol")) { + clearTimeout(t); + } + this.emit("error", er); + this.close(); + }); + w.on("change", changed); + this.watchers.push(w); + } // Fire initial time if file exists. - setImmediate(() => changed("rename", base)); + setImmediate(() => { + if (this.watchers.length > 0) { + // First watcher will correspond to the directory of the first filename. + const w = this.watchers[0]; + w.emit("change", "initial", path.basename([...resolved][0])); + } + }); } /** @@ -71,16 +92,31 @@ class Watcher extends EventEmitter { * @returns {Promise} Always resolves. */ close() { - return new Promise(resolve => { - if (this.watcher) { - this.watcher.once("close", resolve); - this.watcher.close(); - } else { - resolve(); + // Stop any more events from firing, immediately + const t = this.timeout; + + if (t) { + if (typeof t !== "symbol") { + this.timeout = CLOSING; + clearTimeout(t); + } + } + + const p = []; + for (const w of this.watchers) { + p.push(new Promise(resolve => { + w.once("close", resolve); + })); + w.close(); + } + return Promise.all(p).then(() => { + this.watchers = []; + if (t !== ERROR) { + this.emit("close"); } - this.watcher = null; }); } } + Watcher.interval = DEBOUNCE_MS; module.exports = Watcher; diff --git a/lib/compiler/asts.js b/lib/compiler/asts.js index da324c17..c72502e0 100644 --- a/lib/compiler/asts.js +++ b/lib/compiler/asts.js @@ -86,6 +86,50 @@ const asts = { return consumes(node); }, + + combine(asts) { + let combined = null; + for (const ast of asts) { + if (combined) { + // Note: Change topLevelInitializer and initializer to be an array in + // order to keep the location along with. + if (ast.topLevelInitializer) { + if (combined.topLevelInitializer) { + // Try this and see if it's better than the other way in practice. + const init + = ast.topLevelInitializer + + "\n" + + combined.topLevelInitializer; + + // Move imports to the top + let imports = ""; + let nonImports = ""; + for (const line of init.split(/[\r\n]+/)) { + if (/^\s*import/.test(line)) { + imports += line; + } else { + nonImports += line; + } + } + combined.topLevelInitializer = imports + "\n" + nonImports; + } else { + combined.topLevelInitializer = ast.topLevelInitializer; + } + } + if (ast.initializer) { + if (combined.initializer) { + combined.initializer = ast.initializer + "\n" + combined.initializer; + } else { + combined.initializer = ast.initializer; + } + } + combined.rules = combined.rules.concat(ast.rules); + } else { + combined = ast; + } + } + return combined; + }, }; module.exports = asts; diff --git a/lib/peg.js b/lib/peg.js index 1cc4327c..0e986f59 100644 --- a/lib/peg.js +++ b/lib/peg.js @@ -2,6 +2,7 @@ const GrammarError = require("./grammar-error"); const GrammarLocation = require("./grammar-location"); +const asts = require("./compiler/asts.js"); const compiler = require("./compiler"); const parser = require("./parser"); const VERSION = require("./version"); @@ -146,11 +147,22 @@ const peg = { plugins.forEach(p => { p.use(config, options); }); - return peg.compiler.compile( - config.parser.parse(grammar, { - grammarSource: options.grammarSource, + if (!Array.isArray(grammar)) { + grammar = [{ + source: options.grammarSource, + text: grammar, + }]; + } + + const combined = asts.combine( + grammar.map(({ source, text }) => config.parser.parse(text, { + grammarSource: source, reservedWords: config.reservedWords, - }), + })) + ); + + return peg.compiler.compile( + combined, config.passes, options ); diff --git a/test/cli/run.spec.ts b/test/cli/run.spec.ts index 4361e760..0040e210 100644 --- a/test/cli/run.spec.ts +++ b/test/cli/run.spec.ts @@ -323,11 +323,13 @@ describe("MockStream", () => { describe("Command Line Interface", () => { it("has help", async() => { const HELP = `\ -Usage: peggy [options] [input_file] +Usage: peggy [options] [input_file...] Arguments: - input_file Grammar file to read. Use "-" to read - stdin. (default: "-") + input_file Grammar file(s) to read. Use "-" to read + stdin. If multiple files are given, they + are combined in the given order to produce a + single output. (default: ["-"]) Options: -v, --version output the version number @@ -624,7 +626,7 @@ Options: stdin: "foo = '1'", errorCode: "peggy.cli", exitCode: 1, - error: "Error reading input stream", + error: "Error reading input \"____ERROR____FILE_DOES_NOT_EXIST\"", }); await exec({ @@ -1009,13 +1011,6 @@ Options: exitCode: 1, error: /no such file or directory, open '[^']*--trace'/, }); - - await exec({ - args: ["--", "--trace", "--format"], - errorCode: "commander.excessArguments", - exitCode: 1, - error: "too many arguments.", - }); }); it("handles input tests", async() => { @@ -1218,7 +1213,7 @@ error: Rule "unknownRule" is not defined error: "Fake error", onstdout(s, cli) { if (++count === 3) { - cli.watcher?.watcher.emit("error", new Error("Fake error")); + cli.watcher?.watchers[0].emit("error", new Error("Fake error")); } }, }); diff --git a/test/cli/watcher.spec.ts b/test/cli/watcher.spec.ts index 1ca2f106..b7582a56 100644 --- a/test/cli/watcher.spec.ts +++ b/test/cli/watcher.spec.ts @@ -17,7 +17,7 @@ describe("watches files", () => { resolve(); }); }); - w.watcher.emit("error", new Error("Fake error")); + w.watchers[0].emit("error", new Error("Fake error")); return p; }); @@ -27,14 +27,41 @@ describe("watches files", () => { const base = path.basename(fn); const w = new Watcher(fn); w.on("change", () => count++); - w.watcher.emit("change", "rename", base + "_ANOTHER"); - w.watcher.emit("change", "rename", base); + w.watchers[0].emit("change", "rename", base + "_ANOTHER"); + w.watchers[0].emit("change", "rename", base); setTimeout(() => { - w.watcher.emit("change", "rename", base); + w.watchers[0].emit("change", "rename", base); }, Watcher.interval * 0.25); setTimeout(() => { expect(count).toBe(1); + w.watchers[0].emit("change", "rename", base); + }, Watcher.interval * 1.5); + setTimeout(() => { + expect(count).toBe(2); w.close().then(done); - }, Watcher.interval * 1.25); + }, Watcher.interval * 2); + }); + + it("closes after an error", done => { + let count = 0; + const fn = __filename; + const base = path.basename(fn); + const w = new Watcher(fn); + w.on("change", () => count++); + w.on("error", () => { + // Ignored + }); + const firstWatcher = w.watchers[0]; + // Emits error, then closes. w.timeout = ERROR + firstWatcher.emit("error", new Error("Fake error")); + setTimeout(() => { + // Simulate an out-of-order file change while multiple watchers are + // closing. + firstWatcher.emit("change", "rename", base); + }, Watcher.interval * 0.25); + setTimeout(() => { + expect(count).toBe(0); + done(); + }, Watcher.interval * 0.5); }); });