Added --exclude-regex and --exclude-file

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@joernio/astgen",
-  "version": "3.20.0",
+  "version": "3.21.0",
   "description": "Generate JS/TS AST in json format with Babel",
   "exports": "./index.js",
   "keywords": [

src/Defaults.ts

@@ -31,7 +31,9 @@ export const IGNORE_DIRS: string[] = [
     "build",
 ];
 
-export const IGNORE_FILE_PATTERN: RegExp = new RegExp("(conf|test|spec|[.-]min|\\.d)\\.(js|ts|jsx|tsx)$", "i");
+export const IGNORE_FILE_PATTERN: RegExp =
+    new RegExp("(chunk-vendors|app~|mock|e2e|conf|test|spec|[.-]min|\\.d)\\.(js|ts|jsx|tsx)$", "i");
+
 export const MAX_LOC_IN_FILE: number = 50000;
 
 export const BABEL_PARSER_OPTIONS: babelParser.ParserOptions = {

src/FileUtils.ts

@@ -1,8 +1,10 @@
+import Options from "./Options";
 import * as Defaults from "./Defaults";
 
 import {readdirpPromise} from 'readdirp';
 import * as fs from "node:fs";
 import nReadlines from "n-readlines";
+import * as path from "node:path"
 
 function countFileLines(filePath: string): number {
     const broadbandLines = new nReadlines(filePath);
@@ -13,9 +15,29 @@ function countFileLines(filePath: string): number {
     return lineNumber
 }
 
-function ignoreDirectory(dirName: string): boolean {
+function dirIsInIgnorePath(options: Options, fullPath: string, ignorePath: string): boolean {
+    if (path.isAbsolute(ignorePath)) {
+        return fullPath.startsWith(ignorePath)
+    } else {
+        const absIgnorePath = path.join(options.src, ignorePath)
+        return fullPath.startsWith(absIgnorePath)
+    }
+}
+
+function fileIsInIgnorePath(options: Options, fullPath: string, ignorePath: string): boolean {
+    if (path.isAbsolute(ignorePath)) {
+        return fullPath == ignorePath
+    } else {
+        const absIgnorePath = path.join(options.src, ignorePath)
+        return fullPath == absIgnorePath
+    }
+}
+
+function ignoreDirectory(options: Options, dirName: string, fullPath: string): boolean {
     return dirName.startsWith(".") ||
         dirName.startsWith("__") ||
+        options["exclude-file"].some((e: string) => dirIsInIgnorePath(options, fullPath, e)) ||
+        options["exclude-regex"]?.test(fullPath) ||
         Defaults.IGNORE_DIRS.includes(dirName.toLowerCase())
 }
 
@@ -35,19 +57,22 @@ function isTooLarge(fileWithDir: string): boolean {
     return false;
 }
 
-function ignoreFile(fileName: string, fileWithDir: string, extensions: string[]): boolean {
+function ignoreFile(options: Options, fileName: string, fullPath: string, extensions: string[]): boolean {
     return !extensions.some((e: string) => fileName.endsWith(e)) ||
         fileName.startsWith(".") ||
         fileName.startsWith("__") ||
         Defaults.IGNORE_FILE_PATTERN.test(fileName) ||
-        isEmscripten(fileWithDir) ||
-        isTooLarge(fileWithDir)
+        options["exclude-file"].some((e: string) => fileIsInIgnorePath(options, fullPath, e)) ||
+        options["exclude-regex"]?.test(fullPath) ||
+        isEmscripten(fullPath) ||
+        isTooLarge(fullPath)
 }
 
-export async function filesWithExtensions(dir: string, extensions: string[]): Promise<string[]> {
+export async function filesWithExtensions(options: Options, extensions: string[]): Promise<string[]> {
+    const dir = options.src
     const files = await readdirpPromise(dir, {
-        fileFilter: (f) => !ignoreFile(f.basename, f.fullPath, extensions),
-        directoryFilter: (d) => !ignoreDirectory(d.basename),
+        fileFilter: (f) => !ignoreFile(options, f.basename, f.fullPath, extensions),
+        directoryFilter: (d) => !ignoreDirectory(options, d.basename, d.fullPath),
         lstat: true
     });
     // @ts-ignore

src/Options.ts

@@ -3,5 +3,7 @@ export default interface Options {
     output: string,
     type?: string,
     recurse: boolean,
-    tsTypes: boolean
+    tsTypes: boolean,
+    "exclude-file": string[],
+    "exclude-regex"?: RegExp
 }

src/astgen.ts

@@ -12,6 +12,9 @@ async function main(argv: string[]) {
         .option("src", {
             alias: "i",
             default: ".",
+            coerce: (arg: any): string => {
+                return path.resolve(arg.toString())
+            },
             description: "Source directory",
         })
         .option("output", {
@@ -35,6 +38,22 @@ async function main(argv: string[]) {
             type: "boolean",
             description: "Generate type mappings using the Typescript Compiler API",
         })
+        .option("exclude-file", {
+            default: [],
+            type: "string",
+            array: true,
+            description: "Exclude this file. Can be specified multiple times. Default is empty."
+        })
+        .option("exclude-regex", {
+            coerce: (arg: any): RegExp | undefined => {
+                try {
+                    return new RegExp(arg.toString(), "i")
+                } catch (err) {
+                    return undefined;
+                }
+            },
+            description: "Exclude files matching this regex (matches the absolute path)."
+        })
         .version()
         .help("h").parseSync();
 

src/index.ts

@@ -58,7 +58,7 @@ function createTsc(srcFiles: string[]): TscUtils.TscResult | undefined {
  */
 async function createJSAst(options: Options) {
     try {
-        const srcFiles: string[] = await FileUtils.filesWithExtensions(options.src, Defaults.JS_EXTENSIONS);
+        const srcFiles: string[] = await FileUtils.filesWithExtensions(options, Defaults.JS_EXTENSIONS);
         let ts: TscUtils.TscResult | undefined;
         if (options.tsTypes) {
             ts = createTsc(srcFiles);
@@ -95,7 +95,7 @@ async function createJSAst(options: Options) {
  * Generate AST for .vue files
  */
 async function createVueAst(options: Options) {
-    const srcFiles: string[] = await FileUtils.filesWithExtensions(options.src, [".vue"]);
+    const srcFiles: string[] = await FileUtils.filesWithExtensions(options, [".vue"]);
     for (const file of srcFiles) {
         try {
             const ast = toVueAst(file);
@@ -138,20 +138,20 @@ function writeTypesFile(file: string, seenTypes: Map<number, string>, options: O
 }
 
 async function createXAst(options: Options) {
-    const src_dir = options.src;
+    const srcDir = options.src;
     try {
-        fs.accessSync(src_dir, fs.constants.R_OK);
+        fs.accessSync(srcDir, fs.constants.R_OK);
     } catch (err) {
-        console.error(src_dir, "is invalid");
+        console.error(srcDir, "is invalid");
         process.exit(1);
     }
     if (
-        fs.existsSync(path.join(src_dir, "package.json")) ||
-        fs.existsSync(path.join(src_dir, "rush.json"))
+        fs.existsSync(path.join(srcDir, "package.json")) ||
+        fs.existsSync(path.join(srcDir, "rush.json"))
     ) {
         return await createJSAst(options);
     }
-    console.error(src_dir, "unknown project type");
+    console.error(srcDir, "unknown project type");
     process.exit(1);
 }
 

test/astgen.test.ts

@@ -14,7 +14,8 @@ describe('astgen basic functionality', () => {
             type: "js",
             output: path.join(tmpDir, "ast_out"),
             recurse: true,
-            tsTypes: true
+            tsTypes: true,
+            "exclude-file": []
         });
         const resultAst = fs.readFileSync(path.join(tmpDir, "ast_out", "main.js.json")).toString();
         expect(resultAst).toContain("\"fullName\":\"" + testFile.replaceAll("\\", "\\\\") + "\"");
@@ -24,4 +25,111 @@ describe('astgen basic functionality', () => {
 
         fs.rmSync(tmpDir, {recursive: true});
     });
+
+    it('should exclude files by relative file path correctly', async () => {
+        const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
+        const testFile = path.join(tmpDir, "main.js");
+        fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
+        await start({
+            src: tmpDir,
+            type: "js",
+            output: path.join(tmpDir, "ast_out"),
+            recurse: true,
+            tsTypes: false,
+            "exclude-file": ["main.js"]
+        });
+        expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy()
+
+        fs.rmSync(tmpDir, {recursive: true});
+    });
+
+    it('should exclude files by absolute file path correctly', async () => {
+        const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
+        const testFile = path.join(tmpDir, "main.js");
+        fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
+        await start({
+            src: tmpDir,
+            type: "js",
+            output: path.join(tmpDir, "ast_out"),
+            recurse: true,
+            tsTypes: false,
+            "exclude-file": [testFile]
+        });
+        expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy()
+
+        fs.rmSync(tmpDir, {recursive: true});
+    });
+
+    it('should exclude files by relative file path with dir correctly', async () => {
+        const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
+        const testFile = path.join(tmpDir, "src", "main.js");
+        fs.mkdirSync(path.join(tmpDir, "src"))
+        fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
+        await start({
+            src: tmpDir,
+            type: "js",
+            output: path.join(tmpDir, "ast_out"),
+            recurse: true,
+            tsTypes: false,
+            "exclude-file": [path.join("src", "main.js")]
+        });
+        expect(fs.existsSync(path.join(tmpDir, "ast_out", "src", "main.js.json"))).toBeFalsy()
+
+        fs.rmSync(tmpDir, {recursive: true});
+    });
+
+    it('should exclude files by relative dir path correctly', async () => {
+        const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
+        const testFile = path.join(tmpDir, "src", "main.js");
+        fs.mkdirSync(path.join(tmpDir, "src"))
+        fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
+        await start({
+            src: tmpDir,
+            type: "js",
+            output: path.join(tmpDir, "ast_out"),
+            recurse: true,
+            tsTypes: false,
+            "exclude-file": ["src"]
+        });
+        expect(fs.existsSync(path.join(tmpDir, "ast_out", "src", "main.js.json"))).toBeFalsy()
+
+        fs.rmSync(tmpDir, {recursive: true});
+    });
+
+    it('should exclude files by absolute dir path correctly', async () => {
+        const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
+        const testFile = path.join(tmpDir, "src", "main.js");
+        fs.mkdirSync(path.join(tmpDir, "src"))
+        fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
+        await start({
+            src: tmpDir,
+            type: "js",
+            output: path.join(tmpDir, "ast_out"),
+            recurse: true,
+            tsTypes: false,
+            "exclude-file": [path.join(tmpDir, "src")]
+        });
+        expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy()
+
+        fs.rmSync(tmpDir, {recursive: true});
+    });
+
+    it('should exclude files by regex correctly', async () => {
+        const tmpDir: string = fs.mkdtempSync(path.join(os.tmpdir(), "astgen-tests"));
+        const testFile = path.join(tmpDir, "main.js");
+        fs.writeFileSync(testFile, "console.log(\"Hello, world!\");");
+        await start({
+            src: tmpDir,
+            type: "js",
+            output: path.join(tmpDir, "ast_out"),
+            recurse: true,
+            tsTypes: false,
+            "exclude-file": [],
+            "exclude-regex": new RegExp(".*main.*", "i")
+        });
+        expect(fs.existsSync(path.join(tmpDir, "ast_out", "main.js.json"))).toBeFalsy()
+
+        fs.rmSync(tmpDir, {recursive: true});
+    });
+
 });