feat(typegen): watch mode (#324)

* file watcher

* Make `watch` a function returned by initial run

* undo accidental changes

* update lockfile

* Broaden negative assertion

* Update readme

* .value

* isolate watch logs

* cli --watch test and lazy test

* improved type, not sure why tbh

* undo unrelated cli change

* try to avoid anonymous tags

* Add a comma

packages/typegen/package.json

@@ -28,16 +28,19 @@
   },
   "dependencies": {
     "@rushstack/ts-command-line": "^4.7.8",
+    "chokidar": "^3.5.2",
     "execa": "^5.0.0",
     "find-up": "^5.0.0",
     "glob": "^7.1.7",
     "io-ts-extra": "^0.11.4",
     "lodash": "^4.17.21",
+    "memoizee": "^0.4.15",
     "pgsql-ast-parser": "^8.0.0",
     "pluralize": "^8.0.0"
   },
   "devDependencies": {
     "@types/glob": "7.1.3",
+    "@types/memoizee": "0.4.5",
     "@types/pluralize": "0.0.29",
     "fs-syncer": "0.3.4-next.2"
   }

packages/typegen/readme.md

@@ -155,7 +155,7 @@ usage: slonik-typegen generate [-h] [--config PATH] [--root-dir PATH]
                                [--connection-uri URI] [--psql COMMAND]
                                [--default-type TYPESCRIPT] [--glob PATTERN]
                                [--since REF] [--migrate {<=0.8.0}]
-                               [--skip-check-clean]
+                               [--skip-check-clean] [--watch] [--lazy]
                                
 
 Generates a directory containing with a 'sql' tag wrapper based on found 
@@ -208,10 +208,16 @@ Optional arguments:
 
   --skip-check-clean    If enabled, the tool will not check the git status to 
                         ensure changes are checked in.
+
+  --watch               Run the type checker in watch mode. Files will be run 
+                        through the code generator when changed or added.
+
+  --lazy                Skip initial processing of input files. Only useful 
+                        with '--watch'.
 ```
 <!-- codegen:end -->
 
-There are some more configuration options [documented in code](./src/types.ts) but these should be considered experimental, and might change without warning. You can try them out as documented below, but please start a [discussion](https://github.com/mmkal/slonik-tools/discussions) on this library's project page with some info about your use case so the API can be stabilised in a sensible way.
+There are some more configuration options [documented in code](./src/types.ts), but these should be considered experimental, and might change without warning. You can try them out as documented below, but please start a [discussion](https://github.com/mmkal/slonik-tools/discussions) on this library's project page with some info about your use case so the API can be stabilised in a sensible way.
 
 ### writeTypes
 

packages/typegen/src/cli.ts

@@ -17,6 +17,7 @@ export class SlonikTypegenCLI extends cli.CommandLineParser {
 
   onDefineParameters() {}
 }
+
 export class GenerateAction extends cli.CommandLineAction {
   private _params!: ReturnType<typeof GenerateAction._defineParameters>
 
@@ -96,6 +97,14 @@ export class GenerateAction extends cli.CommandLineAction {
         parameterLongName: '--skip-check-clean',
         description: `If enabled, the tool will not check the git status to ensure changes are checked in.`,
       }),
+      watch: action.defineFlagParameter({
+        parameterLongName: '--watch',
+        description: `Run the type checker in watch mode. Files will be run through the code generator when changed or added.`,
+      }),
+      lazy: action.defineFlagParameter({
+        parameterLongName: '--lazy',
+        description: `Skip initial processing of input files. Only useful with '--watch'.`,
+      }),
     }
   }
 
@@ -110,7 +119,7 @@ export class GenerateAction extends cli.CommandLineAction {
 
     const options = optionsModule?.default || optionsModule
 
-    return generate(
+    const run = await generate(
       lodash.merge({}, options, {
         rootDir: this._params.rootDir.value,
         connectionURI: this._params.connectionURI.value,
@@ -119,8 +128,13 @@ export class GenerateAction extends cli.CommandLineAction {
         glob: this._params.since.value ? {since: this._params.since.value} : this._params.glob.value,
         migrate: this._params.migrate.value as Options['migrate'],
         checkClean: this._params.skipCheckClean.value ? ['none'] : undefined,
+        lazy: this._params.lazy.value,
       } as Partial<Options>),
     )
+
+    if (this._params.watch.value) {
+      run.watch()
+    }
   }
 }
 

packages/typegen/src/defaults.ts

@@ -56,6 +56,7 @@ export const getParams = (partial: Partial<Options>): Options => {
     typeParsers = defaultTypeParsers(poolConfig.typeParsers || []),
     migrate = undefined,
     checkClean = defaultCheckClean,
+    lazy = false,
     ...rest
   } = partial
 
@@ -77,5 +78,6 @@ export const getParams = (partial: Partial<Options>): Options => {
     logger,
     migrate,
     checkClean,
+    lazy,
   }
 }

packages/typegen/src/extract/typescript.ts

@@ -55,4 +55,4 @@ const rawExtractWithTypeScript: Options['extractQueries'] = file => {
   }
 }
 
-export const extractWithTypeScript: Options['extractQueries'] = lodash.memoize(rawExtractWithTypeScript)
+export const extractWithTypeScript: Options['extractQueries'] = rawExtractWithTypeScript

packages/typegen/src/index.ts

@@ -6,18 +6,21 @@ import {Options, QueryField, DescribedQuery, ExtractedQuery, QueryParameter} fro
 import {columnInfoGetter, isUntypeable, removeSimpleComments, simplifySql} from './query'
 import * as assert from 'assert'
 import * as path from 'path'
+import * as fs from 'fs'
 import {parameterTypesGetter} from './query/parameters'
 import {migrateLegacyCode} from './migrate'
 import * as write from './write'
 import {createPool} from 'slonik'
+import memoizee = require('memoizee')
+import chokidar = require('chokidar')
 
 export {Options} from './types'
 
 export {defaults}
 
 export {write}
 
-export const generate = (params: Partial<Options>) => {
+export const generate = async (params: Partial<Options>) => {
   const {
     psqlCommand,
     connectionURI,
@@ -32,13 +35,14 @@ export const generate = (params: Partial<Options>) => {
     logger,
     migrate,
     checkClean: checkCleanWhen,
+    lazy,
   } = defaults.getParams(params)
 
   const pool = createPool(connectionURI, poolConfig)
 
-  const {psql, getEnumTypes, getRegtypeToPGType} = psqlClient(`${psqlCommand} "${connectionURI}"`, pool)
+  const {psql: _psql, getEnumTypes, getRegtypeToPGType} = psqlClient(`${psqlCommand} "${connectionURI}"`, pool)
 
-  const gdesc = async (sql: string) => {
+  const _gdesc = async (sql: string) => {
     try {
       return await psql(`${sql} \\gdesc`)
     } catch (e) {
@@ -51,6 +55,9 @@ export const generate = (params: Partial<Options>) => {
     }
   }
 
+  const psql = memoizee(_psql, {max: 1000})
+  const gdesc = memoizee(_gdesc, {max: 1000})
+
   const getFields = async (query: ExtractedQuery): Promise<QueryField[]> => {
     const rows = await gdesc(query.sql)
     const fields = await Promise.all(
@@ -114,7 +121,6 @@ export const generate = (params: Partial<Options>) => {
   }
 
   const findAll = async () => {
-    await maybeDo(checkCleanWhen.includes('before'), checkClean)
     const getColumnInfo = columnInfoGetter(pool)
 
     const globParams: Parameters<typeof globAsync> =
@@ -124,59 +130,109 @@ export const generate = (params: Partial<Options>) => {
         ? [globList(changedFiles({since: glob.since, cwd: path.resolve(rootDir)})), {}]
         : glob
 
-    logger.info(`Searching for files matching ${globParams[0]} in ${rootDir}.`)
-
-    const getFiles = () =>
-      globAsync(globParams[0], {
+    const getFiles = () => {
+      logger.info(`Searching for files matching ${globParams[0]} in ${rootDir}.`)
+      return globAsync(globParams[0], {
         ...globParams[1],
         cwd: path.resolve(process.cwd(), rootDir),
         absolute: true,
       })
+    }
 
     if (migrate) {
       await maybeDo(checkCleanWhen.includes('before-migrate'), checkClean)
       migrateLegacyCode(migrate)({files: await getFiles(), logger})
       await maybeDo(checkCleanWhen.includes('after-migrate'), checkClean)
     }
 
-    const files = await getFiles() // Migration may have deleted some, get files from fresh.
-    const extracted = files.flatMap(extractQueries)
-
-    logger.info(`Found ${files.length} files and ${extracted.length} queries.`)
-
-    const promises = extracted.map(async (query): Promise<DescribedQuery | null> => {
-      try {
-        if (isUntypeable(query.template)) {
-          logger.debug(`Query \`${truncateQuery(query.sql)}\` in file ${query.file} is not typeable`)
+    async function generateForFiles(files: string[]) {
+      const extracted = files.flatMap(extractQueries)
+
+      logger.info(`Found ${files.length} files and ${extracted.length} queries.`)
+
+      const promises = extracted.map(async (query): Promise<DescribedQuery | null> => {
+        try {
+          if (isUntypeable(query.template)) {
+            logger.debug(`Query \`${truncateQuery(query.sql)}\` in file ${query.file} is not typeable`)
+            return null
+          }
+          return {
+            ...query,
+            fields: await getFields(query),
+            parameters: query.file.endsWith('.sql') ? await getParameters(query) : [],
+          }
+        } catch (e) {
+          let message = `${query.file}:${query.line} Describing query failed: ${e}.`
+          if (query.sql.includes('--')) {
+            message += ' Try moving comments to dedicated lines.'
+          }
+          logger.warn(message)
           return null
         }
-        return {
-          ...query,
-          fields: await getFields(query),
-          parameters: query.file.endsWith('.sql') ? await getParameters(query) : [],
-        }
-      } catch (e) {
-        let message = `${query.file}:${query.line} Describing query failed: ${e}.`
-        if (query.sql.includes('--')) {
-          message += ' Try moving comments to dedicated lines.'
-        }
-        logger.warn(message)
-        return null
-      }
-    })
+      })
 
-    const describedQueries = lodash.compact(await Promise.all(promises))
+      const describedQueries = lodash.compact(await Promise.all(promises))
 
-    const uniqueFiles = [...new Set(describedQueries.map(q => q.file))]
-    logger.info(
-      `Succesfully processed ${describedQueries.length} out of ${promises.length} queries in files ${uniqueFiles} .`,
-    )
+      const uniqueFiles = [...new Set(describedQueries.map(q => q.file))]
+      logger.info(
+        `Succesfully processed ${describedQueries.length} out of ${promises.length} queries in files ${uniqueFiles} .`,
+      )
 
-    const analysedQueries = await Promise.all(describedQueries.map(getColumnInfo))
+      const analysedQueries = await Promise.all(describedQueries.map(getColumnInfo))
 
-    await writeTypes(analysedQueries)
-    await maybeDo(checkCleanWhen.includes('after'), checkClean)
+      await writeTypes(analysedQueries)
+    }
+
+    if (!lazy) {
+      await generateForFiles(await getFiles())
+    }
+
+    const watch = () => {
+      const cwd = path.resolve(rootDir)
+      logger.info({message: 'Waiting for files to change', globParams, cwd})
+      const watcher = chokidar.watch(globParams[0], {
+        ignored: globParams[1]?.ignore,
+        cwd,
+        ignoreInitial: true,
+      })
+      const runOne = memoizee((filepath: string, _existingContent: string) => generateForFiles([filepath]), {
+        max: 1000,
+      })
+      let promises: Promise<void>[] = []
+      /**
+       * memoized logger. We're memoizing several layers deep, so when the codegen runs on a file, it memoizes
+       * the file path and content, and the queries inside the file. So when a file updates, it's processed and
+       * re-edited inline, which triggers another file change handler. It's then _re-processed_ but since everything
+       * is memoized the resultant content is unchanged from query cache hits. The file is written to with the same
+       * content one more time before the `runOne` cache is hit. Memoizing the log for one second ensure we don't see
+       * unnecessary `x.ts was changed, running codegen` entries. It's hacky, but there's not much overhead at runtime
+       * or in code.
+       */
+      const log = memoizee((msg: unknown) => logger.info(msg), {max: 1000, maxAge: 5000})
+      const handler = async (filepath: string) => {
+        log(filepath + ' was changed, running codegen')
+        const fullpath = path.join(cwd, filepath)
+        const existingContent = fs.readFileSync(fullpath).toString()
+        const promise = runOne(fullpath, existingContent)
+        promises.push(promise)
+        await promise
+        log(filepath + ' updated.')
+      }
+      watcher.on('add', handler)
+      watcher.on('change', handler)
+      return {
+        close: async () => {
+          await watcher.close()
+          await Promise.all(promises)
+        },
+      }
+    }
+    return watch
   }
 
-  return findAll()
+  await maybeDo(checkCleanWhen.includes('before'), checkClean)
+  const watch = await findAll()
+  await maybeDo(checkCleanWhen.includes('after'), checkClean)
+
+  return {watch}
 }

packages/typegen/src/query/tag.ts

@@ -29,8 +29,8 @@ export function addTags(queries: AnalysedQuery[]): TaggedQuery[] {
       const matchesFirstTag = arr[firstWithTagIndex].identifier === q.identifier
       return {
         ...q,
-        tag: matchesFirstTag ? q.tag : q.tag + '_' + firstWithTagIndex,
-        priority: matchesFirstTag ? 0 : 1,
+        tag: matchesFirstTag ? q.tag : `${q.tag}_${firstWithTagIndex}`,
+        priority: q.tag.startsWith('Anonymous') ? 2 : matchesFirstTag ? 0 : 1,
       }
     })
     .sortBy(q => q.priority)

packages/typegen/src/types.ts

@@ -145,6 +145,10 @@ export interface Options {
    * By default mimics the behavior of `slonik.createTypeParserPreset()`, so if you're only using the defaults (or you don't know!), you can leave this undefined.
    */
   typeParsers: Array<TypeParserInfo>
+  /**
+   * Skip initial processing of input files. Only useful with `watch`.
+   */
+  lazy?: boolean
 }
 
 export type Logger = Record<'error' | 'warn' | 'info' | 'debug', (msg: unknown) => void>

packages/typegen/test/cli.test.ts

@@ -3,6 +3,7 @@ import * as fsSyncer from 'fs-syncer'
 import * as slonik from 'slonik'
 import {psqlCommand} from './helper'
 import * as child_process from 'child_process'
+import * as chokidar from 'chokidar'
 
 beforeEach(() => {
   jest.clearAllMocks()
@@ -26,6 +27,13 @@ jest.mock('child_process', () => ({
   execSync: jest.fn().mockImplementation(() => ''),
 }))
 
+jest.mock('chokidar', () => ({
+  watch: jest.fn().mockReturnValue({
+    on: jest.fn(),
+    close: jest.fn(),
+  }),
+}))
+
 afterAll(async () => {
   await Promise.all(pools.map(p => p.end()))
 })
@@ -90,7 +98,7 @@ test('can skip checking git status', async () => {
   expect(child_process.execSync).not.toHaveBeenCalled()
 }, 20000)
 
-const fixPsqlCommand = (content: string) => content.split(process.env.POSTGRES_PSQL_COMMAND!).join('<<psql>>')
+const fixPsqlCommand = (content: string) => content.split(psqlCommand).join('<<psql>>')
 
 test('typegen.config.js is used by default', async () => {
   const cli = new SlonikTypegenCLI()
@@ -253,3 +261,19 @@ test('use git to get changed files', async () => {
 
   expect(child_process.execSync).toHaveBeenCalledWith(`git diff --relative --name-only main`, {cwd: expect.any(String)})
 }, 20000)
+
+test('use chokidar to watch', async () => {
+  // this only tests that we start using chokidar, watch.test.ts checks the actual functionality
+  const cli = new SlonikTypegenCLI()
+
+  const syncer = fsSyncer.jestFixture({
+    targetState: {},
+  })
+
+  syncer.sync()
+
+  await cli.executeWithoutErrorHandling(['generate', '--root-dir', syncer.baseDir, '--skip-check-clean', '--watch'])
+
+  expect(chokidar.watch).toHaveBeenCalledTimes(1)
+  expect(chokidar.watch([]).on).toHaveBeenCalledTimes(2)
+}, 20000)