From f3ca4370b5f37e4e2d07fcbeea4165cee7a300b0 Mon Sep 17 00:00:00 2001 From: jrea Date: Tue, 21 May 2024 10:02:25 -0400 Subject: [PATCH] fix(server): make eviction work --- packages/server/src/Server.ts | 6 ++-- packages/server/src/db/DBManager.ts | 35 +++++++++++++++++------ packages/server/src/db/NileInstance.ts | 17 ++++++++++- packages/server/src/db/db.test.ts | 8 +++++- packages/server/src/utils/Config/index.ts | 17 +++++++++-- packages/server/src/utils/Event/index.ts | 11 +++++++ 6 files changed, 80 insertions(+), 14 deletions(-) diff --git a/packages/server/src/Server.ts b/packages/server/src/Server.ts index 3c182122..43521c14 100644 --- a/packages/server/src/Server.ts +++ b/packages/server/src/Server.ts @@ -29,7 +29,7 @@ const init = (config: Config): Api => { export class Server { config: Config; api: Api; - private manager: DbManager; + private manager!: DbManager; private servers: Map; constructor(config?: ServerConfig) { @@ -61,8 +61,10 @@ export class Server { ...cfg, }); this.setConfig(updatedConfig); + + this.manager.clear(this.config); this.manager = new DbManager(this.config); - this.api = init(updatedConfig); + this.api = init(this.config); return this; } diff --git a/packages/server/src/db/DBManager.ts b/packages/server/src/db/DBManager.ts index 09d225b2..63244d71 100644 --- a/packages/server/src/db/DBManager.ts +++ b/packages/server/src/db/DBManager.ts @@ -1,7 +1,7 @@ import { Pool } from 'pg'; import { Config } from '../utils/Config'; -import { watchEvictPool } from '../utils/Event'; +import { closeEvictPool, watchEvictPool } from '../utils/Event'; import Logger from '../utils/Logger'; import { ServerConfig } from '../types'; @@ -9,6 +9,7 @@ import NileDatabase from './NileInstance'; export default class DBManager { connections: Map; + cleared: boolean; private makeId( tenantId?: string | undefined | null, @@ -24,31 +25,49 @@ export default class DBManager { } constructor(config: ServerConfig) { const { info } = Logger(config, '[DBManager]'); + this.cleared = false; this.connections = new Map(); // add the base one, so you can at least query const id = this.makeId(); info('constructor', id); this.connections.set(id, new NileDatabase(new Config(config), id)); - watchEvictPool((id) => { - if (id && this.connections.has(id)) { - this.connections.delete(id); - } - }); + watchEvictPool(this.poolWatcher(config)); } + poolWatcher = (config: ServerConfig) => (id: undefined | null | string) => { + const { info } = Logger(config, '[DBManager]'); + if (id && this.connections.has(id)) { + info('Removing', id, 'from db connection pool.'); + this.connections.delete(id); + } + }; - getConnection(config: ServerConfig): Pool { + getConnection = (config: ServerConfig): Pool => { const { info } = Logger(config, '[DBManager]'); const id = this.makeId(config.tenantId, config.userId); const existing = this.connections.get(id); info('# of instances:', this.connections.size); if (existing) { info('returning existing', id); + existing.startTimeout(); return existing.pool; } const newOne = new NileDatabase(new Config(config), id); this.connections.set(id, newOne); info('created new', id); info('# of instances:', this.connections.size); + // resume listening to the evict pool if a connection is requested. + if (this.cleared) { + this.cleared = false; + watchEvictPool(this.poolWatcher(config)); + } return newOne.pool; - } + }; + + clear = (config: ServerConfig) => { + const { info } = Logger(config, '[DBManager]'); + info('Clearing all connections', this.connections.size); + closeEvictPool(this.poolWatcher(config)); + this.cleared = true; + this.connections.clear(); + }; } diff --git a/packages/server/src/db/NileInstance.ts b/packages/server/src/db/NileInstance.ts index 5dd7f01c..4c02a654 100644 --- a/packages/server/src/db/NileInstance.ts +++ b/packages/server/src/db/NileInstance.ts @@ -49,19 +49,34 @@ class NileDatabase { _client.release(); } }); + this.startTimeout(); }); this.pool.on('error', async (e) => { info('pool failed', e); + if (this.timer) { + clearTimeout(this.timer); + } + evictPool(this.id); }); } startTimeout() { + const { info } = Logger(this.config, '[NileInstance]'); if (this.timer) { clearTimeout(this.timer); } this.timer = setTimeout(async () => { - await this.pool.end(); + info( + 'Pool reached idleTimeoutMillis.', + this.id, + 'evicted after', + this.config.db.idleTimeoutMillis, + 'ms' + ); + await this.pool.end(() => { + // something odd going on here. Without the callback, pool.end() is flakey + }); evictPool(this.id); }, this.config.db.idleTimeoutMillis); } diff --git a/packages/server/src/db/db.test.ts b/packages/server/src/db/db.test.ts index 8a858481..08cf7e40 100644 --- a/packages/server/src/db/db.test.ts +++ b/packages/server/src/db/db.test.ts @@ -1,6 +1,12 @@ import NileDB from './index'; -const properties = ['connections']; +const properties = [ + 'connections', + 'clear', + 'cleared', + 'getConnection', + 'poolWatcher', +]; describe('db', () => { it('has expected properties', () => { const db = new NileDB({ diff --git a/packages/server/src/utils/Config/index.ts b/packages/server/src/utils/Config/index.ts index 52a1d9cc..12c12001 100644 --- a/packages/server/src/utils/Config/index.ts +++ b/packages/server/src/utils/Config/index.ts @@ -85,9 +85,22 @@ export class Config { constructor(config?: ServerConfig, logger?: string) { const envVarConfig: EnvConfig = { config, logger }; - this.databaseId = getDatbaseId(envVarConfig) as string; this.user = getUsername(envVarConfig) as string; this.password = getPassword(envVarConfig) as string; + if (process.env.NODE_ENV !== 'TEST') { + if (!this.user) { + throw new Error( + 'User is required. Set NILEDB_USER as an environment variable or set `user` in the config options.' + ); + } + if (!this.password) { + throw new Error( + 'Password is required. Set NILEDB_PASSWORD as an environment variable or set `password` in the config options.' + ); + } + } + + this.databaseId = getDatbaseId(envVarConfig) as string; this.databaseName = getDatabaseName(envVarConfig) as string; this._tenantId = getTenantId(envVarConfig); this.debug = Boolean(config?.debug); @@ -169,7 +182,7 @@ export class Config { throw new Error('HTTP error has occured'); } else { throw new Error( - 'Unable to auto-configure. Please set or remove NILEDB_API, NILEDB_NAME, and NILEDB_HOST in your .env file.' + 'Unable to auto-configure. Please remove NILEDB_NAME, NILEDB_API_URL, NILEDB_POSTGRES_URL, and/or NILEDB_HOST from your environment variables.' ); } } diff --git a/packages/server/src/utils/Event/index.ts b/packages/server/src/utils/Event/index.ts index 3a467676..283925d0 100644 --- a/packages/server/src/utils/Event/index.ts +++ b/packages/server/src/utils/Event/index.ts @@ -32,6 +32,13 @@ class Eventer { // store the callback function of the subscriber this.events[eventName].push(callback); } + + unsubscribe(eventName: string, callback: EventFn) { + const toRemove = this.events[eventName].findIndex((cb) => cb === callback); + if (toRemove !== -1) { + this.events[eventName].splice(toRemove, 1); + } + } } // tenantId manager @@ -58,6 +65,10 @@ export const watchToken = (cb: EventFn) => eventer.subscribe(Events.Token, cb); export const watchEvictPool = (cb: EventFn) => eventer.subscribe(Events.EvictPool, cb); + +export const closeEvictPool = (cb: EventFn) => + eventer.unsubscribe(Events.EvictPool, cb); + export const evictPool = (val: BusValues) => { eventer.publish(Events.EvictPool, val); };