From 32dfa57aaaedf5872ab56dfb45f86ea019f29be1 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sat, 4 Jan 2025 17:20:24 -0800 Subject: [PATCH 1/3] Remove all postgres code --- .env | 6 - .github/workflows/deploy.yml | 25 - .github/workflows/pr-build.yml | 25 - api/db/apps-queries.test.ts | 52 -- api/db/apps-queries.ts | 42 -- api/db/index.test.ts | 96 ---- api/db/index.ts | 75 --- api/db/item-annotations-queries.test.ts | 207 -------- api/db/item-annotations-queries.ts | 202 -------- api/db/item-hash-tags-queries.test.ts | 113 ----- api/db/item-hash-tags-queries.ts | 115 ----- api/db/loadouts-queries.test.ts | 111 ----- api/db/loadouts-queries.ts | 195 -------- api/db/migration-state-queries.ts | 228 --------- api/db/searches-queries.test.ts | 164 ------- api/db/searches-queries.ts | 229 --------- api/db/settings-queries.test.ts | 54 -- api/db/settings-queries.ts | 70 --- api/db/triumphs-queries.test.ts | 75 --- api/db/triumphs-queries.ts | 106 ---- .../20191103052046-settings-table.js | 39 -- .../20191221235741-global-settings-table.js | 43 -- api/migrations/20200113062324-apps.js | 35 -- api/migrations/20200208225933-loadouts.js | 51 -- .../20200208233039-item-annotations.js | 51 -- .../20200212061106-apps-add-origin.js | 34 -- api/migrations/20200222234251-audit-log.js | 47 -- .../20200318051307-add-dim-refresh-setting.js | 33 -- .../20200529053904-tracked-triumphs.js | 45 -- ...618-add-issue-banner-to-global-settings.js | 33 -- api/migrations/20200704024612-searches.js | 48 -- api/migrations/20200707023029-hash-tags.js | 49 -- .../20200912183523-add-loadout-params.js | 27 - .../20201027035807-drop-audit-log.js | 46 -- ...015627-change-item-annotation-to-bigint.js | 33 -- ...05015836-smaller-item-annotations-index.js | 34 -- .../20211104204501-add-loadout-notes.js | 27 - .../20220110060009-loadout-share.js | 49 -- .../20220627015027-global-settings-flavor.js | 27 - .../20220627015325-global-settings-bust.js | 33 -- .../20221016230211-searches-hash.js | 30 -- .../20221016230508-searches-hash-uniq.js | 31 -- .../20221016231000-searches-remove-pk.js | 30 -- ...017055645-item-annotations-crafted-date.js | 27 - ...20221224022935-item-annotations-variant.js | 27 - api/migrations/20230524173111-cron-nulls.js | 31 -- .../20230524173723-cron-searches.js | 31 -- .../20230524173837-cron-loadout-shares.js | 31 -- .../20230524174240-cron-hashtag-nulls.js | 31 -- .../20240521044000-add-search-type.js | 27 - .../20240912182338-migration-state.js | 39 -- api/migrations/package.json | 6 - api/routes/delete-all-data.ts | 54 +- api/routes/export.ts | 49 +- api/routes/import.ts | 40 +- api/routes/profile.ts | 159 +----- api/routes/update.ts | 346 +------------ api/server.test.ts | 3 - api/stately/init/migrate-loadout-shares.ts | 6 - api/stately/init/migrate-users.ts | 34 -- api/stately/migrator/loadout-shares.ts | 15 - api/stately/migrator/user.ts | 44 -- build/.env.travis | 6 - kubernetes/dim-api-configmap.yaml | 7 +- kubernetes/dim-api-deployment.yaml | 5 - package.json | 5 - pnpm-lock.yaml | 461 +----------------- 67 files changed, 29 insertions(+), 4520 deletions(-) delete mode 100644 api/db/apps-queries.test.ts delete mode 100644 api/db/apps-queries.ts delete mode 100644 api/db/index.test.ts delete mode 100644 api/db/index.ts delete mode 100644 api/db/item-annotations-queries.test.ts delete mode 100644 api/db/item-annotations-queries.ts delete mode 100644 api/db/item-hash-tags-queries.test.ts delete mode 100644 api/db/item-hash-tags-queries.ts delete mode 100644 api/db/loadouts-queries.test.ts delete mode 100644 api/db/loadouts-queries.ts delete mode 100644 api/db/migration-state-queries.ts delete mode 100644 api/db/searches-queries.test.ts delete mode 100644 api/db/searches-queries.ts delete mode 100644 api/db/settings-queries.test.ts delete mode 100644 api/db/settings-queries.ts delete mode 100644 api/db/triumphs-queries.test.ts delete mode 100644 api/db/triumphs-queries.ts delete mode 100644 api/migrations/20191103052046-settings-table.js delete mode 100644 api/migrations/20191221235741-global-settings-table.js delete mode 100644 api/migrations/20200113062324-apps.js delete mode 100644 api/migrations/20200208225933-loadouts.js delete mode 100644 api/migrations/20200208233039-item-annotations.js delete mode 100644 api/migrations/20200212061106-apps-add-origin.js delete mode 100644 api/migrations/20200222234251-audit-log.js delete mode 100644 api/migrations/20200318051307-add-dim-refresh-setting.js delete mode 100644 api/migrations/20200529053904-tracked-triumphs.js delete mode 100644 api/migrations/20200703220618-add-issue-banner-to-global-settings.js delete mode 100644 api/migrations/20200704024612-searches.js delete mode 100644 api/migrations/20200707023029-hash-tags.js delete mode 100644 api/migrations/20200912183523-add-loadout-params.js delete mode 100644 api/migrations/20201027035807-drop-audit-log.js delete mode 100644 api/migrations/20210105015627-change-item-annotation-to-bigint.js delete mode 100644 api/migrations/20210105015836-smaller-item-annotations-index.js delete mode 100644 api/migrations/20211104204501-add-loadout-notes.js delete mode 100644 api/migrations/20220110060009-loadout-share.js delete mode 100644 api/migrations/20220627015027-global-settings-flavor.js delete mode 100644 api/migrations/20220627015325-global-settings-bust.js delete mode 100644 api/migrations/20221016230211-searches-hash.js delete mode 100644 api/migrations/20221016230508-searches-hash-uniq.js delete mode 100644 api/migrations/20221016231000-searches-remove-pk.js delete mode 100644 api/migrations/20221017055645-item-annotations-crafted-date.js delete mode 100644 api/migrations/20221224022935-item-annotations-variant.js delete mode 100644 api/migrations/20230524173111-cron-nulls.js delete mode 100644 api/migrations/20230524173723-cron-searches.js delete mode 100644 api/migrations/20230524173837-cron-loadout-shares.js delete mode 100644 api/migrations/20230524174240-cron-hashtag-nulls.js delete mode 100644 api/migrations/20240521044000-add-search-type.js delete mode 100644 api/migrations/20240912182338-migration-state.js delete mode 100644 api/migrations/package.json delete mode 100644 api/stately/init/migrate-loadout-shares.ts delete mode 100644 api/stately/init/migrate-users.ts delete mode 100644 api/stately/migrator/loadout-shares.ts delete mode 100644 api/stately/migrator/user.ts diff --git a/.env b/.env index 04ed51d..f00d805 100644 --- a/.env +++ b/.env @@ -1,8 +1,2 @@ -PGHOST=localhost -PGPORT=31744 -PGDATABASE=postgres -PGUSER=postgresadmin -PGPASSWORD=dsjdsFwqklqwkbj JWT_SECRET=dummysecret -PGSSL=false VHOST=api.destinyitemmanager.com \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3de3252..156d237 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -50,31 +50,6 @@ jobs: - name: Save DigitalOcean kubeconfig with short-lived credentials run: doctl kubernetes cluster kubeconfig save --expiry-seconds 600 ${{secrets.K8S_CLUSTER}} - - name: Add IP address to trusted source (managed database) - uses: GarreauArthur/manage-digital-ocean-managed-database-trusted-sources-gh-action@main - with: - action: 'add' - database_id: ${{ secrets.DATABASE_ID }} - digitalocean_token: ${{ secrets.DIGITALOCEAN_TOKEN }} - - - name: Run DB migrations - run: cd api && npx db-migrate up -e prod - env: - DATABASE_USER: ${{ secrets.DATABASE_USER }} - DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} - DATABASE_HOST: ${{ secrets.DATABASE_HOST }} - DATABASE_NAME: ${{ secrets.DATABASE_NAME }} - DATABASE_PORT: ${{ secrets.DATABASE_PORT }} - - - name: Remove IP address to trusted source (managed database) - if: always() - continue-on-error: true - uses: GarreauArthur/manage-digital-ocean-managed-database-trusted-sources-gh-action@main - with: - action: 'remove' - database_id: ${{ secrets.DATABASE_ID }} - digitalocean_token: ${{ secrets.DIGITALOCEAN_TOKEN }} - - name: Build and deploy run: pnpm run deploy diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 2aa4457..b8bdbe6 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -9,25 +9,6 @@ jobs: runs-on: ubuntu-latest environment: 'test' - services: - postgres: - # Docker Hub image - image: nat212/postgres-cron - # Provide the password for postgres - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: travis_ci_test - POSTGRES_CRON_DB: travis_ci_test - ports: - - 5432:5432 - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - uses: actions/checkout@v4 @@ -42,12 +23,6 @@ jobs: - name: Install run: pnpm install --frozen-lockfile --prefer-offline - - name: Migrate Test DB - run: | - pushd api - npx db-migrate up -e test - popd - - run: cp build/.env.travis .env - name: Build API diff --git a/api/db/apps-queries.test.ts b/api/db/apps-queries.test.ts deleted file mode 100644 index 71d799a..0000000 --- a/api/db/apps-queries.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { DatabaseError } from 'pg-protocol'; -import { v4 as uuid } from 'uuid'; -import { ApiApp } from '../shapes/app.js'; -import { getAllApps, getAppById, insertApp } from './apps-queries.js'; -import { closeDbPool, pool, transaction } from './index.js'; - -const appId = 'apps-queries-test-app'; -const app: ApiApp = { - id: 'apps-queries-test-app', - bungieApiKey: 'foo', - origin: 'https://localhost', - dimApiKey: uuid(), -}; - -beforeEach(() => pool.query({ text: 'delete from apps where id = $1', values: [appId] })); - -afterAll(() => closeDbPool()); - -it('can create a new app', async () => { - await transaction(async (client) => { - expect(await getAppById(client, appId)).toBeNull(); - - await insertApp(client, app); - - const fetchedApp = await getAppById(client, appId); - expect(fetchedApp?.dimApiKey).toEqual(app.dimApiKey); - }); -}); - -it('cannot create a new app with the same name as an existing one', async () => { - await transaction(async (client) => { - await insertApp(client, app); - try { - await insertApp(client, app); - } catch (e) { - if (!(e instanceof DatabaseError)) { - fail('should have thrown a DatabaseError'); - } - expect(e.code).toBe('23505'); - } - }); -}); - -it('can get all apps', async () => { - await transaction(async (client) => { - await insertApp(client, app); - - const apps = await getAllApps(client); - expect(apps.length).toBeGreaterThanOrEqual(1); - expect(apps.find((a) => a.id === appId)?.dimApiKey).toBe(app.dimApiKey); - }); -}); diff --git a/api/db/apps-queries.ts b/api/db/apps-queries.ts deleted file mode 100644 index 5fecc7f..0000000 --- a/api/db/apps-queries.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { ClientBase, QueryResult } from 'pg'; -import { ApiApp } from '../shapes/app.js'; -import { camelize, KeysToSnakeCase, TypesForKeys } from '../utils.js'; - -/** - * Get all registered apps. - */ -export async function getAllApps(client: ClientBase): Promise { - const results = await client.query>({ - name: 'get_all_apps', - text: 'SELECT * FROM apps', - }); - return results.rows.map((row) => camelize(row)); -} - -/** - * Get an app by its ID. - */ -export async function getAppById(client: ClientBase, id: string): Promise { - const results = await client.query>({ - name: 'get_apps', - text: 'SELECT * FROM apps where id = $1', - values: [id], - }); - if (results.rows.length > 0) { - return camelize(results.rows[0]); - } else { - return null; - } -} - -/** - * Insert a new app into the list of registered apps. - */ -export async function insertApp(client: ClientBase, app: ApiApp): Promise { - return client.query>({ - name: 'insert_app', - text: `insert into apps (id, bungie_api_key, dim_api_key, origin) -values ($1, $2, $3, $4)`, - values: [app.id, app.bungieApiKey, app.dimApiKey, app.origin], - }); -} diff --git a/api/db/index.test.ts b/api/db/index.test.ts deleted file mode 100644 index 1a78f5f..0000000 --- a/api/db/index.test.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { closeDbPool, pool, readTransaction, transaction } from './index.js'; - -beforeEach(async () => { - try { - await pool.query(`DROP TABLE transaction_test`); - } catch {} - await pool.query(`CREATE TABLE transaction_test ( - id int PRIMARY KEY NOT NULL, - test text - )`); -}); - -interface TransactionTestRow { - id: number; - test: string; -} - -afterAll(async () => { - try { - await pool.query(`DROP TABLE transaction_test`); - } catch {} - await closeDbPool(); -}); - -describe('transaction', () => { - it('rolls back on errors', async () => { - await pool.query("insert into transaction_test (id, test) values (1, 'testing')"); - - try { - await transaction(async (client) => { - await client.query("insert into transaction_test (id, test) values (2, 'testing')"); - throw new Error('oops'); - }); - fail('should have thrown an error'); - } catch (e) { - expect((e as Error).message).toBe('oops'); - } - - const result = await pool.query('select * from transaction_test'); - expect(result.rows.length).toBe(1); - expect(result.rows[0].id).toBe(1); - }); - - it('commits automatically', async () => { - await transaction(async (client) => { - await client.query("insert into transaction_test (id, test) values (3, 'testing commits')"); - }); - - const result = await pool.query('select * from transaction_test'); - expect(result.rows.length).toBe(1); - expect(result.rows[0].test).toBe('testing commits'); - }); -}); - -describe('readTransaction', () => { - it('has read-committed isolation', async () => { - await pool.query("insert into transaction_test (id, test) values (1, 'testing')"); - - await readTransaction(async (client) => { - // In a different client, update a row - const otherClient = await pool.connect(); - try { - await otherClient.query('BEGIN'); - - await otherClient.query("update transaction_test set test = 'updated' where id = 1"); - - // Now request that info from our original client. - // should be read-committed, so we shouldn't see that update - const result = await client.query( - 'select * from transaction_test where id = 1', - ); - expect(result.rows[0].test).toBe('testing'); - - // Commit the update - await otherClient.query('COMMIT'); - } catch (e) { - await otherClient.query('ROLLBACK'); - throw e; - } finally { - otherClient.release(); - } - - // once that other transaction commits, we'll see its update - const result = await client.query( - 'select * from transaction_test where id = 1', - ); - expect(result.rows[0].test).toBe('updated'); - }); - - // outside, we should still see the transactional update - const result = await pool.query( - 'select * from transaction_test where id = 1', - ); - expect(result.rows[0].test).toBe('updated'); - }); -}); diff --git a/api/db/index.ts b/api/db/index.ts deleted file mode 100644 index 1bcb675..0000000 --- a/api/db/index.ts +++ /dev/null @@ -1,75 +0,0 @@ -import pg, { ClientBase } from 'pg'; -import { metrics } from '../metrics/index.js'; - -// pools will use environment variables -// for connection information (from .env or a ConfigMap) -export const pool = new pg.Pool({ - max: 2, // We get 25 connections per 1GB of RAM (we have 4GB), minus 3 connections for maintenance = 97. We run a variable number of DIM services. - ssl: process.env.PGSSL ? process.env.PGSSL === 'true' : { rejectUnauthorized: false }, - connectionTimeoutMillis: 500, - // Statement query is at the Postgres side, times out any individual query - statement_timeout: 750, - // Query timeout is on the NodeJS side, it times out the an operation on the client - query_timeout: 1000, -}); - -pool.on('connect', () => { - metrics.increment('db.pool.connect.count'); -}); -pool.on('acquire', () => { - metrics.increment('db.pool.acquire.count'); -}); -pool.on('error', (e: Error) => { - metrics.increment('db.pool.error.count'); - metrics.increment(`db.pool.error.${e.name}.count`); -}); -pool.on('remove', () => { - metrics.increment('db.pool.remove.count'); -}); - -const metricsInterval = setInterval(() => { - metrics.gauge('db.pool.total', pool.totalCount); - metrics.gauge('db.pool.idle', pool.idleCount); - metrics.gauge('db.pool.waiting', pool.waitingCount); -}, 10000); - -export async function closeDbPool() { - clearInterval(metricsInterval); - return pool.end(); -} - -/** - * A helper that gets a connection from the pool and then executes fn within a transaction. - */ -export async function transaction(fn: (client: ClientBase) => Promise) { - const client = await pool.connect(); - try { - await client.query('BEGIN'); - - const result = await fn(client); - - await client.query('COMMIT'); - - return result; - } catch (e) { - await client.query('ROLLBACK'); - throw e; - } finally { - client.release(); - } -} - -/** - * A helper that gets a connection from the pool and then executes fn within a transaction that's only meant for reads. - */ -export async function readTransaction(fn: (client: ClientBase) => Promise) { - const client = await pool.connect(); - try { - // We used to wrap multiple reads in a transaction but I'm not sure it matters all that much. - // await client.query('BEGIN'); - return await fn(client); - } finally { - // await client.query('ROLLBACK'); - client.release(); - } -} diff --git a/api/db/item-annotations-queries.test.ts b/api/db/item-annotations-queries.test.ts deleted file mode 100644 index 9142e80..0000000 --- a/api/db/item-annotations-queries.test.ts +++ /dev/null @@ -1,207 +0,0 @@ -import { TagVariant } from '../shapes/item-annotations.js'; -import { closeDbPool, transaction } from './index.js'; -import { - deleteAllItemAnnotations, - deleteItemAnnotation, - deleteItemAnnotationList, - getItemAnnotationsForProfile, - updateItemAnnotation, -} from './item-annotations-queries.js'; - -const appId = 'settings-queries-test-app'; -const platformMembershipId = '213512057'; -const bungieMembershipId = 4321; - -beforeEach(() => - transaction(async (client) => { - await deleteAllItemAnnotations(client, bungieMembershipId); - }), -); - -afterAll(() => closeDbPool()); - -it('can insert tags where none exist before', async () => { - await transaction(async (client) => { - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: 'favorite', - notes: 'the best', - }); - - const annotations = await getItemAnnotationsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - expect(annotations[0]).toEqual({ - id: '123456', - tag: 'favorite', - notes: 'the best', - }); - }); -}); - -it('can update tags where none exist before', async () => { - await transaction(async (client) => { - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: 'favorite', - notes: 'the best', - }); - - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: 'junk', - notes: 'the worst', - }); - - const annotations = await getItemAnnotationsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - expect(annotations[0]).toEqual({ - id: '123456', - tag: 'junk', - notes: 'the worst', - }); - }); -}); - -it('can update tags clearing value', async () => { - await transaction(async (client) => { - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: 'favorite', - notes: 'the best', - }); - - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: null, - }); - - const annotations = await getItemAnnotationsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - expect(annotations[0]).toEqual({ - id: '123456', - notes: 'the best', - }); - }); -}); - -it('can delete tags', async () => { - await transaction(async (client) => { - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: 'favorite', - notes: 'the best', - }); - - await deleteItemAnnotation(client, bungieMembershipId, '123456'); - - const annotations = await getItemAnnotationsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - expect(annotations).toEqual([]); - }); -}); - -it('can delete tags by setting both values to null/empty', async () => { - await transaction(async (client) => { - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: 'favorite', - notes: 'the best', - }); - - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: null, - notes: '', - }); - - const annotations = await getItemAnnotationsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - expect(annotations).toEqual([]); - }); -}); - -it('can insert tags with a variant', async () => { - await transaction(async (client) => { - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: 'keep', - v: TagVariant.PVP, - }); - - const annotations = await getItemAnnotationsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - expect(annotations[0]).toEqual({ - id: '123456', - tag: 'keep', - v: TagVariant.PVP, - }); - - // And updating notes doesn't mess with that: - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - notes: 'pretty cool', - }); - - const annotations2 = await getItemAnnotationsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - expect(annotations2[0]).toEqual({ - id: '123456', - tag: 'keep', - v: TagVariant.PVP, - notes: 'pretty cool', - }); - }); -}); - -it('can clear tags', async () => { - await transaction(async (client) => { - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '123456', - tag: 'favorite', - notes: 'the best', - }); - await updateItemAnnotation(client, appId, bungieMembershipId, platformMembershipId, 2, { - id: '654321', - tag: 'junk', - notes: 'the worst', - }); - - await deleteItemAnnotationList(client, bungieMembershipId, ['123456', '654321']); - - const annotations = await getItemAnnotationsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - expect(annotations).toEqual([]); - }); -}); diff --git a/api/db/item-annotations-queries.ts b/api/db/item-annotations-queries.ts deleted file mode 100644 index 1164f63..0000000 --- a/api/db/item-annotations-queries.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { ClientBase, QueryResult } from 'pg'; -import { metrics } from '../metrics/index.js'; -import { DestinyVersion } from '../shapes/general.js'; -import { ItemAnnotation, TagValue, TagVariant } from '../shapes/item-annotations.js'; - -interface ItemAnnotationRow { - inventory_item_id: string; - tag: TagValue | null; - notes: string | null; - variant: TagVariant | null; - crafted_date: Date | null; -} - -/** - * Get all of the item annotations for a particular platform_membership_id and destiny_version. - */ -export async function getItemAnnotationsForProfile( - client: ClientBase, - bungieMembershipId: number, - platformMembershipId: string, - destinyVersion: DestinyVersion, -): Promise { - const results = await client.query({ - name: 'get_item_annotations', - text: 'SELECT inventory_item_id, tag, notes, variant, crafted_date FROM item_annotations WHERE membership_id = $1 and platform_membership_id = $2 and destiny_version = $3', - values: [bungieMembershipId, platformMembershipId, destinyVersion], - }); - return results.rows.map(convertItemAnnotation); -} - -/** - * Get ALL of the item annotations for a particular user across all platforms. - */ -export async function getAllItemAnnotationsForUser( - client: ClientBase, - bungieMembershipId: number, -): Promise< - { - platformMembershipId: string; - destinyVersion: DestinyVersion; - annotation: ItemAnnotation; - }[] -> { - // TODO: this isn't indexed! - const results = await client.query< - ItemAnnotationRow & { platform_membership_id: string; destiny_version: DestinyVersion } - >({ - name: 'get_all_item_annotations', - text: 'SELECT platform_membership_id, destiny_version, inventory_item_id, tag, notes, variant, crafted_date FROM item_annotations WHERE inventory_item_id != 0 and membership_id = $1', - values: [bungieMembershipId], - }); - return results.rows.map((row) => ({ - platformMembershipId: row.platform_membership_id, - destinyVersion: row.destiny_version, - annotation: convertItemAnnotation(row), - })); -} - -function convertItemAnnotation(row: ItemAnnotationRow): ItemAnnotation { - const result: ItemAnnotation = { - id: row.inventory_item_id, - }; - if (row.tag) { - result.tag = row.tag; - } - if (row.notes) { - result.notes = row.notes; - } - if (row.crafted_date) { - result.craftedDate = row.crafted_date.getTime() / 1000; - } - if (row.variant) { - result.v = row.variant; - } - return result; -} - -/** - * Insert or update (upsert) a single item annotation. - */ -export async function updateItemAnnotation( - client: ClientBase, - appId: string, - bungieMembershipId: number, - platformMembershipId: string, - destinyVersion: DestinyVersion, - itemAnnotation: ItemAnnotation, -): Promise { - const tagValue = clearValue(itemAnnotation.tag); - // Variant will only be set when tag is set and only for "keep" values - const variant = variantValue(tagValue, itemAnnotation.v); - const notesValue = clearValue(itemAnnotation.notes); - - if (tagValue === 'clear' && notesValue === 'clear') { - return deleteItemAnnotation(client, bungieMembershipId, itemAnnotation.id); - } - const response = await client.query({ - name: 'upsert_item_annotation', - text: `insert INTO item_annotations (membership_id, platform_membership_id, destiny_version, inventory_item_id, tag, notes, variant, crafted_date, created_by, last_updated_by) -values ($1, $2, $3, $4, (CASE WHEN $5 = 'clear'::item_tag THEN NULL ELSE $5 END)::item_tag, (CASE WHEN $6 = 'clear' THEN NULL ELSE $6 END), $9, $8, $7, $7) -on conflict (membership_id, inventory_item_id) -do update set (tag, notes, variant, last_updated_at, last_updated_by) = ((CASE WHEN $5 = 'clear' THEN NULL WHEN $5 IS NULL THEN item_annotations.tag ELSE $5 END), (CASE WHEN $6 = 'clear' THEN NULL WHEN $6 IS NULL THEN item_annotations.notes ELSE $6 END), (CASE WHEN $9 = 0 THEN NULL WHEN $9 IS NULL THEN item_annotations.variant ELSE $9 END), current_timestamp, $7)`, - values: [ - bungieMembershipId, - platformMembershipId, - destinyVersion, - itemAnnotation.id, - tagValue, - notesValue, - appId, - itemAnnotation.craftedDate ? new Date(itemAnnotation.craftedDate * 1000) : null, - variant, - ], - }); - - if (response.rowCount! < 1) { - // This should never happen! - metrics.increment('db.itemAnnotations.noRowUpdated.count', 1); - throw new Error('tags - No row was updated'); - } - - return response; -} - -/** - * If the value is explicitly set to null or empty string, we return "clear" which will remove the value from the database. - * If it's undefined we return null, which will preserve the existing value. - * If it's set, we'll return the input which will update the existing value. - */ -function clearValue(val: T | null | undefined): T | 'clear' | null { - if (val === null || (val !== undefined && val.length === 0)) { - return 'clear'; - } else if (!val) { - return null; - } else { - return val; - } -} - -/** - * Like clearValue, this decides whether the variant should be set, cleared, or left alone. - * Returning null preserves the existing value. - * Returning 0, removes the existing value, - */ -function variantValue( - tag: TagValue | 'clear' | null, - v: TagVariant | undefined, -): TagVariant | 0 | null { - if (tag === 'keep') { - return v ?? 0; - } else if (tag !== null) { - // If tag is being cleared or set to a non-keep value, remove the variant - return 0; - } else { - // Otherwise leave it be - return null; - } -} - -/** - * Delete an item annotation. - */ -export async function deleteItemAnnotation( - client: ClientBase, - bungieMembershipId: number, - inventoryItemId: string, -): Promise { - return client.query({ - name: 'delete_item_annotation', - text: `delete from item_annotations where membership_id = $1 and inventory_item_id = $2`, - values: [bungieMembershipId, inventoryItemId], - }); -} - -/** - * Delete an item annotation. - */ -export async function deleteItemAnnotationList( - client: ClientBase, - bungieMembershipId: number, - inventoryItemIds: string[], -): Promise { - return client.query({ - name: 'delete_item_annotation_list', - text: `delete from item_annotations where membership_id = $1 and inventory_item_id::bigint = ANY($2::bigint[])`, - values: [bungieMembershipId, inventoryItemIds], - }); -} - -/** - * Delete all item annotations for a user (on all platforms). - */ -export async function deleteAllItemAnnotations( - client: ClientBase, - bungieMembershipId: number, -): Promise { - return client.query({ - name: 'delete_all_item_annotations', - text: `delete from item_annotations where membership_id = $1`, - values: [bungieMembershipId], - }); -} diff --git a/api/db/item-hash-tags-queries.test.ts b/api/db/item-hash-tags-queries.test.ts deleted file mode 100644 index 3b29d20..0000000 --- a/api/db/item-hash-tags-queries.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { closeDbPool, transaction } from './index.js'; -import { - deleteAllItemHashTags, - deleteItemHashTag, - getItemHashTagsForProfile, - updateItemHashTag, -} from './item-hash-tags-queries.js'; - -const appId = 'settings-queries-test-app'; -const bungieMembershipId = 4321; - -beforeEach(() => - transaction(async (client) => { - await deleteAllItemHashTags(client, bungieMembershipId); - }), -); - -afterAll(() => closeDbPool()); - -it('can insert item hash tags where none exist before', async () => { - await transaction(async (client) => { - await updateItemHashTag(client, appId, bungieMembershipId, { - hash: 2926662838, - tag: 'favorite', - notes: 'the best', - }); - - const annotations = await getItemHashTagsForProfile(client, bungieMembershipId); - expect(annotations[0]).toEqual({ - hash: 2926662838, - tag: 'favorite', - notes: 'the best', - }); - }); -}); - -it('can update item hash tags where none exist before', async () => { - await transaction(async (client) => { - await updateItemHashTag(client, appId, bungieMembershipId, { - hash: 2926662838, - tag: 'favorite', - notes: 'the best', - }); - - await updateItemHashTag(client, appId, bungieMembershipId, { - hash: 2926662838, - tag: 'junk', - notes: 'the worst', - }); - - const annotations = await getItemHashTagsForProfile(client, bungieMembershipId); - expect(annotations[0]).toEqual({ - hash: 2926662838, - tag: 'junk', - notes: 'the worst', - }); - }); -}); - -it('can update item hash tags clearing value', async () => { - await transaction(async (client) => { - await updateItemHashTag(client, appId, bungieMembershipId, { - hash: 2926662838, - tag: 'favorite', - notes: 'the best', - }); - - await updateItemHashTag(client, appId, bungieMembershipId, { - hash: 2926662838, - tag: null, - }); - - const annotations = await getItemHashTagsForProfile(client, bungieMembershipId); - expect(annotations[0]).toEqual({ - hash: 2926662838, - notes: 'the best', - }); - }); -}); - -it('can delete item hash tags', async () => { - await transaction(async (client) => { - await updateItemHashTag(client, appId, bungieMembershipId, { - hash: 2926662838, - tag: 'favorite', - notes: 'the best', - }); - - await deleteItemHashTag(client, bungieMembershipId, 2926662838); - - const annotations = await getItemHashTagsForProfile(client, bungieMembershipId); - expect(annotations).toEqual([]); - }); -}); - -it('can delete item hash tags by setting both values to null/empty', async () => { - await transaction(async (client) => { - await updateItemHashTag(client, appId, bungieMembershipId, { - hash: 2926662838, - tag: 'favorite', - notes: 'the best', - }); - - await updateItemHashTag(client, appId, bungieMembershipId, { - hash: 2926662838, - tag: null, - notes: '', - }); - - const annotations = await getItemHashTagsForProfile(client, bungieMembershipId); - expect(annotations).toEqual([]); - }); -}); diff --git a/api/db/item-hash-tags-queries.ts b/api/db/item-hash-tags-queries.ts deleted file mode 100644 index e04a601..0000000 --- a/api/db/item-hash-tags-queries.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { ClientBase, QueryResult } from 'pg'; -import { metrics } from '../metrics/index.js'; -import { ItemHashTag, TagValue } from '../shapes/item-annotations.js'; - -interface ItemHashTagRow { - item_hash: string; - tag: TagValue | null; - notes: string | null; -} - -/** - * Get all of the hash tags for a particular platform_membership_id and destiny_version. - */ -export async function getItemHashTagsForProfile( - client: ClientBase, - bungieMembershipId: number, -): Promise { - const results = await client.query({ - name: 'get_item_hash_tags', - text: 'SELECT item_hash, tag, notes FROM item_hash_tags WHERE membership_id = $1', - values: [bungieMembershipId], - }); - return results.rows.map(convertItemHashTag); -} - -function convertItemHashTag(row: ItemHashTagRow): ItemHashTag { - const result: ItemHashTag = { - hash: parseInt(row.item_hash, 10), - }; - if (row.tag) { - result.tag = row.tag; - } - if (row.notes) { - result.notes = row.notes; - } - return result; -} - -/** - * Insert or update (upsert) a single item annotation. Loadouts are totally replaced when updated. - */ -export async function updateItemHashTag( - client: ClientBase, - appId: string, - bungieMembershipId: number, - itemHashTag: ItemHashTag, -): Promise { - const tagValue = clearValue(itemHashTag.tag); - const notesValue = clearValue(itemHashTag.notes); - - if (tagValue === 'clear' && notesValue === 'clear') { - return deleteItemHashTag(client, bungieMembershipId, itemHashTag.hash); - } - - const response = await client.query({ - name: 'upsert_hash_tag', - text: `insert INTO item_hash_tags (membership_id, item_hash, tag, notes, created_by, last_updated_by) -values ($1, $2, (CASE WHEN $3 = 'clear'::item_tag THEN NULL ELSE $3 END)::item_tag, (CASE WHEN $4 = 'clear' THEN NULL ELSE $4 END), $5, $5) -on conflict (membership_id, item_hash) -do update set (tag, notes, last_updated_at, last_updated_by) = ((CASE WHEN $3 = 'clear' THEN NULL WHEN $3 IS NULL THEN item_hash_tags.tag ELSE $3 END), (CASE WHEN $4 = 'clear' THEN NULL WHEN $4 IS NULL THEN item_hash_tags.notes ELSE $4 END), current_timestamp, $5)`, - values: [bungieMembershipId, itemHashTag.hash, tagValue, notesValue, appId], - }); - - if (response.rowCount! < 1) { - // This should never happen! - metrics.increment('db.itemHashTags.noRowUpdated.count', 1); - throw new Error('hash tags - No row was updated'); - } - - return response; -} - -/** - * If the value is explicitly set to null or empty string, we return "clear" which will remove the value from the database. - * If it's undefined we return null, which will preserve the existing value. - * If it's set, we'll return the input which will update the existing value. - */ -function clearValue(val: string | null | undefined) { - if (val === null || (val !== undefined && val.length === 0)) { - return 'clear'; - } else if (!val) { - return null; - } else { - return val; - } -} - -/** - * Delete an item hash tags. - */ -export async function deleteItemHashTag( - client: ClientBase, - bungieMembershipId: number, - itemHash: number, -): Promise { - return client.query({ - name: 'delete_item_hash_tag', - text: `delete from item_hash_tags where membership_id = $1 and item_hash = $2`, - values: [bungieMembershipId, itemHash], - }); -} - -/** - * Delete all item hash tags for a user. - */ -export async function deleteAllItemHashTags( - client: ClientBase, - bungieMembershipId: number, -): Promise { - return client.query({ - name: 'delete_all_item_hash_tags', - text: `delete from item_hash_tags where membership_id = $1`, - values: [bungieMembershipId], - }); -} diff --git a/api/db/loadouts-queries.test.ts b/api/db/loadouts-queries.test.ts deleted file mode 100644 index 6e5608c..0000000 --- a/api/db/loadouts-queries.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { v4 as uuid } from 'uuid'; -import { Loadout, LoadoutItem } from '../shapes/loadouts.js'; -import { closeDbPool, transaction } from './index.js'; -import { deleteLoadout, getLoadoutsForProfile, updateLoadout } from './loadouts-queries.js'; - -const appId = 'settings-queries-test-app'; -const bungieMembershipId = 4321; -const platformMembershipId = '213512057'; - -beforeEach(() => - transaction(async (client) => { - await client.query(`delete from loadouts where membership_id = ${bungieMembershipId}`); - }), -); - -afterAll(() => closeDbPool()); - -const loadout: Loadout = { - id: uuid(), - name: 'Test Loadout', - classType: 1, - clearSpace: false, - equipped: [ - { - hash: 100, - id: '1234', - socketOverrides: { 7: 9 }, - }, - { - hash: 200, - id: '4567', - craftedDate: 1000, - }, - ], - unequipped: [ - // This item has an extra property which shouldn't be saved - { - hash: 200, - id: '5678', - amount: 10, - fizbuzz: 11, - } as any as LoadoutItem, - ], -}; - -it('can record a loadout', async () => { - await transaction(async (client) => { - await updateLoadout(client, appId, bungieMembershipId, platformMembershipId, 2, loadout); - - const loadouts = await getLoadoutsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - - expect(loadouts.length).toBe(1); - - const firstLoadout = loadouts[0]; - expect(firstLoadout.createdAt).toBeDefined(); - delete firstLoadout.createdAt; - expect(firstLoadout.lastUpdatedAt).toBeDefined(); - delete firstLoadout.lastUpdatedAt; - expect(firstLoadout.unequipped.length).toBe(1); - expect((firstLoadout.unequipped[0] as { fizbuzz?: number }).fizbuzz).toBeUndefined(); - (firstLoadout.unequipped[0] as { fizbuzz?: number }).fizbuzz = 11; - expect(firstLoadout).toEqual(loadout); - }); -}); - -it('can update a loadout', async () => { - await transaction(async (client) => { - await updateLoadout(client, appId, bungieMembershipId, platformMembershipId, 2, loadout); - - await updateLoadout(client, appId, bungieMembershipId, platformMembershipId, 2, { - ...loadout, - name: 'Updated', - unequipped: [], - }); - - const loadouts = await getLoadoutsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - - expect(loadouts.length).toBe(1); - expect(loadouts[0].name).toEqual('Updated'); - expect(loadouts[0].unequipped.length).toBe(0); - expect(loadouts[0].equipped).toEqual(loadout.equipped); - }); -}); - -it('can delete a loadout', async () => { - await transaction(async (client) => { - await updateLoadout(client, appId, bungieMembershipId, platformMembershipId, 2, loadout); - - const success = await deleteLoadout(client, bungieMembershipId, loadout.id); - expect(success).toBe(true); - - const loadouts = await getLoadoutsForProfile( - client, - bungieMembershipId, - platformMembershipId, - 2, - ); - - expect(loadouts.length).toBe(0); - }); -}); diff --git a/api/db/loadouts-queries.ts b/api/db/loadouts-queries.ts deleted file mode 100644 index 98c68a9..0000000 --- a/api/db/loadouts-queries.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { ClientBase, QueryResult } from 'pg'; -import { metrics } from '../metrics/index.js'; -import { DestinyVersion } from '../shapes/general.js'; -import { Loadout, LoadoutItem } from '../shapes/loadouts.js'; -import { isValidItemId, KeysToSnakeCase } from '../utils.js'; - -export interface LoadoutRow - extends KeysToSnakeCase< - Omit - > { - created_at: Date; - last_updated_at: Date | null; - items: { equipped: LoadoutItem[]; unequipped: LoadoutItem[] }; -} - -/** - * Get all of the loadouts for a particular platform_membership_id and destiny_version. - */ -export async function getLoadoutsForProfile( - client: ClientBase, - bungieMembershipId: number, - platformMembershipId: string, - destinyVersion: DestinyVersion, -): Promise { - const results = await client.query({ - name: 'get_loadouts_for_platform_membership_id', - text: 'SELECT id, name, notes, class_type, emblem_hash, clear_space, items, parameters, created_at, last_updated_at FROM loadouts WHERE membership_id = $1 and platform_membership_id = $2 and destiny_version = $3', - values: [bungieMembershipId, platformMembershipId, destinyVersion], - }); - return results.rows.map(convertLoadout); -} - -/** - * Get ALL of loadouts for a particular user across all platforms. - */ -export async function getAllLoadoutsForUser( - client: ClientBase, - bungieMembershipId: number, -): Promise< - { - platformMembershipId: string; - destinyVersion: DestinyVersion; - loadout: Loadout; - }[] -> { - const results = await client.query< - LoadoutRow & { platform_membership_id: string; destiny_version: DestinyVersion } - >({ - name: 'get_all_loadouts_for_user', - text: 'SELECT membership_id, platform_membership_id, destiny_version, id, name, notes, class_type, emblem_hash, clear_space, items, parameters, created_at, last_updated_at FROM loadouts WHERE membership_id = $1', - values: [bungieMembershipId], - }); - return results.rows.map((row) => { - const loadout = convertLoadout(row); - return { - platformMembershipId: row.platform_membership_id, - destinyVersion: row.destiny_version, - loadout, - }; - }); -} - -export function convertLoadout(row: LoadoutRow): Loadout { - const loadout: Loadout = { - id: row.id, - name: row.name, - classType: row.class_type, - clearSpace: row.clear_space, - equipped: row.items.equipped || [], - unequipped: row.items.unequipped || [], - createdAt: row.created_at.getTime(), - lastUpdatedAt: row.last_updated_at?.getTime(), - }; - if (row.notes) { - loadout.notes = row.notes; - } - if (row.emblem_hash) { - loadout.emblemHash = row.emblem_hash; - } - if (row.parameters) { - loadout.parameters = row.parameters; - } - return loadout; -} - -/** - * Insert or update (upsert) a loadout. Loadouts are totally replaced when updated. - */ -export async function updateLoadout( - client: ClientBase, - appId: string, - bungieMembershipId: number, - platformMembershipId: string, - destinyVersion: DestinyVersion, - loadout: Loadout, -): Promise { - const response = await client.query({ - name: 'upsert_loadout', - text: `insert into loadouts (id, membership_id, platform_membership_id, destiny_version, name, notes, class_type, emblem_hash, clear_space, items, parameters, created_by, last_updated_by) -values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $12) -on conflict (membership_id, id) -do update set (name, notes, class_type, emblem_hash, clear_space, items, parameters, last_updated_at, last_updated_by) = ($5, $6, $7, $8, $9, $10, $11, current_timestamp, $12)`, - values: [ - loadout.id, - bungieMembershipId, - platformMembershipId, - destinyVersion, - loadout.name, - loadout.notes, - loadout.classType, - loadout.emblemHash || null, - loadout.clearSpace, - { - equipped: loadout.equipped.map(cleanItem), - unequipped: loadout.unequipped.map(cleanItem), - }, - loadout.parameters, - appId, - ], - }); - - if (response.rowCount! < 1) { - // This should never happen! - metrics.increment('db.loadouts.noRowUpdated.count', 1); - throw new Error('loadouts - No row was updated'); - } - - return response; -} - -/** - * Make sure items are stored minimally and extra properties don't sneak in - */ -export function cleanItem(item: LoadoutItem): LoadoutItem { - const hash = item.hash; - if (!Number.isFinite(hash)) { - throw new Error('hash must be a number'); - } - - const result: LoadoutItem = { - hash, - }; - - if (item.amount && Number.isFinite(item.amount)) { - result.amount = item.amount; - } - - if (item.id) { - if (!isValidItemId(item.id)) { - throw new Error(`item ID ${item.id} is not in the right format`); - } - result.id = item.id; - } - - if (item.socketOverrides) { - result.socketOverrides = item.socketOverrides; - } - - if (item.craftedDate && Number.isFinite(item.craftedDate)) { - result.craftedDate = item.craftedDate; - } - - return result; -} - -/** - * Delete a loadout. Loadouts are totally replaced when updated. - */ -export async function deleteLoadout( - client: ClientBase, - bungieMembershipId: number, - loadoutId: string, -): Promise { - const response = await client.query({ - name: 'delete_loadout', - text: `delete from loadouts where membership_id = $1 and id = $2`, - values: [bungieMembershipId, loadoutId], - }); - - return response.rowCount! >= 1; -} - -/** - * Delete all loadouts for a user (on all platforms). - */ -export async function deleteAllLoadouts( - client: ClientBase, - bungieMembershipId: number, -): Promise { - return client.query({ - name: 'delete_all_loadouts', - text: `delete from loadouts where membership_id = $1`, - values: [bungieMembershipId], - }); -} diff --git a/api/db/migration-state-queries.ts b/api/db/migration-state-queries.ts deleted file mode 100644 index 377dac7..0000000 --- a/api/db/migration-state-queries.ts +++ /dev/null @@ -1,228 +0,0 @@ -import { ClientBase } from 'pg'; -import { metrics } from '../metrics/index.js'; -import { transaction } from './index.js'; - -export const MAX_MIGRATION_ATTEMPTS = 3; - -export const enum MigrationState { - Invalid = 0, - Postgres = 1, - MigratingToStately = 2, - Stately = 3, -} - -export interface MigrationStateInfo { - bungieMembershipId: number; - state: MigrationState; - lastStateChangeAt: number; - attemptCount: number; - lastError?: string; -} - -interface MigrationStateRow { - membership_id: number; - state: number; - last_state_change_at: Date; - attempt_count: number; - last_error: string | null; -} - -export async function getUsersToMigrate(client: ClientBase): Promise { - const results = await client.query({ - name: 'get_users_to_migrate', - text: 'select membership_id from migration_state where state != 3 limit 1000', - }); - return results.rows.map((row) => row.membership_id); -} - -export async function getMigrationState( - client: ClientBase, - bungieMembershipId: number, -): Promise { - const results = await client.query({ - name: 'get_migration_state', - text: 'SELECT membership_id, state, last_state_change_at, attempt_count, last_error FROM migration_state WHERE membership_id = $1', - values: [bungieMembershipId], - }); - if (results.rows.length > 0) { - return convert(results.rows[0]); - } else { - return { - bungieMembershipId, - state: MigrationState.Stately, - lastStateChangeAt: 0, - attemptCount: 0, - }; - } -} - -function convert(row: MigrationStateRow): MigrationStateInfo { - return { - bungieMembershipId: row.membership_id, - state: row.state, - lastStateChangeAt: row.last_state_change_at.getTime(), - attemptCount: row.attempt_count, - lastError: row.last_error ?? undefined, - }; -} - -export function startMigrationToStately( - client: ClientBase, - bungieMembershipId: number, -): Promise { - return updateMigrationState( - client, - bungieMembershipId, - MigrationState.MigratingToStately, - MigrationState.Postgres, - true, - ); -} - -export function finishMigrationToStately( - client: ClientBase, - bungieMembershipId: number, -): Promise { - return updateMigrationState( - client, - bungieMembershipId, - MigrationState.Stately, - MigrationState.MigratingToStately, - false, - ); -} - -export function abortMigrationToStately( - client: ClientBase, - bungieMembershipId: number, - err: string, -): Promise { - return updateMigrationState( - client, - bungieMembershipId, - MigrationState.Postgres, - MigrationState.MigratingToStately, - false, - err, - ); -} - -async function updateMigrationState( - client: ClientBase, - bungieMembershipId: number, - state: MigrationState, - expectedState: MigrationState, - incrementAttempt = true, - err?: string, -): Promise { - // Postgres upserts are awkward but nice to have - const response = await client.query({ - name: 'update_migration_state', - text: `insert into migration_state (membership_id, state, last_state_change_at, attempt_count, last_error) VALUES ($1, $2, current_timestamp, $3, $4) -on conflict (membership_id) -do update set state = $2, last_state_change_at = current_timestamp, attempt_count = migration_state.attempt_count + $3, last_error = coalesce($4, migration_state.last_error) -where migration_state.state = $5`, - values: [bungieMembershipId, state, incrementAttempt ? 1 : 0, err ?? null, expectedState], - }); - if (response.rowCount === 0) { - throw new Error('Migration state was not in expected state'); - } -} - -// Mostly for tests and delete-my-data -export async function deleteMigrationState( - client: ClientBase, - bungieMembershipId: number, -): Promise { - await client.query({ - name: 'delete_migration_state', - text: 'DELETE FROM migration_state WHERE membership_id = $1', - values: [bungieMembershipId], - }); -} - -const forceStatelyMembershipIds = new Set([ - // Ben - 7094, - // Test user - 1234, -]); - -const dialPercentage = 1.0; // 0 - 1.0 - -// This would be better as a uniform hash but this is good enough for now -function isUserDialedIn(bungieMembershipId: number) { - return (bungieMembershipId % 10000) / 10000 < dialPercentage; -} - -export async function getDesiredMigrationState(migrationState: MigrationStateInfo) { - // TODO: use a uniform hash and a percentage dial to control this - const desiredState = - forceStatelyMembershipIds.has(migrationState.bungieMembershipId) || - isUserDialedIn(migrationState.bungieMembershipId) - ? MigrationState.Stately - : MigrationState.Postgres; - - if (desiredState === migrationState.state) { - return migrationState.state; - } - - if ( - desiredState === MigrationState.Stately && - migrationState.state === MigrationState.Postgres && - migrationState.attemptCount >= MAX_MIGRATION_ATTEMPTS - ) { - return MigrationState.Postgres; - } - - if ( - migrationState.state === MigrationState.MigratingToStately && - // If we've been in this state for more than 15 minutes, just move on - migrationState.lastStateChangeAt < Date.now() - 1000 * 60 * 15 - ) { - await transaction(async (client) => { - abortMigrationToStately(client, migrationState.bungieMembershipId, 'Migration timed out'); - }); - return MigrationState.Postgres; - } - - if (migrationState.state === MigrationState.MigratingToStately) { - throw new Error('Unable to update - please wait a bit and try again.'); - } - - return desiredState; -} - -/** - * Wrap the migration process - start a migration, run fn(), finish the - * migration. Abort on failure. - */ -export async function doMigration( - bungieMembershipId: number, - fn: () => Promise, - onBeforeFinish?: (client: ClientBase) => Promise, -): Promise { - try { - metrics.increment('migration.start.count'); - await transaction(async (client) => { - await startMigrationToStately(client, bungieMembershipId); - }); - await fn(); - await transaction(async (client) => { - await onBeforeFinish?.(client); - await finishMigrationToStately(client, bungieMembershipId); - }); - metrics.increment('migration.finish.count'); - } catch (e) { - console.error(`Stately migration failed for ${bungieMembershipId}`, e); - await transaction(async (client) => { - await abortMigrationToStately( - client, - bungieMembershipId, - e instanceof Error ? e.message : 'Unknown error', - ); - }); - metrics.increment('migration.abort.count'); - throw e; - } -} diff --git a/api/db/searches-queries.test.ts b/api/db/searches-queries.test.ts deleted file mode 100644 index 1c63591..0000000 --- a/api/db/searches-queries.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { SearchType } from '../shapes/search.js'; -import { closeDbPool, transaction } from './index.js'; -import { - deleteAllSearches, - deleteSearch, - getSearchesForProfile, - getSearchesForUser, - importSearch, - saveSearch, - updateUsedSearch, -} from './searches-queries.js'; - -const appId = 'settings-queries-test-app'; -const bungieMembershipId = 4321; - -beforeEach(() => - transaction(async (client) => { - await deleteAllSearches(client, bungieMembershipId); - }), -); - -afterAll(() => closeDbPool()); - -it('can record a used search where none was recorded before', async () => { - await transaction(async (client) => { - await updateUsedSearch(client, appId, bungieMembershipId, 2, 'tag:junk', SearchType.Item); - - const searches = (await getSearchesForProfile(client, bungieMembershipId, 2)).filter( - (s) => s.usageCount > 0, - ); - expect(searches[0].query).toBe('tag:junk'); - expect(searches[0].saved).toBe(false); - expect(searches[0].usageCount).toBe(1); - }); -}); - -it('can track search multiple times', async () => { - await transaction(async (client) => { - await updateUsedSearch(client, appId, bungieMembershipId, 2, 'tag:junk', SearchType.Item); - await updateUsedSearch(client, appId, bungieMembershipId, 2, 'tag:junk', SearchType.Item); - - const searches = (await getSearchesForProfile(client, bungieMembershipId, 2)).filter( - (s) => s.usageCount > 0, - ); - expect(searches[0].query).toBe('tag:junk'); - expect(searches[0].saved).toBe(false); - expect(searches[0].usageCount).toBe(2); - }); -}); - -it('can mark a search as favorite', async () => { - await transaction(async (client) => { - await updateUsedSearch(client, appId, bungieMembershipId, 2, 'tag:junk', SearchType.Item); - await saveSearch(client, appId, bungieMembershipId, 2, 'tag:junk', SearchType.Item, true); - - const searches = (await getSearchesForProfile(client, bungieMembershipId, 2)).filter( - (s) => s.usageCount > 0, - ); - expect(searches[0].query).toBe('tag:junk'); - expect(searches[0].saved).toBe(true); - expect(searches[0].usageCount).toBe(1); - - await saveSearch(client, appId, bungieMembershipId, 2, 'tag:junk', SearchType.Item, false); - - const searches2 = await getSearchesForProfile(client, bungieMembershipId, 2); - expect(searches2[0].query).toBe('tag:junk'); - expect(searches2[0].saved).toBe(false); - // Save/unsave doesn't modify usage count - expect(searches2[0].usageCount).toBe(1); - expect(searches2[0].lastUsage).toBe(searches2[0].lastUsage); - }); -}); -it('can mark a search as favorite even when it hasnt been used', async () => { - await transaction(async (client) => { - await saveSearch(client, appId, bungieMembershipId, 2, 'tag:junk', SearchType.Item, true); - - const searches = (await getSearchesForProfile(client, bungieMembershipId, 2)).filter( - (s) => s.usageCount > 0, - ); - expect(searches[0].query).toBe('tag:junk'); - expect(searches[0].saved).toBe(true); - expect(searches[0].usageCount).toBe(1); - }); -}); - -it('can get all searches across profiles', async () => { - await transaction(async (client) => { - await updateUsedSearch(client, appId, bungieMembershipId, 2, 'tag:junk', SearchType.Item); - await updateUsedSearch(client, appId, bungieMembershipId, 1, 'is:tagged', SearchType.Item); - - const searches = await getSearchesForUser(client, bungieMembershipId); - expect(searches.length).toEqual(2); - }); -}); - -it('can increment usage for one of the built-in searches', async () => { - await transaction(async (client) => { - const searches = await getSearchesForProfile(client, bungieMembershipId, 2); - const query = searches[searches.length - 1].query; - - await updateUsedSearch(client, appId, bungieMembershipId, 2, query, SearchType.Item); - - const searches2 = await getSearchesForProfile(client, bungieMembershipId, 2); - const search = searches2.find((s) => s.query === query); - expect(search?.usageCount).toBe(1); - expect(searches2.length).toBe(searches.length); - }); -}); - -it('can delete a search', async () => { - await transaction(async (client) => { - await updateUsedSearch(client, appId, bungieMembershipId, 2, 'tag:junk', SearchType.Item); - await deleteSearch(client, bungieMembershipId, 2, 'tag:junk', SearchType.Item); - - const searches = (await getSearchesForProfile(client, bungieMembershipId, 2)).filter( - (s) => s.usageCount > 0, - ); - expect(searches.length).toBe(0); - }); -}); - -it('can import a search', async () => { - await transaction(async (client) => { - await importSearch( - client, - appId, - bungieMembershipId, - 2, - 'tag:junk', - true, - 1598199188576, - 5, - SearchType.Item, - ); - - const searches = (await getSearchesForProfile(client, bungieMembershipId, 2)).filter( - (s) => s.usageCount > 0, - ); - expect(searches[0].query).toBe('tag:junk'); - expect(searches[0].saved).toBe(true); - expect(searches[0].usageCount).toBe(5); - }); -}); - -it('can record searches for loadouts', async () => { - await transaction(async (client) => { - await updateUsedSearch( - client, - appId, - bungieMembershipId, - 2, - 'subclass:void', - SearchType.Loadout, - ); - - const searches = (await getSearchesForProfile(client, bungieMembershipId, 2)).filter( - (s) => s.usageCount > 0, - ); - expect(searches[0].query).toBe('subclass:void'); - expect(searches[0].saved).toBe(false); - expect(searches[0].usageCount).toBe(1); - expect(searches[0].type).toBe(SearchType.Loadout); - }); -}); diff --git a/api/db/searches-queries.ts b/api/db/searches-queries.ts deleted file mode 100644 index b2ac1b4..0000000 --- a/api/db/searches-queries.ts +++ /dev/null @@ -1,229 +0,0 @@ -import { uniqBy } from 'es-toolkit'; -import { ClientBase, QueryResult } from 'pg'; -import { metrics } from '../metrics/index.js'; -import { ExportResponse } from '../shapes/export.js'; -import { DestinyVersion } from '../shapes/general.js'; -import { Search, SearchType } from '../shapes/search.js'; -import { KeysToSnakeCase } from '../utils.js'; - -interface SearchRow extends KeysToSnakeCase> { - last_updated_at: Date; - search_type: SearchType; -} - -/* - * These "canned searches" get sent to everyone as a "starter pack" of example searches that'll show up in the recent search dropdown and autocomplete. - */ -const cannedSearchesForD2: Search[] = [ - 'is:blue is:haspower -is:maxpower', - '-is:equipped is:haspower is:incurrentchar', - '-is:exotic -is:locked -is:maxpower -is:tagged stat:total:<55', -].map((query) => ({ - query, - saved: false, - usageCount: 0, - lastUsage: 0, - type: SearchType.Item, -})); - -const cannedSearchesForD1: Search[] = ['-is:equipped is:haslight is:incurrentchar'].map( - (query) => ({ - query, - saved: false, - usageCount: 0, - lastUsage: 0, - type: SearchType.Item, - }), -); -/* - * Searches are stored in a single table, scoped by Bungie.net account and destiny version (D1 searches are separate from D2 searches). - * Favorites and recent searches are stored the same - there's just a favorite flag for saved searches. There is also a usage count - * and a last_updated_at time, so we can order by both frequency and recency (or a combination of both) and we can age out less-used - * searches. For the best results, searches should be normalized so they match up more often. - * - * We can merge this with a list of global suggested searches to avoid an empty menu. - */ - -/** - * Get all of the searches for a particular destiny_version. - */ -export async function getSearchesForProfile( - client: ClientBase, - bungieMembershipId: number, - destinyVersion: DestinyVersion, -): Promise { - const results = await client.query({ - name: 'get_searches', - // TODO: order by frecency - text: 'SELECT query, saved, usage_count, search_type, last_updated_at FROM searches WHERE membership_id = $1 and destiny_version = $2 order by last_updated_at DESC, usage_count DESC LIMIT 500', - values: [bungieMembershipId, destinyVersion], - }); - return uniqBy( - results.rows - .map(convertSearch) - .concat(destinyVersion === 2 ? cannedSearchesForD2 : cannedSearchesForD1), - (s) => s.query, - ); -} - -/** - * Get ALL of the searches for a particular user across all destiny versions. - */ -export async function getSearchesForUser( - client: ClientBase, - bungieMembershipId: number, -): Promise { - // TODO: this isn't indexed! - const results = await client.query({ - name: 'get_all_searches', - text: 'SELECT destiny_version, query, saved, usage_count, search_type, last_updated_at FROM searches WHERE membership_id = $1', - values: [bungieMembershipId], - }); - return results.rows.map((row) => ({ - destinyVersion: row.destiny_version, - search: convertSearch(row), - })); -} - -function convertSearch(row: SearchRow): Search { - return { - query: row.query, - usageCount: row.usage_count, - saved: row.saved, - lastUsage: row.last_updated_at.getTime(), - type: row.search_type, - }; -} - -/** - * Insert or update (upsert) a single search. - * - * It's a bit odd that saving/unsaving a search counts as a "usage" but that's probably OK - */ -export async function updateUsedSearch( - client: ClientBase, - appId: string, - bungieMembershipId: number, - destinyVersion: DestinyVersion, - query: string, - type: SearchType, -): Promise { - const response = await client.query({ - name: 'upsert_search', - text: `insert INTO searches (membership_id, destiny_version, query, search_type, created_by, last_updated_by) -values ($1, $2, $3, $5, $4, $4) -on conflict (membership_id, destiny_version, qhash) -do update set (usage_count, last_used, last_updated_at, last_updated_by) = (searches.usage_count + 1, current_timestamp, current_timestamp, $4)`, - values: [bungieMembershipId, destinyVersion, query, appId, type], - }); - - if (response.rowCount! < 1) { - // This should never happen! - metrics.increment('db.searches.noRowUpdated.count', 1); - throw new Error('searches - No row was updated'); - } - - return response; -} - -/** - * Save/unsave a search. This assumes the search exists. - */ -export async function saveSearch( - client: ClientBase, - appId: string, - bungieMembershipId: number, - destinyVersion: DestinyVersion, - query: string, - type: SearchType, - saved?: boolean, -): Promise { - const response = await client.query({ - name: 'save_search', - text: `UPDATE searches SET (saved, last_updated_by) = ($4, $5) WHERE membership_id = $1 AND destiny_version = $2 AND qhash = decode(md5($3), 'hex') AND query = $3`, - values: [bungieMembershipId, destinyVersion, query, saved, appId], - }); - - if (response.rowCount! < 1) { - // Someone saved a search they haven't used! - metrics.increment('db.searches.noRowUpdated.count', 1); - const insertSavedResponse = await client.query({ - name: 'insert_search_fallback', - text: `insert INTO searches (membership_id, destiny_version, query, search_type, saved, created_by, last_updated_by) - values ($1, $2, $3, $5, true, $4, $4)`, - values: [bungieMembershipId, destinyVersion, query, appId, type], - }); - return insertSavedResponse; - } - - return response; -} -/** - * Insert a single search as part of an import. - */ -export async function importSearch( - client: ClientBase, - appId: string, - bungieMembershipId: number, - destinyVersion: DestinyVersion, - query: string, - saved: boolean, - lastUsage: number, - usageCount: number, - type: SearchType, -): Promise { - const response = await client.query({ - name: 'insert_search', - text: `insert INTO searches (membership_id, destiny_version, query, saved, search_type, usage_count, last_used, created_by, last_updated_by) -values ($1, $2, $3, $4, $8, $5, $6, $7, $7)`, - values: [ - bungieMembershipId, - destinyVersion, - query, - saved, - usageCount, - new Date(lastUsage), - appId, - type, - ], - }); - - if (response.rowCount! < 1) { - // This should never happen! - metrics.increment('db.searches.noRowUpdated.count', 1); - throw new Error('searches - No row was updated'); - } - - return response; -} - -/** - * Delete a single search - */ -export async function deleteSearch( - client: ClientBase, - bungieMembershipId: number, - destinyVersion: DestinyVersion, - query: string, - type: SearchType, -): Promise { - return client.query({ - name: 'delete_search', - text: `delete from searches where membership_id = $1 and destiny_version = $2 and qhash = decode(md5($3), 'hex') and query = $3 and search_type = $4`, - values: [bungieMembershipId, destinyVersion, query, type], - }); -} - -/** - * Delete all searches for a user (for all destiny versions). - */ -export async function deleteAllSearches( - client: ClientBase, - bungieMembershipId: number, -): Promise { - return client.query({ - name: 'delete_all_searches', - text: `delete from searches where membership_id = $1`, - values: [bungieMembershipId], - }); -} diff --git a/api/db/settings-queries.test.ts b/api/db/settings-queries.test.ts deleted file mode 100644 index 494b03d..0000000 --- a/api/db/settings-queries.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { closeDbPool, transaction } from './index.js'; -import { getSettings, setSetting } from './settings-queries.js'; - -const appId = 'settings-queries-test-app'; -const bungieMembershipId = 4321; - -afterAll(() => closeDbPool()); - -it('can insert settings where none exist before', async () => { - await transaction(async (client) => { - await setSetting(client, appId, bungieMembershipId, { - showNewItems: true, - }); - - const settings = await getSettings(client, bungieMembershipId); - expect(settings.showNewItems).toBe(true); - }); -}); - -it('can update settings', async () => { - await transaction(async (client) => { - await setSetting(client, appId, bungieMembershipId, { - showNewItems: true, - }); - - const settings = await getSettings(client, bungieMembershipId); - expect(settings.showNewItems).toBe(true); - - await setSetting(client, appId, bungieMembershipId, { - showNewItems: false, - }); - - const settings2 = await getSettings(client, bungieMembershipId); - expect(settings2.showNewItems).toBe(false); - }); -}); - -it('can partially update settings', async () => { - await transaction(async (client) => { - await setSetting(client, appId, bungieMembershipId, { - showNewItems: true, - }); - - const settings = await getSettings(client, bungieMembershipId); - expect(settings.showNewItems).toBe(true); - - await setSetting(client, appId, bungieMembershipId, { - singleCharacter: true, - }); - - const settings2 = await getSettings(client, bungieMembershipId); - expect(settings2.showNewItems).toBe(true); - }); -}); diff --git a/api/db/settings-queries.ts b/api/db/settings-queries.ts deleted file mode 100644 index b005726..0000000 --- a/api/db/settings-queries.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { ClientBase, QueryResult } from 'pg'; -import { Settings } from '../shapes/settings.js'; - -/** - * Get settings for a particular account. - */ -export async function getSettings( - client: ClientBase, - bungieMembershipId: number, -): Promise> { - const results = await client.query<{ settings: Settings }>({ - name: 'get_settings', - text: 'SELECT settings FROM settings WHERE membership_id = $1', - values: [bungieMembershipId], - }); - return results.rows.length > 0 ? results.rows[0].settings : {}; -} - -/** - * Insert or update (upsert) an entire settings tree, totally replacing whatever's there. - */ -export async function replaceSettings( - client: ClientBase, - appId: string, - bungieMembershipId: number, - settings: Partial, -): Promise { - const result = await client.query({ - name: 'upsert_settings', - text: `insert into settings (membership_id, settings, created_by, last_updated_by) -values ($1, $2, $3, $3) -on conflict (membership_id) -do update set (settings, last_updated_at, last_updated_by) = ($2, current_timestamp, $3)`, - values: [bungieMembershipId, settings, appId], - }); - return result; -} - -/** - * Update specific key/value pairs within settings, leaving the rest alone. Creates the settings row if it doesn't exist. - */ -export async function setSetting( - client: ClientBase, - appId: string, - bungieMembershipId: number, - settings: Partial, -): Promise { - return client.query({ - name: 'set_setting', - text: `insert into settings (membership_id, settings, created_by, last_updated_by) -values ($1, $2, $3, $3) -on conflict (membership_id) -do update set (settings, last_updated_at, last_updated_by) = (settings.settings || $2, current_timestamp, $3)`, - values: [bungieMembershipId, settings, appId], - }); -} - -/** - * Delete the settings row for a particular user. - */ -export async function deleteSettings( - client: ClientBase, - bungieMembershipId: number, -): Promise { - return client.query({ - name: 'delete_settings', - text: `delete FROM settings WHERE membership_id = $1`, - values: [bungieMembershipId], - }); -} diff --git a/api/db/triumphs-queries.test.ts b/api/db/triumphs-queries.test.ts deleted file mode 100644 index 0ce8f54..0000000 --- a/api/db/triumphs-queries.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { closeDbPool, transaction } from './index.js'; -import { - deleteAllTrackedTriumphs, - getAllTrackedTriumphsForUser, - getTrackedTriumphsForProfile, - trackTriumph, - unTrackTriumph, -} from './triumphs-queries.js'; - -const appId = 'settings-queries-test-app'; -const platformMembershipId = '213512057'; -const bungieMembershipId = 4321; - -beforeEach(() => - transaction(async (client) => { - await deleteAllTrackedTriumphs(client, bungieMembershipId); - }), -); - -afterAll(() => closeDbPool()); - -it('can track a triumph where none was tracked before', async () => { - await transaction(async (client) => { - await trackTriumph(client, appId, bungieMembershipId, platformMembershipId, 3851137658); - - const triumphs = await getTrackedTriumphsForProfile( - client, - bungieMembershipId, - platformMembershipId, - ); - expect(triumphs[0]).toEqual(3851137658); - }); -}); - -it('can track a triumph that was already tracked', async () => { - await transaction(async (client) => { - await trackTriumph(client, appId, bungieMembershipId, platformMembershipId, 3851137658); - - await trackTriumph(client, appId, bungieMembershipId, platformMembershipId, 3851137658); - - const triumphs = await getTrackedTriumphsForProfile( - client, - bungieMembershipId, - platformMembershipId, - ); - expect(triumphs[0]).toEqual(3851137658); - }); -}); - -it('can untrack a triumph', async () => { - await transaction(async (client) => { - await trackTriumph(client, appId, bungieMembershipId, platformMembershipId, 3851137658); - - await unTrackTriumph(client, bungieMembershipId, platformMembershipId, 3851137658); - - const triumphs = await getTrackedTriumphsForProfile( - client, - bungieMembershipId, - platformMembershipId, - ); - expect(triumphs.length).toEqual(0); - }); -}); - -it('can get all tracked triumphs across profiles', async () => { - await transaction(async (client) => { - await trackTriumph(client, appId, bungieMembershipId, platformMembershipId, 3851137658); - await trackTriumph(client, appId, bungieMembershipId, '54321', 3851137658); - - await trackTriumph(client, appId, bungieMembershipId, platformMembershipId, 3851137658); - - const triumphs = await getAllTrackedTriumphsForUser(client, bungieMembershipId); - expect(triumphs.length).toEqual(2); - }); -}); diff --git a/api/db/triumphs-queries.ts b/api/db/triumphs-queries.ts deleted file mode 100644 index e9718d4..0000000 --- a/api/db/triumphs-queries.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { ClientBase, QueryResult } from 'pg'; -import { metrics } from '../metrics/index.js'; - -/** - * Get all of the tracked triumphs for a particular platform_membership_id. - */ -export async function getTrackedTriumphsForProfile( - client: ClientBase, - bungieMembershipId: number, - platformMembershipId: string, -): Promise { - const results = await client.query<{ record_hash: string }>({ - name: 'get_tracked_triumphs', - text: 'SELECT record_hash FROM tracked_triumphs WHERE membership_id = $1 and platform_membership_id = $2', - values: [bungieMembershipId, platformMembershipId], - }); - return results.rows.map((row) => parseInt(row.record_hash, 10)); -} - -/** - * Get ALL of the tracked triumphs for a particular user across all platforms. - */ -export async function getAllTrackedTriumphsForUser( - client: ClientBase, - bungieMembershipId: number, -): Promise< - { - platformMembershipId: string; - triumphs: number[]; - }[] -> { - const results = await client.query<{ platform_membership_id: string; record_hash: string }>({ - name: 'get_all_tracked_triumphs', - text: 'SELECT platform_membership_id, record_hash FROM tracked_triumphs WHERE membership_id = $1', - values: [bungieMembershipId], - }); - - const triumphsByAccount: { [platformMembershipId: string]: number[] } = {}; - - for (const row of results.rows) { - (triumphsByAccount[row.platform_membership_id] ||= []).push(parseInt(row.record_hash, 10)); - } - - return Object.entries(triumphsByAccount).map(([platformMembershipId, triumphs]) => ({ - platformMembershipId, - triumphs, - })); -} - -/** - * Add a tracked triumph. - */ -export async function trackTriumph( - client: ClientBase, - appId: string, - bungieMembershipId: number, - platformMembershipId: string, - recordHash: number, -): Promise { - const response = await client.query({ - name: 'insert_tracked_triumph', - text: `insert INTO tracked_triumphs (membership_id, platform_membership_id, record_hash, created_by) -values ($1, $2, $3, $4) -on conflict do nothing`, - values: [bungieMembershipId, platformMembershipId, recordHash, appId], - }); - - return response; -} - -/** - * Remove a tracked triumph. - */ -export async function unTrackTriumph( - client: ClientBase, - bungieMembershipId: number, - platformMembershipId: string, - recordHash: number, -): Promise { - const response = await client.query({ - name: 'delete_tracked_triumph', - text: `delete from tracked_triumphs where membership_id = $1 and platform_membership_id = $2 and record_hash = $3`, - values: [bungieMembershipId, platformMembershipId, recordHash], - }); - - if (response.rowCount! < 1) { - // This should never happen but it's OK - metrics.increment('db.triumphs.noRowDeleted.count', 1); - } - - return response; -} - -/** - * Delete all item annotations for a user (on all platforms). - */ -export async function deleteAllTrackedTriumphs( - client: ClientBase, - bungieMembershipId: number, -): Promise { - return client.query({ - name: 'delete_all_tracked_triumphs', - text: `delete from tracked_triumphs where membership_id = $1`, - values: [bungieMembershipId], - }); -} diff --git a/api/migrations/20191103052046-settings-table.js b/api/migrations/20191103052046-settings-table.js deleted file mode 100644 index 1ec95a2..0000000 --- a/api/migrations/20191103052046-settings-table.js +++ /dev/null @@ -1,39 +0,0 @@ -let dbm; -let type; -let seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function(options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -/** - * The settings table stores its data in a single JSONB column. This allows us to easily - * add and remove settings, and to only store settings that differ from the defaults. - */ -exports.up = function(db, callback) { - db.runSql( - `CREATE TABLE settings ( - membership_id int PRIMARY KEY NOT NULL, - settings jsonb NOT NULL default '{}'::jsonb, - created_at timestamp NOT NULL default current_timestamp, - created_by text NOT NULL, - last_updated_at timestamp NOT NULL default current_timestamp, - last_updated_by text NOT NULL - )`, - callback - ); -}; - -exports.down = function(db, callback) { - db.dropTable('settings', callback); -}; - -exports._meta = { - version: 1 -}; diff --git a/api/migrations/20191221235741-global-settings-table.js b/api/migrations/20191221235741-global-settings-table.js deleted file mode 100644 index 121ae8c..0000000 --- a/api/migrations/20191221235741-global-settings-table.js +++ /dev/null @@ -1,43 +0,0 @@ -'use strict'; - -let dbm; -let type; -let seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function(options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function(db, callback) { - db.runSql( - `CREATE TABLE global_settings ( - dim_api_enabled boolean, - destiny_profile_minimum_refresh_interval int, - destiny_profile_refresh_interval int, - auto_refresh boolean, - refresh_profile_on_visible boolean, - bust_profile_cache_on_hard_refresh boolean - ); - - INSERT INTO global_settings - (dim_api_enabled, destiny_profile_minimum_refresh_interval, destiny_profile_refresh_interval, auto_refresh, refresh_profile_on_visible, bust_profile_cache_on_hard_refresh) - VALUES - (true, 15, 30, false, true, false); - `, - callback - ); -}; - -exports.down = function(db, callback) { - db.dropTable('global_settings', callback); -}; - -exports._meta = { - version: 1 -}; diff --git a/api/migrations/20200113062324-apps.js b/api/migrations/20200113062324-apps.js deleted file mode 100644 index e9ed5ce..0000000 --- a/api/migrations/20200113062324-apps.js +++ /dev/null @@ -1,35 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function(options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function(db, callback) { - db.runSql( - `CREATE TABLE apps ( - id text PRIMARY KEY NOT NULL, - bungie_api_key text NOT NULL, - dim_api_key UUID NOT NULL - ); - `, - callback - ); -}; - -exports.down = function(db, callback) { - db.dropTable('apps', callback); -}; - -exports._meta = { - version: 1 -}; diff --git a/api/migrations/20200208225933-loadouts.js b/api/migrations/20200208225933-loadouts.js deleted file mode 100644 index 32bd031..0000000 --- a/api/migrations/20200208225933-loadouts.js +++ /dev/null @@ -1,51 +0,0 @@ -'use strict'; - -let dbm; -let type; -let seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function(options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function(db, callback) { - db.runSql( - `CREATE TABLE loadouts ( - id UUID NOT NULL, - membership_id int NOT NULL, - platform_membership_id text NOT NULL, - destiny_version smallint NOT NULL default 2, - name text NOT NULL, - class_type smallint NOT NULL default 3, - emblem_hash int, - clear_space boolean default false, - /* Items in a loadout are just JSON */ - items jsonb NOT NULL default '{}'::jsonb, - created_at timestamp NOT NULL default current_timestamp, - created_by text NOT NULL, - last_updated_at timestamp NOT NULL default current_timestamp, - last_updated_by text NOT NULL, - /* loadouts are unique by membership ID and loadout ID - effectively they're scoped by user */ - PRIMARY KEY(membership_id, id) - ); - - /* The typical query to get all loadouts specifies both platform_membership_id and destiny_version. destiny_version is low-cardinality enough to not need to be indexed. */ - CREATE INDEX loadouts_by_platform_membership ON loadouts (membership_id, platform_membership_id); - `, - callback - ); -}; - -exports.down = function(db, callback) { - db.dropTable('loadouts', callback); -}; - -exports._meta = { - version: 1 -}; diff --git a/api/migrations/20200208233039-item-annotations.js b/api/migrations/20200208233039-item-annotations.js deleted file mode 100644 index 91c7ff5..0000000 --- a/api/migrations/20200208233039-item-annotations.js +++ /dev/null @@ -1,51 +0,0 @@ -'use strict'; - -let dbm; -let type; -let seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function(options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function(db, callback) { - db.runSql( - ` - CREATE TYPE item_tag AS ENUM ('favorite', 'keep', 'infuse', 'junk', 'archive', 'clear'); - CREATE TABLE item_annotations ( - membership_id int NOT NULL, - platform_membership_id text NOT NULL, - destiny_version smallint NOT NULL default 2, - inventory_item_id text NOT NULL, - tag item_tag, - notes text, - created_at timestamp NOT NULL default current_timestamp, - created_by text NOT NULL, - last_updated_at timestamp NOT NULL default current_timestamp, - last_updated_by text NOT NULL, - /* tags are unique by membership ID and inventory item ID - effectively they're scoped by user */ - PRIMARY KEY(membership_id, inventory_item_id) - ); - - /* The typical query to get all item annotations specifies both platform_membership_id and destiny_version. destiny_version is low-cardinality enough to not need to be indexed. */ - CREATE INDEX item_annotations_by_platform_membership ON item_annotations (membership_id, platform_membership_id); - `, - callback - ); -}; - -exports.down = function(db, callback) { - db.dropTable('item_annotations', () => { - db.runSql('drop type item_tag', callback); - }); -}; - -exports._meta = { - version: 1 -}; diff --git a/api/migrations/20200212061106-apps-add-origin.js b/api/migrations/20200212061106-apps-add-origin.js deleted file mode 100644 index 48bf050..0000000 --- a/api/migrations/20200212061106-apps-add-origin.js +++ /dev/null @@ -1,34 +0,0 @@ -'use strict'; - -let dbm; -let type; -let seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function(options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function(db, callback) { - db.runSql(`ALTER TABLE apps ADD COLUMN origin text NOT NULL;`, () => { - db.runSql( - `ALTER TABLE apps ADD COLUMN created_at timestamp NOT NULL default current_timestamp;`, - callback - ); - }); -}; - -exports.down = function(db, callback) { - db.runSql(`ALTER TABLE apps DROP COLUMN created_at;`, () => { - db.runSql(`ALTER TABLE apps DROP COLUMN origin;`, callback); - }); -}; - -exports._meta = { - version: 1 -}; diff --git a/api/migrations/20200222234251-audit-log.js b/api/migrations/20200222234251-audit-log.js deleted file mode 100644 index 196bc20..0000000 --- a/api/migrations/20200222234251-audit-log.js +++ /dev/null @@ -1,47 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function(options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -/** An immutable log of actions per account. Entries are arbitrary JSON. */ -exports.up = function(db, callback) { - db.runSql( - ` - CREATE TABLE audit_log ( - membership_id int NOT NULL, - id SERIAL NOT NULL, - platform_membership_id text, - destiny_version smallint NOT NULL default 2, - type text NOT NULL, - entry jsonb NOT NULL, - created_at timestamp NOT NULL default current_timestamp, - created_by text NOT NULL, - /* tags are unique by membership ID and inventory item ID - effectively they're scoped by user */ - PRIMARY KEY(membership_id, id) - ); - - /* Add an index on date */ - CREATE INDEX audit_log_by_time ON audit_log (membership_id, created_at); - `, - callback - ); -}; - -exports.down = function(db, callback) { - db.dropTable('audit_log', callback); -}; - -exports._meta = { - version: 1 -}; diff --git a/api/migrations/20200318051307-add-dim-refresh-setting.js b/api/migrations/20200318051307-add-dim-refresh-setting.js deleted file mode 100644 index df750b8..0000000 --- a/api/migrations/20200318051307-add-dim-refresh-setting.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function(options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function(db, callback) { - db.runSql( - `ALTER TABLE global_settings ADD COLUMN dim_profile_minimum_refresh_interval int NOT NULL default 300;`, - callback - ); -}; - -exports.down = function(db, callback) { - db.runSql( - `ALTER TABLE global_settings DROP COLUMN dim_profile_minimum_refresh_interval;`, - callback - ); -}; - -exports._meta = { - version: 1 -}; diff --git a/api/migrations/20200529053904-tracked-triumphs.js b/api/migrations/20200529053904-tracked-triumphs.js deleted file mode 100644 index 3ad86ad..0000000 --- a/api/migrations/20200529053904-tracked-triumphs.js +++ /dev/null @@ -1,45 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -/** - * Entries for each triumph tracked by a user. Presence in this table indicates the triumph - * is tracked - otherwise it is simply missing. - */ -exports.up = function (db, callback) { - db.runSql( - ` - CREATE TABLE tracked_triumphs ( - membership_id int NOT NULL, - platform_membership_id text NOT NULL, - /* triumphs are only for D2 so we don't need a destiny_version column */ - record_hash bigint NOT NULL, - created_at timestamp NOT NULL default current_timestamp, - created_by text NOT NULL, - /* Tracked triumphs can be different for different profiles */ - PRIMARY KEY(membership_id, platform_membership_id, record_hash) - ); - `, - callback - ); -}; - -exports.down = function (db, callback) { - db.dropTable('tracked_triumphs', callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20200703220618-add-issue-banner-to-global-settings.js b/api/migrations/20200703220618-add-issue-banner-to-global-settings.js deleted file mode 100644 index 7b82582..0000000 --- a/api/migrations/20200703220618-add-issue-banner-to-global-settings.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - `ALTER TABLE global_settings ADD COLUMN show_issue_banner boolean NOT NULL default false;`, - callback - ); -}; - -exports.down = function (db, callback) { - db.runSql( - `ALTER TABLE global_settings DROP COLUMN show_issue_banner;`, - callback - ); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20200704024612-searches.js b/api/migrations/20200704024612-searches.js deleted file mode 100644 index 25db94c..0000000 --- a/api/migrations/20200704024612-searches.js +++ /dev/null @@ -1,48 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - // Searches are stored per-destiny-version, but not per-account - ` - CREATE TABLE searches ( - membership_id int NOT NULL, - destiny_version smallint NOT NULL default 2, - query text NOT NULL, - saved boolean NOT NULL default false, - usage_count int NOT NULL default 1, - last_used timestamp NOT NULL default current_timestamp, - created_at timestamp NOT NULL default current_timestamp, - created_by text NOT NULL, - last_updated_at timestamp NOT NULL default current_timestamp, - last_updated_by text NOT NULL, - PRIMARY KEY(membership_id, destiny_version, query) - ); - - /* The typical query to get all searches specifies both platform_membership_id and destiny_version. destiny_version is low-cardinality enough to not need to be indexed. */ - CREATE INDEX searches_by_membership ON searches (membership_id); - `, - callback - ); -}; - -exports.down = function (db, callback) { - db.dropTable('searches', callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20200707023029-hash-tags.js b/api/migrations/20200707023029-hash-tags.js deleted file mode 100644 index f069599..0000000 --- a/api/migrations/20200707023029-hash-tags.js +++ /dev/null @@ -1,49 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -// I'll probably regret this name, but it's basically item_annotations keyed by item hash instead of instance ID. For shaders. -// Hash-based tags D2-only and aren't specific to one profile - if you like a shader, you like it everywhere. -exports.up = function (db, callback) { - db.runSql( - ` - CREATE TABLE item_hash_tags ( - membership_id int NOT NULL, - item_hash bigint NOT NULL, - tag item_tag, - notes text, - created_at timestamp NOT NULL default current_timestamp, - created_by text NOT NULL, - last_updated_at timestamp NOT NULL default current_timestamp, - last_updated_by text NOT NULL, - /* tags are unique by membership ID and item hash - effectively they're scoped by user */ - PRIMARY KEY(membership_id, item_hash) - ); - - CREATE INDEX item_hash_tags_by_membership ON item_hash_tags (membership_id); - `, - callback - ); -}; - -exports.down = function (db, callback) { - db.removeIndex('item_hash_tags_by_membership', () => { - db.dropTable('item_hash_tags', callback); - }); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20200912183523-add-loadout-params.js b/api/migrations/20200912183523-add-loadout-params.js deleted file mode 100644 index b3f8ec5..0000000 --- a/api/migrations/20200912183523-add-loadout-params.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql(`ALTER TABLE loadouts ADD COLUMN parameters jsonb;`, callback); -}; - -exports.down = function (db, callback) { - db.runSql(`ALTER TABLE loadouts DROP COLUMN parameters;`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20201027035807-drop-audit-log.js b/api/migrations/20201027035807-drop-audit-log.js deleted file mode 100644 index 9d841bf..0000000 --- a/api/migrations/20201027035807-drop-audit-log.js +++ /dev/null @@ -1,46 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - return db.dropTable('audit_log', callback); -}; - -exports.down = function (db, callback) { - db.runSql( - ` - CREATE TABLE audit_log ( - membership_id int NOT NULL, - id SERIAL NOT NULL, - platform_membership_id text, - destiny_version smallint NOT NULL default 2, - type text NOT NULL, - entry jsonb NOT NULL, - created_at timestamp NOT NULL default current_timestamp, - created_by text NOT NULL, - /* tags are unique by membership ID and inventory item ID - effectively they're scoped by user */ - PRIMARY KEY(membership_id, id) - ); - - /* Add an index on date */ - CREATE INDEX audit_log_by_time ON audit_log (membership_id, created_at); - `, - callback - ); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20210105015627-change-item-annotation-to-bigint.js b/api/migrations/20210105015627-change-item-annotation-to-bigint.js deleted file mode 100644 index f79255b..0000000 --- a/api/migrations/20210105015627-change-item-annotation-to-bigint.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - `ALTER TABLE item_annotations ALTER COLUMN platform_membership_id TYPE bigint USING platform_membership_id::bigint, ALTER COLUMN inventory_item_id TYPE bigint USING inventory_item_id::bigint;`, - callback - ); -}; - -exports.down = function (db, callback) { - db.runSql( - `ALTER TABLE item_annotations ALTER COLUMN platform_membership_id TYPE bigint USING platform_membership_id::text, ALTER COLUMN inventory_item_id TYPE text USING inventory_item_id::text;`, - callback - ); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20210105015836-smaller-item-annotations-index.js b/api/migrations/20210105015836-smaller-item-annotations-index.js deleted file mode 100644 index c19aa2f..0000000 --- a/api/migrations/20210105015836-smaller-item-annotations-index.js +++ /dev/null @@ -1,34 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - `CREATE INDEX item_annotations_by_pm_id ON item_annotations (platform_membership_id);`, - () => - db.runSql(`DROP INDEX item_annotations_by_platform_membership;`, callback) - ); -}; - -exports.down = function (db, callback) { - db.runSql( - `CREATE INDEX item_annotations_by_platform_membership ON item_annotations (membership_id, platform_membership_id);`, - () => db.runSql(`DROP INDEX item_annotations_by_pm_id;`, callback) - ); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20211104204501-add-loadout-notes.js b/api/migrations/20211104204501-add-loadout-notes.js deleted file mode 100644 index 1b35279..0000000 --- a/api/migrations/20211104204501-add-loadout-notes.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql(`ALTER TABLE loadouts ADD COLUMN notes text;`, callback); -}; - -exports.down = function (db, callback) { - db.runSql(`ALTER TABLE loadouts DROP COLUMN notes;`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20220110060009-loadout-share.js b/api/migrations/20220110060009-loadout-share.js deleted file mode 100644 index 286f15e..0000000 --- a/api/migrations/20220110060009-loadout-share.js +++ /dev/null @@ -1,49 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - // Basically the same as loadouts, but indexed by a string ID. These are not intended to be mutable. - ` - CREATE TABLE loadout_shares ( - id text NOT NULL, - membership_id integer NOT NULL, - platform_membership_id text NOT NULL, - name text NOT NULL, - notes text, - class_type smallint NOT NULL DEFAULT 3, - emblem_hash integer, - clear_space boolean DEFAULT false, - items jsonb NOT NULL DEFAULT '{}'::jsonb, - parameters jsonb, - created_at timestamp NOT NULL default current_timestamp, - created_by text NOT NULL, - last_accessed_at timestamp, - visits integer NOT NULL default 0, - PRIMARY KEY (id) - ); - `, - callback - ); -}; - -exports.down = function (db, callback) { - db.dropTable('loadout_shares', callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20220627015027-global-settings-flavor.js b/api/migrations/20220627015027-global-settings-flavor.js deleted file mode 100644 index 8dc51d0..0000000 --- a/api/migrations/20220627015027-global-settings-flavor.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql(`ALTER TABLE global_settings ADD COLUMN flavor text NOT NULL default 'app';`, callback); -}; - -exports.down = function (db, callback) { - db.runSql(`ALTER TABLE global_settings DROP COLUMN flavor;`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20220627015325-global-settings-bust.js b/api/migrations/20220627015325-global-settings-bust.js deleted file mode 100644 index ed4fe23..0000000 --- a/api/migrations/20220627015325-global-settings-bust.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - `ALTER TABLE global_settings DROP COLUMN bust_profile_cache_on_hard_refresh;`, - callback - ); -}; - -exports.down = function (db, callback) { - db.runSql( - `ALTER TABLE global_settings ADD COLUMN bust_profile_cache_on_hard_refresh boolean NOT NULL default false;`, - callback - ); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20221016230211-searches-hash.js b/api/migrations/20221016230211-searches-hash.js deleted file mode 100644 index 1e2ee65..0000000 --- a/api/migrations/20221016230211-searches-hash.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - `ALTER TABLE searches ADD COLUMN qhash bytea GENERATED ALWAYS AS (decode(md5(query), 'hex')) STORED;`, - callback - ); -}; - -exports.down = function (db, callback) { - db.runSql(`ALTER TABLE searches DROP COLUMN qhash;`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20221016230508-searches-hash-uniq.js b/api/migrations/20221016230508-searches-hash-uniq.js deleted file mode 100644 index a8fcc69..0000000 --- a/api/migrations/20221016230508-searches-hash-uniq.js +++ /dev/null @@ -1,31 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - `create unique index searches_uniq on searches (membership_id, destiny_version, qhash);`, - callback - ); -}; - -exports.down = function (db, callback) { - db.runSql(`drop index searches_uniq;`, callback); -}; -exports._meta = { - version: 1, -}; - -// TODO: drop existing indexes after updating queries diff --git a/api/migrations/20221016231000-searches-remove-pk.js b/api/migrations/20221016231000-searches-remove-pk.js deleted file mode 100644 index 3516b69..0000000 --- a/api/migrations/20221016231000-searches-remove-pk.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql(`ALTER TABLE searches drop constraint searches_pkey;`, callback); -}; - -exports.down = function (db, callback) { - db.runSql( - `ALTER TABLE searches add ADD PRIMARY KEY(membership_id, destiny_version, query);`, - callback - ); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20221017055645-item-annotations-crafted-date.js b/api/migrations/20221017055645-item-annotations-crafted-date.js deleted file mode 100644 index dbe6a5e..0000000 --- a/api/migrations/20221017055645-item-annotations-crafted-date.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql(`ALTER TABLE item_annotations ADD COLUMN crafted_date timestamp;`, callback); -}; - -exports.down = function (db, callback) { - db.runSql(`ALTER TABLE item_annotations DROP COLUMN crafted_date;`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20221224022935-item-annotations-variant.js b/api/migrations/20221224022935-item-annotations-variant.js deleted file mode 100644 index c568643..0000000 --- a/api/migrations/20221224022935-item-annotations-variant.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql(`ALTER TABLE item_annotations ADD COLUMN variant smallint;`, callback); -}; - -exports.down = function (db, callback) { - db.runSql(`ALTER TABLE item_annotations DROP COLUMN variant;`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20230524173111-cron-nulls.js b/api/migrations/20230524173111-cron-nulls.js deleted file mode 100644 index 1e4cab2..0000000 --- a/api/migrations/20230524173111-cron-nulls.js +++ /dev/null @@ -1,31 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - // Run at 1am Pacific on Monday, delete all tags where both notes and tag is null - `SELECT cron.schedule('null-tags', '0 8 * * 1', $$delete from item_annotations where notes is null and tag is null$$);`, - callback - ); -}; - -exports.down = function (db) { - db.runSql(`cron.unschedule('null-tags')`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20230524173723-cron-searches.js b/api/migrations/20230524173723-cron-searches.js deleted file mode 100644 index 735fcb9..0000000 --- a/api/migrations/20230524173723-cron-searches.js +++ /dev/null @@ -1,31 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - // Run at 1:30am Pacific on Monday, delete loadout shares that were never used - `SELECT cron.schedule('unused-loadout-shares', '30 8 * * 1', $$delete from loadout_shares where visits = 0 and created_at < now() - interval '1 week';$$);`, - callback - ); -}; - -exports.down = function (db) { - db.runSql(`cron.unschedule('unused-loadout-shares')`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20230524173837-cron-loadout-shares.js b/api/migrations/20230524173837-cron-loadout-shares.js deleted file mode 100644 index 7c50d5e..0000000 --- a/api/migrations/20230524173837-cron-loadout-shares.js +++ /dev/null @@ -1,31 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - // Run at 1:10am Pacific on Monday, delete old one-time use searches - `SELECT cron.schedule('old-searches', '10 8 * * 1', $$delete from searches where usage_count = 1 and saved = false and last_used < now() - interval '6 month';$$);`, - callback - ); -}; - -exports.down = function (db) { - db.runSql(`cron.unschedule('old-searches')`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20230524174240-cron-hashtag-nulls.js b/api/migrations/20230524174240-cron-hashtag-nulls.js deleted file mode 100644 index c814fab..0000000 --- a/api/migrations/20230524174240-cron-hashtag-nulls.js +++ /dev/null @@ -1,31 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - // Run at 1am Pacific on Monday, delete all item hash tags where both notes and tag is null - `SELECT cron.schedule('null-tags', '0 8 * * 1', $$delete from item_hash_tags where notes is null and tag is null$$);`, - callback - ); -}; - -exports.down = function (db) { - db.runSql(`cron.unschedule('null-tags')`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20240521044000-add-search-type.js b/api/migrations/20240521044000-add-search-type.js deleted file mode 100644 index 97b7673..0000000 --- a/api/migrations/20240521044000-add-search-type.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql(`ALTER TABLE searches ADD COLUMN search_type smallint NOT NULL DEFAULT 1;`, callback); -}; - -exports.down = function (db, callback) { - db.runSql(`ALTER TABLE searches DROP COLUMN search_type;`, callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/20240912182338-migration-state.js b/api/migrations/20240912182338-migration-state.js deleted file mode 100644 index 0b8fd2a..0000000 --- a/api/migrations/20240912182338-migration-state.js +++ /dev/null @@ -1,39 +0,0 @@ -'use strict'; - -var dbm; -var type; -var seed; - -/** - * We receive the dbmigrate dependency from dbmigrate initially. - * This enables us to not have to rely on NODE_PATH. - */ -exports.setup = function (options, seedLink) { - dbm = options.dbmigrate; - type = dbm.dataType; - seed = seedLink; -}; - -exports.up = function (db, callback) { - db.runSql( - `CREATE TABLE migration_state ( - membership_id int PRIMARY KEY NOT NULL, - state smallint NOT NULL default 1, - last_state_change_at timestamp NOT NULL default current_timestamp, - attempt_count int NOT NULL default 0, - last_error text, - created_at timestamp NOT NULL default current_timestamp, - last_updated_at timestamp NOT NULL default current_timestamp - ); - `, - callback, - ); -}; - -exports.down = function (db, callback) { - db.dropTable('migration_state', callback); -}; - -exports._meta = { - version: 1, -}; diff --git a/api/migrations/package.json b/api/migrations/package.json deleted file mode 100644 index 878e6bf..0000000 --- a/api/migrations/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "dim-api-migrations", - "private": true, - "description": "https://github.com/db-migrate/node-db-migrate/pull/724/files", - "type": "commonjs" -} diff --git a/api/routes/delete-all-data.ts b/api/routes/delete-all-data.ts index d4650bc..3adb688 100644 --- a/api/routes/delete-all-data.ts +++ b/api/routes/delete-all-data.ts @@ -1,14 +1,4 @@ import asyncHandler from 'express-async-handler'; -import { ClientBase } from 'pg'; -import { readTransaction, transaction } from '../db/index.js'; -import { deleteAllItemAnnotations } from '../db/item-annotations-queries.js'; -import { deleteAllItemHashTags } from '../db/item-hash-tags-queries.js'; -import { deleteAllLoadouts } from '../db/loadouts-queries.js'; -import { getMigrationState, MigrationState } from '../db/migration-state-queries.js'; -import { deleteAllSearches } from '../db/searches-queries.js'; -import { deleteSettings } from '../db/settings-queries.js'; -import { deleteAllTrackedTriumphs } from '../db/triumphs-queries.js'; -import { DeleteAllResponse } from '../shapes/delete-all.js'; import { UserInfo } from '../shapes/user.js'; import { deleteAllDataForUser } from '../stately/bulk-queries.js'; @@ -18,52 +8,10 @@ import { deleteAllDataForUser } from '../stately/bulk-queries.js'; export const deleteAllDataHandler = asyncHandler(async (req, res) => { const { bungieMembershipId, profileIds } = req.user as UserInfo; - const migrationState = await readTransaction(async (client) => - getMigrationState(client, bungieMembershipId), - ); - - let result: DeleteAllResponse['deleted']; - switch (migrationState.state) { - case MigrationState.Postgres: - // Also delete from Stately, just to honor the "no data left here" promise - try { - await deleteAllDataForUser(bungieMembershipId, profileIds); - } catch (e) { - console.error('Error deleting data from Stately', e); - } - result = await transaction(async (client) => deleteAllData(client, bungieMembershipId)); - break; - case MigrationState.Stately: - // Also delete from Postgres, just to honor the "no data left here" promise - try { - await transaction(async (client) => deleteAllData(client, bungieMembershipId)); - } catch (e) { - console.error('Error deleting data from Postgres', e); - } - result = await deleteAllDataForUser(bungieMembershipId, profileIds); - break; - default: - // We're in the middle of a migration - throw new Error(`Unable to delete data - please wait a bit and try again.`); - } + const result = await deleteAllDataForUser(bungieMembershipId, profileIds); // default 200 OK res.status(200).send({ deleted: result, }); }); - -/** Postgres delete-all-data implementation just individually deletes from each table */ -export async function deleteAllData( - client: ClientBase, - bungieMembershipId: number, -): Promise { - return { - settings: (await deleteSettings(client, bungieMembershipId)).rowCount!, - loadouts: (await deleteAllLoadouts(client, bungieMembershipId)).rowCount!, - tags: (await deleteAllItemAnnotations(client, bungieMembershipId)).rowCount!, - itemHashTags: (await deleteAllItemHashTags(client, bungieMembershipId)).rowCount!, - triumphs: (await deleteAllTrackedTriumphs(client, bungieMembershipId)).rowCount!, - searches: (await deleteAllSearches(client, bungieMembershipId)).rowCount!, - }; -} diff --git a/api/routes/export.ts b/api/routes/export.ts index 1c1e0f2..9476528 100644 --- a/api/routes/export.ts +++ b/api/routes/export.ts @@ -1,60 +1,13 @@ import asyncHandler from 'express-async-handler'; -import { readTransaction } from '../db/index.js'; -import { getAllItemAnnotationsForUser } from '../db/item-annotations-queries.js'; -import { getItemHashTagsForProfile } from '../db/item-hash-tags-queries.js'; -import { getAllLoadoutsForUser } from '../db/loadouts-queries.js'; -import { getMigrationState, MigrationState } from '../db/migration-state-queries.js'; -import { getSearchesForUser } from '../db/searches-queries.js'; -import { getSettings } from '../db/settings-queries.js'; -import { getAllTrackedTriumphsForUser } from '../db/triumphs-queries.js'; -import { ExportResponse } from '../shapes/export.js'; import { UserInfo } from '../shapes/user.js'; import { exportDataForUser } from '../stately/bulk-queries.js'; export const exportHandler = asyncHandler(async (req, res) => { const { bungieMembershipId, profileIds } = req.user as UserInfo; - const migrationState = await readTransaction(async (client) => - getMigrationState(client, bungieMembershipId), - ); - - let response: ExportResponse; - switch (migrationState.state) { - case MigrationState.Postgres: - case MigrationState.MigratingToStately: // in-progress migration is the same as PG - response = await pgExport(bungieMembershipId); - break; - case MigrationState.Stately: - response = await exportDataForUser(bungieMembershipId, profileIds); - break; - default: - // invalid state - throw new Error(`Unable to export data - please wait a bit and try again.`); - } + const response = await exportDataForUser(bungieMembershipId, profileIds); // Instruct CF not to cache this res.set('Cache-Control', 'no-cache, no-store, max-age=0'); res.send(response); }); - -export async function pgExport(bungieMembershipId: number): Promise { - const response = await readTransaction(async (client) => { - const settings = await getSettings(client, bungieMembershipId); - const loadouts = await getAllLoadoutsForUser(client, bungieMembershipId); - const itemAnnotations = await getAllItemAnnotationsForUser(client, bungieMembershipId); - const itemHashTags = await getItemHashTagsForProfile(client, bungieMembershipId); - const triumphs = await getAllTrackedTriumphsForUser(client, bungieMembershipId); - const searches = await getSearchesForUser(client, bungieMembershipId); - - const response: ExportResponse = { - settings, - loadouts, - tags: itemAnnotations, - itemHashTags, - triumphs, - searches, - }; - return response; - }); - return response; -} diff --git a/api/routes/import.ts b/api/routes/import.ts index e63e7ad..58472b8 100644 --- a/api/routes/import.ts +++ b/api/routes/import.ts @@ -1,7 +1,5 @@ import { isEmpty } from 'es-toolkit/compat'; import asyncHandler from 'express-async-handler'; -import { readTransaction } from '../db/index.js'; -import { doMigration, getMigrationState, MigrationState } from '../db/migration-state-queries.js'; import { ExportResponse } from '../shapes/export.js'; import { DestinyVersion } from '../shapes/general.js'; import { ImportResponse } from '../shapes/import.js'; @@ -41,37 +39,17 @@ export const importHandler = asyncHandler(async (req, res) => { return; } - const migrationState = await readTransaction(async (client) => - getMigrationState(client, bungieMembershipId), + const numTriumphs = await statelyImport( + bungieMembershipId, + profileIds, + settings, + loadouts, + itemAnnotations, + triumphs, + searches, + itemHashTags, ); - let numTriumphs = 0; - const importToStately = async () => { - numTriumphs = await statelyImport( - bungieMembershipId, - profileIds, - settings, - loadouts, - itemAnnotations, - triumphs, - searches, - itemHashTags, - ); - }; - - switch (migrationState.state) { - case MigrationState.Postgres: - await doMigration(bungieMembershipId, importToStately); - break; - case MigrationState.Stately: - await importToStately(); - break; - default: - // in-progress migration - badRequest(res, `Unable to import data - please wait a bit and try again.`); - return; - } - const response: ImportResponse = { loadouts: loadouts.length, tags: itemAnnotations.length, diff --git a/api/routes/profile.ts b/api/routes/profile.ts index 1332123..71cb327 100644 --- a/api/routes/profile.ts +++ b/api/routes/profile.ts @@ -1,19 +1,9 @@ -import * as Sentry from '@sentry/node'; import express from 'express'; import asyncHandler from 'express-async-handler'; -import { readTransaction } from '../db/index.js'; -import { getItemAnnotationsForProfile } from '../db/item-annotations-queries.js'; -import { getItemHashTagsForProfile } from '../db/item-hash-tags-queries.js'; -import { getLoadoutsForProfile } from '../db/loadouts-queries.js'; -import { getMigrationState, MigrationState } from '../db/migration-state-queries.js'; -import { getSearchesForProfile } from '../db/searches-queries.js'; -import { getSettings } from '../db/settings-queries.js'; -import { getTrackedTriumphsForProfile } from '../db/triumphs-queries.js'; import { metrics } from '../metrics/index.js'; import { ApiApp } from '../shapes/app.js'; import { DestinyVersion } from '../shapes/general.js'; import { ProfileResponse } from '../shapes/profile.js'; -import { defaultSettings } from '../shapes/settings.js'; import { UserInfo } from '../shapes/user.js'; import { getProfile } from '../stately/bulk-queries.js'; import { getItemAnnotationsForProfile as getItemAnnotationsForProfileStately } from '../stately/item-annotations-queries.js'; @@ -83,35 +73,13 @@ export const profileHandler = asyncHandler(async (req, res) => { return; } - const migrationState = await readTransaction(async (client) => - getMigrationState(client, bungieMembershipId), + const response = await statelyProfile( + res, + components, + bungieMembershipId, + platformMembershipId, + destinyVersion, ); - let response: ProfileResponse | undefined; - switch (migrationState.state) { - case MigrationState.Postgres: - case MigrationState.MigratingToStately: // in-progress migration is the same as PG - response = await pgProfile( - res, - components, - bungieMembershipId, - platformMembershipId, - destinyVersion, - appId, - ); - break; - case MigrationState.Stately: - response = await statelyProfile( - res, - components, - bungieMembershipId, - platformMembershipId, - destinyVersion, - ); - break; - default: - // invalid state - throw new Error(`Unable to get profile - please wait a bit and try again.`); - } if (!response) { return; // we've already responded @@ -123,121 +91,6 @@ export const profileHandler = asyncHandler(async (req, res) => { res.send(response); }); -async function pgProfile( - res: express.Response, - components: string[], - bungieMembershipId: number, - platformMembershipId: string | undefined, - destinyVersion: DestinyVersion, - appId: string, -) { - return readTransaction(async (client) => { - const response: ProfileResponse = {}; - - if (components.includes('settings')) { - // TODO: should settings be stored under profile too?? maybe primary profile ID? - const start = new Date(); - const storedSettings = await getSettings(client, bungieMembershipId); - - // Clean out deprecated settings (TODO purge from DB) - delete (storedSettings as Record).allowIdPostToDtr; - delete (storedSettings as Record).colorA11y; - delete (storedSettings as Record).itemDetails; - delete (storedSettings as Record).itemPickerEquip; - delete (storedSettings as Record).itemSort; - delete (storedSettings as Record).loAssumeMasterwork; - delete (storedSettings as Record).loLockItemEnergyType; - delete (storedSettings as Record).loMinPower; - delete (storedSettings as Record).loMinStatTotal; - delete (storedSettings as Record).loStatSortOrder; - delete (storedSettings as Record).loUpgradeSpendTier; - delete (storedSettings as Record).reviewsModeSelection; - delete (storedSettings as Record).reviewsPlatformSelectionV2; - delete (storedSettings as Record).showReviews; - - response.settings = { - ...defaultSettings, - ...storedSettings, - }; - metrics.timing('profile.settings', start); - } - - if (components.includes('loadouts')) { - if (!platformMembershipId) { - badRequest(res, 'Need a platformMembershipId to return loadouts'); - return; - } - const start = new Date(); - response.loadouts = await getLoadoutsForProfile( - client, - bungieMembershipId, - platformMembershipId, - destinyVersion, - ); - metrics.timing('profile.loadouts.numReturned', response.loadouts.length); - metrics.timing('profile.loadouts', start); - } - - if (components.includes('tags')) { - if (!platformMembershipId) { - badRequest(res, 'Need a platformMembershipId to return item annotations'); - return; - } - const start = new Date(); - response.tags = await getItemAnnotationsForProfile( - client, - bungieMembershipId, - platformMembershipId, - destinyVersion, - ); - metrics.timing('profile.tags.numReturned', response.tags.length); - metrics.timing('profile.tags', start); - } - - if (components.includes('hashtags')) { - const start = new Date(); - response.itemHashTags = await getItemHashTagsForProfile(client, bungieMembershipId); - metrics.timing('profile.hashtags.numReturned', response.itemHashTags.length); - metrics.timing('profile.hashtags', start); - } - - if (destinyVersion === 2 && components.includes('triumphs')) { - if (!platformMembershipId) { - badRequest(res, 'Need a platformMembershipId to return triumphs'); - return; - } - const start = new Date(); - response.triumphs = await getTrackedTriumphsForProfile( - client, - bungieMembershipId, - platformMembershipId, - ); - metrics.timing('profile.triumphs.numReturned', response.triumphs.length); - metrics.timing('profile.triumphs', start); - } - - if (components.includes('searches')) { - const start = new Date(); - response.searches = await getSearchesForProfile(client, bungieMembershipId, destinyVersion); - metrics.timing('profile.searches.numReturned', response.searches.length); - metrics.timing('profile.searches', start); - } - - if ((response.tags?.length ?? 0) > 1000) { - Sentry.captureMessage('User with a lot of tags', { - extra: { - bungieMembershipId, - destinyVersion, - appId, - tagsLength: response.tags?.length, - }, - }); - } - - return response; - }); -} - // TODO: Probably could enable allowStale, since profiles are cached anyway // TODO: It'd be nice to pass a signal in so we can abort all the parallel fetches async function statelyProfile( diff --git a/api/routes/update.ts b/api/routes/update.ts index 3bbe8a0..f383062 100644 --- a/api/routes/update.ts +++ b/api/routes/update.ts @@ -1,32 +1,7 @@ import { captureMessage } from '@sentry/node'; import { chunk, groupBy, partition, sortBy } from 'es-toolkit'; -import { isEmpty } from 'es-toolkit/compat'; import express from 'express'; import asyncHandler from 'express-async-handler'; -import { ClientBase } from 'pg'; -import { readTransaction, transaction } from '../db/index.js'; -import { - deleteItemAnnotationList, - updateItemAnnotation as updateItemAnnotationInDb, -} from '../db/item-annotations-queries.js'; -import { updateItemHashTag as updateItemHashTagInDb } from '../db/item-hash-tags-queries.js'; -import { - deleteLoadout as deleteLoadoutInDb, - updateLoadout as updateLoadoutInDb, -} from '../db/loadouts-queries.js'; -import { - doMigration, - getDesiredMigrationState, - getMigrationState, - MigrationState, -} from '../db/migration-state-queries.js'; -import { - deleteSearch as deleteSearchInDb, - saveSearch as saveSearchInDb, - updateUsedSearch, -} from '../db/searches-queries.js'; -import { setSetting as setSettingInDb } from '../db/settings-queries.js'; -import { trackTriumph as trackTriumphInDb, unTrackTriumph } from '../db/triumphs-queries.js'; import { metrics } from '../metrics/index.js'; import { ApiApp } from '../shapes/app.js'; import { DestinyVersion } from '../shapes/general.js'; @@ -74,8 +49,6 @@ import { isValidItemId, isValidPlatformMembershipId, } from '../utils.js'; -import { pgExport } from './export.js'; -import { extractImportData, statelyImport } from './import.js'; /** * Update profile information. This accepts a list of update operations and @@ -115,83 +88,16 @@ export const updateHandler = asyncHandler(async (req, res) => { return; } - const migrationState = await readTransaction(async (client) => - getMigrationState(client, bungieMembershipId), - ); - - const desiredMigrationState = await getDesiredMigrationState(migrationState); - const shouldMigrateToStately = - desiredMigrationState === MigrationState.Stately && - migrationState.state !== desiredMigrationState; - const results: ProfileUpdateResult[] = validateUpdates(req, updates, platformMembershipId, appId); // Only attempt updates that pass validation const updatesToApply = updates.filter((_, index) => results[index].status === 'Success'); - const importToStately = async () => { - // Export from Postgres - const exportResponse = await pgExport(bungieMembershipId); - - const { settings, loadouts, itemAnnotations, triumphs, searches, itemHashTags } = - extractImportData(exportResponse); - - if ( - isEmpty(settings) && - loadouts.length === 0 && - itemAnnotations.length === 0 && - triumphs.length === 0 && - searches.length === 0 - ) { - // Nothing to import! - return; - } - await statelyImport( - bungieMembershipId, - profileIds, - settings, - loadouts, - itemAnnotations, - triumphs, - searches, - itemHashTags, - migrationState.attemptCount > 0 || Boolean(migrationState.lastError), - ); - }; - - switch (migrationState.state) { - case MigrationState.Postgres: - if (shouldMigrateToStately) { - // For now let's leave the old data in Postgres as a backup - await doMigration(bungieMembershipId, importToStately); - await statelyUpdate( - updatesToApply, - bungieMembershipId, - platformMembershipId ?? profileIds[0], - destinyVersion, - ); - } else { - await pgUpdate( - updatesToApply, - bungieMembershipId, - platformMembershipId, - destinyVersion, - appId, - ); - } - break; - case MigrationState.Stately: - await statelyUpdate( - updatesToApply, - bungieMembershipId, - platformMembershipId ?? profileIds[0], - destinyVersion, - ); - break; - default: - // in-progress migration - badRequest(res, `Unable to import data - please wait a bit and try again.`); - return; - } + await statelyUpdate( + updatesToApply, + bungieMembershipId, + platformMembershipId ?? profileIds[0], + destinyVersion, + ); res.send({ results, @@ -408,113 +314,6 @@ async function statelyUpdate( } } -async function pgUpdate( - updates: ProfileUpdate[], - bungieMembershipId: number, - platformMembershipId: string | undefined, - destinyVersion: DestinyVersion, - appId: string, -) { - return transaction(async (client) => { - for (const update of updates) { - switch (update.action) { - case 'setting': - await updateSetting(client, appId, bungieMembershipId, update.payload); - break; - - case 'loadout': - await updateLoadout( - client, - appId, - bungieMembershipId, - platformMembershipId!, - destinyVersion, - update.payload, - ); - break; - - case 'delete_loadout': - await deleteLoadout(client, bungieMembershipId, update.payload); - break; - - case 'tag': - await updateItemAnnotation( - client, - appId, - bungieMembershipId, - platformMembershipId!, - destinyVersion, - update.payload, - ); - break; - - case 'tag_cleanup': - await tagCleanup(client, bungieMembershipId, update.payload); - break; - - case 'item_hash_tag': - await updateItemHashTag(client, appId, bungieMembershipId, update.payload); - break; - - case 'track_triumph': - await trackTriumph( - client, - appId, - bungieMembershipId, - platformMembershipId!, - update.payload, - ); - break; - - case 'search': - await recordSearch(client, appId, bungieMembershipId, destinyVersion, update.payload); - break; - - case 'save_search': - await saveSearch(client, appId, bungieMembershipId, destinyVersion, update.payload); - break; - - case 'delete_search': - await deleteSearch(client, bungieMembershipId, destinyVersion, update.payload); - break; - } - } - }); -} - -async function updateSetting( - client: ClientBase, - appId: string, - bungieMembershipId: number, - settings: Partial, -): Promise { - // TODO: how do we set settings back to the default? Maybe just load and replace the whole settings object. - - const start = new Date(); - await setSettingInDb(client, appId, bungieMembershipId, settings); - metrics.timing('update.setting', start); -} - -async function updateLoadout( - client: ClientBase, - appId: string, - bungieMembershipId: number, - platformMembershipId: string, - destinyVersion: DestinyVersion, - loadout: Loadout, -): Promise { - const start = new Date(); - await updateLoadoutInDb( - client, - appId, - bungieMembershipId, - platformMembershipId, - destinyVersion, - loadout, - ); - metrics.timing('update.loadout', start); -} - function validateUpdateLoadout(loadout: Loadout): ProfileUpdateResult { return validateLoadout('update', loadout) ?? { status: 'Success' }; } @@ -604,36 +403,6 @@ export function validateLoadout(metricPrefix: string, loadout: Loadout) { return undefined; } -async function deleteLoadout( - client: ClientBase, - bungieMembershipId: number, - loadoutId: string, -): Promise { - const start = new Date(); - await deleteLoadoutInDb(client, bungieMembershipId, loadoutId); - metrics.timing('update.deleteLoadout', start); -} - -async function updateItemAnnotation( - client: ClientBase, - appId: string, - bungieMembershipId: number, - platformMembershipId: string, - destinyVersion: DestinyVersion, - itemAnnotation: ItemAnnotation, -): Promise { - const start = new Date(); - await updateItemAnnotationInDb( - client, - appId, - bungieMembershipId, - platformMembershipId, - destinyVersion, - itemAnnotation, - ); - metrics.timing('update.tag', start); -} - function validateUpdateItemAnnotation(itemAnnotation: ItemAnnotation): ProfileUpdateResult { if (!isValidItemId(itemAnnotation.id)) { metrics.increment('update.validation.badItemId.count'); @@ -694,81 +463,6 @@ function validateUpdateItemHashTag(itemAnnotation: ItemHashTag): ProfileUpdateRe return { status: 'Success' }; } -async function tagCleanup( - client: ClientBase, - bungieMembershipId: number, - inventoryItemIds: string[], -): Promise { - const start = new Date(); - await deleteItemAnnotationList( - client, - bungieMembershipId, - inventoryItemIds.filter(isValidItemId), - ); - metrics.timing('update.tagCleanup', start); - - return { status: 'Success' }; -} - -async function trackTriumph( - client: ClientBase, - appId: string, - bungieMembershipId: number, - platformMembershipId: string, - payload: TrackTriumphUpdate['payload'], -): Promise { - const start = new Date(); - payload.tracked - ? await trackTriumphInDb( - client, - appId, - bungieMembershipId, - platformMembershipId, - payload.recordHash, - ) - : await unTrackTriumph(client, bungieMembershipId, platformMembershipId, payload.recordHash); - metrics.timing('update.trackTriumph', start); -} - -async function recordSearch( - client: ClientBase, - appId: string, - bungieMembershipId: number, - destinyVersion: DestinyVersion, - payload: UsedSearchUpdate['payload'], -): Promise { - const start = new Date(); - await updateUsedSearch( - client, - appId, - bungieMembershipId, - destinyVersion, - payload.query, - payload.type ?? SearchType.Item, - ); - metrics.timing('update.recordSearch', start); -} - -async function saveSearch( - client: ClientBase, - appId: string, - bungieMembershipId: number, - destinyVersion: DestinyVersion, - payload: SavedSearchUpdate['payload'], -): Promise { - const start = new Date(); - await saveSearchInDb( - client, - appId, - bungieMembershipId, - destinyVersion, - payload.query, - payload.type ?? SearchType.Item, - payload.saved, - ); - metrics.timing('update.saveSearch', start); -} - function validateSearch(payload: UsedSearchUpdate['payload']): ProfileUpdateResult { if (payload.query.length > 2048) { metrics.increment('update.validation.searchTooLong.count'); @@ -786,34 +480,6 @@ function validateSearch(payload: UsedSearchUpdate['payload']): ProfileUpdateResu return { status: 'Success' }; } -async function deleteSearch( - client: ClientBase, - bungieMembershipId: number, - destinyVersion: DestinyVersion, - payload: DeleteSearchUpdate['payload'], -): Promise { - const start = new Date(); - await deleteSearchInDb( - client, - bungieMembershipId, - destinyVersion, - payload.query, - payload.type ?? SearchType.Item, - ); - metrics.timing('update.deleteSearch', start); -} - -async function updateItemHashTag( - client: ClientBase, - appId: string, - bungieMembershipId: number, - payload: ItemHashTagUpdate['payload'], -): Promise { - const start = new Date(); - await updateItemHashTagInDb(client, appId, bungieMembershipId, payload); - metrics.timing('update.updateItemHashTag', start); -} - function consolidateSearchUpdates( updates: (UsedSearchUpdate | SavedSearchUpdate | DeleteSearchUpdate)[], ) { diff --git a/api/server.test.ts b/api/server.test.ts index 3830ea4..d524aef 100644 --- a/api/server.test.ts +++ b/api/server.test.ts @@ -4,7 +4,6 @@ import { makeFetch } from 'supertest-fetch'; import { promisify } from 'util'; import { v4 as uuid } from 'uuid'; import { refreshApps } from './apps/index.js'; -import { closeDbPool } from './db/index.js'; import { app } from './server.js'; import { ApiApp } from './shapes/app.js'; import { DeleteAllResponse } from './shapes/delete-all.js'; @@ -59,8 +58,6 @@ beforeAll(async () => { await client.putBatch(...globalSettings); }); -afterAll(() => closeDbPool()); - it('returns basic info from GET /', async () => { // Sends GET Request to / endpoint const response = await fetch('/'); diff --git a/api/stately/init/migrate-loadout-shares.ts b/api/stately/init/migrate-loadout-shares.ts deleted file mode 100644 index a5c8108..0000000 --- a/api/stately/init/migrate-loadout-shares.ts +++ /dev/null @@ -1,6 +0,0 @@ -// import { migrateLoadoutShareChunk } from '../migrator/loadout-shares.js'; - -// while (true) { -// await migrateLoadoutShareChunk(); -// console.log('Migrated loadout shares'); -// } diff --git a/api/stately/init/migrate-users.ts b/api/stately/init/migrate-users.ts deleted file mode 100644 index 6a6f88a..0000000 --- a/api/stately/init/migrate-users.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { chunk } from 'es-toolkit'; -import { readTransaction } from '../../db/index.js'; -import { getUsersToMigrate } from '../../db/migration-state-queries.js'; -import { delay } from '../../utils.js'; -import { migrateUser } from '../migrator/user.js'; - -while (true) { - try { - const bungieMembershipIds = await readTransaction(async (client) => getUsersToMigrate(client)); - if (bungieMembershipIds.length === 0) { - console.log('No users to migrate'); - break; - } - for (const idChunk of chunk(bungieMembershipIds, 10)) { - await Promise.all( - idChunk.map(async (bungieMembershipId) => { - try { - await migrateUser(bungieMembershipId); - console.log(`Migrated user ${bungieMembershipId}`); - } catch (e) { - if (e instanceof Error) { - console.error(`Error migrating user ${bungieMembershipId}: ${e}`); - } - } - }), - ); - } - } catch (e) { - if (e instanceof Error) { - console.error(`Error getting users to migrate: ${e}`); - } - await delay(1000); - } -} diff --git a/api/stately/migrator/loadout-shares.ts b/api/stately/migrator/loadout-shares.ts deleted file mode 100644 index 9940d32..0000000 --- a/api/stately/migrator/loadout-shares.ts +++ /dev/null @@ -1,15 +0,0 @@ -// import { transaction } from '../../db/index.js'; -// import { deleteLoadoutShares, getLoadoutShares } from '../../db/loadout-share-queries.js'; -// import { addLoadoutSharesForMigration } from '../loadout-share-queries.js'; - -// export async function migrateLoadoutShareChunk() { -// await transaction(async (db) => { -// const loadouts = await getLoadoutShares(db, 50); -// await Promise.all(loadouts.map((loadout) => addLoadoutSharesForMigration([loadout]))); -// console.log('Added to stately'); -// await deleteLoadoutShares( -// db, -// loadouts.map((loadout) => loadout.shareId), -// ); -// }); -// } diff --git a/api/stately/migrator/user.ts b/api/stately/migrator/user.ts deleted file mode 100644 index 31ada9b..0000000 --- a/api/stately/migrator/user.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { isEmpty } from 'es-toolkit/compat'; -import { doMigration } from '../../db/migration-state-queries.js'; -import { pgExport } from '../../routes/export.js'; -import { extractImportData, statelyImport } from '../../routes/import.js'; - -export async function migrateUser(bungieMembershipId: number): Promise { - const importToStately = async () => { - // Export from Postgres - const exportResponse = await pgExport(bungieMembershipId); - - const { settings, loadouts, itemAnnotations, triumphs, searches, itemHashTags } = - extractImportData(exportResponse); - - const profileIds = new Set(); - exportResponse.loadouts.forEach((l) => profileIds.add(l.platformMembershipId)); - exportResponse.tags.forEach((t) => profileIds.add(t.platformMembershipId)); - exportResponse.triumphs.forEach((t) => profileIds.add(t.platformMembershipId)); - - if ( - isEmpty(settings) && - loadouts.length === 0 && - itemAnnotations.length === 0 && - triumphs.length === 0 && - searches.length === 0 - ) { - // Nothing to import! - return; - } - await statelyImport( - bungieMembershipId, - [...profileIds], - settings, - loadouts, - itemAnnotations, - triumphs, - searches, - itemHashTags, - false, - ); - }; - - // For now let's leave the old data in Postgres as a backup - await doMigration(bungieMembershipId, importToStately); -} diff --git a/build/.env.travis b/build/.env.travis index dd9fb3d..2336de6 100644 --- a/build/.env.travis +++ b/build/.env.travis @@ -1,7 +1 @@ -PGHOST=localhost -PGPORT=5432 -PGDATABASE=travis_ci_test -PGUSER=postgres -PGPASSWORD=postgres JWT_SECRET=dummysecret -PGSSL=false \ No newline at end of file diff --git a/kubernetes/dim-api-configmap.yaml b/kubernetes/dim-api-configmap.yaml index e2e588c..de76d63 100644 --- a/kubernetes/dim-api-configmap.yaml +++ b/kubernetes/dim-api-configmap.yaml @@ -5,10 +5,5 @@ metadata: labels: app: dim-api data: - PGHOST: placeholder - PGPORT: placeholder - PGDATABASE: placeholder - PGUSER: placeholder - PGSSLMODE: require SENTRY_DSN: placeholder - SENTRY_STORE_ID: placeholder + STATELY_STORE_ID: placeholder diff --git a/kubernetes/dim-api-deployment.yaml b/kubernetes/dim-api-deployment.yaml index c8db3c1..e24c803 100644 --- a/kubernetes/dim-api-deployment.yaml +++ b/kubernetes/dim-api-deployment.yaml @@ -49,11 +49,6 @@ spec: cpu: "150m" memory: "50Mi" env: - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: dim-api-secret - key: pg_password - name: JWT_SECRET valueFrom: secretKeyRef: diff --git a/package.json b/package.json index 22b94c7..b50df58 100644 --- a/package.json +++ b/package.json @@ -41,13 +41,10 @@ "@types/jest": "^29.5.14", "@types/jsonwebtoken": "^9.0.7", "@types/morgan": "^1.9.9", - "@types/pg": "^8.11.10", "@types/uuid": "^10.0.0", "@types/vhost": "3.0.7", "@typescript-eslint/eslint-plugin": "^8.14.0", "@typescript-eslint/parser": "^8.14.0", - "db-migrate": "^0.11.14", - "db-migrate-pg": "^1.5.2", "eslint": "^9.14.0", "eslint-plugin-regexp": "^2.7.0", "eslint-plugin-sonarjs": "^1.0.4", @@ -84,8 +81,6 @@ "hot-shots": "^10.2.1", "jsonwebtoken": "^9.0.2", "morgan": "^1.10.0", - "pg": "^8.13.1", - "pg-protocol": "^1.7.0", "slugify": "^1.6.6", "uuid": "^11.0.3", "vhost": "^3.0.2" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0de1cf0..a2a1c03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -62,12 +62,6 @@ dependencies: morgan: specifier: ^1.10.0 version: 1.10.0 - pg: - specifier: ^8.13.1 - version: 8.13.1 - pg-protocol: - specifier: ^1.7.0 - version: 1.7.0 slugify: specifier: ^1.6.6 version: 1.6.6 @@ -127,9 +121,6 @@ devDependencies: '@types/morgan': specifier: ^1.9.9 version: 1.9.9 - '@types/pg': - specifier: ^8.11.10 - version: 8.11.10 '@types/uuid': specifier: ^10.0.0 version: 10.0.0 @@ -142,12 +133,6 @@ devDependencies: '@typescript-eslint/parser': specifier: ^8.14.0 version: 8.14.0(eslint@9.14.0)(typescript@5.6.3) - db-migrate: - specifier: ^0.11.14 - version: 0.11.14 - db-migrate-pg: - specifier: ^1.5.2 - version: 1.5.2 eslint: specifier: ^9.14.0 version: 9.14.0 @@ -1452,11 +1437,6 @@ packages: /@bufbuild/protobuf@2.2.2: resolution: {integrity: sha512-UNtPCbrwrenpmrXuRwn9jYpPoweNXj8X5sMvYgsqYyaH8jQ6LfUJSk3dJLnBK+6sfYPrF4iAIo5sd5HQ+tg75A==} - /@colors/colors@1.5.0: - resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} - engines: {node: '>=0.1.90'} - dev: true - /@connectrpc/connect-node@2.0.0(@bufbuild/protobuf@2.2.2)(@connectrpc/connect@2.0.0): resolution: {integrity: sha512-DoI5T+SUvlS/8QBsxt2iDoUg15dSxqhckegrgZpWOtADtmGohBIVbx1UjtWmjLBrP4RdD0FeBw+XyRUSbpKnJQ==} engines: {node: '>=18.14.1'} @@ -2837,14 +2817,6 @@ packages: dependencies: undici-types: 6.19.8 - /@types/pg@8.11.10: - resolution: {integrity: sha512-LczQUW4dbOQzsH2RQ5qoeJ6qJPdrcM/DcMLoqWQkMLMsq83J5lAX3LXjdkWdpscFy67JSOWDnh7Ny/sPFykmkg==} - dependencies: - '@types/node': 22.9.0 - pg-protocol: 1.7.0 - pg-types: 4.0.2 - dev: true - /@types/qs@6.9.17: resolution: {integrity: sha512-rX4/bPcfmvxHDv0XjfJELTTr+iB+tn032nPILqHm5wbthUUUuVtNGGqzhya9XUxjTP8Fpr0qYgSZZKxGY++svQ==} dev: true @@ -3179,22 +3151,6 @@ packages: engines: {node: '>=8'} dev: false - /asn1@0.2.6: - resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} - dependencies: - safer-buffer: 2.1.2 - dev: true - - /async@2.6.4: - resolution: {integrity: sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==} - dependencies: - lodash: 4.17.21 - dev: true - - /async@3.2.3: - resolution: {integrity: sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==} - dev: true - /async@3.2.6: resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} @@ -3327,12 +3283,6 @@ packages: safe-buffer: 5.1.2 dev: false - /bcrypt-pbkdf@1.0.2: - resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} - dependencies: - tweetnacl: 0.14.5 - dev: true - /bignumber.js@9.1.2: resolution: {integrity: sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==} dev: false @@ -3349,10 +3299,6 @@ packages: file-uri-to-path: 1.0.0 dev: false - /bluebird@3.7.2: - resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} - dev: true - /body-parser@1.20.3: resolution: {integrity: sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==} engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} @@ -3504,14 +3450,6 @@ packages: resolution: {integrity: sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA==} dev: true - /cliui@6.0.0: - resolution: {integrity: sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==} - dependencies: - string-width: 4.2.3 - strip-ansi: 6.0.1 - wrap-ansi: 6.2.0 - dev: true - /cliui@8.0.1: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} @@ -3543,11 +3481,6 @@ packages: hasBin: true dev: false - /colors@1.0.3: - resolution: {integrity: sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==} - engines: {node: '>=0.1.90'} - dev: true - /combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -3610,15 +3543,6 @@ packages: vary: 1.1.2 dev: false - /cpu-features@0.0.2: - resolution: {integrity: sha512-/2yieBqvMcRj8McNzkycjW2v3OIUOibBfd2dLEJ0nWts8NobAxwiyw9phVNS6oDL8x8tz9F7uNVFEVpJncQpeA==} - engines: {node: '>=8.0.0'} - requiresBuild: true - dependencies: - nan: 2.22.0 - dev: true - optional: true - /create-jest@29.7.0(@types/node@22.9.0)(ts-node@10.9.2): resolution: {integrity: sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} @@ -3651,56 +3575,6 @@ packages: which: 2.0.2 dev: true - /cycle@1.0.3: - resolution: {integrity: sha512-TVF6svNzeQCOpjCqsy0/CSy8VgObG3wXusJ73xW2GbG5rGx7lC8zxDSURicsXI2UsGdi2L0QNRCi745/wUDvsA==} - engines: {node: '>=0.4.0'} - dev: true - - /db-migrate-base@2.3.1: - resolution: {integrity: sha512-HewYQ3HPmy7NOWmhhMLg9TzN1StEtSqGL3w8IbBRCxEsJ+oM3bDUQ/z5fqpYKfIUK07mMXieCmZYwFpwWkSHDw==} - dependencies: - bluebird: 3.7.2 - dev: true - - /db-migrate-pg@1.5.2: - resolution: {integrity: sha512-agbT9biJi43E7wld9JgnpMKadYgIobMlRXdtRO8JLRWHI1Jc7mObl9pM7iv4AQ4UTLDgjtkqUqtXlfeWtRuRbA==} - dependencies: - bluebird: 3.7.2 - db-migrate-base: 2.3.1 - pg: 8.13.1 - semver: 7.6.3 - transitivePeerDependencies: - - pg-native - dev: true - - /db-migrate-shared@1.2.0: - resolution: {integrity: sha512-65k86bVeHaMxb2L0Gw3y5V+CgZSRwhVQMwDMydmw5MvIpHHwD6SmBciqIwHsZfzJ9yzV/yYhdRefRM6FV5/siw==} - dev: true - - /db-migrate@0.11.14: - resolution: {integrity: sha512-8e+/YsIlM3d69hj+cb6qG6WyubR8nwXfDf9gGLWl4AxHpI6mTomcqhRLNfPrs7Le7AZv2eEsgK8hkXDSQqfIvg==} - engines: {node: '>=8.0.0'} - hasBin: true - dependencies: - balanced-match: 1.0.2 - bluebird: 3.7.2 - db-migrate-shared: 1.2.0 - deep-extend: 0.6.0 - dotenv: 5.0.1 - final-fs: 1.6.1 - inflection: 1.13.4 - mkdirp: 0.5.6 - parse-database-url: 0.3.0 - prompt: 1.3.0 - rc: 1.2.8 - resolve: 1.22.8 - semver: 5.7.2 - tunnel-ssh: 4.1.6 - yargs: 15.4.1 - transitivePeerDependencies: - - supports-color - dev: true - /debug@2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} peerDependencies: @@ -3710,6 +3584,7 @@ packages: optional: true dependencies: ms: 2.0.0 + dev: false /debug@4.3.7: resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==} @@ -3722,11 +3597,6 @@ packages: dependencies: ms: 2.1.3 - /decamelize@1.2.0: - resolution: {integrity: sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==} - engines: {node: '>=0.10.0'} - dev: true - /dedent@1.5.3: resolution: {integrity: sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==} peerDependencies: @@ -3736,11 +3606,6 @@ packages: optional: true dev: true - /deep-extend@0.6.0: - resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} - engines: {node: '>=4.0.0'} - dev: true - /deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} dev: true @@ -3815,11 +3680,6 @@ packages: engines: {node: '>=12'} dev: false - /dotenv@5.0.1: - resolution: {integrity: sha512-4As8uPrjfwb7VXC+WnLCbXK7y+Ueb2B3zgNCePYfhxS1PYeaO1YTeplffTEcbfLhvFNGLAz90VvJs9yomG7bow==} - engines: {node: '>=4.6.0'} - dev: true - /duplexify@4.1.3: resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==} dependencies: @@ -4202,11 +4062,6 @@ packages: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} dev: false - /eyes@0.1.8: - resolution: {integrity: sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==} - engines: {node: '> 0.1.90'} - dev: true - /fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: true @@ -4271,13 +4126,6 @@ packages: to-regex-range: 5.0.1 dev: true - /final-fs@1.6.1: - resolution: {integrity: sha512-r5dgz23H8qh1LxKVJK84zet2PhWSWkIOgbLVUd5PlNFAULD/kCDBH9JEMwJt9dpdTnLsSD4rEqS56p2MH7Wbvw==} - dependencies: - node-fs: 0.1.7 - when: 2.0.1 - dev: true - /finalhandler@1.3.1: resolution: {integrity: sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==} engines: {node: '>= 0.8'} @@ -4683,11 +4531,6 @@ packages: engines: {node: '>=0.8.19'} dev: true - /inflection@1.13.4: - resolution: {integrity: sha512-6I/HUDeYFfuNCVS3td055BaXBwKYuzw7K3ExVMStBowKo9oOAMJIXIHvdyR3iboTCp1b+1i5DSkIZTcwIktuDw==} - engines: {'0': node >= 0.4.0} - dev: true - /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. @@ -4698,10 +4541,6 @@ packages: /inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - /ini@1.3.8: - resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - dev: true - /ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} @@ -4768,10 +4607,6 @@ packages: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} dev: true - /isstream@0.1.2: - resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} - dev: true - /istanbul-lib-coverage@3.2.2: resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} engines: {node: '>=8'} @@ -5418,10 +5253,6 @@ packages: resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} dev: true - /lodash.defaults@4.2.0: - resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} - dev: true - /lodash.includes@4.3.0: resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} dev: false @@ -5462,10 +5293,6 @@ packages: resolution: {integrity: sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==} dev: false - /lodash@4.17.21: - resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} - dev: true - /long@5.2.3: resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==} dev: false @@ -5596,23 +5423,11 @@ packages: yallist: 4.0.0 dev: false - /mkdirp@0.5.6: - resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} - hasBin: true - dependencies: - minimist: 1.2.8 - dev: true - /mkdirp@1.0.4: resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} engines: {node: '>=10'} hasBin: true - /mongodb-uri@0.9.7: - resolution: {integrity: sha512-s6BdnqNoEYfViPJgkH85X5Nw5NpzxN8hoflKLweNa7vBxt2V7kaS06d74pAtqDxde8fn4r9h4dNdLiFGoNV0KA==} - engines: {node: '>= 0.6.0'} - dev: true - /morgan@1.10.0: resolution: {integrity: sha512-AbegBVI4sh6El+1gNwvD5YIck7nSA36weD7xvIxG4in80j/UoK8AEGaWnnz8v1GxonMCltmlNs5ZKbGvl9b1XQ==} engines: {node: '>= 0.8.0'} @@ -5628,17 +5443,15 @@ packages: /ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + dev: false /ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - /mute-stream@0.0.8: - resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==} - dev: true - /nan@2.22.0: resolution: {integrity: sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==} requiresBuild: true + dev: false /natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -5660,12 +5473,6 @@ packages: dependencies: whatwg-url: 5.0.0 - /node-fs@0.1.7: - resolution: {integrity: sha512-XqDBlmUKgDGe76+lZ/0sRBF3XW2vVcK07+ZPvdpUTK8jrvtPahUd0aBqJ9+ZjB01ANjZLuvK3O/eoMVmz62rpA==} - engines: {node: '>=0.1.97'} - os: [linux, darwin, freebsd, win32, smartos, sunos] - dev: true - /node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} dev: true @@ -5723,10 +5530,6 @@ packages: engines: {node: '>= 0.4'} dev: false - /obuf@1.1.2: - resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} - dev: true - /on-finished@2.3.0: resolution: {integrity: sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==} engines: {node: '>= 0.8'} @@ -5809,13 +5612,6 @@ packages: callsites: 3.1.0 dev: true - /parse-database-url@0.3.0: - resolution: {integrity: sha512-YRxDoVBAUk3ksGF9pud+aqWwXmThZzhX9Z1PPxKU03BB3/gu2RcgyMA4rktMYhkIJ9KxwW7lIj00U+TSNz80wg==} - engines: {node: '>= 0.6'} - dependencies: - mongodb-uri: 0.9.7 - dev: true - /parse-duration@1.1.0: resolution: {integrity: sha512-z6t9dvSJYaPoQq7quMzdEagSFtpGu+utzHqqxmpVWNNZRIXnvqyCvn9XsTdh7c/w0Bqmdz3RB3YnRaKtpRtEXQ==} dev: false @@ -5862,78 +5658,6 @@ packages: resolution: {integrity: sha512-7lf7qcQidTku0Gu3YDPc8DJ1q7OOucfa/BSsIwjuh56VU7katFvuM8hULfkwB3Fns/rsVF7PwPKVw1sl5KQS9w==} dev: false - /pg-cloudflare@1.1.1: - resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==} - requiresBuild: true - optional: true - - /pg-connection-string@2.7.0: - resolution: {integrity: sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==} - - /pg-int8@1.0.1: - resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} - engines: {node: '>=4.0.0'} - - /pg-numeric@1.0.2: - resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} - engines: {node: '>=4'} - dev: true - - /pg-pool@3.7.0(pg@8.13.1): - resolution: {integrity: sha512-ZOBQForurqh4zZWjrgSwwAtzJ7QiRX0ovFkZr2klsen3Nm0aoh33Ls0fzfv3imeH/nw/O27cjdz5kzYJfeGp/g==} - peerDependencies: - pg: '>=8.0' - dependencies: - pg: 8.13.1 - - /pg-protocol@1.7.0: - resolution: {integrity: sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ==} - - /pg-types@2.2.0: - resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} - engines: {node: '>=4'} - dependencies: - pg-int8: 1.0.1 - postgres-array: 2.0.0 - postgres-bytea: 1.0.0 - postgres-date: 1.0.7 - postgres-interval: 1.2.0 - - /pg-types@4.0.2: - resolution: {integrity: sha512-cRL3JpS3lKMGsKaWndugWQoLOCoP+Cic8oseVcbr0qhPzYD5DWXK+RZ9LY9wxRf7RQia4SCwQlXk0q6FCPrVng==} - engines: {node: '>=10'} - dependencies: - pg-int8: 1.0.1 - pg-numeric: 1.0.2 - postgres-array: 3.0.2 - postgres-bytea: 3.0.0 - postgres-date: 2.1.0 - postgres-interval: 3.0.0 - postgres-range: 1.1.4 - dev: true - - /pg@8.13.1: - resolution: {integrity: sha512-OUir1A0rPNZlX//c7ksiu7crsGZTKSOXJPgtNiHGIlC9H0lO+NC6ZDYksSgBYY/thSWhnSRBv8w1lieNNGATNQ==} - engines: {node: '>= 8.0.0'} - peerDependencies: - pg-native: '>=3.0.1' - peerDependenciesMeta: - pg-native: - optional: true - dependencies: - pg-connection-string: 2.7.0 - pg-pool: 3.7.0(pg@8.13.1) - pg-protocol: 1.7.0 - pg-types: 2.2.0 - pgpass: 1.0.5 - optionalDependencies: - pg-cloudflare: 1.1.1 - - /pgpass@1.0.5: - resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} - dependencies: - split2: 4.2.0 - /picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} dev: true @@ -5960,50 +5684,6 @@ packages: find-up: 4.1.0 dev: true - /postgres-array@2.0.0: - resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} - engines: {node: '>=4'} - - /postgres-array@3.0.2: - resolution: {integrity: sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==} - engines: {node: '>=12'} - dev: true - - /postgres-bytea@1.0.0: - resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} - engines: {node: '>=0.10.0'} - - /postgres-bytea@3.0.0: - resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} - engines: {node: '>= 6'} - dependencies: - obuf: 1.1.2 - dev: true - - /postgres-date@1.0.7: - resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} - engines: {node: '>=0.10.0'} - - /postgres-date@2.1.0: - resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} - engines: {node: '>=12'} - dev: true - - /postgres-interval@1.2.0: - resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} - engines: {node: '>=0.10.0'} - dependencies: - xtend: 4.0.2 - - /postgres-interval@3.0.0: - resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} - engines: {node: '>=12'} - dev: true - - /postgres-range@1.1.4: - resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} - dev: true - /pprof@4.0.0: resolution: {integrity: sha512-Yhfk7Y0G1MYsy97oXxmSG5nvbM1sCz9EALiNhW/isAv5Xf7svzP+1RfGeBlS6mLSgRJvgSLh6Mi5DaisQuPttw==} engines: {node: '>=14.0.0'} @@ -6069,17 +5749,6 @@ packages: engines: {node: '>=0.4.0'} dev: true - /prompt@1.3.0: - resolution: {integrity: sha512-ZkaRWtaLBZl7KKAKndKYUL8WqNT+cQHKRZnT4RYYms48jQkFw3rrBL+/N5K/KtdEveHkxs982MX2BkDKub2ZMg==} - engines: {node: '>= 6.0.0'} - dependencies: - '@colors/colors': 1.5.0 - async: 3.2.3 - read: 1.0.7 - revalidator: 0.1.8 - winston: 2.4.7 - dev: true - /prompts@2.4.2: resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} engines: {node: '>= 6'} @@ -6194,27 +5863,10 @@ packages: unpipe: 1.0.0 dev: false - /rc@1.2.8: - resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} - hasBin: true - dependencies: - deep-extend: 0.6.0 - ini: 1.3.8 - minimist: 1.2.8 - strip-json-comments: 2.0.1 - dev: true - /react-is@18.3.1: resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} dev: true - /read@1.0.7: - resolution: {integrity: sha512-rSOKNYUmaxy0om1BNjMN4ezNT6VKK+2xF4GBhc81mkH7L60i6dp8qPYrkndNLT3QPphoII3maL9PVC9XmhHwVQ==} - engines: {node: '>=0.8'} - dependencies: - mute-stream: 0.0.8 - dev: true - /readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} @@ -6294,10 +5946,6 @@ packages: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} - /require-main-filename@2.0.0: - resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} - dev: true - /resolve-cwd@3.0.0: resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} engines: {node: '>=8'} @@ -6350,11 +5998,6 @@ packages: engines: {iojs: '>=1.0.0', node: '>=0.10.0'} dev: true - /revalidator@0.1.8: - resolution: {integrity: sha512-xcBILK2pA9oh4SiinPEZfhP8HfrB/ha+a2fTMyl7Om2WjlDVrOQy99N2MXXlUHqGJz4qEu2duXxHJjDWuK/0xg==} - engines: {node: '>= 0.4.0'} - dev: true - /rimraf@2.7.1: resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==} deprecated: Rimraf versions prior to v4 are no longer supported @@ -6415,6 +6058,7 @@ packages: /safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + dev: false /scslre@0.3.0: resolution: {integrity: sha512-3A6sD0WYP7+QrjbfNA2FN3FsOaGGFoekCVgTyypy53gPxhbkCIjtO6YWgdrfM+n/8sI8JeXZOIxsHjMTNxQ4nQ==} @@ -6425,11 +6069,6 @@ packages: regexp-ast-analysis: 0.7.1 dev: true - /semver@5.7.2: - resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} - hasBin: true - dev: true - /semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -6474,6 +6113,7 @@ packages: /set-blocking@2.0.0: resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} + dev: false /set-function-length@1.2.2: resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==} @@ -6556,10 +6196,6 @@ packages: whatwg-url: 7.1.0 dev: false - /split2@4.2.0: - resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} - engines: {node: '>= 10.x'} - /split@1.0.1: resolution: {integrity: sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==} dependencies: @@ -6570,22 +6206,6 @@ packages: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} dev: true - /ssh2@1.4.0: - resolution: {integrity: sha512-XvXwcXKvS452DyQvCa6Ct+chpucwc/UyxgliYz+rWXJ3jDHdtBb9xgmxJdMmnIn5bpgGAEV3KaEsH98ZGPHqwg==} - engines: {node: '>=10.16.0'} - requiresBuild: true - dependencies: - asn1: 0.2.6 - bcrypt-pbkdf: 1.0.2 - optionalDependencies: - cpu-features: 0.0.2 - nan: 2.22.0 - dev: true - - /stack-trace@0.0.10: - resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==} - dev: true - /stack-utils@2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} @@ -6892,20 +6512,6 @@ packages: fsevents: 2.3.3 dev: true - /tunnel-ssh@4.1.6: - resolution: {integrity: sha512-y7+x+T3F3rkx2Zov5Tk9DGfeEBVAdWU3A/91E0Dk5rrZ/VFIlpV2uhhRuaISJUdyG0N+Lcp1fXZMXz+ovPt5vA==} - dependencies: - debug: 2.6.9 - lodash.defaults: 4.2.0 - ssh2: 1.4.0 - transitivePeerDependencies: - - supports-color - dev: true - - /tweetnacl@0.14.5: - resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} - dev: true - /type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -7087,14 +6693,6 @@ packages: webidl-conversions: 4.0.2 dev: false - /when@2.0.1: - resolution: {integrity: sha512-h0l57vFJ4YQe1/U+C+oqBfAoopxXABUm6VqWM0x2gg4pARru4IUWo/PAxyawWgbGtndXrZbA41EzsfxacZVEXQ==} - dev: true - - /which-module@2.0.1: - resolution: {integrity: sha512-iBdZ57RDvnOR9AGBhML2vFZf7h8vmBjhoaZqODJBFWHVtKkDmKuHai3cx5PgVMrX5YDNp27AofYbAwctSS+vhQ==} - dev: true - /which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -7109,32 +6707,11 @@ packages: string-width: 4.2.3 dev: false - /winston@2.4.7: - resolution: {integrity: sha512-vLB4BqzCKDnnZH9PHGoS2ycawueX4HLqENXQitvFHczhgW2vFpSOn31LZtVr1KU8YTw7DS4tM+cqyovxo8taVg==} - engines: {node: '>= 0.10.0'} - dependencies: - async: 2.6.4 - colors: 1.0.3 - cycle: 1.0.3 - eyes: 0.1.8 - isstream: 0.1.2 - stack-trace: 0.0.10 - dev: true - /word-wrap@1.2.5: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} dev: true - /wrap-ansi@6.2.0: - resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} - engines: {node: '>=8'} - dependencies: - ansi-styles: 4.3.0 - string-width: 4.2.3 - strip-ansi: 6.0.1 - dev: true - /wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -7157,9 +6734,6 @@ packages: /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} - - /y18n@4.0.3: - resolution: {integrity: sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==} dev: true /y18n@5.0.8: @@ -7174,35 +6748,10 @@ packages: resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} dev: false - /yargs-parser@18.1.3: - resolution: {integrity: sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==} - engines: {node: '>=6'} - dependencies: - camelcase: 5.3.1 - decamelize: 1.2.0 - dev: true - /yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} - /yargs@15.4.1: - resolution: {integrity: sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==} - engines: {node: '>=8'} - dependencies: - cliui: 6.0.0 - decamelize: 1.2.0 - find-up: 4.1.0 - get-caller-file: 2.0.5 - require-directory: 2.1.1 - require-main-filename: 2.0.0 - set-blocking: 2.0.0 - string-width: 4.2.3 - which-module: 2.0.1 - y18n: 4.0.3 - yargs-parser: 18.1.3 - dev: true - /yargs@17.7.2: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} From 7bde8b87bf258ca801d061516a41dc57fd732e9b Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Sat, 4 Jan 2025 17:34:44 -0800 Subject: [PATCH 2/3] Found some more --- api/apps/index.ts | 24 ------------------------ api/index.ts | 2 -- api/routes/create-app.ts | 17 ----------------- api/routes/platform-info.ts | 24 ++---------------------- 4 files changed, 2 insertions(+), 65 deletions(-) diff --git a/api/apps/index.ts b/api/apps/index.ts index 701dcb2..66e2c2f 100644 --- a/api/apps/index.ts +++ b/api/apps/index.ts @@ -2,8 +2,6 @@ import * as Sentry from '@sentry/node'; import { ListToken } from '@stately-cloud/client'; import { keyBy } from 'es-toolkit'; import { RequestHandler } from 'express'; -import { getAllApps as getAllAppsPostgres } from '../db/apps-queries.js'; -import { pool } from '../db/index.js'; import { metrics } from '../metrics/index.js'; import { ApiApp } from '../shapes/app.js'; import { getAllApps, updateApps } from '../stately/apps-queries.js'; @@ -72,13 +70,6 @@ export async function refreshApps(): Promise { stopAppsRefresh(); try { - if (apps.length === 0) { - // Start off with a copy from postgres, just in case StatelyDB is having - // problems. - await fetchAppsFromPostgres(); - digestApps(); - } - if (!token) { // First time, get 'em all const [appsFromStately, newToken] = await getAllApps(); @@ -110,21 +101,6 @@ export async function refreshApps(): Promise { } } -async function fetchAppsFromPostgres() { - const client = await pool.connect(); - try { - apps = await getAllAppsPostgres(client); - appsByApiKey = keyBy(apps, (a) => a.dimApiKey.toLowerCase()); - origins = new Set(); - for (const app of apps) { - origins.add(app.origin); - } - return apps; - } finally { - client.release(); - } -} - function digestApps() { appsByApiKey = keyBy(apps, (a) => a.dimApiKey.toLowerCase()); origins = new Set(); diff --git a/api/index.ts b/api/index.ts index 10dc6db..f5fa28e 100644 --- a/api/index.ts +++ b/api/index.ts @@ -7,7 +7,6 @@ import http from 'http'; import morgan from 'morgan'; import vhost from 'vhost'; import { refreshApps, stopAppsRefresh } from './apps/index.js'; -import { closeDbPool } from './db/index.js'; import { app as dimGgApp } from './dim-gg/server.js'; import { metrics } from './metrics/index.js'; import { app as dimApiApp } from './server.js'; @@ -137,7 +136,6 @@ createTerminus(server, { onShutdown: async () => { console.log('Shutting down'); stopAppsRefresh(); - closeDbPool(); }, }); diff --git a/api/routes/create-app.ts b/api/routes/create-app.ts index 81a6ff4..812b0c9 100644 --- a/api/routes/create-app.ts +++ b/api/routes/create-app.ts @@ -1,8 +1,5 @@ import asyncHandler from 'express-async-handler'; -import { DatabaseError } from 'pg-protocol'; import { v4 as uuid } from 'uuid'; -import { insertApp as insertAppPostgres } from '../db/apps-queries.js'; -import { transaction } from '../db/index.js'; import { ApiApp, CreateAppRequest } from '../shapes/app.js'; import { insertApp } from '../stately/apps-queries.js'; import { badRequest } from '../utils.js'; @@ -52,20 +49,6 @@ export const createAppHandler = asyncHandler(async (req, res) => { // Put it in StatelyDB app = await insertApp(app); - // Also put it in Postgres, for now! - await transaction(async (client) => { - try { - await insertAppPostgres(client, app); - } catch (e) { - // This is a unique constraint violation, so just get the app! - if (e instanceof DatabaseError && e.code === '23505') { - await client.query('ROLLBACK'); - } else { - throw e; - } - } - }); - // Only return the recovered app if it's for the same origin and key if (app.origin === originUrl.origin && app.bungieApiKey === request.bungieApiKey) { res.send({ diff --git a/api/routes/platform-info.ts b/api/routes/platform-info.ts index bee6162..c53330a 100644 --- a/api/routes/platform-info.ts +++ b/api/routes/platform-info.ts @@ -1,31 +1,11 @@ import asyncHandler from 'express-async-handler'; -import { pool } from '../db/index.js'; -import { defaultGlobalSettings, GlobalSettings } from '../shapes/global-settings.js'; import { getGlobalSettings } from '../stately/global-settings.js'; -import { camelize } from '../utils.js'; export const platformInfoHandler = asyncHandler(async (req, res) => { const flavor = (req.query.flavor as string) ?? 'app'; - let settings: GlobalSettings | undefined = undefined; - try { - // Try StatelyDB first, then fall back to Postgres - settings = await getGlobalSettings(flavor); - } catch (e) { - console.error('Error loading global settings from Stately:', flavor, e); - } - - if (!settings) { - const result = await pool.query({ - name: 'get_global_settings', - text: 'SELECT * FROM global_settings where flavor = $1 LIMIT 1', - values: [flavor], - }); - settings = - result.rowCount! > 0 - ? { ...defaultGlobalSettings, ...camelize(result.rows[0]) } - : defaultGlobalSettings; - } + // Try StatelyDB first, then fall back to Postgres + const settings = await getGlobalSettings(flavor); // Instruct CF to cache for 15 minutes res.set('Cache-Control', 'public, max-age=900'); From 5befaa2dccd31e743366d78b88867107a5bc4d75 Mon Sep 17 00:00:00 2001 From: Ben Hollis Date: Wed, 8 Jan 2025 19:27:35 -0800 Subject: [PATCH 3/3] Missing default --- api/routes/platform-info.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/routes/platform-info.ts b/api/routes/platform-info.ts index c53330a..86f68ba 100644 --- a/api/routes/platform-info.ts +++ b/api/routes/platform-info.ts @@ -1,11 +1,12 @@ import asyncHandler from 'express-async-handler'; +import { defaultGlobalSettings } from '../shapes/global-settings.js'; import { getGlobalSettings } from '../stately/global-settings.js'; export const platformInfoHandler = asyncHandler(async (req, res) => { const flavor = (req.query.flavor as string) ?? 'app'; // Try StatelyDB first, then fall back to Postgres - const settings = await getGlobalSettings(flavor); + const settings = (await getGlobalSettings(flavor)) ?? defaultGlobalSettings; // Instruct CF to cache for 15 minutes res.set('Cache-Control', 'public, max-age=900');