From 7b7aad0413e69f15f35a037df3b92c348b4fd989 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Wed, 18 Dec 2024 15:56:58 +0100 Subject: [PATCH 01/22] Moderation subjectStatusView with statistics --- lexicons/tools/ozone/moderation/defs.json | 55 ++++++++++++++++ packages/api/src/client/lexicons.ts | 63 +++++++++++++++++++ .../types/tools/ozone/moderation/defs.ts | 52 +++++++++++++++ packages/ozone/src/lexicon/lexicons.ts | 63 +++++++++++++++++++ .../types/tools/ozone/moderation/defs.ts | 52 +++++++++++++++ packages/pds/src/lexicon/lexicons.ts | 63 +++++++++++++++++++ .../types/tools/ozone/moderation/defs.ts | 52 +++++++++++++++ 7 files changed, 400 insertions(+) diff --git a/lexicons/tools/ozone/moderation/defs.json b/lexicons/tools/ozone/moderation/defs.json index 7c590cddb3a..67dcb688f34 100644 --- a/lexicons/tools/ozone/moderation/defs.json +++ b/lexicons/tools/ozone/moderation/defs.json @@ -182,6 +182,61 @@ "tags": { "type": "array", "items": { "type": "string" } + }, + "accountStats": { + "description": "Statistics related to the account subject", + "type": "ref", + "ref": "#accountStats" + }, + "recordsStats": { + "description": "Statistics related to the record subjects authored by the subject's account", + "type": "ref", + "ref": "#recordsStats" + } + } + }, + "accountStats": { + "description": "Statistics about a particular account subject on the labeller", + "type": "object", + "properties": { + "suspendedCount": { + "description": "Number of times the account was suspended", + "type": "integer" + }, + "takedownCount": { + "description": "Number of times the account was taken down", + "type": "integer" + }, + "labels": { + "description": "List of labels currently applied on the account", + "type": "array", + "items": { "type": "string" } + } + } + }, + "recordsStats": { + "description": "Statistics about a set of record subjects on the labeller", + "type": "object", + "properties": { + "subjectCount": { + "description": "Total number of record subjects in the set", + "type": "integer" + }, + "pendingCount": { + "description": "Number of record subjects currently in \"reviewOpen\" or \"reviewEscalated\" state", + "type": "integer" + }, + "processedCount": { + "description": "Number of record subjects currently in \"reviewNone\" or \"reviewClosed\" state", + "type": "integer" + }, + "takendownCount": { + "description": "Number of record subjects currently taken down", + "type": "integer" + }, + "labeledCount": { + "description": "Number of record subjects currently having at least one label applied", + "type": "integer" } } }, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index ececcfe4481..3b00a81d213 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -11261,6 +11261,69 @@ export const schemaDict = { type: 'string', }, }, + accountStats: { + description: 'Statistics related to the account subject', + type: 'ref', + ref: 'lex:tools.ozone.moderation.defs#accountStats', + }, + recordsStats: { + description: + "Statistics related to the record subjects authored by the subject's account", + type: 'ref', + ref: 'lex:tools.ozone.moderation.defs#recordsStats', + }, + }, + }, + accountStats: { + description: + 'Statistics about a particular account subject on the labeller', + type: 'object', + properties: { + suspendedCount: { + description: 'Number of times the account was suspended', + type: 'integer', + }, + takedownCount: { + description: 'Number of times the account was taken down', + type: 'integer', + }, + labels: { + description: 'List of labels currently applied on the account', + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + recordsStats: { + description: + 'Statistics about a set of record subjects on the labeller', + type: 'object', + properties: { + subjectCount: { + description: 'Total number of record subjects in the set', + type: 'integer', + }, + pendingCount: { + description: + 'Number of record subjects currently in "reviewOpen" or "reviewEscalated" state', + type: 'integer', + }, + processedCount: { + description: + 'Number of record subjects currently in "reviewNone" or "reviewClosed" state', + type: 'integer', + }, + takendownCount: { + description: 'Number of record subjects currently taken down', + type: 'integer', + }, + labeledCount: { + description: + 'Number of record subjects currently having at least one label applied', + type: 'integer', + }, }, }, subjectReviewState: { diff --git a/packages/api/src/client/types/tools/ozone/moderation/defs.ts b/packages/api/src/client/types/tools/ozone/moderation/defs.ts index fd58278374b..df5fd795be7 100644 --- a/packages/api/src/client/types/tools/ozone/moderation/defs.ts +++ b/packages/api/src/client/types/tools/ozone/moderation/defs.ts @@ -136,6 +136,8 @@ export interface SubjectStatusView { appealed?: boolean suspendUntil?: string tags?: string[] + accountStats?: AccountStats + recordsStats?: RecordsStats [k: string]: unknown } @@ -151,6 +153,56 @@ export function validateSubjectStatusView(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#subjectStatusView', v) } +/** Statistics about a particular account subject on the labeller */ +export interface AccountStats { + /** Number of times the account was suspended */ + suspendedCount?: number + /** Number of times the account was taken down */ + takedownCount?: number + /** List of labels currently applied on the account */ + labels?: string[] + [k: string]: unknown +} + +export function isAccountStats(v: unknown): v is AccountStats { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'tools.ozone.moderation.defs#accountStats' + ) +} + +export function validateAccountStats(v: unknown): ValidationResult { + return lexicons.validate('tools.ozone.moderation.defs#accountStats', v) +} + +/** Statistics about a set of record subjects on the labeller */ +export interface RecordsStats { + /** Total number of record subjects in the set */ + subjectCount?: number + /** Number of record subjects currently in "reviewOpen" or "reviewEscalated" state */ + pendingCount?: number + /** Number of record subjects currently in "reviewNone" or "reviewClosed" state */ + processedCount?: number + /** Number of record subjects currently taken down */ + takendownCount?: number + /** Number of record subjects currently having at least one label applied */ + labeledCount?: number + [k: string]: unknown +} + +export function isRecordsStats(v: unknown): v is RecordsStats { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'tools.ozone.moderation.defs#recordsStats' + ) +} + +export function validateRecordsStats(v: unknown): ValidationResult { + return lexicons.validate('tools.ozone.moderation.defs#recordsStats', v) +} + export type SubjectReviewState = | 'lex:tools.ozone.moderation.defs#reviewOpen' | 'lex:tools.ozone.moderation.defs#reviewEscalated' diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index ececcfe4481..3b00a81d213 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -11261,6 +11261,69 @@ export const schemaDict = { type: 'string', }, }, + accountStats: { + description: 'Statistics related to the account subject', + type: 'ref', + ref: 'lex:tools.ozone.moderation.defs#accountStats', + }, + recordsStats: { + description: + "Statistics related to the record subjects authored by the subject's account", + type: 'ref', + ref: 'lex:tools.ozone.moderation.defs#recordsStats', + }, + }, + }, + accountStats: { + description: + 'Statistics about a particular account subject on the labeller', + type: 'object', + properties: { + suspendedCount: { + description: 'Number of times the account was suspended', + type: 'integer', + }, + takedownCount: { + description: 'Number of times the account was taken down', + type: 'integer', + }, + labels: { + description: 'List of labels currently applied on the account', + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + recordsStats: { + description: + 'Statistics about a set of record subjects on the labeller', + type: 'object', + properties: { + subjectCount: { + description: 'Total number of record subjects in the set', + type: 'integer', + }, + pendingCount: { + description: + 'Number of record subjects currently in "reviewOpen" or "reviewEscalated" state', + type: 'integer', + }, + processedCount: { + description: + 'Number of record subjects currently in "reviewNone" or "reviewClosed" state', + type: 'integer', + }, + takendownCount: { + description: 'Number of record subjects currently taken down', + type: 'integer', + }, + labeledCount: { + description: + 'Number of record subjects currently having at least one label applied', + type: 'integer', + }, }, }, subjectReviewState: { diff --git a/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts b/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts index 7397fafc19d..5886028f594 100644 --- a/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts +++ b/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts @@ -136,6 +136,8 @@ export interface SubjectStatusView { appealed?: boolean suspendUntil?: string tags?: string[] + accountStats?: AccountStats + recordsStats?: RecordsStats [k: string]: unknown } @@ -151,6 +153,56 @@ export function validateSubjectStatusView(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#subjectStatusView', v) } +/** Statistics about a particular account subject on the labeller */ +export interface AccountStats { + /** Number of times the account was suspended */ + suspendedCount?: number + /** Number of times the account was taken down */ + takedownCount?: number + /** List of labels currently applied on the account */ + labels?: string[] + [k: string]: unknown +} + +export function isAccountStats(v: unknown): v is AccountStats { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'tools.ozone.moderation.defs#accountStats' + ) +} + +export function validateAccountStats(v: unknown): ValidationResult { + return lexicons.validate('tools.ozone.moderation.defs#accountStats', v) +} + +/** Statistics about a set of record subjects on the labeller */ +export interface RecordsStats { + /** Total number of record subjects in the set */ + subjectCount?: number + /** Number of record subjects currently in "reviewOpen" or "reviewEscalated" state */ + pendingCount?: number + /** Number of record subjects currently in "reviewNone" or "reviewClosed" state */ + processedCount?: number + /** Number of record subjects currently taken down */ + takendownCount?: number + /** Number of record subjects currently having at least one label applied */ + labeledCount?: number + [k: string]: unknown +} + +export function isRecordsStats(v: unknown): v is RecordsStats { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'tools.ozone.moderation.defs#recordsStats' + ) +} + +export function validateRecordsStats(v: unknown): ValidationResult { + return lexicons.validate('tools.ozone.moderation.defs#recordsStats', v) +} + export type SubjectReviewState = | 'lex:tools.ozone.moderation.defs#reviewOpen' | 'lex:tools.ozone.moderation.defs#reviewEscalated' diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index ececcfe4481..3b00a81d213 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -11261,6 +11261,69 @@ export const schemaDict = { type: 'string', }, }, + accountStats: { + description: 'Statistics related to the account subject', + type: 'ref', + ref: 'lex:tools.ozone.moderation.defs#accountStats', + }, + recordsStats: { + description: + "Statistics related to the record subjects authored by the subject's account", + type: 'ref', + ref: 'lex:tools.ozone.moderation.defs#recordsStats', + }, + }, + }, + accountStats: { + description: + 'Statistics about a particular account subject on the labeller', + type: 'object', + properties: { + suspendedCount: { + description: 'Number of times the account was suspended', + type: 'integer', + }, + takedownCount: { + description: 'Number of times the account was taken down', + type: 'integer', + }, + labels: { + description: 'List of labels currently applied on the account', + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + recordsStats: { + description: + 'Statistics about a set of record subjects on the labeller', + type: 'object', + properties: { + subjectCount: { + description: 'Total number of record subjects in the set', + type: 'integer', + }, + pendingCount: { + description: + 'Number of record subjects currently in "reviewOpen" or "reviewEscalated" state', + type: 'integer', + }, + processedCount: { + description: + 'Number of record subjects currently in "reviewNone" or "reviewClosed" state', + type: 'integer', + }, + takendownCount: { + description: 'Number of record subjects currently taken down', + type: 'integer', + }, + labeledCount: { + description: + 'Number of record subjects currently having at least one label applied', + type: 'integer', + }, }, }, subjectReviewState: { diff --git a/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts b/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts index 7397fafc19d..5886028f594 100644 --- a/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts +++ b/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts @@ -136,6 +136,8 @@ export interface SubjectStatusView { appealed?: boolean suspendUntil?: string tags?: string[] + accountStats?: AccountStats + recordsStats?: RecordsStats [k: string]: unknown } @@ -151,6 +153,56 @@ export function validateSubjectStatusView(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#subjectStatusView', v) } +/** Statistics about a particular account subject on the labeller */ +export interface AccountStats { + /** Number of times the account was suspended */ + suspendedCount?: number + /** Number of times the account was taken down */ + takedownCount?: number + /** List of labels currently applied on the account */ + labels?: string[] + [k: string]: unknown +} + +export function isAccountStats(v: unknown): v is AccountStats { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'tools.ozone.moderation.defs#accountStats' + ) +} + +export function validateAccountStats(v: unknown): ValidationResult { + return lexicons.validate('tools.ozone.moderation.defs#accountStats', v) +} + +/** Statistics about a set of record subjects on the labeller */ +export interface RecordsStats { + /** Total number of record subjects in the set */ + subjectCount?: number + /** Number of record subjects currently in "reviewOpen" or "reviewEscalated" state */ + pendingCount?: number + /** Number of record subjects currently in "reviewNone" or "reviewClosed" state */ + processedCount?: number + /** Number of record subjects currently taken down */ + takendownCount?: number + /** Number of record subjects currently having at least one label applied */ + labeledCount?: number + [k: string]: unknown +} + +export function isRecordsStats(v: unknown): v is RecordsStats { + return ( + isObj(v) && + hasProp(v, '$type') && + v.$type === 'tools.ozone.moderation.defs#recordsStats' + ) +} + +export function validateRecordsStats(v: unknown): ValidationResult { + return lexicons.validate('tools.ozone.moderation.defs#recordsStats', v) +} + export type SubjectReviewState = | 'lex:tools.ozone.moderation.defs#reviewOpen' | 'lex:tools.ozone.moderation.defs#reviewEscalated' From 3f2e708bd6b5062b1b28b29400adc0d82fef224e Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 20 Dec 2024 19:34:34 +0100 Subject: [PATCH 02/22] add account stats to queue entries --- lexicons/tools/ozone/moderation/defs.json | 45 +++-- packages/api/src/client/lexicons.ts | 50 +++-- .../types/tools/ozone/moderation/defs.ts | 28 ++- .../oauth/oauth-client-browser/package.json | 4 +- .../ozone/src/api/moderation/queryStatuses.ts | 6 +- ...220T144630860Z-stats-materialized-views.ts | 187 ++++++++++++++++++ packages/ozone/src/db/migrations/index.ts | 1 + .../src/db/schema/account_events_stats.ts | 16 ++ .../db/schema/account_record_events_stats.ts | 15 ++ .../db/schema/account_record_status_stats.ts | 15 ++ packages/ozone/src/db/schema/index.ts | 11 +- .../src/db/schema/record_events_stats.ts | 15 ++ packages/ozone/src/lexicon/lexicons.ts | 50 +++-- .../types/tools/ozone/moderation/defs.ts | 28 ++- packages/ozone/src/mod-service/index.ts | 123 +++++------- packages/ozone/src/mod-service/status.ts | 51 +++-- packages/ozone/src/mod-service/types.ts | 22 ++- packages/ozone/src/mod-service/views.ts | 109 +++++----- packages/pds/src/lexicon/lexicons.ts | 50 +++-- .../types/tools/ozone/moderation/defs.ts | 28 ++- 20 files changed, 595 insertions(+), 259 deletions(-) create mode 100644 packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts create mode 100644 packages/ozone/src/db/schema/account_events_stats.ts create mode 100644 packages/ozone/src/db/schema/account_record_events_stats.ts create mode 100644 packages/ozone/src/db/schema/account_record_status_stats.ts create mode 100644 packages/ozone/src/db/schema/record_events_stats.ts diff --git a/lexicons/tools/ozone/moderation/defs.json b/lexicons/tools/ozone/moderation/defs.json index 67dcb688f34..2f637025d9b 100644 --- a/lexicons/tools/ozone/moderation/defs.json +++ b/lexicons/tools/ozone/moderation/defs.json @@ -199,43 +199,58 @@ "description": "Statistics about a particular account subject on the labeller", "type": "object", "properties": { - "suspendedCount": { + "reportCount": { + "description": "Total number of reports on the account", + "type": "integer" + }, + "appealCount": { + "description": "Total number of appeals against a moderation action on the account", + "type": "integer" + }, + "suspendCount": { "description": "Number of times the account was suspended", "type": "integer" }, "takedownCount": { "description": "Number of times the account was taken down", "type": "integer" - }, - "labels": { - "description": "List of labels currently applied on the account", - "type": "array", - "items": { "type": "string" } } } }, "recordsStats": { - "description": "Statistics about a set of record subjects on the labeller", + "description": "Statistics about a set of record subject items on the labeller", "type": "object", "properties": { + "totalReports": { + "description": "Cumulative sum of the number of reports on the items in the set", + "type": "integer" + }, + "reportedCount": { + "description": "Number of items that were reported at least once", + "type": "integer" + }, + "escalatedCount": { + "description": "Number of items that were escalated at least once", + "type": "integer" + }, + "appealedCount": { + "description": "Number of items that were appealed at least once", + "type": "integer" + }, "subjectCount": { - "description": "Total number of record subjects in the set", + "description": "Total number of item in the set", "type": "integer" }, "pendingCount": { - "description": "Number of record subjects currently in \"reviewOpen\" or \"reviewEscalated\" state", + "description": "Number of item currently in \"reviewOpen\" or \"reviewEscalated\" state", "type": "integer" }, "processedCount": { - "description": "Number of record subjects currently in \"reviewNone\" or \"reviewClosed\" state", + "description": "Number of item currently in \"reviewNone\" or \"reviewClosed\" state", "type": "integer" }, "takendownCount": { - "description": "Number of record subjects currently taken down", - "type": "integer" - }, - "labeledCount": { - "description": "Number of record subjects currently having at least one label applied", + "description": "Number of item currently taken down", "type": "integer" } } diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 3b00a81d213..60ce6cdb800 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -11279,7 +11279,16 @@ export const schemaDict = { 'Statistics about a particular account subject on the labeller', type: 'object', properties: { - suspendedCount: { + reportCount: { + description: 'Total number of reports on the account', + type: 'integer', + }, + appealCount: { + description: + 'Total number of appeals against a moderation action on the account', + type: 'integer', + }, + suspendCount: { description: 'Number of times the account was suspended', type: 'integer', }, @@ -11287,41 +11296,46 @@ export const schemaDict = { description: 'Number of times the account was taken down', type: 'integer', }, - labels: { - description: 'List of labels currently applied on the account', - type: 'array', - items: { - type: 'string', - }, - }, }, }, recordsStats: { description: - 'Statistics about a set of record subjects on the labeller', + 'Statistics about a set of record subject items on the labeller', type: 'object', properties: { + totalReports: { + description: + 'Cumulative sum of the number of reports on the items in the set', + type: 'integer', + }, + reportedCount: { + description: 'Number of items that were reported at least once', + type: 'integer', + }, + escalatedCount: { + description: 'Number of items that were escalated at least once', + type: 'integer', + }, + appealedCount: { + description: 'Number of items that were appealed at least once', + type: 'integer', + }, subjectCount: { - description: 'Total number of record subjects in the set', + description: 'Total number of item in the set', type: 'integer', }, pendingCount: { description: - 'Number of record subjects currently in "reviewOpen" or "reviewEscalated" state', + 'Number of item currently in "reviewOpen" or "reviewEscalated" state', type: 'integer', }, processedCount: { description: - 'Number of record subjects currently in "reviewNone" or "reviewClosed" state', + 'Number of item currently in "reviewNone" or "reviewClosed" state', type: 'integer', }, takendownCount: { - description: 'Number of record subjects currently taken down', - type: 'integer', - }, - labeledCount: { - description: - 'Number of record subjects currently having at least one label applied', + description: 'Number of item currently taken down', type: 'integer', }, }, diff --git a/packages/api/src/client/types/tools/ozone/moderation/defs.ts b/packages/api/src/client/types/tools/ozone/moderation/defs.ts index df5fd795be7..4c339f0b3e5 100644 --- a/packages/api/src/client/types/tools/ozone/moderation/defs.ts +++ b/packages/api/src/client/types/tools/ozone/moderation/defs.ts @@ -155,12 +155,14 @@ export function validateSubjectStatusView(v: unknown): ValidationResult { /** Statistics about a particular account subject on the labeller */ export interface AccountStats { + /** Total number of reports on the account */ + reportCount?: number + /** Total number of appeals against a moderation action on the account */ + appealCount?: number /** Number of times the account was suspended */ - suspendedCount?: number + suspendCount?: number /** Number of times the account was taken down */ takedownCount?: number - /** List of labels currently applied on the account */ - labels?: string[] [k: string]: unknown } @@ -176,18 +178,24 @@ export function validateAccountStats(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#accountStats', v) } -/** Statistics about a set of record subjects on the labeller */ +/** Statistics about a set of record subject items on the labeller */ export interface RecordsStats { - /** Total number of record subjects in the set */ + /** Cumulative sum of the number of reports on the items in the set */ + totalReports?: number + /** Number of items that were reported at least once */ + reportedCount?: number + /** Number of items that were escalated at least once */ + escalatedCount?: number + /** Number of items that were appealed at least once */ + appealedCount?: number + /** Total number of item in the set */ subjectCount?: number - /** Number of record subjects currently in "reviewOpen" or "reviewEscalated" state */ + /** Number of item currently in "reviewOpen" or "reviewEscalated" state */ pendingCount?: number - /** Number of record subjects currently in "reviewNone" or "reviewClosed" state */ + /** Number of item currently in "reviewNone" or "reviewClosed" state */ processedCount?: number - /** Number of record subjects currently taken down */ + /** Number of item currently taken down */ takendownCount?: number - /** Number of record subjects currently having at least one label applied */ - labeledCount?: number [k: string]: unknown } diff --git a/packages/oauth/oauth-client-browser/package.json b/packages/oauth/oauth-client-browser/package.json index da86bc296bf..290dad9ed3c 100644 --- a/packages/oauth/oauth-client-browser/package.json +++ b/packages/oauth/oauth-client-browser/package.json @@ -19,12 +19,12 @@ "directory": "packages/oauth/oauth-client-browser" }, "type": "commonjs", - "main": "dist/index.js", + "main": "dist/index.cjs", "types": "dist/index.d.ts", "exports": { ".": { "types": "./dist/index.d.ts", - "default": "./dist/index.js" + "default": "./dist/index.cjs" } }, "files": [ diff --git a/packages/ozone/src/api/moderation/queryStatuses.ts b/packages/ozone/src/api/moderation/queryStatuses.ts index c82fdee7091..305a483ad4f 100644 --- a/packages/ozone/src/api/moderation/queryStatuses.ts +++ b/packages/ozone/src/api/moderation/queryStatuses.ts @@ -29,9 +29,9 @@ export default function (server: Server, ctx: AppContext) { onlyMuted = false, limit = 50, cursor, - tags = [], - excludeTags = [], - collections = [], + tags, + excludeTags, + collections, subjectType, queueCount, queueIndex, diff --git a/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts new file mode 100644 index 00000000000..288164b66d0 --- /dev/null +++ b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts @@ -0,0 +1,187 @@ +import { Kysely, sql } from 'kysely' + +import { REASONAPPEAL } from '../../lexicon/types/com/atproto/moderation/defs' +import { DatabaseSchemaType } from '../schema' + +import { + REVIEWESCALATED, + REVIEWOPEN, +} from '../../lexicon/types/tools/ozone/moderation/defs' +import * as modEvent from '../schema/moderation_event' +import * as modStatus from '../schema/moderation_subject_status' +import * as recordEventsStats from '../schema/record_events_stats' + +export async function up(db: Kysely): Promise { + // ~6sec for 16M events + await db.schema + .createView('account_events_stats') + .materialized() + .ifNotExists() + .as( + (db as Kysely) + .selectFrom('moderation_event') + .where('subjectType', '=', 'com.atproto.admin.defs#repoRef') + .where('subjectUri', 'is', null) + .select('subjectDid') + .select([ + (eb) => + sql`COUNT(*) FILTER( + WHERE ${eb.ref('action')} = 'tools.ozone.moderation.defs#modEventTakedown' + AND ${eb.ref('durationInHours')} IS NULL + )`.as('takedownCount'), + (eb) => + sql`COUNT(*) FILTER( + WHERE ${eb.ref('action')} = 'tools.ozone.moderation.defs#modEventTakedown' + AND ${eb.ref('durationInHours')} IS NOT NULL + )`.as('suspendCount'), + (eb) => + sql`COUNT(*) FILTER( + WHERE ${eb.ref('action')} = 'tools.ozone.moderation.defs#modEventEscalate' + )`.as('escalateCount'), + (eb) => + sql`COUNT(*) FILTER( + WHERE ${eb.ref('action')} = 'tools.ozone.moderation.defs#modEventReport' + AND ${eb.ref('meta')} ->> 'reportType' != ${REASONAPPEAL} + )`.as('reportCount'), + (eb) => + sql`COUNT(*) FILTER( + WHERE ${eb.ref('action')} = 'tools.ozone.moderation.defs#modEventReport' + AND ${eb.ref('meta')} ->> 'reportType' = ${REASONAPPEAL} + )`.as('appealCount'), + ]) + .groupBy('subjectDid'), + ) + .execute() + + // TODO try/catch to ignore existing + await db.schema + .createIndex('account_events_stats_did_idx') + // .ifNotExists() // REquires newer version of kysely + .unique() + .on('account_events_stats') + .column('subjectDid') + .execute() + + // ~50sec for 16M events + await db.schema + .createView('record_events_stats') + .materialized() + .ifNotExists() + .as( + (db as Kysely) + .selectFrom('moderation_event') + .select([ + 'subjectDid', + 'subjectUri', + (eb) => + sql`COUNT(*) FILTER (WHERE ${eb.ref('action')} = 'tools.ozone.moderation.defs#modEventEscalate')`.as( + 'escalateCount', + ), + (eb) => + sql`COUNT(*) FILTER (WHERE ${eb.ref('action')} = 'tools.ozone.moderation.defs#modEventReport' AND ${eb.ref('meta')} ->> 'reportType' != 'com.atproto.moderation.defs#reasonAppeal')`.as( + 'reportCount', + ), + (eb) => + sql`COUNT(*) FILTER (WHERE ${eb.ref('action')} = 'tools.ozone.moderation.defs#modEventReport' AND ${eb.ref('meta')} ->> 'reportType' = 'com.atproto.moderation.defs#reasonAppeal')`.as( + 'appealCount', + ), + ]) + .where('subjectType', '=', 'com.atproto.repo.strongRef') + .where('subjectUri', 'is not', null) + .groupBy(['subjectDid', 'subjectUri']), + ) + .execute() + + await db.schema + .createIndex('record_events_stats_uri_idx') + // .ifNotExists() + .unique() + .on('record_events_stats') + .column('subjectUri') + .execute() + + await db.schema + .createIndex('record_events_stats_did_idx') + // .ifNotExists() + .on('record_events_stats') + .column('subjectDid') + .execute() + + await db.schema + .createView('account_record_events_stats') + .materialized() + .ifNotExists() + .as( + (db as Kysely) + .selectFrom('record_events_stats') + .select([ + 'subjectDid', + (eb) => sql`SUM(${eb.ref('reportCount')})`.as('totalReports'), + (eb) => + sql`COUNT(*) FILTER (WHERE ${eb.ref('reportCount')} > 0)`.as( + 'reportedCount', + ), + (eb) => + sql`COUNT(*) FILTER (WHERE ${eb.ref('escalateCount')} > 0)`.as( + 'escalatedCount', + ), + (eb) => + sql`COUNT(*) FILTER (WHERE ${eb.ref('appealCount')} > 0)`.as( + 'appealedCount', + ), + ]) + .groupBy('subjectDid'), + ) + .execute() + + await db.schema + .createIndex('account_record_events_stats_did_idx') + // .ifNotExists() + .unique() + .on('account_record_events_stats') + .column('subjectDid') + .execute() + + await db.schema + .createView('account_record_status_stats') + .materialized() + .ifNotExists() + .as( + (db as Kysely) + .selectFrom('moderation_subject_status') + .select('did') + .select([ + sql`COUNT(*)`.as('subjectCount'), + (eb) => + sql`COUNT(*) FILTER (WHERE ${eb.ref('reviewState')} IN (${REVIEWOPEN}, ${REVIEWESCALATED}))`.as( + 'pendingCount', + ), + (eb) => + sql`COUNT(*) FILTER (WHERE ${eb.ref('reviewState')} NOT IN (${REVIEWOPEN}, ${REVIEWESCALATED}))`.as( + 'processedCount', + ), + (eb) => + sql`COUNT(*) FILTER (WHERE ${eb.ref('takendown')})`.as( + 'takendownCount', + ), + ]) + .where('recordPath', '!=', '') + .groupBy('did'), + ) + .execute() + + await db.schema + .createIndex('account_record_status_stats_did_idx') + // .ifNotExists() + .unique() + .on('account_record_status_stats') + .column('did') + .execute() +} + +export async function down(db: Kysely): Promise { + db.schema.dropView('account_record_status_stats').materialized().execute() + db.schema.dropView('account_record_events_stats').materialized().execute() + db.schema.dropView('record_events_stats').materialized().execute() + db.schema.dropView('account_events_stats').materialized().execute() +} diff --git a/packages/ozone/src/db/migrations/index.ts b/packages/ozone/src/db/migrations/index.ts index be299b76edb..08103249701 100644 --- a/packages/ozone/src/db/migrations/index.ts +++ b/packages/ozone/src/db/migrations/index.ts @@ -17,3 +17,4 @@ export * as _20241001T205730722Z from './20241001T205730722Z-subject-status-revi export * as _20241008T205730722Z from './20241008T205730722Z-sets' export * as _20241018T205730722Z from './20241018T205730722Z-setting' export * as _20241026T205730722Z from './20241026T205730722Z-add-hosting-status-to-subject-status' +export * as _20241220T144630860Z from './20241220T144630860Z-stats-materialized-views' diff --git a/packages/ozone/src/db/schema/account_events_stats.ts b/packages/ozone/src/db/schema/account_events_stats.ts new file mode 100644 index 00000000000..e0848c29a9f --- /dev/null +++ b/packages/ozone/src/db/schema/account_events_stats.ts @@ -0,0 +1,16 @@ +import { GeneratedAlways, Selectable } from 'kysely' + +export const tableName = 'account_events_stats' + +export type AccountEventsStats = { + subjectDid: GeneratedAlways + takedownCount: GeneratedAlways + suspendCount: GeneratedAlways + escalateCount: GeneratedAlways + reportCount: GeneratedAlways + appealCount: GeneratedAlways +} + +export type AccountEventsStatsRow = Selectable + +export type PartialDB = { [tableName]: AccountEventsStats } diff --git a/packages/ozone/src/db/schema/account_record_events_stats.ts b/packages/ozone/src/db/schema/account_record_events_stats.ts new file mode 100644 index 00000000000..0af4d5b2aaa --- /dev/null +++ b/packages/ozone/src/db/schema/account_record_events_stats.ts @@ -0,0 +1,15 @@ +import { GeneratedAlways, Selectable } from 'kysely' + +export const tableName = 'account_record_events_stats' + +type AccountRecordEventsStats = { + subjectDid: GeneratedAlways + totalReports: GeneratedAlways + reportedCount: GeneratedAlways + escalatedCount: GeneratedAlways + appealedCount: GeneratedAlways +} + +export type AccountRecordEventsStatsRow = Selectable + +export type PartialDB = { [tableName]: AccountRecordEventsStats } diff --git a/packages/ozone/src/db/schema/account_record_status_stats.ts b/packages/ozone/src/db/schema/account_record_status_stats.ts new file mode 100644 index 00000000000..7c333209007 --- /dev/null +++ b/packages/ozone/src/db/schema/account_record_status_stats.ts @@ -0,0 +1,15 @@ +import { GeneratedAlways, Selectable } from 'kysely' + +export const tableName = 'account_record_status_stats' + +type AccountRecordStatusStats = { + did: GeneratedAlways + subjectCount: GeneratedAlways + pendingCount: GeneratedAlways + processedCount: GeneratedAlways + takendownCount: GeneratedAlways +} + +export type AccountRecordStatusStatsRow = Selectable + +export type PartialDB = { [tableName]: AccountRecordStatusStats } diff --git a/packages/ozone/src/db/schema/index.ts b/packages/ozone/src/db/schema/index.ts index ba403c802b8..36a1b3ae376 100644 --- a/packages/ozone/src/db/schema/index.ts +++ b/packages/ozone/src/db/schema/index.ts @@ -11,6 +11,11 @@ import * as set from './ozone_set' import * as member from './member' import * as setting from './setting' +import * as recordEventsStats from './record_events_stats' +import * as accountEventsStats from './account_events_stats' +import * as accountRecordEventsStats from './account_record_events_stats' +import * as accountRecordStatusStats from './account_record_status_stats' + export type DatabaseSchemaType = modEvent.PartialDB & modSubjectStatus.PartialDB & label.PartialDB & @@ -21,7 +26,11 @@ export type DatabaseSchemaType = modEvent.PartialDB & communicationTemplate.PartialDB & set.PartialDB & member.PartialDB & - setting.PartialDB + setting.PartialDB & + accountEventsStats.PartialDB & + recordEventsStats.PartialDB & + accountRecordEventsStats.PartialDB & + accountRecordStatusStats.PartialDB export type DatabaseSchema = Kysely diff --git a/packages/ozone/src/db/schema/record_events_stats.ts b/packages/ozone/src/db/schema/record_events_stats.ts new file mode 100644 index 00000000000..951b7773500 --- /dev/null +++ b/packages/ozone/src/db/schema/record_events_stats.ts @@ -0,0 +1,15 @@ +import { GeneratedAlways, Selectable } from 'kysely' + +export const tableName = 'record_events_stats' + +export type RecordEventsStats = { + subjectDid: GeneratedAlways + subjectUri: GeneratedAlways + escalateCount: GeneratedAlways + reportCount: GeneratedAlways + appealCount: GeneratedAlways +} + +export type RecordEventsStatsRow = Selectable + +export type PartialDB = { [tableName]: RecordEventsStats } diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index 3b00a81d213..60ce6cdb800 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -11279,7 +11279,16 @@ export const schemaDict = { 'Statistics about a particular account subject on the labeller', type: 'object', properties: { - suspendedCount: { + reportCount: { + description: 'Total number of reports on the account', + type: 'integer', + }, + appealCount: { + description: + 'Total number of appeals against a moderation action on the account', + type: 'integer', + }, + suspendCount: { description: 'Number of times the account was suspended', type: 'integer', }, @@ -11287,41 +11296,46 @@ export const schemaDict = { description: 'Number of times the account was taken down', type: 'integer', }, - labels: { - description: 'List of labels currently applied on the account', - type: 'array', - items: { - type: 'string', - }, - }, }, }, recordsStats: { description: - 'Statistics about a set of record subjects on the labeller', + 'Statistics about a set of record subject items on the labeller', type: 'object', properties: { + totalReports: { + description: + 'Cumulative sum of the number of reports on the items in the set', + type: 'integer', + }, + reportedCount: { + description: 'Number of items that were reported at least once', + type: 'integer', + }, + escalatedCount: { + description: 'Number of items that were escalated at least once', + type: 'integer', + }, + appealedCount: { + description: 'Number of items that were appealed at least once', + type: 'integer', + }, subjectCount: { - description: 'Total number of record subjects in the set', + description: 'Total number of item in the set', type: 'integer', }, pendingCount: { description: - 'Number of record subjects currently in "reviewOpen" or "reviewEscalated" state', + 'Number of item currently in "reviewOpen" or "reviewEscalated" state', type: 'integer', }, processedCount: { description: - 'Number of record subjects currently in "reviewNone" or "reviewClosed" state', + 'Number of item currently in "reviewNone" or "reviewClosed" state', type: 'integer', }, takendownCount: { - description: 'Number of record subjects currently taken down', - type: 'integer', - }, - labeledCount: { - description: - 'Number of record subjects currently having at least one label applied', + description: 'Number of item currently taken down', type: 'integer', }, }, diff --git a/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts b/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts index 5886028f594..e108ad4203b 100644 --- a/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts +++ b/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts @@ -155,12 +155,14 @@ export function validateSubjectStatusView(v: unknown): ValidationResult { /** Statistics about a particular account subject on the labeller */ export interface AccountStats { + /** Total number of reports on the account */ + reportCount?: number + /** Total number of appeals against a moderation action on the account */ + appealCount?: number /** Number of times the account was suspended */ - suspendedCount?: number + suspendCount?: number /** Number of times the account was taken down */ takedownCount?: number - /** List of labels currently applied on the account */ - labels?: string[] [k: string]: unknown } @@ -176,18 +178,24 @@ export function validateAccountStats(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#accountStats', v) } -/** Statistics about a set of record subjects on the labeller */ +/** Statistics about a set of record subject items on the labeller */ export interface RecordsStats { - /** Total number of record subjects in the set */ + /** Cumulative sum of the number of reports on the items in the set */ + totalReports?: number + /** Number of items that were reported at least once */ + reportedCount?: number + /** Number of items that were escalated at least once */ + escalatedCount?: number + /** Number of items that were appealed at least once */ + appealedCount?: number + /** Total number of item in the set */ subjectCount?: number - /** Number of record subjects currently in "reviewOpen" or "reviewEscalated" state */ + /** Number of item currently in "reviewOpen" or "reviewEscalated" state */ pendingCount?: number - /** Number of record subjects currently in "reviewNone" or "reviewClosed" state */ + /** Number of item currently in "reviewNone" or "reviewClosed" state */ processedCount?: number - /** Number of record subjects currently taken down */ + /** Number of item currently taken down */ takendownCount?: number - /** Number of record subjects currently having at least one label applied */ - labeledCount?: number [k: string]: unknown } diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index cfc4bf76fa9..9fa37a36626 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -1,6 +1,5 @@ import net from 'node:net' import { Insertable, SelectQueryBuilder, sql } from 'kysely' -import { CID } from 'multiformats/cid' import { AtUri, INVALID_HANDLE } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' import { addHoursToDate, chunkArray } from '@atproto/common' @@ -29,11 +28,13 @@ import { RepoRef, RepoBlobRef } from '../lexicon/types/com/atproto/admin/defs' import { adjustModerationSubjectStatus, getStatusIdentifierFromSubject, + moderationSubjectStatusQueryBuilder, } from './status' import { ModEventType, ModerationEventRow, ModerationSubjectStatusRow, + ModerationSubjectStatusRowWithHandle, ReversibleModerationEvent, } from './types' import { ModerationEvent } from '../db/schema/moderation_event' @@ -312,19 +313,6 @@ export class ModerationService { .executeTakeFirst() } - async getCurrentStatus( - subject: { did: string } | { uri: AtUri } | { cids: CID[] }, - ) { - let builder = this.db.db.selectFrom('moderation_subject_status').selectAll() - if ('did' in subject) { - builder = builder.where('did', '=', subject.did) - } else if ('uri' in subject) { - builder = builder.where('recordPath', '=', subject.uri.toString()) - } - // TODO: Handle the cid status - return await builder.execute() - } - async resolveSubjectsForAccount( did: string, createdBy: string, @@ -900,33 +888,33 @@ export class ModerationService { sortDirection: 'asc' | 'desc' lastReviewedBy?: string sortField: 'lastReviewedAt' | 'lastReportedAt' - tags: string[] - excludeTags: string[] - collections: string[] + tags?: string[] + excludeTags?: string[] + collections?: string[] subjectType?: string - }) { - let builder = this.db.db.selectFrom('moderation_subject_status').selectAll() + }): Promise<{ + statuses: ModerationSubjectStatusRowWithHandle[] + cursor?: string + }> { + let builder = moderationSubjectStatusQueryBuilder(this.db.db) + const { ref } = this.db.db.dynamic if (subject) { const subjectInfo = getStatusIdentifierFromSubject(subject) - builder = builder.where( - 'moderation_subject_status.did', - '=', - subjectInfo.did, - ) + builder = builder.where('mss.did', '=', subjectInfo.did) if (!includeAllUserRecords) { builder = builder.where((qb) => subjectInfo.recordPath - ? qb.where('recordPath', '=', subjectInfo.recordPath) - : qb.where('recordPath', '=', ''), + ? qb.where('mss.recordPath', '=', subjectInfo.recordPath) + : qb.where('mss.recordPath', '=', ''), ) } } else if (subjectType === 'account') { - builder = builder.where('recordPath', '=', '') + builder = builder.where('mss.recordPath', '=', '') } else if (subjectType === 'record') { - builder = builder.where('recordPath', '!=', '') + builder = builder.where('mss.recordPath', '!=', '') } // Only fetch items that belongs to the specified queue when specified @@ -948,112 +936,110 @@ export class ModerationService { } // If subjectType is set to 'account' let that take priority and ignore collections filter - if (collections.length && subjectType !== 'account') { - builder = builder.where('recordPath', '!=', '').where((qb) => { - collections.forEach((collection) => { - qb = qb.orWhere('recordPath', 'like', `${collection}/%`) - }) - return qb - }) + if (subjectType !== 'account' && collections?.length) { + builder = builder + .where('mss.recordPath', '!=', '') + .where((qb) => + collections.reduce( + (qb, collection) => + qb.orWhere('mss.recordPath', 'like', `${collection}/%`), + qb.where(sql`false`), + ), + ) } if (ignoreSubjects?.length) { builder = builder - .where('did', 'not in', ignoreSubjects) - .where('recordPath', 'not in', ignoreSubjects) + .where('mss.did', 'not in', ignoreSubjects) + .where('mss.recordPath', 'not in', ignoreSubjects) } if (reviewState) { - builder = builder.where('reviewState', '=', reviewState) + builder = builder.where('mss.reviewState', '=', reviewState) } if (lastReviewedBy) { - builder = builder.where('lastReviewedBy', '=', lastReviewedBy) + builder = builder.where('mss.lastReviewedBy', '=', lastReviewedBy) } if (reviewedAfter) { - builder = builder.where('lastReviewedAt', '>', reviewedAfter) + builder = builder.where('mss.lastReviewedAt', '>', reviewedAfter) } if (reviewedBefore) { - builder = builder.where('lastReviewedAt', '<', reviewedBefore) + builder = builder.where('mss.lastReviewedAt', '<', reviewedBefore) } if (hostingUpdatedAfter) { - builder = builder.where('hostingUpdatedAt', '>', hostingUpdatedAfter) + builder = builder.where('mss.hostingUpdatedAt', '>', hostingUpdatedAfter) } if (hostingUpdatedBefore) { - builder = builder.where('hostingUpdatedAt', '<', hostingUpdatedBefore) + builder = builder.where('mss.hostingUpdatedAt', '<', hostingUpdatedBefore) } if (hostingDeletedAfter) { - builder = builder.where('hostingDeletedAt', '>', hostingDeletedAfter) + builder = builder.where('mss.hostingDeletedAt', '>', hostingDeletedAfter) } if (hostingDeletedBefore) { - builder = builder.where('hostingDeletedAt', '<', hostingDeletedBefore) + builder = builder.where('mss.hostingDeletedAt', '<', hostingDeletedBefore) } if (hostingStatuses?.length) { - builder = builder.where('hostingStatus', 'in', hostingStatuses) + builder = builder.where('mss.hostingStatus', 'in', hostingStatuses) } if (reportedAfter) { - builder = builder.where('lastReviewedAt', '>', reportedAfter) + builder = builder.where('mss.lastReviewedAt', '>', reportedAfter) } if (reportedBefore) { - builder = builder.where('lastReportedAt', '<', reportedBefore) + builder = builder.where('mss.lastReportedAt', '<', reportedBefore) } if (takendown) { - builder = builder.where('takendown', '=', true) + builder = builder.where('mss.takendown', '=', true) } if (appealed !== undefined) { builder = appealed === false - ? builder.where('appealed', 'is', null) - : builder.where('appealed', '=', appealed) + ? builder.where('mss.appealed', 'is', null) + : builder.where('mss.appealed', '=', appealed) } if (!includeMuted) { builder = builder.where((qb) => qb - .where('muteUntil', '<', new Date().toISOString()) - .orWhere('muteUntil', 'is', null), + .where('mss.muteUntil', '<', new Date().toISOString()) + .orWhere('mss.muteUntil', 'is', null), ) } if (onlyMuted) { builder = builder.where((qb) => qb - .where('muteUntil', '>', new Date().toISOString()) - .orWhere('muteReportingUntil', '>', new Date().toISOString()), + .where('mss.muteUntil', '>', new Date().toISOString()) + .orWhere('mss.muteReportingUntil', '>', new Date().toISOString()), ) } - if (tags.length) { + if (tags?.length) { builder = this.applyTagFilter(builder, tags) } - if (excludeTags.length) { + if (excludeTags?.length) { builder = builder.where((qb) => qb .where( - sql`NOT(${ref( - 'moderation_subject_status.tags', - )} ?| array[${sql.join(excludeTags)}]::TEXT[])`, + sql`NOT(${ref('mss.tags')} ?| array[${sql.join(excludeTags)}]::TEXT[])`, ) .orWhere('tags', 'is', null), ) } - const keyset = new StatusKeyset( - ref(`moderation_subject_status.${sortField}`), - ref('moderation_subject_status.id'), - ) + const keyset = new StatusKeyset(ref(`mss.${sortField}`), ref('mss.id')) const paginatedBuilder = paginate(builder, { limit, cursor, @@ -1067,13 +1053,12 @@ export class ModerationService { const infos = await this.views.getAccoutInfosByDid( results.map((r) => r.did), ) - const resultsWithHandles = results.map((r) => ({ - ...r, - handle: infos.get(r.did)?.handle ?? INVALID_HANDLE, - })) return { - statuses: resultsWithHandles, + statuses: results.map((r) => ({ + ...r, + handle: infos.get(r.did)?.handle ?? INVALID_HANDLE, + })), cursor: keyset.packFromResult(results), } } diff --git a/packages/ozone/src/mod-service/status.ts b/packages/ozone/src/mod-service/status.ts index cc3539e7728..deb198fc636 100644 --- a/packages/ozone/src/mod-service/status.ts +++ b/packages/ozone/src/mod-service/status.ts @@ -1,18 +1,18 @@ // This may require better organization but for now, just dumping functions here containing DB queries for moderation status +import { HOUR } from '@atproto/common' import { AtUri } from '@atproto/syntax' import { Database } from '../db' -import { ModerationSubjectStatus } from '../db/schema/moderation_subject_status' +import DatabaseSchema from '../db/schema' +import { jsonb } from '../db/types' +import { REASONAPPEAL } from '../lexicon/types/com/atproto/moderation/defs' import { - REVIEWOPEN, REVIEWCLOSED, REVIEWESCALATED, REVIEWNONE, + REVIEWOPEN, } from '../lexicon/types/tools/ozone/moderation/defs' import { ModerationEventRow, ModerationSubjectStatusRow } from './types' -import { HOUR } from '@atproto/common' -import { REASONAPPEAL } from '../lexicon/types/com/atproto/moderation/defs' -import { jsonb } from '../db/types' const getSubjectStatusForModerationEvent = ({ currentStatus, @@ -203,6 +203,24 @@ const getSubjectStatusForRecordEvent = ({ return {} } +export const moderationSubjectStatusQueryBuilder = (db: DatabaseSchema) => { + return db + .selectFrom('moderation_subject_status as mss') + .selectAll('mss') + .leftJoin('account_events_stats as aes', (join) => + join.onRef('mss.did', '=', 'aes.subjectDid'), + ) + .selectAll('aes') + .leftJoin('account_record_events_stats as ares', (join) => + join.onRef('mss.did', '=', 'ares.subjectDid'), + ) + .selectAll('ares') + .leftJoin('account_record_status_stats as arss', (join) => + join.onRef('mss.did', '=', 'arss.did'), + ) + .selectAll('arss') +} + // Based on a given moderation action event, this function will update the moderation status of the subject // If there's no existing status, it will create one // If the action event does not affect the status, it will do nothing @@ -393,29 +411,6 @@ export const adjustModerationSubjectStatus = async ( return status || null } -type ModerationSubjectStatusFilter = - | Pick - | Pick - | Pick -export const getModerationSubjectStatus = async ( - db: Database, - filters: ModerationSubjectStatusFilter, -) => { - let builder = db.db - .selectFrom('moderation_subject_status') - // DID will always be passed at the very least - .where('did', '=', filters.did) - .where('recordPath', '=', 'recordPath' in filters ? filters.recordPath : '') - - if ('recordCid' in filters) { - builder = builder.where('recordCid', '=', filters.recordCid) - } else { - builder = builder.where('recordCid', 'is', null) - } - - return builder.executeTakeFirst() -} - export const getStatusIdentifierFromSubject = ( subject: string | AtUri, ): { did: string; recordPath: string } => { diff --git a/packages/ozone/src/mod-service/types.ts b/packages/ozone/src/mod-service/types.ts index 29a63d3e019..a2682750487 100644 --- a/packages/ozone/src/mod-service/types.ts +++ b/packages/ozone/src/mod-service/types.ts @@ -18,8 +18,28 @@ export type ModerationEventRowWithHandle = ModerationEventRow & { creatorHandle?: string | null } export type ModerationSubjectStatusRow = Selectable +export type ModerationSubjectStatusRowWithStats = ModerationSubjectStatusRow & { + // account_events_stats + takedownCount: number | null + suspendCount: number | null + escalateCount: number | null + reportCount: number | null + appealCount: number | null + + // account_record_events_stats + totalReports: number | null + reportedCount: number | null + escalatedCount: number | null + appealedCount: number | null + + // account_record_status_stats + subjectCount: number | null + pendingCount: number | null + processedCount: number | null + takendownCount: number | null +} export type ModerationSubjectStatusRowWithHandle = - ModerationSubjectStatusRow & { handle: string | null } + ModerationSubjectStatusRowWithStats & { handle: string | null } export type ModEventType = | ToolsOzoneModerationDefs.ModEventTakedown diff --git a/packages/ozone/src/mod-service/views.ts b/packages/ozone/src/mod-service/views.ts index 7fde08ca470..ad42f8e77fa 100644 --- a/packages/ozone/src/mod-service/views.ts +++ b/packages/ozone/src/mod-service/views.ts @@ -1,10 +1,6 @@ import { sql } from 'kysely' import { AtUri, INVALID_HANDLE, normalizeDatetimeAlways } from '@atproto/syntax' -import { - AtpAgent, - AppBskyFeedDefs, - ToolsOzoneModerationDefs, -} from '@atproto/api' +import { AtpAgent, AppBskyFeedDefs } from '@atproto/api' import { dedupeStrs } from '@atproto/common' import { BlobRef } from '@atproto/lexicon' import { Keypair } from '@atproto/crypto' @@ -12,7 +8,6 @@ import { Database } from '../db' import { ModEventView, RepoView, - RepoViewDetail, RecordView, RecordViewDetail, BlobView, @@ -34,6 +29,7 @@ import { dbLogger } from '../logger' import { httpLogger } from '../logger' import { ParsedLabelers } from '../util' import { ids } from '../lexicon/lexicons' +import { moderationSubjectStatusQueryBuilder } from './status' export type AuthHeaders = { headers: { @@ -481,11 +477,12 @@ export class ModerationViews { async blob(blobs: BlobRef[]): Promise { if (!blobs.length) return [] const { ref } = this.db.db.dynamic - const modStatusResults = await this.db.db - .selectFrom('moderation_subject_status') + const modStatusResults = await moderationSubjectStatusQueryBuilder( + this.db.db, + ) .where( sql`${ref( - 'moderation_subject_status.blobCids', + 'mss.blobCids', )} @> ${JSON.stringify(blobs.map((blob) => blob.ref.toString()))}`, ) .selectAll() @@ -529,10 +526,10 @@ export class ModerationViews { await Promise.all( res.map(async (labelRow) => { const signedLabel = await this.formatLabelAndEnsureSig(labelRow) - if (!labels.has(labelRow.uri)) { - labels.set(labelRow.uri, []) - } - labels.get(labelRow.uri)?.push(signedLabel) + + const current = labels.get(labelRow.uri) + if (current) current.push(signedLabel) + else labels.set(labelRow.uri, [signedLabel]) }), ) return labels @@ -556,54 +553,37 @@ export class ModerationViews { return signed } - async getSubjectStatus( - subjects: string[], - ): Promise> { - const parsedSubjects = subjects.map((subject) => parseSubjectId(subject)) - const filterForSubject = (did: string, recordPath?: string) => { - return (clause: any) => { - clause = clause - .where('moderation_subject_status.did', '=', did) - .where('moderation_subject_status.recordPath', '=', recordPath || '') - return clause - } - // TODO: Fix the typing here? - } - - const builder = this.db.db - .selectFrom('moderation_subject_status') - .where((clause) => { - parsedSubjects.forEach((subject, i) => { - const applySubjectFilter = filterForSubject( - subject.did, - subject.recordPath, - ) - if (i === 0) { - clause = clause.where(applySubjectFilter) - } else { - clause = clause.orWhere(applySubjectFilter) - } - }) - - return clause - }) - .selectAll() + async getSubjectStatus(subjects: string[]) { + const parsedSubjects = subjects.map(parseSubjectId) + + const builder = moderationSubjectStatusQueryBuilder(this.db.db).where( + (clause) => { + return parsedSubjects.reduce( + (clause, sub) => { + return clause.orWhere((qb) => + qb + .where('mss.did', '=', sub.did) + .where('mss.recordPath', '=', sub.recordPath ?? ''), + ) + }, + // If the array is empty, no result should ne returned + clause.where(sql`false`), + ) + }, + ) const [statusRes, accountsByDid] = await Promise.all([ builder.execute(), this.getAccoutInfosByDid(parsedSubjects.map((s) => s.did)), ]) - return statusRes.reduce((acc, cur) => { - const subject = cur.recordPath - ? formatSubjectId(cur.did, cur.recordPath) - : cur.did - const handle = accountsByDid.get(cur.did)?.handle - return acc.set(subject, { - ...cur, - handle: handle ?? INVALID_HANDLE, - }) - }, new Map()) + return new Map( + statusRes.map((status) => { + const subjectId = formatSubjectId(status.did, status.recordPath) + const handle = accountsByDid.get(status.did)?.handle ?? INVALID_HANDLE + return [subjectId, { ...status, handle }] + }), + ) } formatSubjectStatus( @@ -628,6 +608,23 @@ export class ModerationViews { subjectBlobCids: status.blobCids || [], tags: status.tags || [], subject: subjectFromStatusRow(status).lex(), + + accountStats: { + reportCount: status.reportCount ?? undefined, + appealCount: status.appealCount ?? undefined, + suspendCount: status.suspendCount ?? undefined, + takedownCount: status.takedownCount ?? undefined, + }, + recordsStats: { + totalReports: status.totalReports ?? undefined, + reportedCount: status.reportedCount ?? undefined, + escalatedCount: status.escalatedCount ?? undefined, + appealedCount: status.appealedCount ?? undefined, + subjectCount: status.subjectCount ?? undefined, + pendingCount: status.pendingCount ?? undefined, + processedCount: status.processedCount ?? undefined, + takendownCount: status.takendownCount ?? undefined, + }, } if (status.recordPath !== '') { @@ -677,7 +674,7 @@ type RecordInfo = { indexedAt: string } -function parseSubjectId(subject: string) { +function parseSubjectId(subject: string): { did: string; recordPath?: string } { if (subject.startsWith('did:')) { return { did: subject } } diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 3b00a81d213..60ce6cdb800 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -11279,7 +11279,16 @@ export const schemaDict = { 'Statistics about a particular account subject on the labeller', type: 'object', properties: { - suspendedCount: { + reportCount: { + description: 'Total number of reports on the account', + type: 'integer', + }, + appealCount: { + description: + 'Total number of appeals against a moderation action on the account', + type: 'integer', + }, + suspendCount: { description: 'Number of times the account was suspended', type: 'integer', }, @@ -11287,41 +11296,46 @@ export const schemaDict = { description: 'Number of times the account was taken down', type: 'integer', }, - labels: { - description: 'List of labels currently applied on the account', - type: 'array', - items: { - type: 'string', - }, - }, }, }, recordsStats: { description: - 'Statistics about a set of record subjects on the labeller', + 'Statistics about a set of record subject items on the labeller', type: 'object', properties: { + totalReports: { + description: + 'Cumulative sum of the number of reports on the items in the set', + type: 'integer', + }, + reportedCount: { + description: 'Number of items that were reported at least once', + type: 'integer', + }, + escalatedCount: { + description: 'Number of items that were escalated at least once', + type: 'integer', + }, + appealedCount: { + description: 'Number of items that were appealed at least once', + type: 'integer', + }, subjectCount: { - description: 'Total number of record subjects in the set', + description: 'Total number of item in the set', type: 'integer', }, pendingCount: { description: - 'Number of record subjects currently in "reviewOpen" or "reviewEscalated" state', + 'Number of item currently in "reviewOpen" or "reviewEscalated" state', type: 'integer', }, processedCount: { description: - 'Number of record subjects currently in "reviewNone" or "reviewClosed" state', + 'Number of item currently in "reviewNone" or "reviewClosed" state', type: 'integer', }, takendownCount: { - description: 'Number of record subjects currently taken down', - type: 'integer', - }, - labeledCount: { - description: - 'Number of record subjects currently having at least one label applied', + description: 'Number of item currently taken down', type: 'integer', }, }, diff --git a/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts b/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts index 5886028f594..e108ad4203b 100644 --- a/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts +++ b/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts @@ -155,12 +155,14 @@ export function validateSubjectStatusView(v: unknown): ValidationResult { /** Statistics about a particular account subject on the labeller */ export interface AccountStats { + /** Total number of reports on the account */ + reportCount?: number + /** Total number of appeals against a moderation action on the account */ + appealCount?: number /** Number of times the account was suspended */ - suspendedCount?: number + suspendCount?: number /** Number of times the account was taken down */ takedownCount?: number - /** List of labels currently applied on the account */ - labels?: string[] [k: string]: unknown } @@ -176,18 +178,24 @@ export function validateAccountStats(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#accountStats', v) } -/** Statistics about a set of record subjects on the labeller */ +/** Statistics about a set of record subject items on the labeller */ export interface RecordsStats { - /** Total number of record subjects in the set */ + /** Cumulative sum of the number of reports on the items in the set */ + totalReports?: number + /** Number of items that were reported at least once */ + reportedCount?: number + /** Number of items that were escalated at least once */ + escalatedCount?: number + /** Number of items that were appealed at least once */ + appealedCount?: number + /** Total number of item in the set */ subjectCount?: number - /** Number of record subjects currently in "reviewOpen" or "reviewEscalated" state */ + /** Number of item currently in "reviewOpen" or "reviewEscalated" state */ pendingCount?: number - /** Number of record subjects currently in "reviewNone" or "reviewClosed" state */ + /** Number of item currently in "reviewNone" or "reviewClosed" state */ processedCount?: number - /** Number of record subjects currently taken down */ + /** Number of item currently taken down */ takendownCount?: number - /** Number of record subjects currently having at least one label applied */ - labeledCount?: number [k: string]: unknown } From da81bb8c20f7a0e9a4d9867581fdbfefc64f4af4 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 10 Jan 2025 14:43:07 +0100 Subject: [PATCH 03/22] fix --- packages/oauth/oauth-client-browser/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/oauth/oauth-client-browser/package.json b/packages/oauth/oauth-client-browser/package.json index 290dad9ed3c..da86bc296bf 100644 --- a/packages/oauth/oauth-client-browser/package.json +++ b/packages/oauth/oauth-client-browser/package.json @@ -19,12 +19,12 @@ "directory": "packages/oauth/oauth-client-browser" }, "type": "commonjs", - "main": "dist/index.cjs", + "main": "dist/index.js", "types": "dist/index.d.ts", "exports": { ".": { "types": "./dist/index.d.ts", - "default": "./dist/index.cjs" + "default": "./dist/index.js" } }, "files": [ From 70052ef386e68ca4a92862cd31270c45e6a5a0b8 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 10 Jan 2025 16:21:55 +0100 Subject: [PATCH 04/22] adapt --- packages/ozone/src/mod-service/index.ts | 73 +++++++++++-------------- 1 file changed, 31 insertions(+), 42 deletions(-) diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 9fa37a36626..01b1ce38292 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -1,5 +1,5 @@ import net from 'node:net' -import { Insertable, SelectQueryBuilder, sql } from 'kysely' +import { Insertable, sql } from 'kysely' import { AtUri, INVALID_HANDLE } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' import { addHoursToDate, chunkArray } from '@atproto/common' @@ -793,45 +793,6 @@ export class ModerationService { return result } - applyTagFilter = ( - builder: SelectQueryBuilder, - tags: string[], - ) => { - const { ref } = this.db.db.dynamic - // Build an array of conditions - const conditions = tags - .map((tag) => { - if (tag.includes('&&')) { - // Split by '&&' for AND logic - const subTags = tag - .split('&&') - // Make sure spaces on either sides of '&&' are trimmed - .map((subTag) => subTag.trim()) - // Remove empty strings after trimming is applied - .filter(Boolean) - - if (!subTags.length) return null - - return sql`(${sql.join( - subTags.map( - (subTag) => - sql`${ref('moderation_subject_status.tags')} ? ${subTag}`, - ), - sql` AND `, - )})` - } else { - // Single tag condition - return sql`${ref('moderation_subject_status.tags')} ? ${tag}` - } - }) - .filter(Boolean) - - if (!conditions.length) return builder - - // Combine all conditions with OR - return builder.where(sql`(${sql.join(conditions, sql` OR `)})`) - } - async getSubjectStatuses({ queueCount, queueIndex, @@ -1025,8 +986,24 @@ export class ModerationService { ) } - if (tags?.length) { - builder = this.applyTagFilter(builder, tags) + // ["tag1", "tag2 && tag3", "tag4"] => [["tag1"], ["tag2", "tag3"], ["tag4"]] + const conditions = parseTags(tags) + if (conditions?.length) { + // [["tag1"], ["tag2", "tag3"], ["tag4"]] => (tags ? 'tag1') OR (tags ? 'tag2' AND tags ? 'tag3') OR (tags ? 'tag4') + builder = builder.where((qb) => + conditions.reduce( + (qb, subTags, i) => + // OR every conditions + qb[i === 0 ? 'where' : 'orWhere']((qb) => + subTags.reduce( + // AND every sub subTags + (qb, subTag) => qb.where(sql`${ref('mss.tags')} ? ${subTag}`), + qb, + ), + ), + qb, + ), + ) } if (excludeTags?.length) { @@ -1180,6 +1157,18 @@ export class ModerationService { } } +const parseTags = (tags?: string[]) => + tags + ?.map((tag) => + tag + .split(/\s*&&\s*/g) + .map((subTag) => subTag.trim()) + // Ignore invalid syntax ("", "tag1 &&", "&& tag2", "tag1 && && tag2", etc.) + .filter(Boolean), + ) + // Ignore invalid items + .filter((subTags): subTags is [string, ...string[]] => subTags.length > 0) + const isSafeUrl = (url: URL) => { if (url.protocol !== 'https:') return false if (!url.hostname || url.hostname === 'localhost') return false From 76b1f856a7428c3c3dcdda1c55dd979fe9dd177f Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Tue, 14 Jan 2025 11:32:08 +0100 Subject: [PATCH 05/22] review changes --- lexicons/tools/ozone/moderation/defs.json | 4 +- packages/api/src/client/lexicons.ts | 21 ++- .../types/tools/ozone/moderation/defs.ts | 4 +- packages/bsky/src/lexicon/lexicons.ts | 15 ++ packages/ozone/src/lexicon/lexicons.ts | 21 ++- .../types/tools/ozone/moderation/defs.ts | 4 +- packages/ozone/src/mod-service/index.ts | 148 ++++++++++++++---- packages/ozone/src/mod-service/status.ts | 10 +- packages/ozone/src/mod-service/views.ts | 41 ++--- packages/pds/src/lexicon/lexicons.ts | 21 ++- .../types/tools/ozone/moderation/defs.ts | 4 +- 11 files changed, 220 insertions(+), 73 deletions(-) diff --git a/lexicons/tools/ozone/moderation/defs.json b/lexicons/tools/ozone/moderation/defs.json index 2f637025d9b..a0faf974515 100644 --- a/lexicons/tools/ozone/moderation/defs.json +++ b/lexicons/tools/ozone/moderation/defs.json @@ -196,7 +196,7 @@ } }, "accountStats": { - "description": "Statistics about a particular account subject on the labeller", + "description": "Statistics about a particular account subject", "type": "object", "properties": { "reportCount": { @@ -218,7 +218,7 @@ } }, "recordsStats": { - "description": "Statistics about a set of record subject items on the labeller", + "description": "Statistics about a set of record subject items", "type": "object", "properties": { "totalReports": { diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 60ce6cdb800..df7bf01b474 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -4828,6 +4828,11 @@ export const schemaDict = { ref: 'lex:app.bsky.actor.defs#profileView', }, }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -8335,6 +8340,11 @@ export const schemaDict = { 'If true, response has fallen-back to generic results, and is not scoped using relativeToDid', default: false, }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -9406,6 +9416,11 @@ export const schemaDict = { description: 'DID of the account these suggestions are relative to. If this is returned undefined, suggestions are based on the viewer.', }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -11275,8 +11290,7 @@ export const schemaDict = { }, }, accountStats: { - description: - 'Statistics about a particular account subject on the labeller', + description: 'Statistics about a particular account subject', type: 'object', properties: { reportCount: { @@ -11299,8 +11313,7 @@ export const schemaDict = { }, }, recordsStats: { - description: - 'Statistics about a set of record subject items on the labeller', + description: 'Statistics about a set of record subject items', type: 'object', properties: { totalReports: { diff --git a/packages/api/src/client/types/tools/ozone/moderation/defs.ts b/packages/api/src/client/types/tools/ozone/moderation/defs.ts index 4c339f0b3e5..590e966d359 100644 --- a/packages/api/src/client/types/tools/ozone/moderation/defs.ts +++ b/packages/api/src/client/types/tools/ozone/moderation/defs.ts @@ -153,7 +153,7 @@ export function validateSubjectStatusView(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#subjectStatusView', v) } -/** Statistics about a particular account subject on the labeller */ +/** Statistics about a particular account subject */ export interface AccountStats { /** Total number of reports on the account */ reportCount?: number @@ -178,7 +178,7 @@ export function validateAccountStats(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#accountStats', v) } -/** Statistics about a set of record subject items on the labeller */ +/** Statistics about a set of record subject items */ export interface RecordsStats { /** Cumulative sum of the number of reports on the items in the set */ totalReports?: number diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 28191074e8c..bafb2697927 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -4828,6 +4828,11 @@ export const schemaDict = { ref: 'lex:app.bsky.actor.defs#profileView', }, }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -8335,6 +8340,11 @@ export const schemaDict = { 'If true, response has fallen-back to generic results, and is not scoped using relativeToDid', default: false, }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -9406,6 +9416,11 @@ export const schemaDict = { description: 'DID of the account these suggestions are relative to. If this is returned undefined, suggestions are based on the viewer.', }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index 60ce6cdb800..df7bf01b474 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -4828,6 +4828,11 @@ export const schemaDict = { ref: 'lex:app.bsky.actor.defs#profileView', }, }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -8335,6 +8340,11 @@ export const schemaDict = { 'If true, response has fallen-back to generic results, and is not scoped using relativeToDid', default: false, }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -9406,6 +9416,11 @@ export const schemaDict = { description: 'DID of the account these suggestions are relative to. If this is returned undefined, suggestions are based on the viewer.', }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -11275,8 +11290,7 @@ export const schemaDict = { }, }, accountStats: { - description: - 'Statistics about a particular account subject on the labeller', + description: 'Statistics about a particular account subject', type: 'object', properties: { reportCount: { @@ -11299,8 +11313,7 @@ export const schemaDict = { }, }, recordsStats: { - description: - 'Statistics about a set of record subject items on the labeller', + description: 'Statistics about a set of record subject items', type: 'object', properties: { totalReports: { diff --git a/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts b/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts index e108ad4203b..06e6d2cfaa9 100644 --- a/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts +++ b/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts @@ -153,7 +153,7 @@ export function validateSubjectStatusView(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#subjectStatusView', v) } -/** Statistics about a particular account subject on the labeller */ +/** Statistics about a particular account subject */ export interface AccountStats { /** Total number of reports on the account */ reportCount?: number @@ -178,7 +178,7 @@ export function validateAccountStats(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#accountStats', v) } -/** Statistics about a set of record subject items on the labeller */ +/** Statistics about a set of record subject items */ export interface RecordsStats { /** Cumulative sum of the number of reports on the items in the set */ totalReports?: number diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 01b1ce38292..9468021d603 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -1,5 +1,6 @@ import net from 'node:net' import { Insertable, sql } from 'kysely' +import { CID } from 'multiformats/cid' import { AtUri, INVALID_HANDLE } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' import { addHoursToDate, chunkArray } from '@atproto/common' @@ -313,6 +314,19 @@ export class ModerationService { .executeTakeFirst() } + async getCurrentStatus( + subject: { did: string } | { uri: AtUri } | { cids: CID[] }, + ) { + let builder = this.db.db.selectFrom('moderation_subject_status').selectAll() + if ('did' in subject) { + builder = builder.where('did', '=', subject.did) + } else if ('uri' in subject) { + builder = builder.where('recordPath', '=', subject.uri.toString()) + } + // TODO: Handle the cid status + return await builder.execute() + } + async resolveSubjectsForAccount( did: string, createdBy: string, @@ -863,19 +877,27 @@ export class ModerationService { if (subject) { const subjectInfo = getStatusIdentifierFromSubject(subject) - builder = builder.where('mss.did', '=', subjectInfo.did) + builder = builder.where( + 'moderation_subject_status.did', + '=', + subjectInfo.did, + ) if (!includeAllUserRecords) { builder = builder.where((qb) => subjectInfo.recordPath - ? qb.where('mss.recordPath', '=', subjectInfo.recordPath) - : qb.where('mss.recordPath', '=', ''), + ? qb.where( + 'moderation_subject_status.recordPath', + '=', + subjectInfo.recordPath, + ) + : qb.where('moderation_subject_status.recordPath', '=', ''), ) } } else if (subjectType === 'account') { - builder = builder.where('mss.recordPath', '=', '') + builder = builder.where('moderation_subject_status.recordPath', '=', '') } else if (subjectType === 'record') { - builder = builder.where('mss.recordPath', '!=', '') + builder = builder.where('moderation_subject_status.recordPath', '!=', '') } // Only fetch items that belongs to the specified queue when specified @@ -899,11 +921,15 @@ export class ModerationService { // If subjectType is set to 'account' let that take priority and ignore collections filter if (subjectType !== 'account' && collections?.length) { builder = builder - .where('mss.recordPath', '!=', '') + .where('moderation_subject_status.recordPath', '!=', '') .where((qb) => collections.reduce( (qb, collection) => - qb.orWhere('mss.recordPath', 'like', `${collection}/%`), + qb.orWhere( + 'moderation_subject_status.recordPath', + 'like', + `${collection}/%`, + ), qb.where(sql`false`), ), ) @@ -911,78 +937,134 @@ export class ModerationService { if (ignoreSubjects?.length) { builder = builder - .where('mss.did', 'not in', ignoreSubjects) - .where('mss.recordPath', 'not in', ignoreSubjects) + .where('moderation_subject_status.did', 'not in', ignoreSubjects) + .where('moderation_subject_status.recordPath', 'not in', ignoreSubjects) } if (reviewState) { - builder = builder.where('mss.reviewState', '=', reviewState) + builder = builder.where( + 'moderation_subject_status.reviewState', + '=', + reviewState, + ) } if (lastReviewedBy) { - builder = builder.where('mss.lastReviewedBy', '=', lastReviewedBy) + builder = builder.where( + 'moderation_subject_status.lastReviewedBy', + '=', + lastReviewedBy, + ) } if (reviewedAfter) { - builder = builder.where('mss.lastReviewedAt', '>', reviewedAfter) + builder = builder.where( + 'moderation_subject_status.lastReviewedAt', + '>', + reviewedAfter, + ) } if (reviewedBefore) { - builder = builder.where('mss.lastReviewedAt', '<', reviewedBefore) + builder = builder.where( + 'moderation_subject_status.lastReviewedAt', + '<', + reviewedBefore, + ) } if (hostingUpdatedAfter) { - builder = builder.where('mss.hostingUpdatedAt', '>', hostingUpdatedAfter) + builder = builder.where( + 'moderation_subject_status.hostingUpdatedAt', + '>', + hostingUpdatedAfter, + ) } if (hostingUpdatedBefore) { - builder = builder.where('mss.hostingUpdatedAt', '<', hostingUpdatedBefore) + builder = builder.where( + 'moderation_subject_status.hostingUpdatedAt', + '<', + hostingUpdatedBefore, + ) } if (hostingDeletedAfter) { - builder = builder.where('mss.hostingDeletedAt', '>', hostingDeletedAfter) + builder = builder.where( + 'moderation_subject_status.hostingDeletedAt', + '>', + hostingDeletedAfter, + ) } if (hostingDeletedBefore) { - builder = builder.where('mss.hostingDeletedAt', '<', hostingDeletedBefore) + builder = builder.where( + 'moderation_subject_status.hostingDeletedAt', + '<', + hostingDeletedBefore, + ) } if (hostingStatuses?.length) { - builder = builder.where('mss.hostingStatus', 'in', hostingStatuses) + builder = builder.where( + 'moderation_subject_status.hostingStatus', + 'in', + hostingStatuses, + ) } if (reportedAfter) { - builder = builder.where('mss.lastReviewedAt', '>', reportedAfter) + builder = builder.where( + 'moderation_subject_status.lastReviewedAt', + '>', + reportedAfter, + ) } if (reportedBefore) { - builder = builder.where('mss.lastReportedAt', '<', reportedBefore) + builder = builder.where( + 'moderation_subject_status.lastReportedAt', + '<', + reportedBefore, + ) } if (takendown) { - builder = builder.where('mss.takendown', '=', true) + builder = builder.where('moderation_subject_status.takendown', '=', true) } if (appealed !== undefined) { builder = appealed === false - ? builder.where('mss.appealed', 'is', null) - : builder.where('mss.appealed', '=', appealed) + ? builder.where('moderation_subject_status.appealed', 'is', null) + : builder.where('moderation_subject_status.appealed', '=', appealed) } if (!includeMuted) { builder = builder.where((qb) => qb - .where('mss.muteUntil', '<', new Date().toISOString()) - .orWhere('mss.muteUntil', 'is', null), + .where( + 'moderation_subject_status.muteUntil', + '<', + new Date().toISOString(), + ) + .orWhere('moderation_subject_status.muteUntil', 'is', null), ) } if (onlyMuted) { builder = builder.where((qb) => qb - .where('mss.muteUntil', '>', new Date().toISOString()) - .orWhere('mss.muteReportingUntil', '>', new Date().toISOString()), + .where( + 'moderation_subject_status.muteUntil', + '>', + new Date().toISOString(), + ) + .orWhere( + 'moderation_subject_status.muteReportingUntil', + '>', + new Date().toISOString(), + ), ) } @@ -997,7 +1079,10 @@ export class ModerationService { qb[i === 0 ? 'where' : 'orWhere']((qb) => subTags.reduce( // AND every sub subTags - (qb, subTag) => qb.where(sql`${ref('mss.tags')} ? ${subTag}`), + (qb, subTag) => + qb.where( + sql`${ref('moderation_subject_status.tags')} ? ${subTag}`, + ), qb, ), ), @@ -1010,13 +1095,16 @@ export class ModerationService { builder = builder.where((qb) => qb .where( - sql`NOT(${ref('mss.tags')} ?| array[${sql.join(excludeTags)}]::TEXT[])`, + sql`NOT(${ref('moderation_subject_status.tags')} ?| array[${sql.join(excludeTags)}]::TEXT[])`, ) .orWhere('tags', 'is', null), ) } - const keyset = new StatusKeyset(ref(`mss.${sortField}`), ref('mss.id')) + const keyset = new StatusKeyset( + ref(`moderation_subject_status.${sortField}`), + ref('moderation_subject_status.id'), + ) const paginatedBuilder = paginate(builder, { limit, cursor, diff --git a/packages/ozone/src/mod-service/status.ts b/packages/ozone/src/mod-service/status.ts index deb198fc636..f1ea811bc16 100644 --- a/packages/ozone/src/mod-service/status.ts +++ b/packages/ozone/src/mod-service/status.ts @@ -205,18 +205,18 @@ const getSubjectStatusForRecordEvent = ({ export const moderationSubjectStatusQueryBuilder = (db: DatabaseSchema) => { return db - .selectFrom('moderation_subject_status as mss') - .selectAll('mss') + .selectFrom('moderation_subject_status') + .selectAll('moderation_subject_status') .leftJoin('account_events_stats as aes', (join) => - join.onRef('mss.did', '=', 'aes.subjectDid'), + join.onRef('moderation_subject_status.did', '=', 'aes.subjectDid'), ) .selectAll('aes') .leftJoin('account_record_events_stats as ares', (join) => - join.onRef('mss.did', '=', 'ares.subjectDid'), + join.onRef('moderation_subject_status.did', '=', 'ares.subjectDid'), ) .selectAll('ares') .leftJoin('account_record_status_stats as arss', (join) => - join.onRef('mss.did', '=', 'arss.did'), + join.onRef('moderation_subject_status.did', '=', 'arss.did'), ) .selectAll('arss') } diff --git a/packages/ozone/src/mod-service/views.ts b/packages/ozone/src/mod-service/views.ts index ad42f8e77fa..f2b79fc0709 100644 --- a/packages/ozone/src/mod-service/views.ts +++ b/packages/ozone/src/mod-service/views.ts @@ -482,7 +482,7 @@ export class ModerationViews { ) .where( sql`${ref( - 'mss.blobCids', + 'moderation_subject_status.blobCids', )} @> ${JSON.stringify(blobs.map((blob) => blob.ref.toString()))}`, ) .selectAll() @@ -553,31 +553,36 @@ export class ModerationViews { return signed } - async getSubjectStatus(subjects: string[]) { + async getSubjectStatus( + subjects: string[], + ): Promise> { + if (!subjects.length) return new Map() + const parsedSubjects = subjects.map(parseSubjectId) - const builder = moderationSubjectStatusQueryBuilder(this.db.db).where( - (clause) => { - return parsedSubjects.reduce( - (clause, sub) => { - return clause.orWhere((qb) => - qb - .where('mss.did', '=', sub.did) - .where('mss.recordPath', '=', sub.recordPath ?? ''), - ) - }, - // If the array is empty, no result should ne returned - clause.where(sql`false`), - ) - }, - ) + const builder = moderationSubjectStatusQueryBuilder(this.db.db) + // + .where((qb) => { + for (const sub of parsedSubjects) { + qb = qb.orWhere((qb) => + qb + .where('moderation_subject_status.did', '=', sub.did) + .where( + 'moderation_subject_status.recordPath', + '=', + sub.recordPath ?? '', + ), + ) + } + return qb + }) const [statusRes, accountsByDid] = await Promise.all([ builder.execute(), this.getAccoutInfosByDid(parsedSubjects.map((s) => s.did)), ]) - return new Map( + return new Map( statusRes.map((status) => { const subjectId = formatSubjectId(status.did, status.recordPath) const handle = accountsByDid.get(status.did)?.handle ?? INVALID_HANDLE diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 60ce6cdb800..df7bf01b474 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -4828,6 +4828,11 @@ export const schemaDict = { ref: 'lex:app.bsky.actor.defs#profileView', }, }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -8335,6 +8340,11 @@ export const schemaDict = { 'If true, response has fallen-back to generic results, and is not scoped using relativeToDid', default: false, }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -9406,6 +9416,11 @@ export const schemaDict = { description: 'DID of the account these suggestions are relative to. If this is returned undefined, suggestions are based on the viewer.', }, + recId: { + type: 'integer', + description: + 'Snowflake for this recommendation, use when submitting recommendation events.', + }, }, }, }, @@ -11275,8 +11290,7 @@ export const schemaDict = { }, }, accountStats: { - description: - 'Statistics about a particular account subject on the labeller', + description: 'Statistics about a particular account subject', type: 'object', properties: { reportCount: { @@ -11299,8 +11313,7 @@ export const schemaDict = { }, }, recordsStats: { - description: - 'Statistics about a set of record subject items on the labeller', + description: 'Statistics about a set of record subject items', type: 'object', properties: { totalReports: { diff --git a/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts b/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts index e108ad4203b..06e6d2cfaa9 100644 --- a/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts +++ b/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts @@ -153,7 +153,7 @@ export function validateSubjectStatusView(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#subjectStatusView', v) } -/** Statistics about a particular account subject on the labeller */ +/** Statistics about a particular account subject */ export interface AccountStats { /** Total number of reports on the account */ reportCount?: number @@ -178,7 +178,7 @@ export function validateAccountStats(v: unknown): ValidationResult { return lexicons.validate('tools.ozone.moderation.defs#accountStats', v) } -/** Statistics about a set of record subject items on the labeller */ +/** Statistics about a set of record subject items */ export interface RecordsStats { /** Cumulative sum of the number of reports on the items in the set */ totalReports?: number From 8ac6d9b894aeffa60f03532a80b8d8d8cb65e158 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Tue, 14 Jan 2025 11:37:28 +0100 Subject: [PATCH 06/22] style --- packages/ozone/src/mod-service/index.ts | 53 ++++++++++++------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 9468021d603..62b421fb512 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -922,17 +922,16 @@ export class ModerationService { if (subjectType !== 'account' && collections?.length) { builder = builder .where('moderation_subject_status.recordPath', '!=', '') - .where((qb) => - collections.reduce( - (qb, collection) => - qb.orWhere( - 'moderation_subject_status.recordPath', - 'like', - `${collection}/%`, - ), - qb.where(sql`false`), - ), - ) + .where((qb) => { + for (const collection of collections) { + qb = qb.orWhere( + 'moderation_subject_status.recordPath', + 'like', + `${collection}/%`, + ) + } + return qb + }) } if (ignoreSubjects?.length) { @@ -1072,23 +1071,21 @@ export class ModerationService { const conditions = parseTags(tags) if (conditions?.length) { // [["tag1"], ["tag2", "tag3"], ["tag4"]] => (tags ? 'tag1') OR (tags ? 'tag2' AND tags ? 'tag3') OR (tags ? 'tag4') - builder = builder.where((qb) => - conditions.reduce( - (qb, subTags, i) => - // OR every conditions - qb[i === 0 ? 'where' : 'orWhere']((qb) => - subTags.reduce( - // AND every sub subTags - (qb, subTag) => - qb.where( - sql`${ref('moderation_subject_status.tags')} ? ${subTag}`, - ), - qb, - ), - ), - qb, - ), - ) + builder = builder.where((qb) => { + for (const subTags of conditions) { + // OR between every conditions items (subTags) + qb = qb.orWhere((qb) => { + // AND between every subTags items (subTag) + for (const subTag of subTags) { + qb = qb.where( + sql`${ref('moderation_subject_status.tags')} ? ${subTag}`, + ) + } + return qb + }) + } + return qb + }) } if (excludeTags?.length) { From 30fd511770de03236b30aab477226b8446f4afd3 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Tue, 14 Jan 2025 11:40:04 +0100 Subject: [PATCH 07/22] review comments --- .../migrations/20241220T144630860Z-stats-materialized-views.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts index 288164b66d0..4fee310a9ed 100644 --- a/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts +++ b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts @@ -53,7 +53,6 @@ export async function up(db: Kysely): Promise { ) .execute() - // TODO try/catch to ignore existing await db.schema .createIndex('account_events_stats_did_idx') // .ifNotExists() // REquires newer version of kysely From 2cd6036a65f0971656be19ec424e29aa9fd547d0 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Thu, 16 Jan 2025 12:16:06 +0100 Subject: [PATCH 08/22] wip --- lexicons/tools/ozone/moderation/defs.json | 4 ++ .../tools/ozone/moderation/queryStatuses.json | 15 +++- packages/api/src/client/lexicons.ts | 21 +++++- .../types/tools/ozone/moderation/defs.ts | 2 + .../tools/ozone/moderation/queryStatuses.ts | 10 ++- .../ozone/src/api/moderation/queryStatuses.ts | 64 +---------------- packages/ozone/src/db/index.ts | 1 + packages/ozone/src/lexicon/lexicons.ts | 21 +++++- .../types/tools/ozone/moderation/defs.ts | 2 + .../tools/ozone/moderation/queryStatuses.ts | 10 ++- packages/ozone/src/mod-service/index.ts | 71 +++++++++---------- packages/ozone/src/mod-service/status.ts | 30 +++++--- packages/ozone/src/mod-service/views.ts | 6 ++ packages/pds/src/index.ts | 8 ++- packages/pds/src/lexicon/lexicons.ts | 21 +++++- .../types/tools/ozone/moderation/defs.ts | 2 + .../tools/ozone/moderation/queryStatuses.ts | 10 ++- 17 files changed, 180 insertions(+), 118 deletions(-) diff --git a/lexicons/tools/ozone/moderation/defs.json b/lexicons/tools/ozone/moderation/defs.json index a0faf974515..23bba198f24 100644 --- a/lexicons/tools/ozone/moderation/defs.json +++ b/lexicons/tools/ozone/moderation/defs.json @@ -211,6 +211,10 @@ "description": "Number of times the account was suspended", "type": "integer" }, + "escalateCount": { + "description": "Number of times the account was escalated", + "type": "integer" + }, "takedownCount": { "description": "Number of times the account was taken down", "type": "integer" diff --git a/lexicons/tools/ozone/moderation/queryStatuses.json b/lexicons/tools/ozone/moderation/queryStatuses.json index a8ff110db84..89efcf76020 100644 --- a/lexicons/tools/ozone/moderation/queryStatuses.json +++ b/lexicons/tools/ozone/moderation/queryStatuses.json @@ -107,7 +107,12 @@ "sortField": { "type": "string", "default": "lastReportedAt", - "enum": ["lastReviewedAt", "lastReportedAt"] + "enum": [ + "lastReviewedAt", + "lastReportedAt", + "reportedRecordsCount", + "takendownRecordsCount" + ] }, "sortDirection": { "type": "string", @@ -158,6 +163,14 @@ "type": "string", "description": "If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.", "knownValues": ["account", "record"] + }, + "minReportedRecordsCount": { + "type": "integer", + "description": "If specified, only subjects that belong to an account that has at least this many reported records will be returned." + }, + "minTakendownRecordsCount": { + "type": "integer", + "description": "If specified, only subjects that belong to an account that has at least this many taken down records will be returned." } } }, diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 78b5d7ab661..1e6f86577f9 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -11329,6 +11329,10 @@ export const schemaDict = { description: 'Number of times the account was suspended', type: 'integer', }, + escalateCount: { + description: 'Number of times the account was escalated', + type: 'integer', + }, takedownCount: { description: 'Number of times the account was taken down', type: 'integer', @@ -12622,7 +12626,12 @@ export const schemaDict = { sortField: { type: 'string', default: 'lastReportedAt', - enum: ['lastReviewedAt', 'lastReportedAt'], + enum: [ + 'lastReviewedAt', + 'lastReportedAt', + 'reportedRecordsCount', + 'takendownRecordsCount', + ], }, sortDirection: { type: 'string', @@ -12677,6 +12686,16 @@ export const schemaDict = { "If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.", knownValues: ['account', 'record'], }, + minReportedRecordsCount: { + type: 'integer', + description: + 'If specified, only subjects that belong to an account that has at least this many reported records will be returned.', + }, + minTakendownRecordsCount: { + type: 'integer', + description: + 'If specified, only subjects that belong to an account that has at least this many taken down records will be returned.', + }, }, }, output: { diff --git a/packages/api/src/client/types/tools/ozone/moderation/defs.ts b/packages/api/src/client/types/tools/ozone/moderation/defs.ts index 590e966d359..df753c9cfc4 100644 --- a/packages/api/src/client/types/tools/ozone/moderation/defs.ts +++ b/packages/api/src/client/types/tools/ozone/moderation/defs.ts @@ -161,6 +161,8 @@ export interface AccountStats { appealCount?: number /** Number of times the account was suspended */ suspendCount?: number + /** Number of times the account was escalated */ + escalateCount?: number /** Number of times the account was taken down */ takedownCount?: number [k: string]: unknown diff --git a/packages/api/src/client/types/tools/ozone/moderation/queryStatuses.ts b/packages/api/src/client/types/tools/ozone/moderation/queryStatuses.ts index 2af6808ed19..d129a737225 100644 --- a/packages/api/src/client/types/tools/ozone/moderation/queryStatuses.ts +++ b/packages/api/src/client/types/tools/ozone/moderation/queryStatuses.ts @@ -48,7 +48,11 @@ export interface QueryParams { ignoreSubjects?: string[] /** Get all subject statuses that were reviewed by a specific moderator */ lastReviewedBy?: string - sortField?: 'lastReviewedAt' | 'lastReportedAt' + sortField?: + | 'lastReviewedAt' + | 'lastReportedAt' + | 'reportedRecordsCount' + | 'takendownRecordsCount' sortDirection?: 'asc' | 'desc' /** Get subjects that were taken down */ takendown?: boolean @@ -62,6 +66,10 @@ export interface QueryParams { collections?: string[] /** If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. */ subjectType?: 'account' | 'record' | (string & {}) + /** If specified, only subjects that belong to an account that has at least this many reported records will be returned. */ + minReportedRecordsCount?: number + /** If specified, only subjects that belong to an account that has at least this many taken down records will be returned. */ + minTakendownRecordsCount?: number } export type InputSchema = undefined diff --git a/packages/ozone/src/api/moderation/queryStatuses.ts b/packages/ozone/src/api/moderation/queryStatuses.ts index 305a483ad4f..a2f77560436 100644 --- a/packages/ozone/src/api/moderation/queryStatuses.ts +++ b/packages/ozone/src/api/moderation/queryStatuses.ts @@ -1,75 +1,13 @@ import { Server } from '../../lexicon' import AppContext from '../../context' -import { getReviewState } from '../util' export default function (server: Server, ctx: AppContext) { server.tools.ozone.moderation.queryStatuses({ auth: ctx.authVerifier.modOrAdminToken, handler: async ({ params }) => { - const { - includeAllUserRecords, - subject, - takendown, - appealed, - reviewState, - reviewedAfter, - reviewedBefore, - reportedAfter, - reportedBefore, - ignoreSubjects, - lastReviewedBy, - hostingDeletedBefore, - hostingDeletedAfter, - hostingUpdatedBefore, - hostingUpdatedAfter, - hostingStatuses, - sortDirection = 'desc', - sortField = 'lastReportedAt', - includeMuted = false, - onlyMuted = false, - limit = 50, - cursor, - tags, - excludeTags, - collections, - subjectType, - queueCount, - queueIndex, - queueSeed, - } = params const db = ctx.db const modService = ctx.modService(db) - const results = await modService.getSubjectStatuses({ - reviewState: getReviewState(reviewState), - includeAllUserRecords, - subject, - takendown, - appealed, - reviewedAfter, - reviewedBefore, - reportedAfter, - reportedBefore, - includeMuted, - hostingDeletedBefore, - hostingDeletedAfter, - hostingUpdatedBefore, - hostingUpdatedAfter, - hostingStatuses, - onlyMuted, - ignoreSubjects, - sortDirection, - lastReviewedBy, - sortField, - limit, - cursor, - tags, - excludeTags, - collections, - subjectType, - queueCount, - queueIndex, - queueSeed, - }) + const results = await modService.getSubjectStatuses(params) const subjectStatuses = results.statuses.map((status) => modService.views.formatSubjectStatus(status), ) diff --git a/packages/ozone/src/db/index.ts b/packages/ozone/src/db/index.ts index b5b66542583..4485a342a8b 100644 --- a/packages/ozone/src/db/index.ts +++ b/packages/ozone/src/db/index.ts @@ -71,6 +71,7 @@ export class Database { this.pool = pool this.db = new Kysely({ dialect: new PostgresDialect({ pool }), + log: ['error', 'query'], }) } diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index 78b5d7ab661..1e6f86577f9 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -11329,6 +11329,10 @@ export const schemaDict = { description: 'Number of times the account was suspended', type: 'integer', }, + escalateCount: { + description: 'Number of times the account was escalated', + type: 'integer', + }, takedownCount: { description: 'Number of times the account was taken down', type: 'integer', @@ -12622,7 +12626,12 @@ export const schemaDict = { sortField: { type: 'string', default: 'lastReportedAt', - enum: ['lastReviewedAt', 'lastReportedAt'], + enum: [ + 'lastReviewedAt', + 'lastReportedAt', + 'reportedRecordsCount', + 'takendownRecordsCount', + ], }, sortDirection: { type: 'string', @@ -12677,6 +12686,16 @@ export const schemaDict = { "If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.", knownValues: ['account', 'record'], }, + minReportedRecordsCount: { + type: 'integer', + description: + 'If specified, only subjects that belong to an account that has at least this many reported records will be returned.', + }, + minTakendownRecordsCount: { + type: 'integer', + description: + 'If specified, only subjects that belong to an account that has at least this many taken down records will be returned.', + }, }, }, output: { diff --git a/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts b/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts index 06e6d2cfaa9..572d323a298 100644 --- a/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts +++ b/packages/ozone/src/lexicon/types/tools/ozone/moderation/defs.ts @@ -161,6 +161,8 @@ export interface AccountStats { appealCount?: number /** Number of times the account was suspended */ suspendCount?: number + /** Number of times the account was escalated */ + escalateCount?: number /** Number of times the account was taken down */ takedownCount?: number [k: string]: unknown diff --git a/packages/ozone/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts b/packages/ozone/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts index 0ccd1030c0b..fed507cf703 100644 --- a/packages/ozone/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts +++ b/packages/ozone/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts @@ -49,7 +49,11 @@ export interface QueryParams { ignoreSubjects?: string[] /** Get all subject statuses that were reviewed by a specific moderator */ lastReviewedBy?: string - sortField: 'lastReviewedAt' | 'lastReportedAt' + sortField: + | 'lastReviewedAt' + | 'lastReportedAt' + | 'reportedRecordsCount' + | 'takendownRecordsCount' sortDirection: 'asc' | 'desc' /** Get subjects that were taken down */ takendown?: boolean @@ -63,6 +67,10 @@ export interface QueryParams { collections?: string[] /** If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. */ subjectType?: 'account' | 'record' | (string & {}) + /** If specified, only subjects that belong to an account that has at least this many reported records will be returned. */ + minReportedRecordsCount?: number + /** If specified, only subjects that belong to an account that has at least this many taken down records will be returned. */ + minTakendownRecordsCount?: number } export type InputSchema = undefined diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 62b421fb512..77ba5cc6198 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -41,6 +41,7 @@ import { import { ModerationEvent } from '../db/schema/moderation_event' import { StatusKeyset, TimeIdKeyset, paginate } from '../db/pagination' import { Label } from '../lexicon/types/com/atproto/label/defs' +import { QueryParams as QueryStatusParams } from '../lexicon/types/tools/ozone/moderation/queryStatuses' import { ModSubject, RecordSubject, @@ -58,6 +59,7 @@ import { httpLogger as log } from '../logger' import { OzoneConfig } from '../config' import { LABELER_HEADER_NAME, ParsedLabelers } from '../util' import { ids } from '../lexicon/lexicons' +import { getReviewState } from '../api/util' export type ModerationServiceCreator = (db: Database) => ModerationService @@ -821,53 +823,25 @@ export class ModerationService { reviewedBefore, reportedAfter, reportedBefore, - includeMuted, + includeMuted = false, hostingDeletedBefore, hostingDeletedAfter, hostingUpdatedBefore, hostingUpdatedAfter, hostingStatuses, - onlyMuted, + onlyMuted = false, ignoreSubjects, - sortDirection, + sortDirection = 'desc', lastReviewedBy, - sortField, + sortField = 'lastReportedAt', subject, tags, excludeTags, collections, subjectType, - }: { - queueCount?: number - queueIndex?: number - queueSeed?: string - includeAllUserRecords?: boolean - cursor?: string - limit?: number - takendown?: boolean - appealed?: boolean - reviewedBefore?: string - reviewState?: ModerationSubjectStatusRow['reviewState'] - reviewedAfter?: string - reportedAfter?: string - reportedBefore?: string - includeMuted?: boolean - hostingDeletedBefore?: string - hostingDeletedAfter?: string - hostingUpdatedBefore?: string - hostingUpdatedAfter?: string - hostingStatuses?: string[] - onlyMuted?: boolean - subject?: string - ignoreSubjects?: string[] - sortDirection: 'asc' | 'desc' - lastReviewedBy?: string - sortField: 'lastReviewedAt' | 'lastReportedAt' - tags?: string[] - excludeTags?: string[] - collections?: string[] - subjectType?: string - }): Promise<{ + minReportedRecordsCount = 0, + minTakendownRecordsCount = 0, + }: QueryStatusParams): Promise<{ statuses: ModerationSubjectStatusRowWithHandle[] cursor?: string }> { @@ -940,11 +914,12 @@ export class ModerationService { .where('moderation_subject_status.recordPath', 'not in', ignoreSubjects) } - if (reviewState) { + const reviewStateNormalized = getReviewState(reviewState) + if (reviewStateNormalized) { builder = builder.where( 'moderation_subject_status.reviewState', '=', - reviewState, + reviewStateNormalized, ) } @@ -1098,8 +1073,28 @@ export class ModerationService { ) } + if (minTakendownRecordsCount > 0) { + builder = builder.where( + 'account_record_status_stats.takendownCount', + '>=', + minTakendownRecordsCount, + ) + } + + if (minReportedRecordsCount > 0) { + builder = builder.where( + 'account_record_events_stats.reportedCount', + '>=', + minReportedRecordsCount, + ) + } + const keyset = new StatusKeyset( - ref(`moderation_subject_status.${sortField}`), + sortField === 'reportedRecordsCount' + ? ref(`account_record_events_stats.reportedCount`) + : sortField === 'takendownRecordsCount' + ? ref(`account_record_status_stats.takendownCount`) + : ref(`moderation_subject_status.${sortField}`), ref('moderation_subject_status.id'), ) const paginatedBuilder = paginate(builder, { diff --git a/packages/ozone/src/mod-service/status.ts b/packages/ozone/src/mod-service/status.ts index f1ea811bc16..695579a511e 100644 --- a/packages/ozone/src/mod-service/status.ts +++ b/packages/ozone/src/mod-service/status.ts @@ -207,18 +207,30 @@ export const moderationSubjectStatusQueryBuilder = (db: DatabaseSchema) => { return db .selectFrom('moderation_subject_status') .selectAll('moderation_subject_status') - .leftJoin('account_events_stats as aes', (join) => - join.onRef('moderation_subject_status.did', '=', 'aes.subjectDid'), + .leftJoin('account_events_stats', (join) => + join.onRef( + 'moderation_subject_status.did', + '=', + 'account_events_stats.subjectDid', + ), ) - .selectAll('aes') - .leftJoin('account_record_events_stats as ares', (join) => - join.onRef('moderation_subject_status.did', '=', 'ares.subjectDid'), + .selectAll('account_events_stats') + .leftJoin('account_record_events_stats', (join) => + join.onRef( + 'moderation_subject_status.did', + '=', + 'account_record_events_stats.subjectDid', + ), ) - .selectAll('ares') - .leftJoin('account_record_status_stats as arss', (join) => - join.onRef('moderation_subject_status.did', '=', 'arss.did'), + .selectAll('account_record_events_stats') + .leftJoin('account_record_status_stats', (join) => + join.onRef( + 'moderation_subject_status.did', + '=', + 'account_record_status_stats.did', + ), ) - .selectAll('arss') + .selectAll('account_record_status_stats') } // Based on a given moderation action event, this function will update the moderation status of the subject diff --git a/packages/ozone/src/mod-service/views.ts b/packages/ozone/src/mod-service/views.ts index f2b79fc0709..8d8af599fac 100644 --- a/packages/ozone/src/mod-service/views.ts +++ b/packages/ozone/src/mod-service/views.ts @@ -615,16 +615,22 @@ export class ModerationViews { subject: subjectFromStatusRow(status).lex(), accountStats: { + // account_events_stats reportCount: status.reportCount ?? undefined, appealCount: status.appealCount ?? undefined, suspendCount: status.suspendCount ?? undefined, takedownCount: status.takedownCount ?? undefined, + escalateCount: status.escalateCount ?? undefined, }, + recordsStats: { + // account_record_events_stats totalReports: status.totalReports ?? undefined, reportedCount: status.reportedCount ?? undefined, escalatedCount: status.escalatedCount ?? undefined, appealedCount: status.appealedCount ?? undefined, + + // account_record_status_stats subjectCount: status.subjectCount ?? undefined, pendingCount: status.pendingCount ?? undefined, processedCount: status.processedCount ?? undefined, diff --git a/packages/pds/src/index.ts b/packages/pds/src/index.ts index fb73a3c3017..0d74f9a09f7 100644 --- a/packages/pds/src/index.ts +++ b/packages/pds/src/index.ts @@ -99,7 +99,13 @@ export class PDS { app.use(loggerMiddleware) app.use(compression()) app.use(authRoutes.createRouter(ctx)) // Before CORS - app.use(cors({ maxAge: DAY / SECOND })) + app.use( + cors({ + maxAge: DAY / SECOND, + credentials: true, + origin: (origin, callback) => callback(null, origin), + }), + ) app.use(basicRoutes.createRouter(ctx)) app.use(wellKnown.createRouter(ctx)) app.use(server.xrpc.router) diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 78b5d7ab661..1e6f86577f9 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -11329,6 +11329,10 @@ export const schemaDict = { description: 'Number of times the account was suspended', type: 'integer', }, + escalateCount: { + description: 'Number of times the account was escalated', + type: 'integer', + }, takedownCount: { description: 'Number of times the account was taken down', type: 'integer', @@ -12622,7 +12626,12 @@ export const schemaDict = { sortField: { type: 'string', default: 'lastReportedAt', - enum: ['lastReviewedAt', 'lastReportedAt'], + enum: [ + 'lastReviewedAt', + 'lastReportedAt', + 'reportedRecordsCount', + 'takendownRecordsCount', + ], }, sortDirection: { type: 'string', @@ -12677,6 +12686,16 @@ export const schemaDict = { "If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.", knownValues: ['account', 'record'], }, + minReportedRecordsCount: { + type: 'integer', + description: + 'If specified, only subjects that belong to an account that has at least this many reported records will be returned.', + }, + minTakendownRecordsCount: { + type: 'integer', + description: + 'If specified, only subjects that belong to an account that has at least this many taken down records will be returned.', + }, }, }, output: { diff --git a/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts b/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts index 06e6d2cfaa9..572d323a298 100644 --- a/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts +++ b/packages/pds/src/lexicon/types/tools/ozone/moderation/defs.ts @@ -161,6 +161,8 @@ export interface AccountStats { appealCount?: number /** Number of times the account was suspended */ suspendCount?: number + /** Number of times the account was escalated */ + escalateCount?: number /** Number of times the account was taken down */ takedownCount?: number [k: string]: unknown diff --git a/packages/pds/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts b/packages/pds/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts index 0ccd1030c0b..fed507cf703 100644 --- a/packages/pds/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts +++ b/packages/pds/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts @@ -49,7 +49,11 @@ export interface QueryParams { ignoreSubjects?: string[] /** Get all subject statuses that were reviewed by a specific moderator */ lastReviewedBy?: string - sortField: 'lastReviewedAt' | 'lastReportedAt' + sortField: + | 'lastReviewedAt' + | 'lastReportedAt' + | 'reportedRecordsCount' + | 'takendownRecordsCount' sortDirection: 'asc' | 'desc' /** Get subjects that were taken down */ takendown?: boolean @@ -63,6 +67,10 @@ export interface QueryParams { collections?: string[] /** If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. */ subjectType?: 'account' | 'record' | (string & {}) + /** If specified, only subjects that belong to an account that has at least this many reported records will be returned. */ + minReportedRecordsCount?: number + /** If specified, only subjects that belong to an account that has at least this many taken down records will be returned. */ + minTakendownRecordsCount?: number } export type InputSchema = undefined From b6fcd1eeb420aef704a39cd9764dd1a1fcad1997 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Thu, 16 Jan 2025 14:36:27 +0100 Subject: [PATCH 09/22] add indexes --- ...220T144630860Z-stats-materialized-views.ts | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts index 4fee310a9ed..19731020386 100644 --- a/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts +++ b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts @@ -12,6 +12,17 @@ import * as modStatus from '../schema/moderation_subject_status' import * as recordEventsStats from '../schema/record_events_stats' export async function up(db: Kysely): Promise { + // Used by "tools.ozone.moderation.queryStatuses". Reduces query cost by two + // order of magnitudes when sorting using "reportedRecordsCount" or + // "takendownRecordsCount" and filtering by "reviewState". + await db.schema + .createIndex('moderation_subject_status_did_id_review_state_idx') + .on('moderation_subject_status') + .column('did') + .expression(sql`"id" ASC NULLS FIRST`) + .column('reviewState') + .execute() + // ~6sec for 16M events await db.schema .createView('account_events_stats') @@ -55,7 +66,6 @@ export async function up(db: Kysely): Promise { await db.schema .createIndex('account_events_stats_did_idx') - // .ifNotExists() // REquires newer version of kysely .unique() .on('account_events_stats') .column('subjectDid') @@ -93,7 +103,6 @@ export async function up(db: Kysely): Promise { await db.schema .createIndex('record_events_stats_uri_idx') - // .ifNotExists() .unique() .on('record_events_stats') .column('subjectUri') @@ -101,7 +110,6 @@ export async function up(db: Kysely): Promise { await db.schema .createIndex('record_events_stats_did_idx') - // .ifNotExists() .on('record_events_stats') .column('subjectDid') .execute() @@ -135,12 +143,18 @@ export async function up(db: Kysely): Promise { await db.schema .createIndex('account_record_events_stats_did_idx') - // .ifNotExists() .unique() .on('account_record_events_stats') .column('subjectDid') .execute() + await db.schema + .createIndex('account_record_events_stats_reported_count_idx') + .on('account_record_events_stats') + .expression(sql`"reportedCount" ASC NULLS FIRST`) + .column('subjectDid') + .execute() + await db.schema .createView('account_record_status_stats') .materialized() @@ -171,11 +185,17 @@ export async function up(db: Kysely): Promise { await db.schema .createIndex('account_record_status_stats_did_idx') - // .ifNotExists() .unique() .on('account_record_status_stats') .column('did') .execute() + + await db.schema + .createIndex('account_record_status_stats_takendown_count_idx') + .on('account_record_status_stats') + .expression(sql`"takendownCount" ASC NULLS FIRST`) + .column('did') + .execute() } export async function down(db: Kysely): Promise { From 444f072704aac5722796ba33303dea7c0e00acae Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Thu, 16 Jan 2025 16:56:31 +0100 Subject: [PATCH 10/22] refresh materialized view --- packages/ozone/src/background.ts | 141 ++++++++++++++++-- packages/ozone/src/daemon/context.ts | 43 +++++- packages/ozone/src/daemon/index.ts | 9 +- .../src/daemon/materialized-view-refresher.ts | 22 +++ packages/ozone/src/index.ts | 8 +- 5 files changed, 195 insertions(+), 28 deletions(-) create mode 100644 packages/ozone/src/daemon/materialized-view-refresher.ts diff --git a/packages/ozone/src/background.ts b/packages/ozone/src/background.ts index 78cd790e779..d3426da13e5 100644 --- a/packages/ozone/src/background.ts +++ b/packages/ozone/src/background.ts @@ -2,21 +2,45 @@ import PQueue from 'p-queue' import { Database } from './db' import { dbLogger } from './logger' -// A simple queue for in-process, out-of-band/backgrounded work +type Task = (db: Database, signal: AbortSignal) => Promise +/** + * A simple queue for in-process, out-of-band/backgrounded work + */ export class BackgroundQueue { - queue = new PQueue({ concurrency: 20 }) - destroyed = false - constructor(public db: Database) {} + private abortController = new AbortController() + private queue = new PQueue({ concurrency: 20 }) + + protected get signal() { + return this.abortController.signal + } + + public get destroyed() { + return this.signal.aborted + } + + constructor(protected db: Database) {} + + getStats() { + return { + runningCount: this.queue.pending, + waitingCount: this.queue.size, + } + } add(task: Task) { if (this.destroyed) { return } + + const { db, signal } = this + this.queue - .add(() => task(this.db)) + .add(() => task(db, signal)) .catch((err) => { - dbLogger.error(err, 'background queue task failed') + if (!isCausedBySignal(err, signal)) { + dbLogger.error(err, 'background queue task failed') + } }) } @@ -24,12 +48,109 @@ export class BackgroundQueue { await this.queue.onIdle() } - // On destroy we stop accepting new tasks, but complete all pending/in-progress tasks. - // The application calls this only once http connections have drained (tasks no longer being added). + // On destroy we stop accepting new tasks, but complete all + // pending/in-progress tasks. Tasks can decide to abort their current + // operation based on the signal they received. The application calls this + // only once http connections have drained (tasks no longer being added). async destroy() { - this.destroyed = true + this.abortController.abort() await this.queue.onIdle() } } -type Task = (db: Database) => Promise +/** + * A simple periodic background task runner + */ +export class PeriodicBackgroundTask { + private abortController = new AbortController() + private promise?: Promise + + protected get signal() { + return this.abortController.signal + } + + public get destroyed() { + return this.signal.aborted + } + + constructor( + protected db: Database, + protected interval: number, + protected task: Task, + ) {} + + async start() { + this.signal.throwIfAborted() + if (this.promise !== undefined) throw new Error('Already started') + + const { db, task } = this + + this.promise = startInterval( + async (signal) => { + try { + await task(db, signal) + } catch (err) { + if (!isCausedBySignal(err, signal)) { + dbLogger.error(err, 'periodic background task failed') + } + } + }, + this.interval, + this.signal, + ) + } + + async destroy() { + this.signal.throwIfAborted() + this.abortController.abort() + + await this.promise + this.promise = undefined + } +} + +/** + * Determines whether the cause of an error is a signal's reason + */ +function isCausedBySignal(err: unknown, { reason }: AbortSignal) { + return err === reason || (err instanceof Error && err.cause === reason) +} + +function startInterval( + fn: (signal: AbortSignal) => void | Promise, + interval: number, + signal: AbortSignal, +) { + signal.throwIfAborted() + + return new Promise((resolve) => { + let timer: NodeJS.Timeout | undefined + + const run = async () => { + timer = undefined // record that we are running + try { + await fn(signal) + } finally { + if (signal.aborted) resolve() + else schedule() + } + } + + const schedule = () => { + timer = setTimeout(run, interval) + } + + const stop = () => { + if (timer) { + clearTimeout(timer) + resolve() + } else { + // fn is running, resolve() will be called + } + } + + signal.addEventListener('abort', stop, { once: true }) + + schedule() + }) +} diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index d81f3f4d78b..0063cc11683 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -6,17 +6,20 @@ import { OzoneConfig, OzoneSecrets } from '../config' import { Database } from '../db' import { EventPusher } from './event-pusher' import { EventReverser } from './event-reverser' -import { ModerationService, ModerationServiceCreator } from '../mod-service' +import { ModerationService } from '../mod-service' import { BackgroundQueue } from '../background' import { getSigningKeyId } from '../util' +import { MaterializedViewRefresher } from './materialized-view-refresher' +import { allFulfilled } from '@atproto/common' export type DaemonContextOptions = { db: Database cfg: OzoneConfig - modService: ModerationServiceCreator + backgroundQueue: BackgroundQueue signingKey: Keypair eventPusher: EventPusher eventReverser: EventReverser + materializedViewRefresher: MaterializedViewRefresher } export class DaemonContext { @@ -67,13 +70,16 @@ export class DaemonContext { const eventReverser = new EventReverser(db, modService) + const materializedViewRefresher = new MaterializedViewRefresher(db) + return new DaemonContext({ db, cfg, - modService, + backgroundQueue, signingKey, eventPusher, eventReverser, + materializedViewRefresher, ...(overrides ?? {}), }) } @@ -86,8 +92,8 @@ export class DaemonContext { return this.opts.cfg } - get modService(): ModerationServiceCreator { - return this.opts.modService + get backgroundQueue(): BackgroundQueue { + return this.opts.backgroundQueue } get eventPusher(): EventPusher { @@ -97,6 +103,33 @@ export class DaemonContext { get eventReverser(): EventReverser { return this.opts.eventReverser } + + get materializedViewRefresher(): MaterializedViewRefresher { + return this.opts.materializedViewRefresher + } + + async start() { + this.eventPusher.start() + this.eventReverser.start() + this.materializedViewRefresher.start() + } + + async processAll() { + await this.eventPusher.processAll() + } + + async destroy() { + try { + await allFulfilled([ + this.eventReverser.destroy(), + this.eventPusher.destroy(), + this.materializedViewRefresher.destroy(), + ]) + } finally { + await this.backgroundQueue.destroy() + await this.db.close() + } + } } export default DaemonContext diff --git a/packages/ozone/src/daemon/index.ts b/packages/ozone/src/daemon/index.ts index 501b8caad5c..e29305ba91f 100644 --- a/packages/ozone/src/daemon/index.ts +++ b/packages/ozone/src/daemon/index.ts @@ -18,17 +18,14 @@ export class OzoneDaemon { } async start() { - this.ctx.eventPusher.start() - this.ctx.eventReverser.start() + await this.ctx.start() } async processAll() { - await this.ctx.eventPusher.processAll() + await this.ctx.processAll() } async destroy() { - await this.ctx.eventReverser.destroy() - await this.ctx.eventPusher.destroy() - await this.ctx.db.close() + await this.ctx.destroy() } } diff --git a/packages/ozone/src/daemon/materialized-view-refresher.ts b/packages/ozone/src/daemon/materialized-view-refresher.ts new file mode 100644 index 00000000000..0e7820d9102 --- /dev/null +++ b/packages/ozone/src/daemon/materialized-view-refresher.ts @@ -0,0 +1,22 @@ +import { MINUTE } from '@atproto/common' +import Database from '../db' +import { sql } from 'kysely' +import { PeriodicBackgroundTask } from '../background' + +export class MaterializedViewRefresher extends PeriodicBackgroundTask { + constructor(db: Database) { + super(db, 15 * MINUTE, async ({ db }, signal) => { + for (const view of [ + 'account_events_stats', + 'record_events_stats', + 'account_record_events_stats', + 'account_record_status_stats', + ]) { + if (signal.aborted) break + await sql`REFRESH MATERIALIZED VIEW CONCURRENTLY ${sql.id(view)}`.execute( + db, + ) + } + }) + } +} diff --git a/packages/ozone/src/index.ts b/packages/ozone/src/index.ts index eb31f58b973..743f181bdee 100644 --- a/packages/ozone/src/index.ts +++ b/packages/ozone/src/index.ts @@ -114,13 +114,7 @@ export class OzoneService { }, 'db pool stats', ) - dbLogger.info( - { - runningCount: backgroundQueue.queue.pending, - waitingCount: backgroundQueue.queue.size, - }, - 'background queue stats', - ) + dbLogger.info(backgroundQueue.getStats(), 'background queue stats') }, 10000) await this.ctx.sequencer.start() const server = this.app.listen(this.ctx.cfg.service.port) From 55d90a07712b2a0123b48e5692e1d02f22dc3d7a Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 10:14:58 +0100 Subject: [PATCH 11/22] use the background queue to refresh materialized views --- packages/ozone/src/background.ts | 126 +++++++----------- packages/ozone/src/daemon/context.ts | 4 +- .../src/daemon/materialized-view-refresher.ts | 13 +- packages/ozone/src/util.ts | 90 +++++++++++++ 4 files changed, 150 insertions(+), 83 deletions(-) diff --git a/packages/ozone/src/background.ts b/packages/ozone/src/background.ts index d3426da13e5..288ccbe2be6 100644 --- a/packages/ozone/src/background.ts +++ b/packages/ozone/src/background.ts @@ -1,6 +1,7 @@ import PQueue from 'p-queue' import { Database } from './db' import { dbLogger } from './logger' +import { boundAbortController, isCausedBySignal, startInterval } from './util' type Task = (db: Database, signal: AbortSignal) => Promise @@ -11,7 +12,7 @@ export class BackgroundQueue { private abortController = new AbortController() private queue = new PQueue({ concurrency: 20 }) - protected get signal() { + public get signal() { return this.abortController.signal } @@ -28,30 +29,50 @@ export class BackgroundQueue { } } - add(task: Task) { + /** + * Add a task that will be executed at some point in the future. + * + * The task will be executed even if the backgroundQueue is destroyed, unless + * the provided `signal` is aborted. + * + * The `signal` provided to the task will be aborted whenever either the + * backgroundQueue is destroyed or the provided `signal` is aborted. + */ + async add(task: Task, signal?: AbortSignal): Promise { if (this.destroyed) { return } - const { db, signal } = this + const abortController = boundAbortController(this.signal, signal) - this.queue - .add(() => task(db, signal)) - .catch((err) => { - if (!isCausedBySignal(err, signal)) { + return this.queue.add(async () => { + try { + // Do not run the task if the signal provided to the task has become + // aborted. Do not use `abortController.signal` here since we do not + // want to abort the task if the backgroundQueue is being destroyed. + if (signal?.aborted) return + + await task(this.db, abortController.signal) + } catch (err) { + if (!isCausedBySignal(err, abortController.signal)) { dbLogger.error(err, 'background queue task failed') } - }) + } finally { + abortController.abort() + } + }) } async processAll() { await this.queue.onIdle() } - // On destroy we stop accepting new tasks, but complete all - // pending/in-progress tasks. Tasks can decide to abort their current - // operation based on the signal they received. The application calls this - // only once http connections have drained (tasks no longer being added). + /** + * On destroy we stop accepting new tasks, but complete all + * pending/in-progress tasks. Tasks can decide to abort their current + * operation based on the signal they received. The application calls this + * only once http connections have drained (tasks no longer being added). + */ async destroy() { this.abortController.abort() await this.queue.onIdle() @@ -62,10 +83,10 @@ export class BackgroundQueue { * A simple periodic background task runner */ export class PeriodicBackgroundTask { - private abortController = new AbortController() + private abortController: AbortController private promise?: Promise - protected get signal() { + public get signal() { return this.abortController.signal } @@ -74,83 +95,32 @@ export class PeriodicBackgroundTask { } constructor( - protected db: Database, + protected backgroundQueue: BackgroundQueue, protected interval: number, protected task: Task, - ) {} + ) { + if (!Number.isFinite(interval) || interval <= 0) { + throw new TypeError('interval must be a positive number') + } + + // Bind this class's signal to the backgroundQueue's signal (destroying this + // instance if the backgroundQueue is destroyed) + this.abortController = boundAbortController(backgroundQueue.signal) + } async start() { - this.signal.throwIfAborted() - if (this.promise !== undefined) throw new Error('Already started') - - const { db, task } = this - - this.promise = startInterval( - async (signal) => { - try { - await task(db, signal) - } catch (err) { - if (!isCausedBySignal(err, signal)) { - dbLogger.error(err, 'periodic background task failed') - } - } - }, + // Noop if already started. Throws if the signal is aborted (destroyed). + this.promise ||= startInterval( + async (signal) => this.backgroundQueue.add(this.task, signal), this.interval, this.signal, ) } async destroy() { - this.signal.throwIfAborted() this.abortController.abort() await this.promise this.promise = undefined } } - -/** - * Determines whether the cause of an error is a signal's reason - */ -function isCausedBySignal(err: unknown, { reason }: AbortSignal) { - return err === reason || (err instanceof Error && err.cause === reason) -} - -function startInterval( - fn: (signal: AbortSignal) => void | Promise, - interval: number, - signal: AbortSignal, -) { - signal.throwIfAborted() - - return new Promise((resolve) => { - let timer: NodeJS.Timeout | undefined - - const run = async () => { - timer = undefined // record that we are running - try { - await fn(signal) - } finally { - if (signal.aborted) resolve() - else schedule() - } - } - - const schedule = () => { - timer = setTimeout(run, interval) - } - - const stop = () => { - if (timer) { - clearTimeout(timer) - resolve() - } else { - // fn is running, resolve() will be called - } - } - - signal.addEventListener('abort', stop, { once: true }) - - schedule() - }) -} diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 0063cc11683..eb9952f85a2 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -70,7 +70,9 @@ export class DaemonContext { const eventReverser = new EventReverser(db, modService) - const materializedViewRefresher = new MaterializedViewRefresher(db) + const materializedViewRefresher = new MaterializedViewRefresher( + backgroundQueue, + ) return new DaemonContext({ db, diff --git a/packages/ozone/src/daemon/materialized-view-refresher.ts b/packages/ozone/src/daemon/materialized-view-refresher.ts index 0e7820d9102..22c84cd8b53 100644 --- a/packages/ozone/src/daemon/materialized-view-refresher.ts +++ b/packages/ozone/src/daemon/materialized-view-refresher.ts @@ -1,11 +1,10 @@ import { MINUTE } from '@atproto/common' -import Database from '../db' import { sql } from 'kysely' -import { PeriodicBackgroundTask } from '../background' +import { BackgroundQueue, PeriodicBackgroundTask } from '../background' export class MaterializedViewRefresher extends PeriodicBackgroundTask { - constructor(db: Database) { - super(db, 15 * MINUTE, async ({ db }, signal) => { + constructor(backgroundQueue: BackgroundQueue, interval = 30 * MINUTE) { + super(backgroundQueue, interval, async ({ db }, signal) => { for (const view of [ 'account_events_stats', 'record_events_stats', @@ -13,6 +12,12 @@ export class MaterializedViewRefresher extends PeriodicBackgroundTask { 'account_record_status_stats', ]) { if (signal.aborted) break + + // Kysely does not provide a way to cancel a running query. Because of + // this, killing the process during a refresh will cause the process to + // wait for the current refresh to finish before exiting. This is not + // ideal, but it is the best we can do until Kysely provides a way to + // cancel a query. await sql`REFRESH MATERIALIZED VIEW CONCURRENTLY ${sql.id(view)}`.execute( db, ) diff --git a/packages/ozone/src/util.ts b/packages/ozone/src/util.ts index 717316f5c96..97546b5dd5f 100644 --- a/packages/ozone/src/util.ts +++ b/packages/ozone/src/util.ts @@ -83,3 +83,93 @@ export const formatLabelerHeader = (parsed: ParsedLabelers): string => { ) return parts.join(',') } + +/** + * Utility function similar to `setInterval()`. The main difference is that the + * execution is controlled through a signal and that the function will wait for + * `interval` milliseconds *between* the end of the previous execution and the + * start of the next one (instead of starting the execution every `interval` + * milliseconds), ensuring that the function is not running concurrently. + * + * @returns A promise that resolves when the signal is aborted, and the last + * execution is done. + * + * @throws if the signal is already aborted. + */ +export function startInterval( + fn: (signal: AbortSignal) => void | Promise, + interval: number, + signal: AbortSignal, +) { + signal.throwIfAborted() + + return new Promise((resolve) => { + let timer: NodeJS.Timeout | undefined + + const run = async () => { + timer = undefined // record that we are running + + // Cloning the signal for this particular run to prevent memory leaks + const abortController = boundAbortController(signal) + try { + await fn(abortController.signal) + } finally { + abortController.abort() + if (signal.aborted) resolve() + else schedule() + } + } + + const schedule = () => { + timer = setTimeout(run, interval) + } + + const stop = () => { + if (timer) { + clearTimeout(timer) + resolve() + } else { + // fn is running, resolve() will be called + } + } + + signal.addEventListener('abort', stop, { once: true }) + + schedule() + }) +} + +/** + * Determines whether the cause of an error is a signal's reason + */ +export function isCausedBySignal(err: unknown, { reason }: AbortSignal) { + return err === reason || (err instanceof Error && err.cause === reason) +} + +/** + * Creates an AbortController that will be aborted when any of the given signals + * is aborted. + * + * @note Make sure to call `abortController.abort()` when you are done with + * the controller to avoid memory leaks. + * + * @throws if any of the input signals is already aborted. + */ +export function boundAbortController( + ...signals: readonly (AbortSignal | undefined | null)[] +): AbortController { + for (const signal of signals) { + signal?.throwIfAborted() + } + + const abortController = new AbortController() + + for (const signal of signals) { + signal?.addEventListener('abort', () => abortController.abort(), { + once: true, + signal: abortController.signal, + }) + } + + return abortController +} From 7759a30c25b260a80e6fca28484917ebcc196f2b Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 10:24:39 +0100 Subject: [PATCH 12/22] tidy --- packages/dev-env/src/bin.ts | 1 + packages/ozone/src/config/config.ts | 2 ++ packages/ozone/src/config/env.ts | 4 ++++ packages/ozone/src/daemon/context.ts | 1 + packages/ozone/src/db/index.ts | 1 - 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/dev-env/src/bin.ts b/packages/dev-env/src/bin.ts index c0b8c5c7e2c..8f367b2cf8f 100644 --- a/packages/dev-env/src/bin.ts +++ b/packages/dev-env/src/bin.ts @@ -30,6 +30,7 @@ const run = async () => { port: 2587, chatUrl: 'http://localhost:2590', // must run separate chat service chatDid: 'did:example:chat', + dbMaterializedViewRefreshIntervalMs: 30_000, }, introspect: { port: 2581 }, }) diff --git a/packages/ozone/src/config/config.ts b/packages/ozone/src/config/config.ts index a5eea6c5c1f..808163c15f1 100644 --- a/packages/ozone/src/config/config.ts +++ b/packages/ozone/src/config/config.ts @@ -24,6 +24,7 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { poolSize: env.dbPoolSize, poolMaxUses: env.dbPoolMaxUses, poolIdleTimeoutMs: env.dbPoolIdleTimeoutMs, + materializedViewRefreshIntervalMs: env.dbMaterializedViewRefreshIntervalMs, } assert(env.appviewUrl, 'appviewUrl is required') @@ -122,6 +123,7 @@ export type DatabaseConfig = { poolSize?: number poolMaxUses?: number poolIdleTimeoutMs?: number + materializedViewRefreshIntervalMs?: number } export type AppviewConfig = { diff --git a/packages/ozone/src/config/env.ts b/packages/ozone/src/config/env.ts index 173b874e78e..6d930c4e59c 100644 --- a/packages/ozone/src/config/env.ts +++ b/packages/ozone/src/config/env.ts @@ -20,6 +20,9 @@ export const readEnv = (): OzoneEnvironment => { dbPoolSize: envInt('OZONE_DB_POOL_SIZE'), dbPoolMaxUses: envInt('OZONE_DB_POOL_MAX_USES'), dbPoolIdleTimeoutMs: envInt('OZONE_DB_POOL_IDLE_TIMEOUT_MS'), + dbMaterializedViewRefreshIntervalMs: envInt( + 'OZONE_DB_MATERIALIZED_VIEW_REFRESH_INTERVAL_MS', + ), didPlcUrl: envStr('OZONE_DID_PLC_URL'), didCacheStaleTTL: envInt('OZONE_DID_CACHE_STALE_TTL'), didCacheMaxTTL: envInt('OZONE_DID_CACHE_MAX_TTL'), @@ -53,6 +56,7 @@ export type OzoneEnvironment = { dbPoolSize?: number dbPoolMaxUses?: number dbPoolIdleTimeoutMs?: number + dbMaterializedViewRefreshIntervalMs?: number didPlcUrl?: string didCacheStaleTTL?: number didCacheMaxTTL?: number diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index eb9952f85a2..7d6f9f20d39 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -72,6 +72,7 @@ export class DaemonContext { const materializedViewRefresher = new MaterializedViewRefresher( backgroundQueue, + cfg.db.materializedViewRefreshIntervalMs, ) return new DaemonContext({ diff --git a/packages/ozone/src/db/index.ts b/packages/ozone/src/db/index.ts index 4485a342a8b..b5b66542583 100644 --- a/packages/ozone/src/db/index.ts +++ b/packages/ozone/src/db/index.ts @@ -71,7 +71,6 @@ export class Database { this.pool = pool this.db = new Kysely({ dialect: new PostgresDialect({ pool }), - log: ['error', 'query'], }) } From b38d597abb17797fdd0afc61c5cb2877f82246d9 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 10:26:42 +0100 Subject: [PATCH 13/22] fix verrify --- packages/ozone/tests/query-labels.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ozone/tests/query-labels.test.ts b/packages/ozone/tests/query-labels.test.ts index f1e47c48fd7..66157fb9ba7 100644 --- a/packages/ozone/tests/query-labels.test.ts +++ b/packages/ozone/tests/query-labels.test.ts @@ -156,6 +156,7 @@ describe('ozone query labels', () => { newSigningKey, newSigningKeyId, ctx.cfg, + // @ts-ignore modSrvc.backgroundQueue, ctx.idResolver, // @ts-ignore From 1d8757d16a58b60b73384a1e123f6cb0e1cef5d9 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 11:33:57 +0100 Subject: [PATCH 14/22] tidy --- packages/ozone/src/background.ts | 14 ++++++++++-- packages/ozone/src/mod-service/types.ts | 1 + packages/ozone/src/mod-service/views.ts | 8 +++---- packages/ozone/src/util.ts | 29 +++++++++++++++++++------ 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/packages/ozone/src/background.ts b/packages/ozone/src/background.ts index 288ccbe2be6..f80e5fc2984 100644 --- a/packages/ozone/src/background.ts +++ b/packages/ozone/src/background.ts @@ -52,6 +52,9 @@ export class BackgroundQueue { // want to abort the task if the backgroundQueue is being destroyed. if (signal?.aborted) return + // The task will receive a "combined signal" allowing it to abort if + // either the backgroundQueue is destroyed or the provided signal is + // aborted. await task(this.db, abortController.signal) } catch (err) { if (!isCausedBySignal(err, abortController.signal)) { @@ -108,16 +111,23 @@ export class PeriodicBackgroundTask { this.abortController = boundAbortController(backgroundQueue.signal) } - async start() { + start() { // Noop if already started. Throws if the signal is aborted (destroyed). this.promise ||= startInterval( async (signal) => this.backgroundQueue.add(this.task, signal), this.interval, this.signal, - ) + ).catch((err) => { + if (!isCausedBySignal(err, this.signal)) { + dbLogger.error(err, 'periodic background task failed') + } + }) } async destroy() { + // @NOTE This instance does not "own" the backgroundQueue, so we do not + // destroy it here. + this.abortController.abort() await this.promise diff --git a/packages/ozone/src/mod-service/types.ts b/packages/ozone/src/mod-service/types.ts index a2682750487..42d6c7115f3 100644 --- a/packages/ozone/src/mod-service/types.ts +++ b/packages/ozone/src/mod-service/types.ts @@ -38,6 +38,7 @@ export type ModerationSubjectStatusRowWithStats = ModerationSubjectStatusRow & { processedCount: number | null takendownCount: number | null } + export type ModerationSubjectStatusRowWithHandle = ModerationSubjectStatusRowWithStats & { handle: string | null } diff --git a/packages/ozone/src/mod-service/views.ts b/packages/ozone/src/mod-service/views.ts index 8d8af599fac..db19dbac4a6 100644 --- a/packages/ozone/src/mod-service/views.ts +++ b/packages/ozone/src/mod-service/views.ts @@ -583,10 +583,10 @@ export class ModerationViews { ]) return new Map( - statusRes.map((status) => { - const subjectId = formatSubjectId(status.did, status.recordPath) - const handle = accountsByDid.get(status.did)?.handle ?? INVALID_HANDLE - return [subjectId, { ...status, handle }] + statusRes.map((row): [string, ModerationSubjectStatusRowWithHandle] => { + const subjectId = formatSubjectId(row.did, row.recordPath) + const handle = accountsByDid.get(row.did)?.handle ?? INVALID_HANDLE + return [subjectId, { ...row, handle }] }), ) } diff --git a/packages/ozone/src/util.ts b/packages/ozone/src/util.ts index 97546b5dd5f..0b6f41a353b 100644 --- a/packages/ozone/src/util.ts +++ b/packages/ozone/src/util.ts @@ -1,3 +1,4 @@ +import assert from 'node:assert' import { createRetryable } from '@atproto/common' import { ResponseType, XRPCError } from '@atproto/xrpc' import { parseList } from 'structured-headers' @@ -91,6 +92,10 @@ export const formatLabelerHeader = (parsed: ParsedLabelers): string => { * start of the next one (instead of starting the execution every `interval` * milliseconds), ensuring that the function is not running concurrently. * + * @param fn The function to execute. That function must not throw any error + * other than `signal.reason` or an {@link Error} that has the `signal.reason` + * as its cause. + * * @returns A promise that resolves when the signal is aborted, and the last * execution is done. * @@ -100,6 +105,7 @@ export function startInterval( fn: (signal: AbortSignal) => void | Promise, interval: number, signal: AbortSignal, + runImmediately = false, ) { signal.throwIfAborted() @@ -107,12 +113,16 @@ export function startInterval( let timer: NodeJS.Timeout | undefined const run = async () => { - timer = undefined // record that we are running - // Cloning the signal for this particular run to prevent memory leaks const abortController = boundAbortController(signal) try { await fn(abortController.signal) + } catch (err) { + if (!isCausedBySignal(err, abortController.signal)) { + // Will cause "unhandledRejection" event to be emitted. This is + // expected. + throw err + } } finally { abortController.abort() if (signal.aborted) resolve() @@ -121,21 +131,26 @@ export function startInterval( } const schedule = () => { - timer = setTimeout(run, interval) + assert(timer === undefined, 'unexpected state') + timer = setTimeout(() => { + timer = undefined // record that we are running + void run() + }, interval) } const stop = () => { - if (timer) { + if (timer === undefined) { + // fn is running, `resolve` will be called from `run`'s finally block + } else { clearTimeout(timer) resolve() - } else { - // fn is running, resolve() will be called } } signal.addEventListener('abort', stop, { once: true }) - schedule() + if (runImmediately) void run() + else schedule() }) } From 5d3d8290c955bb57c21d2b64c3a9a2e971f82311 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 11:58:12 +0100 Subject: [PATCH 15/22] Add filtering based on "minAccountSuspendCount" --- .../tools/ozone/moderation/queryStatuses.json | 4 ++++ packages/api/src/client/lexicons.ts | 5 +++++ .../tools/ozone/moderation/queryStatuses.ts | 2 ++ ...41220T144630860Z-stats-materialized-views.ts | 7 +++++++ packages/ozone/src/lexicon/lexicons.ts | 5 +++++ .../tools/ozone/moderation/queryStatuses.ts | 2 ++ packages/ozone/src/mod-service/index.ts | 17 +++++++++++++---- packages/pds/src/lexicon/lexicons.ts | 5 +++++ .../tools/ozone/moderation/queryStatuses.ts | 2 ++ 9 files changed, 45 insertions(+), 4 deletions(-) diff --git a/lexicons/tools/ozone/moderation/queryStatuses.json b/lexicons/tools/ozone/moderation/queryStatuses.json index 89efcf76020..c280b2ed61e 100644 --- a/lexicons/tools/ozone/moderation/queryStatuses.json +++ b/lexicons/tools/ozone/moderation/queryStatuses.json @@ -164,6 +164,10 @@ "description": "If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.", "knownValues": ["account", "record"] }, + "minAccountSuspendCount": { + "type": "integer", + "description": "If specified, only subjects that belong to an account that has at least this many suspensions will be returned." + }, "minReportedRecordsCount": { "type": "integer", "description": "If specified, only subjects that belong to an account that has at least this many reported records will be returned." diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 1e6f86577f9..bf038065fd7 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -12686,6 +12686,11 @@ export const schemaDict = { "If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.", knownValues: ['account', 'record'], }, + minAccountSuspendCount: { + type: 'integer', + description: + 'If specified, only subjects that belong to an account that has at least this many suspensions will be returned.', + }, minReportedRecordsCount: { type: 'integer', description: diff --git a/packages/api/src/client/types/tools/ozone/moderation/queryStatuses.ts b/packages/api/src/client/types/tools/ozone/moderation/queryStatuses.ts index d129a737225..97e22ff240c 100644 --- a/packages/api/src/client/types/tools/ozone/moderation/queryStatuses.ts +++ b/packages/api/src/client/types/tools/ozone/moderation/queryStatuses.ts @@ -66,6 +66,8 @@ export interface QueryParams { collections?: string[] /** If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. */ subjectType?: 'account' | 'record' | (string & {}) + /** If specified, only subjects that belong to an account that has at least this many suspensions will be returned. */ + minAccountSuspendCount?: number /** If specified, only subjects that belong to an account that has at least this many reported records will be returned. */ minReportedRecordsCount?: number /** If specified, only subjects that belong to an account that has at least this many taken down records will be returned. */ diff --git a/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts index 19731020386..65712115dfc 100644 --- a/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts +++ b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts @@ -71,6 +71,13 @@ export async function up(db: Kysely): Promise { .column('subjectDid') .execute() + await db.schema + .createIndex('account_events_stats_suspend_count_idx') + .on('account_events_stats') + .expression(sql`"suspendCount" ASC NULLS FIRST`) + .column('subjectDid') + .execute() + // ~50sec for 16M events await db.schema .createView('record_events_stats') diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index 1e6f86577f9..bf038065fd7 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -12686,6 +12686,11 @@ export const schemaDict = { "If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.", knownValues: ['account', 'record'], }, + minAccountSuspendCount: { + type: 'integer', + description: + 'If specified, only subjects that belong to an account that has at least this many suspensions will be returned.', + }, minReportedRecordsCount: { type: 'integer', description: diff --git a/packages/ozone/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts b/packages/ozone/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts index fed507cf703..28270425304 100644 --- a/packages/ozone/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts +++ b/packages/ozone/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts @@ -67,6 +67,8 @@ export interface QueryParams { collections?: string[] /** If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. */ subjectType?: 'account' | 'record' | (string & {}) + /** If specified, only subjects that belong to an account that has at least this many suspensions will be returned. */ + minAccountSuspendCount?: number /** If specified, only subjects that belong to an account that has at least this many reported records will be returned. */ minReportedRecordsCount?: number /** If specified, only subjects that belong to an account that has at least this many taken down records will be returned. */ diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index 77ba5cc6198..a1fd403fa2f 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -839,8 +839,9 @@ export class ModerationService { excludeTags, collections, subjectType, - minReportedRecordsCount = 0, - minTakendownRecordsCount = 0, + minAccountSuspendCount, + minReportedRecordsCount, + minTakendownRecordsCount, }: QueryStatusParams): Promise<{ statuses: ModerationSubjectStatusRowWithHandle[] cursor?: string @@ -1073,7 +1074,15 @@ export class ModerationService { ) } - if (minTakendownRecordsCount > 0) { + if (minAccountSuspendCount != null && minAccountSuspendCount > 0) { + builder = builder.where( + 'account_events_stats.suspendCount', + '>=', + minAccountSuspendCount, + ) + } + + if (minTakendownRecordsCount != null && minTakendownRecordsCount > 0) { builder = builder.where( 'account_record_status_stats.takendownCount', '>=', @@ -1081,7 +1090,7 @@ export class ModerationService { ) } - if (minReportedRecordsCount > 0) { + if (minReportedRecordsCount != null && minReportedRecordsCount > 0) { builder = builder.where( 'account_record_events_stats.reportedCount', '>=', diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 1e6f86577f9..bf038065fd7 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -12686,6 +12686,11 @@ export const schemaDict = { "If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored.", knownValues: ['account', 'record'], }, + minAccountSuspendCount: { + type: 'integer', + description: + 'If specified, only subjects that belong to an account that has at least this many suspensions will be returned.', + }, minReportedRecordsCount: { type: 'integer', description: diff --git a/packages/pds/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts b/packages/pds/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts index fed507cf703..28270425304 100644 --- a/packages/pds/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts +++ b/packages/pds/src/lexicon/types/tools/ozone/moderation/queryStatuses.ts @@ -67,6 +67,8 @@ export interface QueryParams { collections?: string[] /** If specified, subjects of the given type (account or record) will be returned. When this is set to 'account' the 'collections' parameter will be ignored. When includeAllUserRecords or subject is set, this will be ignored. */ subjectType?: 'account' | 'record' | (string & {}) + /** If specified, only subjects that belong to an account that has at least this many suspensions will be returned. */ + minAccountSuspendCount?: number /** If specified, only subjects that belong to an account that has at least this many reported records will be returned. */ minReportedRecordsCount?: number /** If specified, only subjects that belong to an account that has at least this many taken down records will be returned. */ From 3b09a4a5db7d59fea21c1d0df0fd236660ba42c3 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 12:48:32 +0100 Subject: [PATCH 16/22] fix tests --- packages/ozone/src/background.ts | 5 +++- packages/ozone/src/mod-service/status.ts | 26 ++++++++++++++++--- .../proxied/__snapshots__/admin.test.ts.snap | 6 +++++ packages/pds/tests/proxied/admin.test.ts | 3 +++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/packages/ozone/src/background.ts b/packages/ozone/src/background.ts index f80e5fc2984..a3fb1f2f619 100644 --- a/packages/ozone/src/background.ts +++ b/packages/ozone/src/background.ts @@ -83,7 +83,10 @@ export class BackgroundQueue { } /** - * A simple periodic background task runner + * A simple periodic background task runner. This class will schedule a task to + * run through a provided {@link BackgroundQueue} at a fixed interval. The task + * will never run more than once concurrently, and will wait at least `interval` + * milliseconds between the end of one run and the start of the next. */ export class PeriodicBackgroundTask { private abortController: AbortController diff --git a/packages/ozone/src/mod-service/status.ts b/packages/ozone/src/mod-service/status.ts index 695579a511e..67fb3b4a20c 100644 --- a/packages/ozone/src/mod-service/status.ts +++ b/packages/ozone/src/mod-service/status.ts @@ -204,6 +204,10 @@ const getSubjectStatusForRecordEvent = ({ } export const moderationSubjectStatusQueryBuilder = (db: DatabaseSchema) => { + // @NOTE: Using select() instead of selectAll() below because the materialized + // views might be incomplete, and we don't want the null `did` columns to + // interfere with the (never null) `did` column from the + // `moderation_subject_status` table in the results return db .selectFrom('moderation_subject_status') .selectAll('moderation_subject_status') @@ -214,7 +218,13 @@ export const moderationSubjectStatusQueryBuilder = (db: DatabaseSchema) => { 'account_events_stats.subjectDid', ), ) - .selectAll('account_events_stats') + .select([ + 'account_events_stats.takedownCount', + 'account_events_stats.suspendCount', + 'account_events_stats.escalateCount', + 'account_events_stats.reportCount', + 'account_events_stats.appealCount', + ]) .leftJoin('account_record_events_stats', (join) => join.onRef( 'moderation_subject_status.did', @@ -222,7 +232,12 @@ export const moderationSubjectStatusQueryBuilder = (db: DatabaseSchema) => { 'account_record_events_stats.subjectDid', ), ) - .selectAll('account_record_events_stats') + .select([ + 'account_record_events_stats.totalReports', + 'account_record_events_stats.reportedCount', + 'account_record_events_stats.escalatedCount', + 'account_record_events_stats.appealedCount', + ]) .leftJoin('account_record_status_stats', (join) => join.onRef( 'moderation_subject_status.did', @@ -230,7 +245,12 @@ export const moderationSubjectStatusQueryBuilder = (db: DatabaseSchema) => { 'account_record_status_stats.did', ), ) - .selectAll('account_record_status_stats') + .select([ + 'account_record_status_stats.subjectCount', + 'account_record_status_stats.pendingCount', + 'account_record_status_stats.processedCount', + 'account_record_status_stats.takendownCount', + ]) } // Based on a given moderation action event, this function will update the moderation status of the subject diff --git a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap index 45f8c6daa02..9bdd2f05b45 100644 --- a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap @@ -142,6 +142,7 @@ Object { "indexedAt": "1970-01-01T00:00:00.000Z", "moderation": Object { "subjectStatus": Object { + "accountStats": Object {}, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -151,6 +152,7 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", + "recordsStats": Object {}, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", @@ -240,6 +242,7 @@ Object { "labels": Array [], "moderation": Object { "subjectStatus": Object { + "accountStats": Object {}, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#recordHosting", @@ -248,6 +251,7 @@ Object { "id": 5, "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", + "recordsStats": Object {}, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -301,6 +305,7 @@ Object { "invitesDisabled": true, "moderation": Object { "subjectStatus": Object { + "accountStats": Object {}, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -310,6 +315,7 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", + "recordsStats": Object {}, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", diff --git a/packages/pds/tests/proxied/admin.test.ts b/packages/pds/tests/proxied/admin.test.ts index 1f51d2df9f5..d9871ea73a3 100644 --- a/packages/pds/tests/proxied/admin.test.ts +++ b/packages/pds/tests/proxied/admin.test.ts @@ -62,6 +62,9 @@ describe('proxies admin requests', () => { password: 'password', inviteCode: invite.code, }) + }) + + beforeEach(async () => { await network.processAll() }) From 77e3924d89a0c725423de2d6ea57e536595503fd Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 13:36:28 +0100 Subject: [PATCH 17/22] Update test snapshots --- packages/ozone/src/mod-service/views.ts | 6 ++++ .../__snapshots__/get-record.test.ts.snap | 12 +++++++ .../__snapshots__/get-records.test.ts.snap | 6 ++++ .../tests/__snapshots__/get-repo.test.ts.snap | 6 ++++ .../__snapshots__/get-repos.test.ts.snap | 6 ++++ .../moderation-events.test.ts.snap | 6 ++++ .../moderation-statuses.test.ts.snap | 36 +++++++++++++++++++ .../takedown-appeal.test.ts.snap | 6 ++++ .../proxied/__snapshots__/admin.test.ts.snap | 24 +++++++++---- 9 files changed, 102 insertions(+), 6 deletions(-) diff --git a/packages/ozone/src/mod-service/views.ts b/packages/ozone/src/mod-service/views.ts index db19dbac4a6..12a3d9f018c 100644 --- a/packages/ozone/src/mod-service/views.ts +++ b/packages/ozone/src/mod-service/views.ts @@ -615,6 +615,9 @@ export class ModerationViews { subject: subjectFromStatusRow(status).lex(), accountStats: { + // Explicitly typing to allow for easy manipulation (e.g. to strip from tests snapshots) + $type: 'tools.ozone.moderation.defs#accountStats', + // account_events_stats reportCount: status.reportCount ?? undefined, appealCount: status.appealCount ?? undefined, @@ -624,6 +627,9 @@ export class ModerationViews { }, recordsStats: { + // Explicitly typing to allow for easy manipulation (e.g. to strip from tests snapshots) + $type: 'tools.ozone.moderation.defs#recordStats', + // account_record_events_stats totalReports: status.totalReports ?? undefined, reportedCount: status.reportedCount ?? undefined, diff --git a/packages/ozone/tests/__snapshots__/get-record.test.ts.snap b/packages/ozone/tests/__snapshots__/get-record.test.ts.snap index d24b9d76106..162e6869864 100644 --- a/packages/ozone/tests/__snapshots__/get-record.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-record.test.ts.snap @@ -28,6 +28,9 @@ Object { ], "moderation": Object { "subjectStatus": Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#recordHosting", @@ -37,6 +40,9 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -134,6 +140,9 @@ Object { ], "moderation": Object { "subjectStatus": Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#recordHosting", @@ -143,6 +152,9 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.repo.strongRef", diff --git a/packages/ozone/tests/__snapshots__/get-records.test.ts.snap b/packages/ozone/tests/__snapshots__/get-records.test.ts.snap index 38d0efdebca..d51f13d14e7 100644 --- a/packages/ozone/tests/__snapshots__/get-records.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-records.test.ts.snap @@ -31,6 +31,9 @@ Object { ], "moderation": Object { "subjectStatus": Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#recordHosting", @@ -40,6 +43,9 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.repo.strongRef", diff --git a/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap b/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap index 397a817ec0b..5f5f9e6a5b1 100644 --- a/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap @@ -22,6 +22,9 @@ Object { ], "moderation": Object { "subjectStatus": Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -31,6 +34,9 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", diff --git a/packages/ozone/tests/__snapshots__/get-repos.test.ts.snap b/packages/ozone/tests/__snapshots__/get-repos.test.ts.snap index d0db4fcf9de..b79338558cf 100644 --- a/packages/ozone/tests/__snapshots__/get-repos.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-repos.test.ts.snap @@ -25,6 +25,9 @@ Object { ], "moderation": Object { "subjectStatus": Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -34,6 +37,9 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", diff --git a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap index 8e4b5a62c84..39d73a950bc 100644 --- a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap @@ -18,6 +18,9 @@ Object { "indexedAt": "1970-01-01T00:00:00.000Z", "moderation": Object { "subjectStatus": Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -27,6 +30,9 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewEscalated", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", diff --git a/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap index 92b4646c120..1d094b55fd9 100644 --- a/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap @@ -3,6 +3,9 @@ exports[`moderation-statuses query statuses returns statuses filtered by subject language 1`] = ` Array [ Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#recordHosting", @@ -10,6 +13,9 @@ Array [ }, "id": 7, "lastReportedAt": "1970-01-01T00:00:00.000Z", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -27,6 +33,9 @@ Array [ "updatedAt": "1970-01-01T00:00:00.000Z", }, Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -34,6 +43,9 @@ Array [ }, "id": 5, "lastReportedAt": "1970-01-01T00:00:00.000Z", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", @@ -55,6 +67,9 @@ Array [ exports[`moderation-statuses query statuses returns statuses for subjects that received moderation events 1`] = ` Array [ Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#recordHosting", @@ -62,6 +77,9 @@ Array [ }, "id": 7, "lastReportedAt": "1970-01-01T00:00:00.000Z", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -79,6 +97,9 @@ Array [ "updatedAt": "1970-01-01T00:00:00.000Z", }, Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -86,6 +107,9 @@ Array [ }, "id": 5, "lastReportedAt": "1970-01-01T00:00:00.000Z", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", @@ -102,6 +126,9 @@ Array [ "updatedAt": "1970-01-01T00:00:00.000Z", }, Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#recordHosting", @@ -109,6 +136,9 @@ Array [ }, "id": 3, "lastReportedAt": "1970-01-01T00:00:00.000Z", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -125,6 +155,9 @@ Array [ "updatedAt": "1970-01-01T00:00:00.000Z", }, Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -132,6 +165,9 @@ Array [ }, "id": 1, "lastReportedAt": "1970-01-01T00:00:00.000Z", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", diff --git a/packages/pds/tests/__snapshots__/takedown-appeal.test.ts.snap b/packages/pds/tests/__snapshots__/takedown-appeal.test.ts.snap index a4dfd5d505c..1e7e1c31845 100644 --- a/packages/pds/tests/__snapshots__/takedown-appeal.test.ts.snap +++ b/packages/pds/tests/__snapshots__/takedown-appeal.test.ts.snap @@ -2,6 +2,9 @@ exports[`appeal account takedown actor takedown allows appeal request. 1`] = ` Object { + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "appealed": true, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -13,6 +16,9 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(0)", + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewEscalated", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", diff --git a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap index 9bdd2f05b45..697a3c465f3 100644 --- a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap @@ -142,7 +142,9 @@ Object { "indexedAt": "1970-01-01T00:00:00.000Z", "moderation": Object { "subjectStatus": Object { - "accountStats": Object {}, + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -152,7 +154,9 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", - "recordsStats": Object {}, + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", @@ -242,7 +246,9 @@ Object { "labels": Array [], "moderation": Object { "subjectStatus": Object { - "accountStats": Object {}, + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#recordHosting", @@ -251,7 +257,9 @@ Object { "id": 5, "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", - "recordsStats": Object {}, + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.repo.strongRef", @@ -305,7 +313,9 @@ Object { "invitesDisabled": true, "moderation": Object { "subjectStatus": Object { - "accountStats": Object {}, + "accountStats": Object { + "$type": "tools.ozone.moderation.defs#accountStats", + }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { "$type": "tools.ozone.moderation.defs#accountHosting", @@ -315,7 +325,9 @@ Object { "lastReportedAt": "1970-01-01T00:00:00.000Z", "lastReviewedAt": "1970-01-01T00:00:00.000Z", "lastReviewedBy": "user(1)", - "recordsStats": Object {}, + "recordsStats": Object { + "$type": "tools.ozone.moderation.defs#recordStats", + }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { "$type": "com.atproto.admin.defs#repoRef", From 41924b76d125dc13294a1cf4cd208051488d1afc Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 14:30:40 +0100 Subject: [PATCH 18/22] update materialized views when running `processAll` --- packages/ozone/src/background.ts | 36 ++++++--- packages/ozone/src/daemon/context.ts | 5 +- ...220T144630860Z-stats-materialized-views.ts | 7 +- .../__snapshots__/get-record.test.ts.snap | 16 ++++ .../__snapshots__/get-records.test.ts.snap | 8 ++ .../tests/__snapshots__/get-repo.test.ts.snap | 5 ++ .../__snapshots__/get-repos.test.ts.snap | 5 ++ .../moderation-events.test.ts.snap | 13 ++++ .../moderation-statuses.test.ts.snap | 78 +++++++++++++++++++ packages/ozone/tests/get-record.test.ts | 4 + packages/ozone/tests/get-records.test.ts | 4 + packages/ozone/tests/get-repo.test.ts | 4 + packages/ozone/tests/get-repos.test.ts | 4 + .../ozone/tests/moderation-events.test.ts | 4 + .../ozone/tests/moderation-statuses.test.ts | 4 + .../proxied/__snapshots__/admin.test.ts.snap | 39 ++++++++++ 16 files changed, 224 insertions(+), 12 deletions(-) diff --git a/packages/ozone/src/background.ts b/packages/ozone/src/background.ts index a3fb1f2f619..ac97e39c932 100644 --- a/packages/ozone/src/background.ts +++ b/packages/ozone/src/background.ts @@ -90,7 +90,9 @@ export class BackgroundQueue { */ export class PeriodicBackgroundTask { private abortController: AbortController - private promise?: Promise + + private intervalPromise?: Promise + private runningPromise?: Promise public get signal() { return this.abortController.signal @@ -114,17 +116,31 @@ export class PeriodicBackgroundTask { this.abortController = boundAbortController(backgroundQueue.signal) } + private run(signal: AbortSignal): Promise { + // Already running + if (this.runningPromise) return this.runningPromise + + const promise = this.backgroundQueue.add(this.task, signal) + + // Store the promise on this instance + return (this.runningPromise = promise).finally(() => { + if (this.runningPromise === promise) this.runningPromise = undefined + }) + } + start() { // Noop if already started. Throws if the signal is aborted (destroyed). - this.promise ||= startInterval( - async (signal) => this.backgroundQueue.add(this.task, signal), + this.intervalPromise ||= startInterval( + async (signal) => this.run(signal), this.interval, this.signal, - ).catch((err) => { - if (!isCausedBySignal(err, this.signal)) { - dbLogger.error(err, 'periodic background task failed') - } - }) + ) + } + + async processAll() { + if (!this.intervalPromise) return // not started + + return this.run(this.signal) } async destroy() { @@ -133,7 +149,7 @@ export class PeriodicBackgroundTask { this.abortController.abort() - await this.promise - this.promise = undefined + await this.intervalPromise + this.intervalPromise = undefined } } diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 7d6f9f20d39..2d9d5a26dde 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -118,7 +118,10 @@ export class DaemonContext { } async processAll() { - await this.eventPusher.processAll() + await allFulfilled([ + this.eventPusher.processAll(), + this.materializedViewRefresher.processAll(), + ]) } async destroy() { diff --git a/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts index 65712115dfc..b961e61a7d0 100644 --- a/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts +++ b/packages/ozone/src/db/migrations/20241220T144630860Z-stats-materialized-views.ts @@ -130,7 +130,12 @@ export async function up(db: Kysely): Promise { .selectFrom('record_events_stats') .select([ 'subjectDid', - (eb) => sql`SUM(${eb.ref('reportCount')})`.as('totalReports'), + (eb) => + // Casting to "bigint" because "numeric" gets casted to a string + // by default by postgres-node. + sql`SUM(${eb.ref('reportCount')})::bigint`.as( + 'totalReports', + ), (eb) => sql`COUNT(*) FILTER (WHERE ${eb.ref('reportCount')} > 0)`.as( 'reportedCount', diff --git a/packages/ozone/tests/__snapshots__/get-record.test.ts.snap b/packages/ozone/tests/__snapshots__/get-record.test.ts.snap index 162e6869864..bed827aced6 100644 --- a/packages/ozone/tests/__snapshots__/get-record.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-record.test.ts.snap @@ -42,6 +42,14 @@ Object { "lastReviewedBy": "user(1)", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 0, + "processedCount": 1, + "reportedCount": 1, + "subjectCount": 1, + "takendownCount": 1, + "totalReports": 2, }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { @@ -154,6 +162,14 @@ Object { "lastReviewedBy": "user(1)", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 0, + "processedCount": 1, + "reportedCount": 1, + "subjectCount": 1, + "takendownCount": 1, + "totalReports": 2, }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { diff --git a/packages/ozone/tests/__snapshots__/get-records.test.ts.snap b/packages/ozone/tests/__snapshots__/get-records.test.ts.snap index d51f13d14e7..05f6f6f3273 100644 --- a/packages/ozone/tests/__snapshots__/get-records.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-records.test.ts.snap @@ -45,6 +45,14 @@ Object { "lastReviewedBy": "user(1)", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 0, + "processedCount": 1, + "reportedCount": 1, + "subjectCount": 1, + "takendownCount": 1, + "totalReports": 2, }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { diff --git a/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap b/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap index 5f5f9e6a5b1..d548cd9d844 100644 --- a/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-repo.test.ts.snap @@ -24,6 +24,11 @@ Object { "subjectStatus": Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 1, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { diff --git a/packages/ozone/tests/__snapshots__/get-repos.test.ts.snap b/packages/ozone/tests/__snapshots__/get-repos.test.ts.snap index b79338558cf..1da0c8ab0fb 100644 --- a/packages/ozone/tests/__snapshots__/get-repos.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/get-repos.test.ts.snap @@ -27,6 +27,11 @@ Object { "subjectStatus": Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 1, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { diff --git a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap index 39d73a950bc..3aaf889373b 100644 --- a/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation-events.test.ts.snap @@ -20,6 +20,11 @@ Object { "subjectStatus": Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 1, + "reportCount": 4, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -32,6 +37,14 @@ Object { "lastReviewedBy": "user(1)", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 2, + "processedCount": 0, + "reportedCount": 2, + "subjectCount": 2, + "takendownCount": 0, + "totalReports": 3, }, "reviewState": "tools.ozone.moderation.defs#reviewEscalated", "subject": Object { diff --git a/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap b/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap index 1d094b55fd9..9e1258212db 100644 --- a/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap +++ b/packages/ozone/tests/__snapshots__/moderation-statuses.test.ts.snap @@ -5,6 +5,11 @@ Array [ Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -15,6 +20,14 @@ Array [ "lastReportedAt": "1970-01-01T00:00:00.000Z", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 1, + "processedCount": 0, + "reportedCount": 1, + "subjectCount": 1, + "takendownCount": 0, + "totalReports": 2, }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { @@ -35,6 +48,11 @@ Array [ Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -45,6 +63,14 @@ Array [ "lastReportedAt": "1970-01-01T00:00:00.000Z", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 1, + "processedCount": 0, + "reportedCount": 1, + "subjectCount": 1, + "takendownCount": 0, + "totalReports": 2, }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { @@ -69,6 +95,11 @@ Array [ Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -79,6 +110,14 @@ Array [ "lastReportedAt": "1970-01-01T00:00:00.000Z", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 1, + "processedCount": 0, + "reportedCount": 1, + "subjectCount": 1, + "takendownCount": 0, + "totalReports": 2, }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { @@ -99,6 +138,11 @@ Array [ Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -109,6 +153,14 @@ Array [ "lastReportedAt": "1970-01-01T00:00:00.000Z", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 1, + "processedCount": 0, + "reportedCount": 1, + "subjectCount": 1, + "takendownCount": 0, + "totalReports": 2, }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { @@ -128,6 +180,11 @@ Array [ Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -138,6 +195,14 @@ Array [ "lastReportedAt": "1970-01-01T00:00:00.000Z", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 1, + "processedCount": 0, + "reportedCount": 1, + "subjectCount": 1, + "takendownCount": 0, + "totalReports": 2, }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { @@ -157,6 +222,11 @@ Array [ Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -167,6 +237,14 @@ Array [ "lastReportedAt": "1970-01-01T00:00:00.000Z", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 1, + "processedCount": 0, + "reportedCount": 1, + "subjectCount": 1, + "takendownCount": 0, + "totalReports": 2, }, "reviewState": "tools.ozone.moderation.defs#reviewOpen", "subject": Object { diff --git a/packages/ozone/tests/get-record.test.ts b/packages/ozone/tests/get-record.test.ts index 8d4dbbb2527..0e289b550fb 100644 --- a/packages/ozone/tests/get-record.test.ts +++ b/packages/ozone/tests/get-record.test.ts @@ -33,6 +33,10 @@ describe('admin get record view', () => { await network.processAll() }) + beforeEach(async () => { + await network.processAll() + }) + afterAll(async () => { await network.close() }) diff --git a/packages/ozone/tests/get-records.test.ts b/packages/ozone/tests/get-records.test.ts index 7098ae6510d..3b3e63e1bd2 100644 --- a/packages/ozone/tests/get-records.test.ts +++ b/packages/ozone/tests/get-records.test.ts @@ -32,6 +32,10 @@ describe('admin get records view', () => { await network.processAll() }) + beforeEach(async () => { + await network.processAll() + }) + afterAll(async () => { await network.close() }) diff --git a/packages/ozone/tests/get-repo.test.ts b/packages/ozone/tests/get-repo.test.ts index c122dc3a001..6ededcbe00e 100644 --- a/packages/ozone/tests/get-repo.test.ts +++ b/packages/ozone/tests/get-repo.test.ts @@ -38,6 +38,10 @@ describe('admin get repo view', () => { await network.processAll() }) + beforeEach(async () => { + await network.processAll() + }) + afterAll(async () => { await network.close() }) diff --git a/packages/ozone/tests/get-repos.test.ts b/packages/ozone/tests/get-repos.test.ts index 52028d9230d..a56c7b54426 100644 --- a/packages/ozone/tests/get-repos.test.ts +++ b/packages/ozone/tests/get-repos.test.ts @@ -38,6 +38,10 @@ describe('admin get multiple repos', () => { await network.processAll() }) + beforeEach(async () => { + await network.processAll() + }) + afterAll(async () => { await network.close() }) diff --git a/packages/ozone/tests/moderation-events.test.ts b/packages/ozone/tests/moderation-events.test.ts index 181f7e9f3f4..9758f7c0759 100644 --- a/packages/ozone/tests/moderation-events.test.ts +++ b/packages/ozone/tests/moderation-events.test.ts @@ -68,6 +68,10 @@ describe('moderation-events', () => { await seedEvents() }) + beforeEach(async () => { + await network.processAll() + }) + afterAll(async () => { await network.close() }) diff --git a/packages/ozone/tests/moderation-statuses.test.ts b/packages/ozone/tests/moderation-statuses.test.ts index a8b4de52d00..dfe3cd9add3 100644 --- a/packages/ozone/tests/moderation-statuses.test.ts +++ b/packages/ozone/tests/moderation-statuses.test.ts @@ -73,6 +73,10 @@ describe('moderation-statuses', () => { await seedEvents() }) + beforeEach(async () => { + await network.processAll() + }) + afterAll(async () => { await network.close() }) diff --git a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap index 697a3c465f3..32f98e2ae62 100644 --- a/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap +++ b/packages/pds/tests/proxied/__snapshots__/admin.test.ts.snap @@ -144,6 +144,11 @@ Object { "subjectStatus": Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -156,6 +161,14 @@ Object { "lastReviewedBy": "user(1)", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 0, + "processedCount": 1, + "reportedCount": 0, + "subjectCount": 1, + "takendownCount": 0, + "totalReports": 0, }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { @@ -248,6 +261,11 @@ Object { "subjectStatus": Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -259,6 +277,14 @@ Object { "lastReviewedBy": "user(1)", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 0, + "processedCount": 1, + "reportedCount": 0, + "subjectCount": 1, + "takendownCount": 0, + "totalReports": 0, }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { @@ -315,6 +341,11 @@ Object { "subjectStatus": Object { "accountStats": Object { "$type": "tools.ozone.moderation.defs#accountStats", + "appealCount": 0, + "escalateCount": 0, + "reportCount": 2, + "suspendCount": 0, + "takedownCount": 0, }, "createdAt": "1970-01-01T00:00:00.000Z", "hosting": Object { @@ -327,6 +358,14 @@ Object { "lastReviewedBy": "user(1)", "recordsStats": Object { "$type": "tools.ozone.moderation.defs#recordStats", + "appealedCount": 0, + "escalatedCount": 0, + "pendingCount": 0, + "processedCount": 1, + "reportedCount": 0, + "subjectCount": 1, + "takendownCount": 0, + "totalReports": 0, }, "reviewState": "tools.ozone.moderation.defs#reviewClosed", "subject": Object { From 5983e5f6d095814da01b851753a7f675e2a0dd7a Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 14:40:32 +0100 Subject: [PATCH 19/22] tidy --- packages/ozone/src/background.ts | 30 ++++++++++++++++------------ packages/ozone/src/daemon/context.ts | 2 +- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/ozone/src/background.ts b/packages/ozone/src/background.ts index ac97e39c932..53aadbf4894 100644 --- a/packages/ozone/src/background.ts +++ b/packages/ozone/src/background.ts @@ -116,20 +116,30 @@ export class PeriodicBackgroundTask { this.abortController = boundAbortController(backgroundQueue.signal) } - private run(signal: AbortSignal): Promise { - // Already running + public run(signal?: AbortSignal): Promise { + // `startInterval` already ensures that only one run is in progress at a + // time. However, we want to be able to expose a `run()` method that can be + // used to force a run, which could cause concurrent executions. We prevent + // this using the `runningPromise` property. + if (this.runningPromise) return this.runningPromise - const promise = this.backgroundQueue.add(this.task, signal) + // Combine the `this.signal` with the provided `signal`, if any. + const abortController = boundAbortController(this.signal, signal) + + const promise = this.backgroundQueue.add(this.task, abortController.signal) - // Store the promise on this instance return (this.runningPromise = promise).finally(() => { if (this.runningPromise === promise) this.runningPromise = undefined + + // Cleanup the listeners added by `boundAbortController` + abortController.abort() }) } - start() { - // Noop if already started. Throws if the signal is aborted (destroyed). + public start() { + // Noop if already started. Throws if this.signal is aborted (instance is + // destroyed). this.intervalPromise ||= startInterval( async (signal) => this.run(signal), this.interval, @@ -137,13 +147,7 @@ export class PeriodicBackgroundTask { ) } - async processAll() { - if (!this.intervalPromise) return // not started - - return this.run(this.signal) - } - - async destroy() { + public async destroy() { // @NOTE This instance does not "own" the backgroundQueue, so we do not // destroy it here. diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 2d9d5a26dde..f197a215fd5 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -120,7 +120,7 @@ export class DaemonContext { async processAll() { await allFulfilled([ this.eventPusher.processAll(), - this.materializedViewRefresher.processAll(), + this.materializedViewRefresher.run(), ]) } From 5fac9728ce02aee32369c79bc9d154ba67158d4e Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 14:44:41 +0100 Subject: [PATCH 20/22] processAll sequentially --- packages/ozone/src/daemon/context.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index f197a215fd5..722ffa4655e 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -118,10 +118,9 @@ export class DaemonContext { } async processAll() { - await allFulfilled([ - this.eventPusher.processAll(), - this.materializedViewRefresher.run(), - ]) + // Sequential because the materialized view values depend on the events. + await this.eventPusher.processAll() + await this.materializedViewRefresher.run() } async destroy() { From 38d22c1c02b8ff39178291f442ed95347b33b65b Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 15:41:27 +0100 Subject: [PATCH 21/22] tidy --- packages/ozone/src/util.ts | 48 ++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/packages/ozone/src/util.ts b/packages/ozone/src/util.ts index 0b6f41a353b..4e0f56b9ff4 100644 --- a/packages/ozone/src/util.ts +++ b/packages/ozone/src/util.ts @@ -109,23 +109,35 @@ export function startInterval( ) { signal.throwIfAborted() - return new Promise((resolve) => { + // Renaming for clarity + const inputSignal = signal + + // Clone the input signal in order to be able to abort the interval in case + // `fn` throws an unexpected error. + const intervalController = boundAbortController(inputSignal) + const intervalSignal = intervalController.signal + + return new Promise((resolve, reject) => { let timer: NodeJS.Timeout | undefined const run = async () => { // Cloning the signal for this particular run to prevent memory leaks - const abortController = boundAbortController(signal) + const runController = boundAbortController(intervalSignal) + const runSignal = runController.signal + try { - await fn(abortController.signal) + await fn(runSignal) } catch (err) { - if (!isCausedBySignal(err, abortController.signal)) { - // Will cause "unhandledRejection" event to be emitted. This is - // expected. - throw err + // Silently ignore the error if it is caused by the signal + if (!isCausedBySignal(err, runSignal)) { + // Invalid behavior: stop the interval and reject the promise. + intervalController.abort() + reject(new Error('Unexpected error', { cause: err })) } } finally { - abortController.abort() - if (signal.aborted) resolve() + runController.abort() + + if (intervalSignal.aborted) resolve() else schedule() } } @@ -133,21 +145,33 @@ export function startInterval( const schedule = () => { assert(timer === undefined, 'unexpected state') timer = setTimeout(() => { - timer = undefined // record that we are running + timer = undefined // "running" state void run() }, interval) } const stop = () => { + // This function will only be called if the `inputSignal` is aborted + // before the interval controller is aborted. + + // Stop the interval + intervalController.abort() + if (timer === undefined) { - // fn is running, `resolve` will be called from `run`'s finally block + // `fn` is currently running; `run`'s finally block will resolve the + // promise. } else { + // The execution was scheduled but not started yet. Clear the timer and + // resolve the promise. clearTimeout(timer) resolve() } } - signal.addEventListener('abort', stop, { once: true }) + inputSignal.addEventListener('abort', stop, { + once: true, + signal: intervalSignal, + }) if (runImmediately) void run() else schedule() From 71f8af25be0223aa8be5d8f4db7871e2471f04a5 Mon Sep 17 00:00:00 2001 From: Matthieu Sieben Date: Fri, 17 Jan 2025 15:52:57 +0100 Subject: [PATCH 22/22] tidy --- packages/ozone/src/util.ts | 3 ++- packages/pds/src/index.ts | 8 +------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/ozone/src/util.ts b/packages/ozone/src/util.ts index 4e0f56b9ff4..8c5f850a543 100644 --- a/packages/ozone/src/util.ts +++ b/packages/ozone/src/util.ts @@ -202,9 +202,10 @@ export function boundAbortController( } const abortController = new AbortController() + const abort = () => abortController.abort() for (const signal of signals) { - signal?.addEventListener('abort', () => abortController.abort(), { + signal?.addEventListener('abort', abort, { once: true, signal: abortController.signal, }) diff --git a/packages/pds/src/index.ts b/packages/pds/src/index.ts index 0d74f9a09f7..fb73a3c3017 100644 --- a/packages/pds/src/index.ts +++ b/packages/pds/src/index.ts @@ -99,13 +99,7 @@ export class PDS { app.use(loggerMiddleware) app.use(compression()) app.use(authRoutes.createRouter(ctx)) // Before CORS - app.use( - cors({ - maxAge: DAY / SECOND, - credentials: true, - origin: (origin, callback) => callback(null, origin), - }), - ) + app.use(cors({ maxAge: DAY / SECOND })) app.use(basicRoutes.createRouter(ctx)) app.use(wellKnown.createRouter(ctx)) app.use(server.xrpc.router)