-
Notifications
You must be signed in to change notification settings - Fork 251
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
engine/cleanupmgr: move alert log operations to job queue (#4211)
* cleanup: implement context-aware sleep and cleanup manager functionality * cleanup: format code for consistency in alert store initialization * cleanup: remove unused ConfigSource field from SetupArgs struct * cleanup: refactor alert cleanup logic and add shift cleanup functionality * cleanup: refactor alert cleanup logic to use whileWork for better control flow * cleanup: add comment * cleanup: remove unused cleanup statements from DB and update logic * test: refactor alert auto-close and cleanup tests for improved reliability * fix: update ShiftArgs Kind to reflect cleanup-manager-shifts * feat: add periodic jobs for schedule data cleanup and update queries * feat: add workers for cleanup of shifts and schedule data * fix: update periodic job interval for schedule data cleanup to 24 hours * feat: add logging support to cleanup manager and engine initialization * refactor: streamline schedule data cleanup by extracting user validation and shift trimming logic * feat: add logging for schedule data updates in cleanup manager * refactor: remove unnecessary checks for empty shifts in user and shift trimming functions * refactor: enhance schedule data cleanup by improving user validation and logging * refactor: improve formatting of schedule data update call in cleanup manager * docs: add comment to clarify cleanupData function purpose in schedule data management * feat: add CleanupAlertLogs function to manage alert log entries for deleted alerts * refactor: remove unused cleanupAlertLogs statement and related logic from cleanup manager * feat: implement timeout for CleanupAlertLogs worker to handle longer job durations * refactor: rename cleanupDays to more descriptive staleThresholdDays in cleanup manager functions * docs: enhance comment for CleanupMgrScheduleData to clarify last_cleanup_at usage * engine/cleanupmgr: refactor schedule data cleanup logic and add job for looking up schedules needing cleanup * engine/cleanupmgr: implement alert log cleanup job scheduling and refactor related queries * engine/initriver: remove unused noopWorker type * engine/cleanupmgr: rename LookForWorkArgs types for consistency * fix(cleanupmgr): handle empty alert_logs by using coalesce for min and max IDs
- Loading branch information
1 parent
11f21f3
commit 22265b2
Showing
7 changed files
with
276 additions
and
62 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| package cleanupmanager | ||
|
|
||
| import ( | ||
| "context" | ||
| "database/sql" | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| "github.com/jackc/pgx/v5" | ||
| "github.com/riverqueue/river" | ||
| "github.com/target/goalert/gadb" | ||
| ) | ||
|
|
||
| type AlertLogLFWArgs struct{} | ||
|
|
||
| func (AlertLogLFWArgs) Kind() string { return "cleanup-manager-alert-logs-lfw" } | ||
|
|
||
| type AlertLogArgs struct { | ||
| StartID int64 | ||
| EndID int64 | ||
| } | ||
|
|
||
| const ( | ||
| batchSize = 5000 | ||
| blockSize = 100000 | ||
| ) | ||
|
|
||
| // LookForWorkAlertLogs will schedule alert log cleanup jobs for blocks of alert log IDs. | ||
| // | ||
| // The strategy here is to look for the minimum and maximum alert log IDs in the database, then schedule jobs for each `blockSize` block of IDs, | ||
| // and those jobs will then cleanup the alert logs in that range `batchSize` at a time. | ||
| func (db *DB) LookForWorkAlertLogs(ctx context.Context, j *river.Job[AlertLogLFWArgs]) error { | ||
| var min, max int64 | ||
| err := db.lock.WithTxShared(ctx, func(ctx context.Context, tx *sql.Tx) error { | ||
| row, err := gadb.New(tx).CleanupMgrAlertLogsMinMax(ctx) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| min, max = row.MinID, row.MaxID | ||
| return nil | ||
| }) | ||
| if min == 0 && max == 0 { | ||
| return nil | ||
| } | ||
| if err != nil { | ||
| return fmt.Errorf("get min/max alert log ID: %w", err) | ||
| } | ||
|
|
||
| max++ | ||
|
|
||
| var params []river.InsertManyParams | ||
| for i := int64(0); i < max; i += blockSize { | ||
| if i < min { | ||
| // skip sparse blocks | ||
| continue | ||
| } | ||
|
|
||
| params = append(params, river.InsertManyParams{ | ||
| Args: AlertLogArgs{StartID: i, EndID: i + blockSize}, | ||
| InsertOpts: &river.InsertOpts{ | ||
| Queue: QueueName, | ||
| Priority: PriorityAlertLogs, | ||
| UniqueOpts: river.UniqueOpts{ByArgs: true}, | ||
| }, | ||
| }) | ||
| } | ||
|
|
||
| if len(params) == 0 { | ||
| return nil | ||
| } | ||
|
|
||
| _, err = river.ClientFromContext[pgx.Tx](ctx).InsertMany(ctx, params) | ||
| if err != nil { | ||
| return fmt.Errorf("insert many: %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func (AlertLogArgs) Kind() string { return "cleanup-manager-alert-logs" } | ||
|
|
||
| // CleanupAlertLogs will remove alert log entries for deleted alerts. | ||
| func (db *DB) CleanupAlertLogs(ctx context.Context, j *river.Job[AlertLogArgs]) error { | ||
| lastID := j.Args.StartID | ||
|
|
||
| err := db.whileWork(ctx, func(ctx context.Context, tx *sql.Tx) (done bool, err error) { | ||
| db.logger.DebugContext(ctx, "Cleaning up alert logs...", "lastID", lastID) | ||
| lastID, err = gadb.New(tx).CleanupAlertLogs(ctx, | ||
| gadb.CleanupAlertLogsParams{ | ||
| BatchSize: batchSize, | ||
| StartID: lastID, | ||
| EndID: j.Args.EndID, | ||
| }) | ||
| if errors.Is(err, sql.ErrNoRows) { | ||
| return true, nil | ||
| } | ||
| if err != nil { | ||
| return false, fmt.Errorf("cleanup alert logs: %w", err) | ||
| } | ||
|
|
||
| return false, nil | ||
| }) | ||
| if err != nil { | ||
| return fmt.Errorf("cleanup alert logs: %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.