From ce9b0d98b04312a3a167cf799ac00e49d7a8880b Mon Sep 17 00:00:00 2001 From: Squeaky Date: Mon, 9 Sep 2024 10:58:54 +0200 Subject: [PATCH] thread pool executor for deletion --- bin/remove-message-attachments.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/remove-message-attachments.py b/bin/remove-message-attachments.py index 04f0e38b0..903bf4302 100755 --- a/bin/remove-message-attachments.py +++ b/bin/remove-message-attachments.py @@ -3,6 +3,7 @@ import enum import logging from collections.abc import Iterable +from concurrent.futures import ThreadPoolExecutor import click from sqlalchemy.orm import Query @@ -122,6 +123,8 @@ def run( assert batch_size > 0 assert delete_batch_size > 0 + delete_executor = ThreadPoolExecutor(max_workers=10) + for repetition in range(repeat): blocks = find_blocks( limit, @@ -163,7 +166,7 @@ def run( delete_sha256s.add(block.data_sha256) if len(delete_sha256s) >= delete_batch_size: - delete_batch(delete_sha256s, dry_run) + delete_executor.submit(delete_batch, delete_sha256s.copy(), dry_run) delete_sha256s.clear() delete_batch(delete_sha256s, dry_run) @@ -173,6 +176,8 @@ def run( after_id = max_id + 1 + delete_executor.shutdown(wait=True) + if __name__ == "__main__": run()