From de8b8cbb05d5e7915600d5b88bd155e985124fa0 Mon Sep 17 00:00:00 2001 From: David Manthey Date: Thu, 9 Jan 2025 09:18:57 -0500 Subject: [PATCH] When clearing all caches, better clear vips and gdal caches We weren't clearing the gdal block cache, and weren't clearing the vips cache completely. Further, the vips operation cache can't be set to zero or it has the potential to cause a future memory error. GDAL also has a dataset pool (also a proxy pool), which I don't think this clears. --- CHANGELOG.md | 1 + .../gdal/large_image_source_gdal/__init__.py | 11 ++++++++++- .../vips/large_image_source_vips/__init__.py | 18 ++++++++++++------ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c327f6e8..fc802847f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Read jp2k compressed associated images in tiff files ([#1754](../../pull/1754)) - Improve writing to zarr sinks from multiple processes ([#1713](../../pull/1713)) - Slightly faster GDAL validateCOG ([#1761](../../pull/1761)) +- Improve clearing caches ([#1765](../../pull/1765)) ### Changes diff --git a/sources/gdal/large_image_source_gdal/__init__.py b/sources/gdal/large_image_source_gdal/__init__.py index 326d30a87..994af4555 100644 --- a/sources/gdal/large_image_source_gdal/__init__.py +++ b/sources/gdal/large_image_source_gdal/__init__.py @@ -26,7 +26,7 @@ import numpy as np import PIL.Image -from large_image.cache_util import LruCacheMetaclass, methodcache +from large_image.cache_util import LruCacheMetaclass, _cacheClearFuncs, methodcache from large_image.constants import (TILE_FORMAT_IMAGE, TILE_FORMAT_NUMPY, TILE_FORMAT_PIL, SourcePriority, TileOutputMimeTypes) @@ -77,6 +77,15 @@ def _lazyImport(): import pyproj # isort: on + + def _clearGDALCache(): + old = gdal.GetCacheMax() + # print('Clearing GDAL cache: size %r, max %r' % ( + # gdal.GetCacheUsed(), old)) + gdal.SetCacheMax(0) + gdal.SetCacheMax(old) + + _cacheClearFuncs.append(_clearGDALCache) except ImportError: msg = 'gdal module not found.' raise TileSourceError(msg) diff --git a/sources/vips/large_image_source_vips/__init__.py b/sources/vips/large_image_source_vips/__init__.py index 099641f14..9f3a49e09 100644 --- a/sources/vips/large_image_source_vips/__init__.py +++ b/sources/vips/large_image_source_vips/__init__.py @@ -26,12 +26,18 @@ def _clearVipsCache(): - old = pyvips.voperation.cache_get_max_files() - pyvips.voperation.cache_set_max_files(0) - pyvips.voperation.cache_set_max_files(old) - old = pyvips.voperation.cache_get_max() - pyvips.voperation.cache_set_max(0) - pyvips.voperation.cache_set_max(old) + oldfiles = pyvips.cache_get_max_files() + oldmax = pyvips.cache_get_max() + oldmem = pyvips.cache_get_max_mem() + # print('Clearing vips cache: size %r, max files %r, max %r, mem %r' % ( + # pyvips.cache_get_size(), oldfiles, oldmax, oldmem)) + pyvips.cache_set_max_files(0) + # This shouldn't ever go to zero + pyvips.cache_set_max(1) + pyvips.cache_set_max_mem(0) + pyvips.cache_set_max_files(oldfiles) + pyvips.cache_set_max(oldmax) + pyvips.cache_set_max_mem(oldmem) _cacheClearFuncs.append(_clearVipsCache)