Skip to content

Commit

Permalink
Refactor mosaic tile fetching and caching logic; improve CPU core det…
Browse files Browse the repository at this point in the history 
…ection
  • Loading branch information
@dqunbp
dqunbp committed Dec 6, 2024
1 parent cb33660 commit c408161
Show file tree
Hide file tree
Showing 2 changed files with 98 additions and 71 deletions.
159 changes: 93 additions & 66 deletions src/mosaic.mjs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import sharp from "sharp";
import * as db from "./db.mjs";
import { cacheGet, cachePut, cacheDelete, cachePurgeMosaic } from "./cache.mjs";
import { cacheGet, cachePut } from "./cache.mjs";
import { Tile, TileImage, constructParentTileFromChildren } from "./tile.mjs";
import { enqueueTileFetching } from "./titiler_fetcher.mjs";
import { getGeotiffMetadata } from "./metadata.mjs";
Expand All @@ -25,11 +25,11 @@ function cachePutTile(tile, key, z, x, y, extension) {
return cachePut(tile, `${key}/${z}/${x}/${y}.${extension}`);
}

// request tile for single image
// uuid -- s3 image url
// z, x, y -- coordinates
// meta -- object that contains minzoom, maxzoom and tile url template
// geojson -- image outline
/**
* Fetches a single source tile from a given image (identified by its S3 key and metadata).
* - If zoom level exceeds maxzoom, returns an empty tile.
* - Checks the cache first; if not found, fetches the tile or reconstructs it from sub-tiles.
*/
async function source(key, z, x, y, meta, geojson) {
if (z > meta.maxzoom) {
return Tile.createEmpty(z, x, y);
Expand All @@ -41,17 +41,18 @@ async function source(key, z, x, y, meta, geojson) {
}

const tileCover = getTileCover(geojson, z);
const intersects = tileCover.find((pos) => {
return pos[0] === x && pos[1] === y && pos[2] === z;
});
const intersects = tileCover.find((pos) => pos[0] === x && pos[1] === y && pos[2] === z);
if (!intersects) {
// If the tile doesn't intersect with image footprint, cache null and return empty
await cachePutTile(null, key, z, x, y, "png");
return Tile.createEmpty(z, x, y);
}

if (z >= meta.minzoom && z <= meta.maxzoom) {
// Fetch tile directly from source if zoom is within the image range
tileBuffer = await enqueueTileFetching(meta.tileUrl, z, x, y);
} else if (z < meta.maxzoom) {
// If zoom is less than maxzoom but less detailed than needed, build tile from children
const tiles = await Promise.all([
source(key, z + 1, x * 2, y * 2, meta, geojson),
source(key, z + 1, x * 2 + 1, y * 2, meta, geojson),
Expand All @@ -66,17 +67,16 @@ async function source(key, z, x, y, meta, geojson) {
}

await cachePutTile(tileBuffer, key, z, x, y, "png");

return new Tile(new TileImage(tileBuffer, "png"), z, x, y);
}

function requestCachedMosaic256px(z, x, y) {
return cachedMosaic256px(z, x, y);
}

// This map ensures that concurrent requests for the same mosaic512px tile are deduplicated
const activeMosaicRequests = new Map();
function requestCachedMosaic512px(z, x, y) {
// wrapper that deduplicates mosaic function calls
const key = JSON.stringify([z, x, y]);
if (activeMosaicRequests.has(key)) {
return activeMosaicRequests.get(key);
Expand All @@ -88,6 +88,10 @@ function requestCachedMosaic512px(z, x, y) {
return request;
}

/**
* Attempts to fetch a cached 256px mosaic tile.
* If not cached, generates it from a 512px mosaic.
*/
async function cachedMosaic256px(z, x, y) {
let tileBuffer = await cacheGetTile("__mosaic256__", z, x, y, "png");
if (tileBuffer) {
Expand All @@ -101,10 +105,13 @@ async function cachedMosaic256px(z, x, y) {

const mosaicTile = await mosaic256px(z, x, y);
await cachePutTile(mosaicTile.image.buffer, "__mosaic256__", z, x, y, mosaicTile.image.extension);

return mosaicTile;
}

/**
* Attempts to fetch a cached 512px mosaic tile.
* If not cached, generates it by calling mosaic512px.
*/
async function cachedMosaic512px(z, x, y) {
let tileBuffer = await cacheGetTile("__mosaic__", z, x, y, "png");
if (tileBuffer) {
Expand All @@ -118,10 +125,14 @@ async function cachedMosaic512px(z, x, y) {

const mosaicTile = await mosaic512px(z, x, y);
await cachePutTile(mosaicTile.image.buffer, "__mosaic__", z, x, y, mosaicTile.image.extension);

return mosaicTile;
}

/**
* Generates a 256px mosaic tile from a 512px tile.
* If z is even, we scale down a 512px tile.
* If z is odd, we extract a quarter of a parent 512px tile.
*/
async function mosaic256px(z, x, y, filters = {}) {
const request512pxFn = Object.keys(filters).length > 0 ? mosaic512px : requestCachedMosaic512px;
let tile256;
Expand All @@ -134,16 +145,23 @@ async function mosaic256px(z, x, y, filters = {}) {
}

tile256.image.transformInJpegIfFullyOpaque();

return tile256;
}

/**
* Generates a 512px mosaic tile:
* 1. Queries DB for images intersecting at the given z, x, y.
* 2. Fetches metadata for each image's UUID.
* 3. If z < 9, also generates a parent tile constructed from children tiles (downsampled).
* 4. Combines all candidate tiles, sorts them by criteria, and blends them into a final mosaic.
*/
async function mosaic512px(z, x, y, filters = {}) {
const request512pxFn = Object.keys(filters).length > 0 ? mosaic512px : requestCachedMosaic512px;

let dbClient;
let rows;

// Build a parametrized SQL query to find all images covering this tile.
const { sqlQuery, sqlQueryParams, queryTag } = buildParametrizedFiltersQuery(
OAM_LAYER_ID,
z,
Expand All @@ -152,6 +170,7 @@ async function mosaic512px(z, x, y, filters = {}) {
filters
);

// Execute the DB query
try {
dbClient = await db.getClient();
const dbResponse = await dbClient.query({
Expand All @@ -168,6 +187,7 @@ async function mosaic512px(z, x, y, filters = {}) {
}
}

// Pre-fetch metadata for all rows
const metadataByUuid = {};
await Promise.all(
rows.map(async (row) => {
Expand All @@ -176,88 +196,95 @@ async function mosaic512px(z, x, y, filters = {}) {
}
})
);
const tilePromises = [];

const rowTiles = [];
let parentTilePromise = null;

// If z < 9, we also consider a parent tile made from children tiles at z+1.
// We first gather tile promises for rows, then handle parent tile separately.
if (z < 9) {
for (const row of rows) {
const meta = metadataByUuid[row.uuid];
if (!meta) {
continue;
}

if (meta.maxzoom < 9) {
// Only consider images that can contribute at low zoom levels
if (meta && meta.maxzoom < 9) {
const key = keyFromS3Url(row.uuid);
const geojson = JSON.parse(row.geojson);
tilePromises.push(source(key, z, x, y, meta, geojson));
rowTiles.push({ row, promise: source(key, z, x, y, meta, geojson) });
}
}

tilePromises.push(
constructParentTileFromChildren(
await Promise.all([
request512pxFn(z + 1, x * 2, y * 2, filters),
request512pxFn(z + 1, x * 2 + 1, y * 2, filters),
request512pxFn(z + 1, x * 2, y * 2 + 1, filters),
request512pxFn(z + 1, x * 2 + 1, y * 2 + 1, filters),
]),
z,
x,
y
)
);
// Parent tile at higher zoom: constructed from 4 sub-tiles (z+1)
const children = await Promise.all([
request512pxFn(z + 1, x * 2, y * 2, filters),
request512pxFn(z + 1, x * 2 + 1, y * 2, filters),
request512pxFn(z + 1, x * 2, y * 2 + 1, filters),
request512pxFn(z + 1, x * 2 + 1, y * 2 + 1, filters),
]);
parentTilePromise = constructParentTileFromChildren(children, z, x, y);
} else {
// For z >= 9, we just request tiles directly from the source images listed in rows
for (const row of rows) {
const meta = metadataByUuid[row.uuid];
if (!meta) {
continue;
}

if (!meta) continue;
const key = keyFromS3Url(row.uuid);
const geojson = JSON.parse(row.geojson);
tilePromises.push(source(key, z, x, y, meta, geojson));
rowTiles.push({ row, promise: source(key, z, x, y, meta, geojson) });
}
}

const tiles = await Promise.all(tilePromises);

// Begin cloudless stitching logic
const filteredTiles = tiles
.filter((tile) => !tile.empty())
.map((tile, index) => ({
tile,
meta: metadataByUuid[rows[index].uuid],
}))
.filter(({ meta, tile }, index) => {
if (!meta) {
console.warn(`Null metadata found for tile at index ${index}, skipping...`);
return false;
}
return true;
// Resolve all row-based tile promises
const resolvedRowTiles = await Promise.all(rowTiles.map((rt) => rt.promise));
let finalTiles = [];

// Map resolved tiles back to their rows to get correct metadata by index
resolvedRowTiles.forEach((tile, index) => {
const { row } = rowTiles[index];
const meta = metadataByUuid[row.uuid];
if (!meta) {
console.warn(`Null metadata found for uuid ${row.uuid}, skipping...`);
return;
}
if (!tile.empty()) {
finalTiles.push({ tile, meta });
}
});

// If we have a parent tile, we can add it to finalTiles.
// It doesn't directly correspond to a row, so we assign neutral metadata
// or use a placeholder. Adjust as needed for your criteria.
if (parentTilePromise) {
const parentTile = await parentTilePromise;
finalTiles.push({
tile: parentTile,
meta: {
// Using placeholder metadata so it can be included in sorting
uploaded_at: new Date(0),
file_size: 0,
gsd: Infinity,
},
});
}

// Sort tiles based on the criteria
filteredTiles.sort((a, b) => {
// 1. Prefer the latest image
// Sort tiles by:
// 1. Most recent uploaded_at
// 2. Larger file_size
// 3. Lower GSD (higher resolution)
finalTiles.sort((a, b) => {
const dateA = new Date(a.meta.uploaded_at);
const dateB = new Date(b.meta.uploaded_at);

// 2. Prefer larger file size
const fileSizeA = a.meta.file_size;
const fileSizeB = b.meta.file_size;

// 3. Prefer higher resolution (lower GSD)
const gsdA = a.meta.gsd;
const gsdB = b.meta.gsd;

// Comparison based on criteria
if (dateA !== dateB) return dateB - dateA;
if (dateA.getTime() !== dateB.getTime()) return dateB - dateA;
if (fileSizeA !== fileSizeB) return fileSizeB - fileSizeA;
return gsdA - gsdB;
});

const tileBuffers = filteredTiles.map(({ tile }) => tile.image.buffer);

// Blend all final candidate tiles together
const tileBuffers = finalTiles.map(({ tile }) => tile.image.buffer);
const tileBuffer = await blendTiles(tileBuffers, 512, 512);

const tile = new Tile(new TileImage(tileBuffer, "png"), z, x, y);
await tile.image.transformInJpegIfFullyOpaque();

Expand Down
10 changes: 5 additions & 5 deletions src/titiler_fetcher.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import PQueue from "p-queue";
import os from "node:os";

// Getting the number of cores
const numCPUs = os.cpus().length;
const numCPUs = process.env.NUM_CPUS ? parseInt(process.env.NUM_CPUS, 10) : os.cpus().length;

console.log("numCPUs:", numCPUs);

Expand Down Expand Up @@ -47,19 +47,19 @@ async function enqueueTileFetching(tileUrl, z, x, y) {
}

const request = tileRequestQueue
.add(() => fetchTile(url), { priority: Math.pow(2, z), timeout: FETCH_QUEUE_TTL_MS })
.add(() => fetchTile(url), { priority: z, timeout: FETCH_QUEUE_TTL_MS })
.catch((error) => {
const logContext = {
url,
zoomLevel: z,
errorType: error.name,
errorMessage: error.message,
timeout: FETCH_QUEUE_TTL_MS
timeout: FETCH_QUEUE_TTL_MS,
};
if (error.name === "TimeoutError") {
console.error('Tile request timeout', logContext);
console.error("Tile request timeout", logContext);
} else {
console.error('Tile request failed', logContext);
console.error("Tile request failed", logContext);
}
})
.finally(() => {
Expand Down

0 comments on commit c408161

Please sign in to comment.