diff --git a/services/feed-requests/migrations/Migration20241231120419.ts b/services/feed-requests/migrations/Migration20241231120419.ts new file mode 100644 index 000000000..54324b424 --- /dev/null +++ b/services/feed-requests/migrations/Migration20241231120419.ts @@ -0,0 +1,19 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20241231120419 extends Migration { + + async up(): Promise { + this.addSql(`ALTER TYPE request_partitioned_status ADD VALUE 'CACHE_CONTROL_SKIPPED';`); + // switch the created_at and status of the index + this.addSql(`DROP INDEX request_partitioned_lookupkey_created_at_status_index;`); + this.addSql(`CREATE INDEX request_partitioned_lookupkey_status_created_at_index ON request_partitioned (lookup_key, status, created_at);`); + } + + async down(): Promise { + this.addSql(`ALTER TYPE request_partitioned_status DROP VALUE 'CACHE_CONTROL_SKIPPED';`); + // switch the created_at and status of the index + this.addSql(`DROP INDEX request_partitioned_lookupkey_status_created_at_index;`); + this.addSql(`CREATE INDEX request_partitioned_lookupkey_created_at_status_index ON request_partitioned (lookup_key, created_at, status);`); + } + +} diff --git a/services/feed-requests/src/feed-fetcher/entities/response.entity.ts b/services/feed-requests/src/feed-fetcher/entities/response.entity.ts index abbca7bf3..df765e4eb 100644 --- a/services/feed-requests/src/feed-fetcher/entities/response.entity.ts +++ b/services/feed-requests/src/feed-fetcher/entities/response.entity.ts @@ -49,10 +49,7 @@ export class Response { type: 'json', nullable: true, }) - headers?: { - etag?: string; - lastModified?: string; - } | null; + headers?: Record | null; @Property({ type: 'timestamp with time zone', diff --git a/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts b/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts index 5795eafae..7e7c0bd75 100644 --- a/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts +++ b/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts @@ -58,7 +58,7 @@ interface FetchOptions { interface FetchResponse { ok: boolean; status: number; - headers: Map<'etag' | 'last-modified' | 'server' | 'content-type', string>; + headers: Map; text: () => Promise; arrayBuffer: () => Promise; } @@ -135,19 +135,11 @@ export class FeedFetcherService { request: Request; decodedResponseText: string | null | undefined; } | null> { - const logDebug = - url === - 'https://www.clanaod.net/forums/external.php?type=RSS2&forumids=102'; - const request = await this.partitionedRequestsStore.getLatestRequest( lookupKey || url, ); if (!request) { - if (logDebug) { - logger.warn(`Running debug on schedule: no request was found`); - } - return null; } @@ -162,13 +154,6 @@ export class FeedFetcherService { ).toString() : ''; - if (logDebug) { - logger.warn( - `Running debug on schedule: got cache key ${request.response.redisCacheKey}`, - { text }, - ); - } - return { request, decodedResponseText: text, @@ -232,20 +217,13 @@ export class FeedFetcherService { request.status = RequestStatus.BAD_STATUS_CODE; } - const etag = res.headers.get('etag'); - const lastModified = res.headers.get('last-modified'); - const response = new Response(); response.createdAt = request.createdAt; response.statusCode = res.status; response.headers = {}; - if (etag) { - response.headers.etag = etag; - } - - if (lastModified) { - response.headers.lastModified = lastModified; + for (const [key, val] of res.headers.entries()) { + response.headers[key] = val; } let text: string | null = null; @@ -459,6 +437,11 @@ export class FeedFetcherService { convertHeaderValue(normalizedHeaders.get('last-modified')), ); headers.set('server', convertHeaderValue(normalizedHeaders.get('server'))); + headers.set('date', convertHeaderValue(normalizedHeaders.get('date'))); + headers.set( + 'cache-control', + convertHeaderValue(normalizedHeaders.get('cache-control')), + ); return { headers, diff --git a/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts b/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts index c514ef0bf..4938a63d8 100644 --- a/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts +++ b/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts @@ -120,11 +120,19 @@ export default class PartitionedRequestsStoreService { async getLatestOkRequest( lookupKey: string, - ): Promise { + opts?: { + fields?: Record<'response_headers', boolean>; + }, + ): Promise; + }> { const em = this.orm.em.getConnection(); const [result] = await em.execute( - `SELECT created_at FROM request_partitioned + `SELECT created_at ${ + opts?.fields?.response_headers ? ', response_headers' : '' + } FROM request_partitioned WHERE lookup_key = ? AND status = 'OK' ORDER BY created_at DESC @@ -136,7 +144,10 @@ export default class PartitionedRequestsStoreService { return null; } - return { createdAt: new Date(result.created_at) }; + return { + createdAt: new Date(result.created_at), + responseHeaders: result.response_headers, + }; } async countFailedRequests(lookupKey: string, since?: Date) {