Skip to content

Commit

Permalink
Use if-modified-since and if-none-match headers (#386)
Browse files Browse the repository at this point in the history
  • Loading branch information
@synzen
synzen authored Jan 1, 2025
1 parent fd2543f commit 4d471c8
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 67 deletions.
74 changes: 55 additions & 19 deletions services/feed-requests/src/feed-fetcher/feed-fetcher-listener.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,10 +212,11 @@ export class FeedFetcherListenerService {
.subtract(Math.round(rateSeconds * 0.75), 'seconds')
.toDate();

const latestRequestAfterTime = await this.getLatestRequestAfterTime(
{ url },
dateToCheck,
);
const latestRequestAfterTime =
await this.partitionedRequestsStoreService.getLatestStatusAfterTime(
lookupKey || url,
dateToCheck,
);

if (latestRequestAfterTime) {
logger.debug(
Expand All @@ -240,6 +241,12 @@ export class FeedFetcherListenerService {
return { successful: false };
}

const latestOkRequest =
await this.partitionedRequestsStoreService.getLatestOkRequestWithResponseBody(
data.lookupKey || data.url,
{ fields: ['response_headers'] },
);

const { request } = await this.feedFetcherService.fetchAndSaveResponse(
url,
{
Expand All @@ -250,7 +257,12 @@ export class FeedFetcherListenerService {
}
: undefined,
source: RequestSource.Schedule,
headers: data.headers,
headers: {
...data.headers,
'If-Modified-Since':
latestOkRequest?.responseHeaders?.['last-modified'] || '',
'If-None-Match': latestOkRequest?.responseHeaders?.etag,
},
},
);

Expand Down Expand Up @@ -452,6 +464,43 @@ export class FeedFetcherListenerService {
}
}

// async isLatestResponseStillFreshInCache({
// lookupKey,
// }: {
// lookupKey: string;
// }) {
// const latestOkRequest =
// await this.partitionedRequestsStoreService.getLatestOkRequestWithResponseBody(lookupKey);

// if (!latestOkRequest) {
// return false;
// }

// const cacheControl = latestOkRequest.responseHeaders?.['cache-control'];

// if (!cacheControl) {
// return false;
// }

// const directives = cacheControl.split(',').map((d) => d.trim());
// const maxAgeDirective = directives.find((d) => d.startsWith('max-age='));
// const publicDirective = directives.includes('public');

// if (!maxAgeDirective || !publicDirective) {
// return false;
// }

// const maxAge = parseInt(maxAgeDirective.split('=')[1]);

// const baseDate = latestOkRequest.responseHeaders?.date
// ? new Date(latestOkRequest.responseHeaders?.date)
// : latestOkRequest.createdAt;

// const expirationDate = baseDate.getTime() + maxAge * 1000;

// return expirationDate > Date.now();
// }

async countFailedRequests({
lookupKey,
url,
Expand All @@ -460,7 +509,7 @@ export class FeedFetcherListenerService {
url: string;
}): Promise<number> {
const latestOkRequest =
await this.partitionedRequestsStoreService.getLatestOkRequest(
await this.partitionedRequestsStoreService.getLatestOkRequestWithResponseBody(
lookupKey || url,
);

Expand All @@ -477,17 +526,4 @@ export class FeedFetcherListenerService {

return dayjs(referenceDate).add(minutesToWait, 'minute').toDate();
}

async getLatestRequestAfterTime(
requestQuery: {
lookupKey?: string;
url: string;
},
time: Date,
) {
return this.partitionedRequestsStoreService.getLatestStatusAfterTime(
requestQuery.lookupKey || requestQuery.url,
time,
);
}
}
82 changes: 40 additions & 42 deletions services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,32 @@ const convertHeaderValue = (val?: string | string[] | null) => {
return val || '';
};

const trimHeadersForStorage = (obj?: HeadersInit) => {
const trimHeadersForStorage = (
obj?: Record<string, string | undefined>,
): Record<string, string> => {
const trimmed = Object.entries(obj || {}).reduce((acc, [key, val]) => {
if (val) {
acc[key] = val;
}

return acc;
}, {} as Record<string, string>);

if (!obj) {
return obj;
return trimmed;
}

const newObj: HeadersInit = {};

for (const key in obj) {
if (obj[key]) {
newObj[key.toLowerCase()] = obj[key];
for (const key in trimmed) {
if (trimmed[key]) {
trimmed[key.toLowerCase()] = trimmed[key];
}
}

if (newObj.authorization) {
newObj.authorization = 'SECRET';
if (trimmed.authorization) {
trimmed.authorization = 'SECRET';
}

return newObj;
return trimmed;
};

interface FetchOptions {
Expand Down Expand Up @@ -134,19 +142,12 @@ export class FeedFetcherService {
request: Request;
decodedResponseText: string | null | undefined;
} | null> {
const logDebug =
url ===
'https://www.clanaod.net/forums/external.php?type=RSS2&forumids=102';

const request = await this.partitionedRequestsStore.getLatestRequest(
lookupKey || url,
);
const request =
await this.partitionedRequestsStore.getLatestRequestWithResponseBody(
lookupKey || url,
);

if (!request) {
if (logDebug) {
logger.warn(`Running debug on schedule: no request was found`);
}

return null;
}

Expand All @@ -161,13 +162,6 @@ export class FeedFetcherService {
).toString()
: '';

if (logDebug) {
logger.warn(
`Running debug on schedule: got cache key ${request.response.redisCacheKey}`,
{ text },
);
}

return {
request,
decodedResponseText: text,
Expand All @@ -187,7 +181,7 @@ export class FeedFetcherService {
| undefined;
flushEntities?: boolean;
saveResponseToObjectStorage?: boolean;
headers?: Record<string, string>;
headers?: Record<string, string | undefined>;
source: RequestSource | undefined;
},
): Promise<{
Expand Down Expand Up @@ -231,21 +225,16 @@ export class FeedFetcherService {
request.status = RequestStatus.BAD_STATUS_CODE;
}

const etag = res.headers.get('etag');
const lastModified = res.headers.get('last-modified');

const response = new Response();
response.createdAt = request.createdAt;
response.statusCode = res.status;
response.headers = {};
const headersToStore: Record<string, string> = {};

if (etag) {
response.headers.etag = etag;
}
res.headers.forEach((val, key) => {
headersToStore[key] = val;
});

if (lastModified) {
response.headers.lastModified = lastModified;
}
response.headers = headersToStore;

let text: string | null = null;

Expand Down Expand Up @@ -409,10 +398,19 @@ export class FeedFetcherService {
controller.abort();
}, this.feedRequestTimeoutMs);

// Necessary since passing If-None-Match header with empty string may cause a 200 when expecting 304
const withoutEmptyHeaderVals = Object.entries(
options?.headers || {},
).reduce((acc, [key, val]) => {
if (val) {
acc[key] = val;
}

return acc;
}, {});

const useOptions = {
headers: {
...options?.headers,
},
headers: withoutEmptyHeaderVals,
redirect: 'follow' as const,
signal: controller.signal,
};
Expand Down
35 changes: 29 additions & 6 deletions services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,15 +118,24 @@ export default class PartitionedRequestsStoreService {
return new Date(result.next_retry_date) || null;
}

async getLatestOkRequest(
async getLatestOkRequestWithResponseBody(
lookupKey: string,
): Promise<null | { createdAt: Date }> {
opts?: {
fields?: Array<'response_headers'>;
},
): Promise<null | {
createdAt: Date;
responseHeaders?: Record<string, string>;
}> {
const em = this.orm.em.getConnection();

const [result] = await em.execute(
`SELECT created_at FROM request_partitioned
`SELECT created_at ${
opts?.fields?.includes('response_headers') ? ', response_headers' : ''
} FROM request_partitioned
WHERE lookup_key = ?
AND status = 'OK'
AND response_status_code != 304
ORDER BY created_at DESC
LIMIT 1`,
[lookupKey],
Expand All @@ -136,7 +145,10 @@ export default class PartitionedRequestsStoreService {
return null;
}

return { createdAt: new Date(result.created_at) };
return {
createdAt: new Date(result.created_at),
responseHeaders: result.response_headers,
};
}

async countFailedRequests(lookupKey: string, since?: Date) {
Expand All @@ -151,7 +163,15 @@ export default class PartitionedRequestsStoreService {
const [result] = await em.execute(
`SELECT COUNT(*) FROM request_partitioned
WHERE lookup_key = ?
AND status != 'OK'
AND status IN (
'INTERNAL_ERROR',
'FETCH_ERROR',
'PARSE_ERROR',
'BAD_STATUS_CODE',
'FETCH_TIMEOUT',
'REFUSED_LARGE_FEED',
'INVALID_SSL_CERTIFICATE'
)
${since ? 'AND created_at >= ?' : ''}
`,
params,
Expand Down Expand Up @@ -218,12 +238,15 @@ export default class PartitionedRequestsStoreService {
}));
}

async getLatestRequest(lookupKey: string): Promise<Request | null> {
async getLatestRequestWithResponseBody(
lookupKey: string,
): Promise<Request | null> {
const em = this.orm.em.getConnection();

const [result] = await em.execute(
`SELECT * FROM request_partitioned
WHERE lookup_key = ?
AND response_status_code != 304
ORDER BY created_at DESC
LIMIT 1`,
[lookupKey],
Expand Down

0 comments on commit 4d471c8

Please sign in to comment.