diff --git a/package-lock.json b/package-lock.json index aef9114..bbdfa39 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@cityssm/expressjs-server-js": "^2.3.3", "@cityssm/font-awesome-v5-iconclasses": "^0.1.0", "@cityssm/green-button-parser": "^0.7.0", - "@cityssm/green-button-subscriber": "^0.4.0", + "@cityssm/green-button-subscriber": "^0.5.0", "@cityssm/is-private-network-address": "^0.2.1", "@cityssm/utils-datetime": "^0.1.1", "@fortawesome/fontawesome-free": "^5.15.4", @@ -2233,30 +2233,18 @@ } }, "node_modules/@cityssm/green-button-subscriber": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@cityssm/green-button-subscriber/-/green-button-subscriber-0.4.0.tgz", - "integrity": "sha512-cCoYHB5PWpBSEN7LQp8ApOQW2Ak9qQQVM7/Id1BCcwQTk4e5JR/Gr2Qj8FLeTpB9dH2EOzNNWC71z7HulUHSNg==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@cityssm/green-button-subscriber/-/green-button-subscriber-0.5.0.tgz", + "integrity": "sha512-Lay7BAUJ6D8HRYLBBBN/HF7D5KvQyfppUuHhQz+ZUZDKsJC/kTjduo8DXGrFUCosBVLfdj5srK9/AA+DFLOLaQ==", "dependencies": { - "@cityssm/green-button-parser": "^0.6.1", - "axios": "^1.5.0", + "@cityssm/green-button-parser": "^0.7.0", + "axios": "^1.5.1", "debug": "^4.3.4" }, "engines": { "node": "^14.13.1 || >=16.0.0" } }, - "node_modules/@cityssm/green-button-subscriber/node_modules/@cityssm/green-button-parser": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/@cityssm/green-button-parser/-/green-button-parser-0.6.1.tgz", - "integrity": "sha512-hEvsyR8kFs8ZeKdzLQfWMRCZlQlwTueQak09J6gAkvr9ZinNGsz5cGDkreS+sYuL70D8eriJFzZT9Jb9ZO30ig==", - "dependencies": { - "core-js": "^3.32.0", - "xml2js": "^0.6.2" - }, - "engines": { - "node": "^14.13.1 || >=16.0.0" - } - }, "node_modules/@cityssm/is-private-network-address": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/@cityssm/is-private-network-address/-/is-private-network-address-0.2.1.tgz", @@ -4402,9 +4390,9 @@ } }, "node_modules/axios": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", - "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", "dependencies": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", diff --git a/package.json b/package.json index 1f81fd2..b0834b5 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "@cityssm/expressjs-server-js": "^2.3.3", "@cityssm/font-awesome-v5-iconclasses": "^0.1.0", "@cityssm/green-button-parser": "^0.7.0", - "@cityssm/green-button-subscriber": "^0.4.0", + "@cityssm/green-button-subscriber": "^0.5.0", "@cityssm/is-private-network-address": "^0.2.1", "@cityssm/utils-datetime": "^0.1.1", "@fortawesome/fontawesome-free": "^5.15.4", diff --git a/tasks/greenButtonCMDProcessor.js b/tasks/greenButtonCMDProcessor.js index f1524e6..37bc0d0 100644 --- a/tasks/greenButtonCMDProcessor.js +++ b/tasks/greenButtonCMDProcessor.js @@ -3,11 +3,12 @@ import { helpers as greenButtonHelpers } from '@cityssm/green-button-parser'; import { GreenButtonSubscriber } from '@cityssm/green-button-subscriber'; import Debug from 'debug'; import exitHook from 'exit-hook'; -import { setIntervalAsync, clearIntervalAsync } from 'set-interval-async'; +import { setIntervalAsync, clearIntervalAsync } from 'set-interval-async/fixed'; import { getConfigProperty } from '../helpers/functions.config.js'; import { recordGreenButtonData } from '../helpers/functions.greenButton.js'; const debug = Debug('emile:tasks:greenButtonCMDProcessor'); process.title = 'EMILE - greenButtonCMDProcessor'; +const taskIntervalMillis = 3600 * 1000; const pollingIntervalMillis = 86400 * 1000 + 60000; const updatedMinsCacheFile = 'data/caches/greenButtonCMDProcessor.json'; let updatedMins = {}; @@ -19,6 +20,7 @@ catch { updatedMins = {}; } let terminateTask = false; +let taskIsRunning = false; function saveCache() { try { fs.writeFileSync(updatedMinsCacheFile, JSON.stringify(updatedMins, undefined, 2)); @@ -29,91 +31,110 @@ function saveCache() { } } async function processGreenButtonSubscriptions() { + if (taskIsRunning) { + return; + } debug('Process started'); - const greenButtonSubscriptions = getConfigProperty('subscriptions.greenButton'); - for (const [subscriptionKey, greenButtonSubscription] of Object.entries(greenButtonSubscriptions)) { - if (terminateTask) { - break; - } - debug(`Loading authorizations for subscription: ${subscriptionKey} ...`); - if (updatedMins[subscriptionKey] === undefined) { - updatedMins[subscriptionKey] = {}; - } - const greenButtonSubscriber = new GreenButtonSubscriber(greenButtonSubscription.configuration); - const authorizations = await greenButtonSubscriber.getAuthorizations(); - if (authorizations === undefined) { - debug(`Unable to retieve authorizations: ${subscriptionKey}`); - continue; - } - const authorizationEntries = greenButtonHelpers.getEntriesByContentType(authorizations, 'Authorization'); - if (authorizationEntries.length === 0) { - debug(`Subscription contains no authorizations: ${subscriptionKey}`); - continue; - } - for (const authorizationEntry of authorizationEntries) { - const authorizationId = authorizationEntry.links.selfUid ?? ''; - if (authorizationId === '' || - (greenButtonSubscription.authorizationIdsToExclude ?? []).includes(authorizationId) || - (greenButtonSubscription.authorizationIdsToInclude !== undefined && - !greenButtonSubscription.authorizationIdsToInclude.includes(authorizationId)) || - authorizationEntry.content.Authorization.status_value !== 'Active') { - debug(`Skipping authorization id: ${subscriptionKey}, ${authorizationId}`); - continue; - } - let timeMillis = updatedMins[subscriptionKey][authorizationId]; - if (timeMillis === undefined) { - timeMillis = { - polledMillis: 0, - updatedMillis: 0 - }; - } - else if (typeof timeMillis === 'number') { - timeMillis = { - polledMillis: timeMillis, - updatedMillis: timeMillis - }; + taskIsRunning = true; + try { + const greenButtonSubscriptions = getConfigProperty('subscriptions.greenButton'); + subscriptionLoop: for (const [subscriptionKey, greenButtonSubscription] of Object.entries(greenButtonSubscriptions)) { + if (terminateTask) { + break; } - if (timeMillis.polledMillis + pollingIntervalMillis > Date.now()) { - debug(`Skipping recently refreshed authorization id: ${subscriptionKey}, ${authorizationId}`); + if ((greenButtonSubscription.pollingHoursToExclude ?? []).includes(new Date().getHours())) { + debug(`Subscription cannot be polled at this hour: ${subscriptionKey}`); continue; } - let updatedMin; - if (timeMillis.updatedMillis === 0) { - updatedMin = new Date(); - updatedMin.setFullYear(updatedMin.getFullYear() - 1); - } - else { - updatedMin = new Date(timeMillis.updatedMillis); + debug(`Loading authorizations for subscription: ${subscriptionKey} ...`); + if (updatedMins[subscriptionKey] === undefined) { + updatedMins[subscriptionKey] = {}; } - const usageData = await greenButtonSubscriber.getBatchSubscriptionsByAuthorization(authorizationId, { - updatedMin - }); - if (usageData === undefined) { - debug(`Unable to retrieve subscription data: ${subscriptionKey}, ${authorizationId}`); + const greenButtonSubscriber = new GreenButtonSubscriber(greenButtonSubscription.configuration); + const authorizations = await greenButtonSubscriber.getAuthorizations(); + if (authorizations === undefined) { + debug(`Unable to retieve authorizations: ${subscriptionKey}`); continue; } - try { - await recordGreenButtonData(usageData, {}); - } - catch (error) { - debug(`Error recording data: ${subscriptionKey}, ${authorizationId}`); - debug(error); + const authorizationEntries = greenButtonHelpers.getEntriesByContentType(authorizations.json, 'Authorization'); + if (authorizationEntries.length === 0) { + debug(`Subscription contains no authorizations: ${subscriptionKey}`); + continue; } - finally { - updatedMins[subscriptionKey][authorizationId] = { - polledMillis: Date.now(), - updatedMillis: usageData.updatedDate?.getTime() ?? timeMillis.updatedMillis ?? 0 - }; - saveCache(); + for (const authorizationEntry of authorizationEntries) { + if ((greenButtonSubscription.pollingHoursToExclude ?? []).includes(new Date().getHours())) { + debug(`Subscription cannot be polled at this hour: ${subscriptionKey}`); + continue subscriptionLoop; + } + const authorizationId = authorizationEntry.links.selfUid ?? ''; + if (authorizationId === '' || + (greenButtonSubscription.authorizationIdsToExclude ?? []).includes(authorizationId) || + (greenButtonSubscription.authorizationIdsToInclude !== undefined && + !greenButtonSubscription.authorizationIdsToInclude.includes(authorizationId)) || + authorizationEntry.content.Authorization.status_value !== 'Active') { + debug(`Skipping authorization id: ${subscriptionKey}, ${authorizationId}`); + continue; + } + let timeMillis = updatedMins[subscriptionKey][authorizationId]; + if (timeMillis === undefined) { + timeMillis = { + polledMillis: 0, + updatedMillis: 0 + }; + } + else if (typeof timeMillis === 'number') { + timeMillis = { + polledMillis: timeMillis, + updatedMillis: timeMillis + }; + } + if (timeMillis.polledMillis + pollingIntervalMillis > Date.now()) { + debug(`Skipping recently refreshed authorization id: ${subscriptionKey}, ${authorizationId}`); + continue; + } + let updatedMin; + if (timeMillis.updatedMillis === 0) { + updatedMin = new Date(); + updatedMin.setFullYear(updatedMin.getFullYear() - 1); + } + else { + updatedMin = new Date(timeMillis.updatedMillis); + } + const usageData = await greenButtonSubscriber.getBatchSubscriptionsByAuthorization(authorizationId, { + updatedMin + }); + if (usageData === undefined) { + debug(`Unable to retrieve subscription data: ${subscriptionKey}, ${authorizationId}`); + continue; + } + try { + await recordGreenButtonData(usageData.json, {}); + } + catch (error) { + debug(`Error recording data: ${subscriptionKey}, ${authorizationId}`); + debug(error); + } + finally { + updatedMins[subscriptionKey][authorizationId] = { + polledMillis: Date.now(), + updatedMillis: usageData.json?.updatedDate?.getTime() ?? + timeMillis.updatedMillis ?? + 0 + }; + saveCache(); + } } } } + finally { + taskIsRunning = false; + } } await processGreenButtonSubscriptions().catch((error) => { debug('Error running task.'); debug(error); }); -const intervalID = setIntervalAsync(processGreenButtonSubscriptions, pollingIntervalMillis); +const intervalID = setIntervalAsync(processGreenButtonSubscriptions, taskIntervalMillis); exitHook(() => { terminateTask = true; try { diff --git a/tasks/greenButtonCMDProcessor.ts b/tasks/greenButtonCMDProcessor.ts index 32adaa9..b7aaaff 100644 --- a/tasks/greenButtonCMDProcessor.ts +++ b/tasks/greenButtonCMDProcessor.ts @@ -8,7 +8,7 @@ import type { GreenButtonJson } from '@cityssm/green-button-parser/types/entryTy import { GreenButtonSubscriber } from '@cityssm/green-button-subscriber' import Debug from 'debug' import exitHook from 'exit-hook' -import { setIntervalAsync, clearIntervalAsync } from 'set-interval-async' +import { setIntervalAsync, clearIntervalAsync } from 'set-interval-async/fixed' import { getConfigProperty } from '../helpers/functions.config.js' import { recordGreenButtonData } from '../helpers/functions.greenButton.js' @@ -21,6 +21,7 @@ process.title = 'EMILE - greenButtonCMDProcessor' * Task */ +const taskIntervalMillis = 3600 * 1000 const pollingIntervalMillis = 86_400 * 1000 + 60_000 const updatedMinsCacheFile = 'data/caches/greenButtonCMDProcessor.json' @@ -40,6 +41,7 @@ try { } let terminateTask = false +let taskIsRunning = false function saveCache(): void { try { @@ -54,125 +56,160 @@ function saveCache(): void { } async function processGreenButtonSubscriptions(): Promise { - debug('Process started') - - const greenButtonSubscriptions = getConfigProperty( - 'subscriptions.greenButton' - ) - - for (const [subscriptionKey, greenButtonSubscription] of Object.entries( - greenButtonSubscriptions - )) { - if (terminateTask) { - break - } - - debug(`Loading authorizations for subscription: ${subscriptionKey} ...`) - - if (updatedMins[subscriptionKey] === undefined) { - updatedMins[subscriptionKey] = {} - } - - const greenButtonSubscriber = new GreenButtonSubscriber( - greenButtonSubscription.configuration - ) - - const authorizations = await greenButtonSubscriber.getAuthorizations() + if (taskIsRunning) { + return + } - if (authorizations === undefined) { - debug(`Unable to retieve authorizations: ${subscriptionKey}`) - continue - } + debug('Process started') + taskIsRunning = true - const authorizationEntries = greenButtonHelpers.getEntriesByContentType( - authorizations as GreenButtonJson, - 'Authorization' + try { + const greenButtonSubscriptions = getConfigProperty( + 'subscriptions.greenButton' ) - if (authorizationEntries.length === 0) { - debug(`Subscription contains no authorizations: ${subscriptionKey}`) - continue - } - - for (const authorizationEntry of authorizationEntries) { - const authorizationId = authorizationEntry.links.selfUid ?? '' + // eslint-disable-next-line no-labels + subscriptionLoop: for (const [ + subscriptionKey, + greenButtonSubscription + ] of Object.entries(greenButtonSubscriptions)) { + if (terminateTask) { + break + } if ( - authorizationId === '' || - (greenButtonSubscription.authorizationIdsToExclude ?? []).includes( - authorizationId - ) || - (greenButtonSubscription.authorizationIdsToInclude !== undefined && - !greenButtonSubscription.authorizationIdsToInclude.includes( - authorizationId - )) || - authorizationEntry.content.Authorization.status_value !== 'Active' - ) { - debug( - `Skipping authorization id: ${subscriptionKey}, ${authorizationId}` + (greenButtonSubscription.pollingHoursToExclude ?? []).includes( + new Date().getHours() ) + ) { + debug(`Subscription cannot be polled at this hour: ${subscriptionKey}`) continue } - let timeMillis = updatedMins[subscriptionKey][authorizationId] + debug(`Loading authorizations for subscription: ${subscriptionKey} ...`) - if (timeMillis === undefined) { - timeMillis = { - polledMillis: 0, - updatedMillis: 0 - } - } else if (typeof timeMillis === 'number') { - timeMillis = { - polledMillis: timeMillis, - updatedMillis: timeMillis - } + if (updatedMins[subscriptionKey] === undefined) { + updatedMins[subscriptionKey] = {} } - if (timeMillis.polledMillis + pollingIntervalMillis > Date.now()) { - debug( - `Skipping recently refreshed authorization id: ${subscriptionKey}, ${authorizationId}` - ) + const greenButtonSubscriber = new GreenButtonSubscriber( + greenButtonSubscription.configuration + ) + + const authorizations = await greenButtonSubscriber.getAuthorizations() + + if (authorizations === undefined) { + debug(`Unable to retieve authorizations: ${subscriptionKey}`) continue } - let updatedMin: Date + const authorizationEntries = greenButtonHelpers.getEntriesByContentType( + authorizations.json as GreenButtonJson, + 'Authorization' + ) - if (timeMillis.updatedMillis === 0) { - updatedMin = new Date() - updatedMin.setFullYear(updatedMin.getFullYear() - 1) - } else { - updatedMin = new Date(timeMillis.updatedMillis) + if (authorizationEntries.length === 0) { + debug(`Subscription contains no authorizations: ${subscriptionKey}`) + continue } - const usageData = - await greenButtonSubscriber.getBatchSubscriptionsByAuthorization( - authorizationId, - { - updatedMin + for (const authorizationEntry of authorizationEntries) { + if ( + (greenButtonSubscription.pollingHoursToExclude ?? []).includes( + new Date().getHours() + ) + ) { + debug( + `Subscription cannot be polled at this hour: ${subscriptionKey}` + ) + + // eslint-disable-next-line no-labels + continue subscriptionLoop + } + + const authorizationId = authorizationEntry.links.selfUid ?? '' + + if ( + authorizationId === '' || + (greenButtonSubscription.authorizationIdsToExclude ?? []).includes( + authorizationId + ) || + (greenButtonSubscription.authorizationIdsToInclude !== undefined && + !greenButtonSubscription.authorizationIdsToInclude.includes( + authorizationId + )) || + authorizationEntry.content.Authorization.status_value !== 'Active' + ) { + debug( + `Skipping authorization id: ${subscriptionKey}, ${authorizationId}` + ) + continue + } + + let timeMillis = updatedMins[subscriptionKey][authorizationId] + + if (timeMillis === undefined) { + timeMillis = { + polledMillis: 0, + updatedMillis: 0 } - ) + } else if (typeof timeMillis === 'number') { + timeMillis = { + polledMillis: timeMillis, + updatedMillis: timeMillis + } + } - if (usageData === undefined) { - debug( - `Unable to retrieve subscription data: ${subscriptionKey}, ${authorizationId}` - ) - continue - } + if (timeMillis.polledMillis + pollingIntervalMillis > Date.now()) { + debug( + `Skipping recently refreshed authorization id: ${subscriptionKey}, ${authorizationId}` + ) + continue + } + + let updatedMin: Date - try { - await recordGreenButtonData(usageData as GreenButtonJson, {}) - } catch (error) { - debug(`Error recording data: ${subscriptionKey}, ${authorizationId}`) - debug(error) - } finally { - updatedMins[subscriptionKey][authorizationId] = { - polledMillis: Date.now(), - updatedMillis: - usageData.updatedDate?.getTime() ?? timeMillis.updatedMillis ?? 0 + if (timeMillis.updatedMillis === 0) { + updatedMin = new Date() + updatedMin.setFullYear(updatedMin.getFullYear() - 1) + } else { + updatedMin = new Date(timeMillis.updatedMillis) + } + + const usageData = + await greenButtonSubscriber.getBatchSubscriptionsByAuthorization( + authorizationId, + { + updatedMin + } + ) + + if (usageData === undefined) { + debug( + `Unable to retrieve subscription data: ${subscriptionKey}, ${authorizationId}` + ) + continue + } + + try { + await recordGreenButtonData(usageData.json as GreenButtonJson, {}) + } catch (error) { + debug(`Error recording data: ${subscriptionKey}, ${authorizationId}`) + debug(error) + } finally { + updatedMins[subscriptionKey][authorizationId] = { + polledMillis: Date.now(), + updatedMillis: + usageData.json?.updatedDate?.getTime() ?? + timeMillis.updatedMillis ?? + 0 + } + saveCache() } - saveCache() } } + } finally { + taskIsRunning = false } } @@ -187,7 +224,7 @@ await processGreenButtonSubscriptions().catch((error) => { const intervalID = setIntervalAsync( processGreenButtonSubscriptions, - pollingIntervalMillis + taskIntervalMillis ) exitHook(() => { diff --git a/types/configTypes.d.ts b/types/configTypes.d.ts index 7a1f5c6..36566d9 100644 --- a/types/configTypes.d.ts +++ b/types/configTypes.d.ts @@ -82,4 +82,5 @@ export interface ConfigGreenButtonSubscription { configuration: GreenButtonSubscriberConfiguration; authorizationIdsToExclude?: string[]; authorizationIdsToInclude?: string[]; + pollingHoursToExclude?: number[]; } diff --git a/types/configTypes.ts b/types/configTypes.ts index d7b3a5e..aba65b5 100644 --- a/types/configTypes.ts +++ b/types/configTypes.ts @@ -105,4 +105,5 @@ export interface ConfigGreenButtonSubscription { configuration: GreenButtonSubscriberConfiguration authorizationIdsToExclude?: string[] authorizationIdsToInclude?: string[] + pollingHoursToExclude?: number[] }