Skip to content

Commit

Permalink
time of day polling limits
Browse files Browse the repository at this point in the history 
for enbridge implementation #16
  • Loading branch information
@dangowans
dangowans committed Oct 5, 2023
1 parent c171262 commit 6db74fe
Show file tree
Hide file tree
Showing 6 changed files with 236 additions and 188 deletions.
30 changes: 9 additions & 21 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"@cityssm/expressjs-server-js": "^2.3.3",
"@cityssm/font-awesome-v5-iconclasses": "^0.1.0",
"@cityssm/green-button-parser": "^0.7.0",
"@cityssm/green-button-subscriber": "^0.4.0",
"@cityssm/green-button-subscriber": "^0.5.0",
"@cityssm/is-private-network-address": "^0.2.1",
"@cityssm/utils-datetime": "^0.1.1",
"@fortawesome/fontawesome-free": "^5.15.4",
Expand Down
161 changes: 91 additions & 70 deletions tasks/greenButtonCMDProcessor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ import { helpers as greenButtonHelpers } from '@cityssm/green-button-parser';
import { GreenButtonSubscriber } from '@cityssm/green-button-subscriber';
import Debug from 'debug';
import exitHook from 'exit-hook';
import { setIntervalAsync, clearIntervalAsync } from 'set-interval-async';
import { setIntervalAsync, clearIntervalAsync } from 'set-interval-async/fixed';
import { getConfigProperty } from '../helpers/functions.config.js';
import { recordGreenButtonData } from '../helpers/functions.greenButton.js';
const debug = Debug('emile:tasks:greenButtonCMDProcessor');
process.title = 'EMILE - greenButtonCMDProcessor';
const taskIntervalMillis = 3600 * 1000;
const pollingIntervalMillis = 86400 * 1000 + 60000;
const updatedMinsCacheFile = 'data/caches/greenButtonCMDProcessor.json';
let updatedMins = {};
Expand All @@ -19,6 +20,7 @@ catch {
updatedMins = {};
}
let terminateTask = false;
let taskIsRunning = false;
function saveCache() {
try {
fs.writeFileSync(updatedMinsCacheFile, JSON.stringify(updatedMins, undefined, 2));
Expand All @@ -29,91 +31,110 @@ function saveCache() {
}
}
async function processGreenButtonSubscriptions() {
if (taskIsRunning) {
return;
}
debug('Process started');
const greenButtonSubscriptions = getConfigProperty('subscriptions.greenButton');
for (const [subscriptionKey, greenButtonSubscription] of Object.entries(greenButtonSubscriptions)) {
if (terminateTask) {
break;
}
debug(`Loading authorizations for subscription: ${subscriptionKey} ...`);
if (updatedMins[subscriptionKey] === undefined) {
updatedMins[subscriptionKey] = {};
}
const greenButtonSubscriber = new GreenButtonSubscriber(greenButtonSubscription.configuration);
const authorizations = await greenButtonSubscriber.getAuthorizations();
if (authorizations === undefined) {
debug(`Unable to retieve authorizations: ${subscriptionKey}`);
continue;
}
const authorizationEntries = greenButtonHelpers.getEntriesByContentType(authorizations, 'Authorization');
if (authorizationEntries.length === 0) {
debug(`Subscription contains no authorizations: ${subscriptionKey}`);
continue;
}
for (const authorizationEntry of authorizationEntries) {
const authorizationId = authorizationEntry.links.selfUid ?? '';
if (authorizationId === '' ||
(greenButtonSubscription.authorizationIdsToExclude ?? []).includes(authorizationId) ||
(greenButtonSubscription.authorizationIdsToInclude !== undefined &&
!greenButtonSubscription.authorizationIdsToInclude.includes(authorizationId)) ||
authorizationEntry.content.Authorization.status_value !== 'Active') {
debug(`Skipping authorization id: ${subscriptionKey}, ${authorizationId}`);
continue;
}
let timeMillis = updatedMins[subscriptionKey][authorizationId];
if (timeMillis === undefined) {
timeMillis = {
polledMillis: 0,
updatedMillis: 0
};
}
else if (typeof timeMillis === 'number') {
timeMillis = {
polledMillis: timeMillis,
updatedMillis: timeMillis
};
taskIsRunning = true;
try {
const greenButtonSubscriptions = getConfigProperty('subscriptions.greenButton');
subscriptionLoop: for (const [subscriptionKey, greenButtonSubscription] of Object.entries(greenButtonSubscriptions)) {
if (terminateTask) {
break;
}
if (timeMillis.polledMillis + pollingIntervalMillis > Date.now()) {
debug(`Skipping recently refreshed authorization id: ${subscriptionKey}, ${authorizationId}`);
if ((greenButtonSubscription.pollingHoursToExclude ?? []).includes(new Date().getHours())) {
debug(`Subscription cannot be polled at this hour: ${subscriptionKey}`);
continue;
}
let updatedMin;
if (timeMillis.updatedMillis === 0) {
updatedMin = new Date();
updatedMin.setFullYear(updatedMin.getFullYear() - 1);
}
else {
updatedMin = new Date(timeMillis.updatedMillis);
debug(`Loading authorizations for subscription: ${subscriptionKey} ...`);
if (updatedMins[subscriptionKey] === undefined) {
updatedMins[subscriptionKey] = {};
}
const usageData = await greenButtonSubscriber.getBatchSubscriptionsByAuthorization(authorizationId, {
updatedMin
});
if (usageData === undefined) {
debug(`Unable to retrieve subscription data: ${subscriptionKey}, ${authorizationId}`);
const greenButtonSubscriber = new GreenButtonSubscriber(greenButtonSubscription.configuration);
const authorizations = await greenButtonSubscriber.getAuthorizations();
if (authorizations === undefined) {
debug(`Unable to retieve authorizations: ${subscriptionKey}`);
continue;
}
try {
await recordGreenButtonData(usageData, {});
}
catch (error) {
debug(`Error recording data: ${subscriptionKey}, ${authorizationId}`);
debug(error);
const authorizationEntries = greenButtonHelpers.getEntriesByContentType(authorizations.json, 'Authorization');
if (authorizationEntries.length === 0) {
debug(`Subscription contains no authorizations: ${subscriptionKey}`);
continue;
}
finally {
updatedMins[subscriptionKey][authorizationId] = {
polledMillis: Date.now(),
updatedMillis: usageData.updatedDate?.getTime() ?? timeMillis.updatedMillis ?? 0
};
saveCache();
for (const authorizationEntry of authorizationEntries) {
if ((greenButtonSubscription.pollingHoursToExclude ?? []).includes(new Date().getHours())) {
debug(`Subscription cannot be polled at this hour: ${subscriptionKey}`);
continue subscriptionLoop;
}
const authorizationId = authorizationEntry.links.selfUid ?? '';
if (authorizationId === '' ||
(greenButtonSubscription.authorizationIdsToExclude ?? []).includes(authorizationId) ||
(greenButtonSubscription.authorizationIdsToInclude !== undefined &&
!greenButtonSubscription.authorizationIdsToInclude.includes(authorizationId)) ||
authorizationEntry.content.Authorization.status_value !== 'Active') {
debug(`Skipping authorization id: ${subscriptionKey}, ${authorizationId}`);
continue;
}
let timeMillis = updatedMins[subscriptionKey][authorizationId];
if (timeMillis === undefined) {
timeMillis = {
polledMillis: 0,
updatedMillis: 0
};
}
else if (typeof timeMillis === 'number') {
timeMillis = {
polledMillis: timeMillis,
updatedMillis: timeMillis
};
}
if (timeMillis.polledMillis + pollingIntervalMillis > Date.now()) {
debug(`Skipping recently refreshed authorization id: ${subscriptionKey}, ${authorizationId}`);
continue;
}
let updatedMin;
if (timeMillis.updatedMillis === 0) {
updatedMin = new Date();
updatedMin.setFullYear(updatedMin.getFullYear() - 1);
}
else {
updatedMin = new Date(timeMillis.updatedMillis);
}
const usageData = await greenButtonSubscriber.getBatchSubscriptionsByAuthorization(authorizationId, {
updatedMin
});
if (usageData === undefined) {
debug(`Unable to retrieve subscription data: ${subscriptionKey}, ${authorizationId}`);
continue;
}
try {
await recordGreenButtonData(usageData.json, {});
}
catch (error) {
debug(`Error recording data: ${subscriptionKey}, ${authorizationId}`);
debug(error);
}
finally {
updatedMins[subscriptionKey][authorizationId] = {
polledMillis: Date.now(),
updatedMillis: usageData.json?.updatedDate?.getTime() ??
timeMillis.updatedMillis ??
0
};
saveCache();
}
}
}
}
finally {
taskIsRunning = false;
}
}
await processGreenButtonSubscriptions().catch((error) => {
debug('Error running task.');
debug(error);
});
const intervalID = setIntervalAsync(processGreenButtonSubscriptions, pollingIntervalMillis);
const intervalID = setIntervalAsync(processGreenButtonSubscriptions, taskIntervalMillis);
exitHook(() => {
terminateTask = true;
try {
Expand Down
Loading

0 comments on commit 6db74fe

Please sign in to comment.