From 25c55697fa360155bbeac126cfe33e281e346b02 Mon Sep 17 00:00:00 2001 From: Drew Volz Date: Sun, 12 May 2024 01:31:18 -0700 Subject: [PATCH 01/10] parse bonapp mealtime data --- source/ccci-stolaf-college/v1/index.js | 4 + source/ccci-stolaf-college/v1/reports.js | 15 ++++ source/reports-bonapp/index.js | 109 +++++++++++++++++++++++ source/reports-bonapp/index.test.js | 0 source/reports-bonapp/types.js | 7 ++ 5 files changed, 135 insertions(+) create mode 100644 source/ccci-stolaf-college/v1/reports.js create mode 100644 source/reports-bonapp/index.js create mode 100644 source/reports-bonapp/index.test.js create mode 100644 source/reports-bonapp/types.js diff --git a/source/ccci-stolaf-college/v1/index.js b/source/ccci-stolaf-college/v1/index.js index b888fa0e..98c775ec 100644 --- a/source/ccci-stolaf-college/v1/index.js +++ b/source/ccci-stolaf-college/v1/index.js @@ -13,6 +13,7 @@ import * as menus from './menu.js' import * as news from './news.js' import * as orgs from './orgs.js' import * as printing from './printing.js' +import * as reports from './reports.js' import * as streams from './streams.js' import * as transit from './transit.js' import * as util from './util.js' @@ -115,6 +116,9 @@ api.get('/streams/upcoming', streams.upcoming) // stoprint api.get('/printing/color-printers', printing.colorPrinters) +// reports +api.get('/reports/stav', reports.stavMealtimeReport) + // utilities api.get('/util/html-to-md', util.htmlToMarkdown) diff --git a/source/ccci-stolaf-college/v1/reports.js b/source/ccci-stolaf-college/v1/reports.js new file mode 100644 index 00000000..a02c514c --- /dev/null +++ b/source/ccci-stolaf-college/v1/reports.js @@ -0,0 +1,15 @@ +import {ONE_HOUR} from '../../ccc-lib/constants.js' +import * as report from '../../reports-bonapp/index.js' +import mem from 'memoize' + +const getReport = mem(report.report, {maxAge: ONE_HOUR}) + +export const REPORT_URLS = { + stav: 'https://www.stolaf.edu/apps/mealtimes/', +} + +export async function stavMealtimeReport(ctx) { + ctx.cacheControl(ONE_HOUR) + + ctx.body = await getReport(REPORT_URLS.stav) +} diff --git a/source/reports-bonapp/index.js b/source/reports-bonapp/index.js new file mode 100644 index 00000000..1b270eae --- /dev/null +++ b/source/reports-bonapp/index.js @@ -0,0 +1,109 @@ +import {get} from '../ccc-lib/http.js' +import {ONE_MINUTE} from '../ccc-lib/constants.js' + +import {JSDOM, VirtualConsole} from 'jsdom' +import mem from 'memoize' +import * as Sentry from '@sentry/node' + +import {StavReportType} from './types.js' + +const getBonAppPage = mem(get, {maxAge: ONE_MINUTE}) + +const NUMBER_OF_CHARTS_TO_PARSE = 7 + +/** + * @param {string|URL} url + * @return {Promise} + */ +async function getBonAppReportWebpage(url) { + const virtualConsole = new VirtualConsole() + virtualConsole.sendTo(console, {omitJSDOMErrors: true}) + virtualConsole.on('jsdomError', (err) => { + let jsdomErrorMessagesToSkip = [ + 'Uncaught [ReferenceError: wp is not defined]', + 'Uncaught [ReferenceError: jQuery is not defined]', + 'Uncaught [Error: Create skia surface failed]', + "Uncaught [TypeError: Cannot read properties of undefined (reading 'slice')]", + "Failed to create chart: can't acquire context from the given item", + 'Not implemented: HTMLCanvasElement.prototype.getContext (without installing the canvas npm package)', + ] + if (jsdomErrorMessagesToSkip.includes(err.message)) { + return + } + console.error(err) + }) + + const body = await getBonAppPage(url).text() + return new JSDOM(body, { + runScripts: 'dangerously', + resources: 'usable', + virtualConsole, + beforeParse(window) { + window.fetch = global.fetch + } + }) +} + +/** + * @param {string|URL} reportUrl + * @returns {Promise} + */ +async function _report(reportUrl) { + let dom = await getBonAppReportWebpage(reportUrl) + + dom.window.console.info = ((info) => { + let infoMessagesToSkip = [ + 'Initialized global navigation scripts', + 'Initialized mobile menu script scripts', + 'Initialized tools navigation scripts', + 'Initialized all javascript that targeted document ready.', + ] + if (infoMessagesToSkip.includes(info)) { + return + } + console.info(info) + }) + + dom.window.onload = () => { + const charts = dom.window.Chart.instances + + const parse = (chart) => { + const {labels, datasets} = chart.data + return { + title: datasets[0].label, + times: labels, + data: datasets[0].data, + } + } + + const payload = [] + + for(let i=0; i} + */ +export function report(reportUrl) { + try { + return _report(reportUrl) + } catch (err) { + console.error(err) + Sentry.isInitialized() && Sentry.captureException(err) + return CustomCafe({message: 'Could not load BonApp report'}) + } +} diff --git a/source/reports-bonapp/index.test.js b/source/reports-bonapp/index.test.js new file mode 100644 index 00000000..e69de29b diff --git a/source/reports-bonapp/types.js b/source/reports-bonapp/types.js new file mode 100644 index 00000000..b06d5cac --- /dev/null +++ b/source/reports-bonapp/types.js @@ -0,0 +1,7 @@ +import {z} from 'zod' + +export const StavReportType = z.array(z.object({ + title: z.string(), + times: z.array(z.string()), + data: z.array(z.number()), +})) From d5905bdb88d90825a9e1bf574262e2878444677a Mon Sep 17 00:00:00 2001 From: Drew Volz Date: Sat, 18 May 2024 19:33:01 -0700 Subject: [PATCH 02/10] add promise to wait for window fetch to finish * wrap and resolve promise and return parsed data * add custom report type in error * exclude chart error from console --- source/reports-bonapp/index.js | 56 +++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/source/reports-bonapp/index.js b/source/reports-bonapp/index.js index 1b270eae..609bdecf 100644 --- a/source/reports-bonapp/index.js +++ b/source/reports-bonapp/index.js @@ -5,6 +5,7 @@ import {JSDOM, VirtualConsole} from 'jsdom' import mem from 'memoize' import * as Sentry from '@sentry/node' +import {CustomReportType} from './helpers.js' import {StavReportType} from './types.js' const getBonAppPage = mem(get, {maxAge: ONE_MINUTE}) @@ -24,7 +25,6 @@ async function getBonAppReportWebpage(url) { 'Uncaught [ReferenceError: jQuery is not defined]', 'Uncaught [Error: Create skia surface failed]', "Uncaught [TypeError: Cannot read properties of undefined (reading 'slice')]", - "Failed to create chart: can't acquire context from the given item", 'Not implemented: HTMLCanvasElement.prototype.getContext (without installing the canvas npm package)', ] if (jsdomErrorMessagesToSkip.includes(err.message)) { @@ -51,6 +51,16 @@ async function getBonAppReportWebpage(url) { async function _report(reportUrl) { let dom = await getBonAppReportWebpage(reportUrl) + dom.window.console.error = ((error) => { + let errorMessagesToSkip = [ + "Failed to create chart: can't acquire context from the given item", + ] + if (errorMessagesToSkip.includes(error)) { + return + } + console.error(error) + }) + dom.window.console.info = ((info) => { let infoMessagesToSkip = [ 'Initialized global navigation scripts', @@ -64,34 +74,32 @@ async function _report(reportUrl) { console.info(info) }) - dom.window.onload = () => { - const charts = dom.window.Chart.instances + return new Promise((resolve, reject) => { + dom.window.onload = () => { + const charts = dom.window.Chart.instances - const parse = (chart) => { - const {labels, datasets} = chart.data - return { - title: datasets[0].label, - times: labels, - data: datasets[0].data, + const parse = (chart) => { + const {labels, datasets} = chart.data + return { + title: datasets[0].label, + times: labels, + data: datasets[0].data, + } } - } - const payload = [] + const payload = [] - for(let i=0; i Date: Sat, 18 May 2024 19:33:56 -0700 Subject: [PATCH 03/10] add custom report helper --- source/reports-bonapp/helpers.js | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 source/reports-bonapp/helpers.js diff --git a/source/reports-bonapp/helpers.js b/source/reports-bonapp/helpers.js new file mode 100644 index 00000000..d459ef00 --- /dev/null +++ b/source/reports-bonapp/helpers.js @@ -0,0 +1,10 @@ +import {StavReportType} from './types.js' + +/** @returns {StavReportType} */ +export function CustomReportType({message}) { + return StavReportType.parse([{ + title: message, + times: [], + data: [], + }]) +} From 792719d42fb8b9442f5bd380819d1ca9615ad02a Mon Sep 17 00:00:00 2001 From: Drew Volz Date: Sun, 19 May 2024 02:39:07 +0000 Subject: [PATCH 04/10] format with prettier --- source/reports-bonapp/helpers.js | 12 +++++++----- source/reports-bonapp/index.js | 14 +++++++------- source/reports-bonapp/types.js | 12 +++++++----- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/source/reports-bonapp/helpers.js b/source/reports-bonapp/helpers.js index d459ef00..c9e3bf74 100644 --- a/source/reports-bonapp/helpers.js +++ b/source/reports-bonapp/helpers.js @@ -2,9 +2,11 @@ import {StavReportType} from './types.js' /** @returns {StavReportType} */ export function CustomReportType({message}) { - return StavReportType.parse([{ - title: message, - times: [], - data: [], - }]) + return StavReportType.parse([ + { + title: message, + times: [], + data: [], + }, + ]) } diff --git a/source/reports-bonapp/index.js b/source/reports-bonapp/index.js index 609bdecf..fc35f211 100644 --- a/source/reports-bonapp/index.js +++ b/source/reports-bonapp/index.js @@ -40,7 +40,7 @@ async function getBonAppReportWebpage(url) { virtualConsole, beforeParse(window) { window.fetch = global.fetch - } + }, }) } @@ -51,7 +51,7 @@ async function getBonAppReportWebpage(url) { async function _report(reportUrl) { let dom = await getBonAppReportWebpage(reportUrl) - dom.window.console.error = ((error) => { + dom.window.console.error = (error) => { let errorMessagesToSkip = [ "Failed to create chart: can't acquire context from the given item", ] @@ -59,9 +59,9 @@ async function _report(reportUrl) { return } console.error(error) - }) + } - dom.window.console.info = ((info) => { + dom.window.console.info = (info) => { let infoMessagesToSkip = [ 'Initialized global navigation scripts', 'Initialized mobile menu script scripts', @@ -72,7 +72,7 @@ async function _report(reportUrl) { return } console.info(info) - }) + } return new Promise((resolve, reject) => { dom.window.onload = () => { @@ -89,7 +89,7 @@ async function _report(reportUrl) { const payload = [] - for(let i=0; i Date: Sat, 18 May 2024 20:11:06 -0700 Subject: [PATCH 05/10] add report sanity and endpoint throw test checks --- source/ccci-stolaf-college/v1/reports.test.js | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 source/ccci-stolaf-college/v1/reports.test.js diff --git a/source/ccci-stolaf-college/v1/reports.test.js b/source/ccci-stolaf-college/v1/reports.test.js new file mode 100644 index 00000000..0d5e9073 --- /dev/null +++ b/source/ccci-stolaf-college/v1/reports.test.js @@ -0,0 +1,31 @@ +import {suite, test} from 'node:test' +import assert from 'node:assert/strict' +import lodash from 'lodash' + +import * as reports from './reports.js' +import {StavReportType} from '../../reports-bonapp/types.js' + +const {noop} = lodash + +const reportFunctions = { + stav: reports.REPORT_URLS.stav, +} + +suite('sanity checks', () => { + test('all report info endpoints are represented in this test file', () => { + assert.deepStrictEqual( + Object.keys(reportFunctions), + Object.keys(reports.REPORT_URLS), + ) + }) +}) + +suite('endpoints should not throw', {concurrency: 4}, () => { + for (const report of Object.keys(reports.REPORT_URLS)) { + test(`${report} report endpoint should return a StavReportType struct`, async () => { + let ctx = {cacheControl: noop, body: null} + await assert.doesNotReject(() => reportFunctions[report](ctx)) + assert.doesNotThrow(() => StavReportType.parse(ctx.body)) + }) + } +}) From 61bf0effca629cd4081987064b0773cb8512377c Mon Sep 17 00:00:00 2001 From: Drew Volz Date: Sat, 18 May 2024 20:12:05 -0700 Subject: [PATCH 06/10] add report info fetch and return assertion tests --- source/reports-bonapp/index.test.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/source/reports-bonapp/index.test.js b/source/reports-bonapp/index.test.js index e69de29b..e082d4f9 100644 --- a/source/reports-bonapp/index.test.js +++ b/source/reports-bonapp/index.test.js @@ -0,0 +1,17 @@ +import {suite, test} from 'node:test' +import assert from 'node:assert/strict' + +import * as bonAppReport from './index.js' +import {StavReportType} from './types.js' +import {REPORT_URLS} from '../ccci-stolaf-college/v1/reports.js' + +suite('report info', {concurrency: true}, () => { + test('fetching the data should not throw', async () => { + await assert.doesNotReject(() => bonAppReport.report(REPORT_URLS.stav)) + }) + + test('it should return a StavReportType struct', async () => { + let data = await bonAppReport.report(REPORT_URLS.stav) + assert.doesNotThrow(() => StavReportType.parse(data)) + }) +}) From 61e16a7edc615e5f79c6ea84cc50b2307fafde49 Mon Sep 17 00:00:00 2001 From: Drew Volz Date: Sat, 18 May 2024 20:15:07 -0700 Subject: [PATCH 07/10] prettier changes From f4eaf1a57418a388434523020d3fd1175e171799 Mon Sep 17 00:00:00 2001 From: Drew Volz Date: Sun, 19 May 2024 03:43:04 +0000 Subject: [PATCH 08/10] reports endpoints tests --- source/ccci-stolaf-college/v1/reports.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/ccci-stolaf-college/v1/reports.test.js b/source/ccci-stolaf-college/v1/reports.test.js index 0d5e9073..057ef4d0 100644 --- a/source/ccci-stolaf-college/v1/reports.test.js +++ b/source/ccci-stolaf-college/v1/reports.test.js @@ -8,7 +8,7 @@ import {StavReportType} from '../../reports-bonapp/types.js' const {noop} = lodash const reportFunctions = { - stav: reports.REPORT_URLS.stav, + stav: reports.stavMealtimeReport, } suite('sanity checks', () => { From 926aaf9c4865a0109acb6eda1de22936e6a433ee Mon Sep 17 00:00:00 2001 From: Drew Volz Date: Sun, 19 May 2024 03:47:16 +0000 Subject: [PATCH 09/10] prettier again --- source/reports-bonapp/index.test.js | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/source/reports-bonapp/index.test.js b/source/reports-bonapp/index.test.js index e082d4f9..f6478562 100644 --- a/source/reports-bonapp/index.test.js +++ b/source/reports-bonapp/index.test.js @@ -1,17 +1,17 @@ -import {suite, test} from 'node:test' -import assert from 'node:assert/strict' - -import * as bonAppReport from './index.js' -import {StavReportType} from './types.js' -import {REPORT_URLS} from '../ccci-stolaf-college/v1/reports.js' - -suite('report info', {concurrency: true}, () => { - test('fetching the data should not throw', async () => { - await assert.doesNotReject(() => bonAppReport.report(REPORT_URLS.stav)) - }) - - test('it should return a StavReportType struct', async () => { - let data = await bonAppReport.report(REPORT_URLS.stav) - assert.doesNotThrow(() => StavReportType.parse(data)) - }) -}) +import {suite, test} from 'node:test' +import assert from 'node:assert/strict' + +import * as bonAppReport from './index.js' +import {StavReportType} from './types.js' +import {REPORT_URLS} from '../ccci-stolaf-college/v1/reports.js' + +suite('report info', {concurrency: true}, () => { + test('fetching the data should not throw', async () => { + await assert.doesNotReject(() => bonAppReport.report(REPORT_URLS.stav)) + }) + + test('it should return a StavReportType struct', async () => { + let data = await bonAppReport.report(REPORT_URLS.stav) + assert.doesNotThrow(() => StavReportType.parse(data)) + }) +}) From 10fa560a0d90fd844d356aeecbd9df786e6035ee Mon Sep 17 00:00:00 2001 From: Drew Volz Date: Sun, 19 May 2024 03:54:04 +0000 Subject: [PATCH 10/10] fix lint with unused reject --- source/reports-bonapp/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/reports-bonapp/index.js b/source/reports-bonapp/index.js index fc35f211..abd17dea 100644 --- a/source/reports-bonapp/index.js +++ b/source/reports-bonapp/index.js @@ -74,7 +74,7 @@ async function _report(reportUrl) { console.info(info) } - return new Promise((resolve, reject) => { + return new Promise((resolve, _reject) => { dom.window.onload = () => { const charts = dom.window.Chart.instances