From b5fe90cfd8d5932902e1b792020ff485db81f5ae Mon Sep 17 00:00:00 2001 From: hhack Date: Tue, 27 Feb 2024 11:31:48 +0100 Subject: [PATCH] fix get all newest and last updated data --- .gitignore | 1 + netlify/functions/slackbot.ts | 244 ++++++++++++++++++++++------------ 2 files changed, 161 insertions(+), 84 deletions(-) diff --git a/.gitignore b/.gitignore index 36e8f9a..8000694 100644 --- a/.gitignore +++ b/.gitignore @@ -271,3 +271,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +test.js \ No newline at end of file diff --git a/netlify/functions/slackbot.ts b/netlify/functions/slackbot.ts index 62a09c6..9c95a49 100644 --- a/netlify/functions/slackbot.ts +++ b/netlify/functions/slackbot.ts @@ -1,12 +1,12 @@ -import { Handler, HandlerEvent, HandlerContext } from 'netlify/functions'; -import { App, ExpressReceiver, ReceiverEvent } from '@slack/bolt' -import * as dotenv from 'dotenv' -import fetch from 'node-fetch'; +import { Handler, HandlerEvent, HandlerContext } from "netlify/functions"; +import { App, ExpressReceiver, ReceiverEvent } from "@slack/bolt"; +import * as dotenv from "dotenv"; +import fetch from "node-fetch"; dotenv.config(); const expressReceiver = new ExpressReceiver({ signingSecret: `${process.env.SLACK_SIGNING_SECRET}`, - processBeforeResponse: true + processBeforeResponse: true, }); const app = new App({ @@ -17,151 +17,224 @@ const app = new App({ appToken: `${process.env.APP_TOKEN}`, }); - // Declare functions that are needed for fetching and analysing date from CKAN API const getJSON = async (url: string) => { const response = await fetch(url); if (!response.ok) { const txt = await response.text(); - throw new Error(txt) + throw new Error(txt); } const json = await response.json(); - return json // get JSON from the response -} + return json; // get JSON from the response +}; -function findNewest(data: { [x: string]: any; }, days: number) { - var today = new Date() - let newestArray: any[] = [] +function findNewest(data: { [x: string]: any }, days: number) { + var today = new Date(); + let newestArray: any[] = []; for (const obj in data) { - let date = new Date(data[obj].date_released) - if ((new Date(today.getFullYear(), today.getMonth(), today.getDate() - days))<=date) { - newestArray = newestArray.concat(data[obj]) + let date = new Date(data[obj].date_released); + if ( + new Date(today.getFullYear(), today.getMonth(), today.getDate() - days) <= + date + ) { + newestArray = newestArray.concat(data[obj]); } } return newestArray; } -function findUpdated(data: { [x: string]: any; }, days: number) { - const today = new Date() - let updatedArray: any[] = [] +function findUpdated(data: { [x: string]: any }, days: number) { + const today = new Date(); + let updatedArray: any[] = []; for (const obj in data) { try { - let date = new Date(data[obj].date_updated) - if ((new Date(today.getFullYear(), today.getMonth(), today.getDate() - days))<=date) { - updatedArray = updatedArray.concat(data[obj]) + let date = new Date(data[obj].date_updated); + if ( + new Date( + today.getFullYear(), + today.getMonth(), + today.getDate() - days + ) <= date + ) { + updatedArray = updatedArray.concat(data[obj]); } } catch (error) { - console.log("err") - console.error(error); - } + console.log("err"); + console.error(error); + } } return updatedArray; } -function generateTextResponse_newest (newestArray: any[], days: number) { - let text = "" +function generateTextResponse_newest(newestArray: any[], days: number) { + let text = ""; if (newestArray.length === 0) { - text = text.concat("*Keine neuen Datensätze!*\nIn den letzten " + days + " Tagen wurden keine neuen Datensätze im Berliner Datenportal veröffentlicht.\n"); - } - else { - text = text.concat("*Neue offene Datensätze!* :star:\nIn den letzten " + days + " Tagen wurden folgende neue Datensätze im Berliner Datenportal veröffentlicht:\n") + text = text.concat( + "*Keine neuen Datensätze!*\nIn den letzten " + + days + + " Tagen wurden keine neuen Datensätze im Berliner Datenportal veröffentlicht.\n" + ); + } else { + text = text.concat( + "*Neue offene Datensätze!* :star:\nIn den letzten " + + days + + " Tagen wurden folgende neue Datensätze im Berliner Datenportal veröffentlicht:\n" + ); for (const obj in newestArray) { - text = text.concat(">**\n>" + newestArray[obj].author.toString() + "\n>_" + newestArray[obj].date_released.toString() + "_\n") + text = text.concat( + ">**\n>" + + newestArray[obj].author.toString() + + "\n>_" + + newestArray[obj].date_released.toString() + + "_\n" + ); } } - return text + return text; } -function generateTextResponse_updated (updatedArray: any[], days: number) { - let text = "" +function generateTextResponse_updated(updatedArray: any[], days: number) { + let text = ""; if (updatedArray.length === 0) { - text = text.concat("\nIn den letzten " + days + " Tagen wurden keine der bereits veröffentlichten Datensätze im Berliner Datenportal geupdated."); - } - else { - text = text.concat("\n*Datensatz-Updates!*\nIn den letzten " + days + " Tagen wurden folgende bereits veröffentlichte Datensätze geupdated:\n") + text = text.concat( + "\nIn den letzten " + + days + + " Tagen wurden keine der bereits veröffentlichten Datensätze im Berliner Datenportal geupdated." + ); + } else { + text = text.concat( + "\n*Datensatz-Updates!*\nIn den letzten " + + days + + " Tagen wurden folgende bereits veröffentlichte Datensätze geupdated:\n" + ); for (const obj in updatedArray) { - text = text.concat(">** _" + updatedArray[obj].date_updated.toString() + " (Erstveröffentlichung: " + updatedArray[obj].date_released.toString() + ")_\n") + text = text.concat( + ">** _" + + updatedArray[obj].date_updated.toString() + + " (Erstveröffentlichung: " + + updatedArray[obj].date_released.toString() + + ")_\n" + ); } } - return text + return text; } -const processData = async (data:any, days: number) => { - let resultsArray: any[] = [] - for (const id in data.result.results){ - resultsArray = resultsArray.concat(data.result.results[id]); +const processData = async (data: any, days: number) => { + const newestArray = findNewest(data, days); + const updatedArray = findUpdated(data, days); + const text_newest = generateTextResponse_newest(newestArray, days); + const text_updated = generateTextResponse_updated(updatedArray, days); + const text: string[] = []; + text.push(text_newest); + text.push(text_updated); + return text; +}; + +const getData = async () => { + let resultsArray: any[] = []; + + const dataUpdated = await getJSON( + "https://datenregister.berlin.de/api/3/action/package_search?start=0&rows=100&sort=date_updated%20desc" + ); + + for (const id in dataUpdated.result.results) { + resultsArray = resultsArray.concat(dataUpdated.result.results[id]); + } + + const dataReleased = await getJSON( + "https://datenregister.berlin.de/api/3/action/package_search?start=0&rows=100&sort=date_released%20desc" + ); + + for (const id in dataReleased.result.results) { + resultsArray = resultsArray.concat(dataReleased.result.results[id]); } - const newestArray = findNewest(resultsArray, days) - const updatedArray = findUpdated(resultsArray, days) - const text_newest = generateTextResponse_newest(newestArray, days) - const text_updated = generateTextResponse_updated(updatedArray, days) - const text : string[] = []; - text.push(text_newest) - text.push(text_updated) - return text -} -async function replyMessage(channelId: string, messageThreadTs: string): Promise { + const data = resultsArray.filter( + (obj, index, self) => index === self.findIndex((t) => t["id"] === obj["id"]) + ); + + return data; +}; + +async function replyMessage( + channelId: string, + messageThreadTs: string +): Promise { try { - let days = 7 - const data = await getJSON("https://datenregister.berlin.de/api/3/action/package_search?start=0&rows=100") + let days = 7; + const data = await getData(); const text = await processData(data, days); await app.client.chat.postMessage({ token: process.env.SLACK_BOT_TOKEN, channel: channelId, thread_ts: messageThreadTs, - text: text[0] + text: text[0], }); await app.client.chat.postMessage({ token: process.env.SLACK_BOT_TOKEN, channel: channelId, thread_ts: messageThreadTs, - text: text[1] + text: text[1], }); } catch (error) { console.error(error); - } - } + } +} -app.message("Was gibt's Neues im Berliner Open Data Portal?", async ({ message }) => { - await replyMessage(message.channel, message.ts); -}); +app.message( + "Was gibt's Neues im Berliner Open Data Portal?", + async ({ message }) => { + await replyMessage(message.channel, message.ts); + } +); // This is the Slash-Command to ask for newest data sets of the last XX days (number is given as an argument with the slash command) app.command("/opendata", async ({ body, ack, say }) => { try { await ack(); - let days = Number.parseInt(body.text) + let days = Number.parseInt(body.text); if (!days) { - days = 7 - } - const data = await getJSON("https://datenregister.berlin.de/api/3/action/package_search?start=0&rows=100") + days = 7; + } + const data = await getData(); const text = await processData(data, days); await app.client.chat.postMessage({ token: `${process.env.SLACK_BOT_TOKEN}`, channel: body.channel_id, - text: text[0] + text[1] - }) - } - catch (error) { + text: text[0] + text[1], + }); + } catch (error) { console.error(error); } }); - -function parseRequestBody(stringBody: string | null, contentType: string | undefined) { +function parseRequestBody( + stringBody: string | null, + contentType: string | undefined +) { try { let inputStringBody: string = stringBody ?? ""; let result: any = {}; - if(contentType && contentType === 'application/x-www-form-urlencoded') { - var keyValuePairs = inputStringBody.split('&'); - keyValuePairs.forEach(function(pair: string): void { - let individualKeyValuePair: string[] = pair.split('='); - result[individualKeyValuePair[0]] = decodeURIComponent(individualKeyValuePair[1] || ''); + if (contentType && contentType === "application/x-www-form-urlencoded") { + var keyValuePairs = inputStringBody.split("&"); + keyValuePairs.forEach(function (pair: string): void { + let individualKeyValuePair: string[] = pair.split("="); + result[individualKeyValuePair[0]] = decodeURIComponent( + individualKeyValuePair[1] || "" + ); }); return JSON.parse(JSON.stringify(result)); } else { @@ -172,12 +245,15 @@ function parseRequestBody(stringBody: string | null, contentType: string | undef } } -const handler: Handler = async (event: HandlerEvent, _context: HandlerContext) => { +const handler: Handler = async ( + event: HandlerEvent, + _context: HandlerContext +) => { const payload = parseRequestBody(event.body, event.headers["content-type"]); - if(payload && payload.type && payload.type === 'url_verification') { + if (payload && payload.type && payload.type === "url_verification") { return { statusCode: 200, - body: payload.challenge + body: payload.challenge, }; } const slackEvent: ReceiverEvent = { @@ -187,12 +263,12 @@ const handler: Handler = async (event: HandlerEvent, _context: HandlerContext) = resolve(); return { statusCode: 200, - body: response ?? "" + body: response ?? "", }; }); }, }; - + await app.processEvent(slackEvent); return { statusCode: 200, @@ -200,4 +276,4 @@ const handler: Handler = async (event: HandlerEvent, _context: HandlerContext) = }; }; -export { handler }; \ No newline at end of file +export { handler };