Skip to content

Commit

Permalink
fix get all newest and last updated data
Browse files Browse the repository at this point in the history
  • Loading branch information
@hanshack
hanshack committed Feb 27, 2024
1 parent aa9a05d commit b5fe90c
Show file tree
Hide file tree
Showing 2 changed files with 161 additions and 84 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,3 +271,4 @@ cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
test.js
244 changes: 160 additions & 84 deletions netlify/functions/slackbot.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import { Handler, HandlerEvent, HandlerContext } from 'netlify/functions';
import { App, ExpressReceiver, ReceiverEvent } from '@slack/bolt'
import * as dotenv from 'dotenv'
import fetch from 'node-fetch';
import { Handler, HandlerEvent, HandlerContext } from "netlify/functions";
import { App, ExpressReceiver, ReceiverEvent } from "@slack/bolt";
import * as dotenv from "dotenv";
import fetch from "node-fetch";
dotenv.config();

const expressReceiver = new ExpressReceiver({
signingSecret: `${process.env.SLACK_SIGNING_SECRET}`,
processBeforeResponse: true
processBeforeResponse: true,
});

const app = new App({
Expand All @@ -17,151 +17,224 @@ const app = new App({
appToken: `${process.env.APP_TOKEN}`,
});


// Declare functions that are needed for fetching and analysing date from CKAN API
const getJSON = async (url: string) => {
const response = await fetch(url);
if (!response.ok) {
const txt = await response.text();
throw new Error(txt)
throw new Error(txt);
}
const json = await response.json();
return json // get JSON from the response
}
return json; // get JSON from the response
};

function findNewest(data: { [x: string]: any; }, days: number) {
var today = new Date()
let newestArray: any[] = []
function findNewest(data: { [x: string]: any }, days: number) {
var today = new Date();
let newestArray: any[] = [];
for (const obj in data) {
let date = new Date(data[obj].date_released)
if ((new Date(today.getFullYear(), today.getMonth(), today.getDate() - days))<=date) {
newestArray = newestArray.concat(data[obj])
let date = new Date(data[obj].date_released);
if (
new Date(today.getFullYear(), today.getMonth(), today.getDate() - days) <=
date
) {
newestArray = newestArray.concat(data[obj]);
}
}
return newestArray;
}

function findUpdated(data: { [x: string]: any; }, days: number) {
const today = new Date()
let updatedArray: any[] = []
function findUpdated(data: { [x: string]: any }, days: number) {
const today = new Date();
let updatedArray: any[] = [];
for (const obj in data) {
try {
let date = new Date(data[obj].date_updated)
if ((new Date(today.getFullYear(), today.getMonth(), today.getDate() - days))<=date) {
updatedArray = updatedArray.concat(data[obj])
let date = new Date(data[obj].date_updated);
if (
new Date(
today.getFullYear(),
today.getMonth(),
today.getDate() - days
) <= date
) {
updatedArray = updatedArray.concat(data[obj]);
}
} catch (error) {
console.log("err")
console.error(error);
}
console.log("err");
console.error(error);
}
}
return updatedArray;
}

function generateTextResponse_newest (newestArray: any[], days: number) {
let text = ""
function generateTextResponse_newest(newestArray: any[], days: number) {
let text = "";
if (newestArray.length === 0) {
text = text.concat("*Keine neuen Datensätze!*\nIn den letzten " + days + " Tagen wurden keine neuen Datensätze im Berliner Datenportal veröffentlicht.\n");
}
else {
text = text.concat("*Neue offene Datensätze!* :star:\nIn den letzten " + days + " Tagen wurden folgende neue Datensätze im Berliner Datenportal veröffentlicht:\n")
text = text.concat(
"*Keine neuen Datensätze!*\nIn den letzten " +
days +
" Tagen wurden keine neuen Datensätze im Berliner Datenportal veröffentlicht.\n"
);
} else {
text = text.concat(
"*Neue offene Datensätze!* :star:\nIn den letzten " +
days +
" Tagen wurden folgende neue Datensätze im Berliner Datenportal veröffentlicht:\n"
);
for (const obj in newestArray) {
text = text.concat(">*<https://daten.berlin.de/search/node/" + newestArray[obj].title.toString().replace(/\s/g, "%20") + "|" + newestArray[obj].title.toString() + ">*\n>" + newestArray[obj].author.toString() + "\n>_" + newestArray[obj].date_released.toString() + "_\n")
text = text.concat(
">*<https://daten.berlin.de/search/node/" +
newestArray[obj].title.toString().replace(/\s/g, "%20") +
"|" +
newestArray[obj].title.toString() +
">*\n>" +
newestArray[obj].author.toString() +
"\n>_" +
newestArray[obj].date_released.toString() +
"_\n"
);
}
}
return text
return text;
}

function generateTextResponse_updated (updatedArray: any[], days: number) {
let text = ""
function generateTextResponse_updated(updatedArray: any[], days: number) {
let text = "";
if (updatedArray.length === 0) {
text = text.concat("\nIn den letzten " + days + " Tagen wurden keine der bereits veröffentlichten Datensätze im Berliner Datenportal geupdated.");
}
else {
text = text.concat("\n*Datensatz-Updates!*\nIn den letzten " + days + " Tagen wurden folgende bereits veröffentlichte Datensätze geupdated:\n")
text = text.concat(
"\nIn den letzten " +
days +
" Tagen wurden keine der bereits veröffentlichten Datensätze im Berliner Datenportal geupdated."
);
} else {
text = text.concat(
"\n*Datensatz-Updates!*\nIn den letzten " +
days +
" Tagen wurden folgende bereits veröffentlichte Datensätze geupdated:\n"
);
for (const obj in updatedArray) {
text = text.concat(">*<https://daten.berlin.de/search/node/" + updatedArray[obj].title.toString().replace(/\s/g, "%20") + "|" + updatedArray[obj].title.toString() + ">* _" + updatedArray[obj].date_updated.toString() + " (Erstveröffentlichung: " + updatedArray[obj].date_released.toString() + ")_\n")
text = text.concat(
">*<https://daten.berlin.de/search/node/" +
updatedArray[obj].title.toString().replace(/\s/g, "%20") +
"|" +
updatedArray[obj].title.toString() +
">* _" +
updatedArray[obj].date_updated.toString() +
" (Erstveröffentlichung: " +
updatedArray[obj].date_released.toString() +
")_\n"
);
}
}
return text
return text;
}

const processData = async (data:any, days: number) => {
let resultsArray: any[] = []
for (const id in data.result.results){
resultsArray = resultsArray.concat(data.result.results[id]);
const processData = async (data: any, days: number) => {
const newestArray = findNewest(data, days);
const updatedArray = findUpdated(data, days);
const text_newest = generateTextResponse_newest(newestArray, days);
const text_updated = generateTextResponse_updated(updatedArray, days);
const text: string[] = [];
text.push(text_newest);
text.push(text_updated);
return text;
};

const getData = async () => {
let resultsArray: any[] = [];

const dataUpdated = await getJSON(
"https://datenregister.berlin.de/api/3/action/package_search?start=0&rows=100&sort=date_updated%20desc"
);

for (const id in dataUpdated.result.results) {
resultsArray = resultsArray.concat(dataUpdated.result.results[id]);
}

const dataReleased = await getJSON(
"https://datenregister.berlin.de/api/3/action/package_search?start=0&rows=100&sort=date_released%20desc"
);

for (const id in dataReleased.result.results) {
resultsArray = resultsArray.concat(dataReleased.result.results[id]);
}
const newestArray = findNewest(resultsArray, days)
const updatedArray = findUpdated(resultsArray, days)
const text_newest = generateTextResponse_newest(newestArray, days)
const text_updated = generateTextResponse_updated(updatedArray, days)
const text : string[] = [];
text.push(text_newest)
text.push(text_updated)
return text
}

async function replyMessage(channelId: string, messageThreadTs: string): Promise<void> {
const data = resultsArray.filter(
(obj, index, self) => index === self.findIndex((t) => t["id"] === obj["id"])
);

return data;
};

async function replyMessage(
channelId: string,
messageThreadTs: string
): Promise<void> {
try {
let days = 7
const data = await getJSON("https://datenregister.berlin.de/api/3/action/package_search?start=0&rows=100")
let days = 7;
const data = await getData();
const text = await processData(data, days);

await app.client.chat.postMessage({
token: process.env.SLACK_BOT_TOKEN,
channel: channelId,
thread_ts: messageThreadTs,
text: text[0]
text: text[0],
});
await app.client.chat.postMessage({
token: process.env.SLACK_BOT_TOKEN,
channel: channelId,
thread_ts: messageThreadTs,
text: text[1]
text: text[1],
});
} catch (error) {
console.error(error);
}
}
}
}

app.message("Was gibt's Neues im Berliner Open Data Portal?", async ({ message }) => {
await replyMessage(message.channel, message.ts);
});
app.message(
"Was gibt's Neues im Berliner Open Data Portal?",
async ({ message }) => {
await replyMessage(message.channel, message.ts);
}
);

// This is the Slash-Command to ask for newest data sets of the last XX days (number is given as an argument with the slash command)
app.command("/opendata", async ({ body, ack, say }) => {
try {
await ack();

let days = Number.parseInt(body.text)
let days = Number.parseInt(body.text);
if (!days) {
days = 7
}
const data = await getJSON("https://datenregister.berlin.de/api/3/action/package_search?start=0&rows=100")
days = 7;
}
const data = await getData();
const text = await processData(data, days);

await app.client.chat.postMessage({
token: `${process.env.SLACK_BOT_TOKEN}`,
channel: body.channel_id,
text: text[0] + text[1]
})
}
catch (error) {
text: text[0] + text[1],
});
} catch (error) {
console.error(error);
}
});


function parseRequestBody(stringBody: string | null, contentType: string | undefined) {
function parseRequestBody(
stringBody: string | null,
contentType: string | undefined
) {
try {
let inputStringBody: string = stringBody ?? "";
let result: any = {};

if(contentType && contentType === 'application/x-www-form-urlencoded') {
var keyValuePairs = inputStringBody.split('&');
keyValuePairs.forEach(function(pair: string): void {
let individualKeyValuePair: string[] = pair.split('=');
result[individualKeyValuePair[0]] = decodeURIComponent(individualKeyValuePair[1] || '');
if (contentType && contentType === "application/x-www-form-urlencoded") {
var keyValuePairs = inputStringBody.split("&");
keyValuePairs.forEach(function (pair: string): void {
let individualKeyValuePair: string[] = pair.split("=");
result[individualKeyValuePair[0]] = decodeURIComponent(
individualKeyValuePair[1] || ""
);
});
return JSON.parse(JSON.stringify(result));
} else {
Expand All @@ -172,12 +245,15 @@ function parseRequestBody(stringBody: string | null, contentType: string | undef
}
}

const handler: Handler = async (event: HandlerEvent, _context: HandlerContext) => {
const handler: Handler = async (
event: HandlerEvent,
_context: HandlerContext
) => {
const payload = parseRequestBody(event.body, event.headers["content-type"]);
if(payload && payload.type && payload.type === 'url_verification') {
if (payload && payload.type && payload.type === "url_verification") {
return {
statusCode: 200,
body: payload.challenge
body: payload.challenge,
};
}
const slackEvent: ReceiverEvent = {
Expand All @@ -187,17 +263,17 @@ const handler: Handler = async (event: HandlerEvent, _context: HandlerContext) =
resolve();
return {
statusCode: 200,
body: response ?? ""
body: response ?? "",
};
});
},
};

await app.processEvent(slackEvent);
return {
statusCode: 200,
body: "",
};
};

export { handler };
export { handler };

0 comments on commit b5fe90c

Please sign in to comment.