From 0ec3bda8b0cb89503200eddbd6171bf9d9c8efe9 Mon Sep 17 00:00:00 2001 From: Vladislav Ivanov Date: Fri, 12 Jul 2024 20:52:25 +0200 Subject: [PATCH 1/3] update telegram provider --- .gitignore | 1 + CONTRIBUTING.md | 5 +- lib/api/routes/notificationAdapterRouter.js | 16 +++ lib/notification/adapter/apprise.js | 2 +- lib/notification/adapter/telegram.js | 130 +++++++++++++----- lib/provider/immoscout.js | 2 + lib/provider/kleinanzeigen.js | 1 + .../NotificationAdapterMutator.jsx | 5 +- 8 files changed, 127 insertions(+), 35 deletions(-) diff --git a/.gitignore b/.gitignore index 45b5ac8..e353ca3 100755 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ db/ npm-debug.log .DS_Store .idea +.vscode/ \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a9ab5ec..10cb0e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,7 @@ function applyBlacklist(o) { const config = { url: null, - //this is the container wrapping the search listings + //this is the container wrapping the search listings crawlContainer: '#result-list-stage .item', crawlFields: { id: '@id', @@ -36,6 +36,9 @@ const config = { title: '.item a img@title', link: 'a[id*="lnkImgToDetails_"]@href', address: '.item .box-25 .ellipsis .text-100 | removeNewline | trim', + image: 'img@src', + //some websites provide image URLs for rendered ads only and lazy URLs for the rest + lazyImage: 'lazyImg@src' }, paginate: '#idResultList .margin-bottom-6.margin-bottom-sm-12 .panel a.pull-right@href', normalize: normalize, diff --git a/lib/api/routes/notificationAdapterRouter.js b/lib/api/routes/notificationAdapterRouter.js index 483fa14..047a81c 100644 --- a/lib/api/routes/notificationAdapterRouter.js +++ b/lib/api/routes/notificationAdapterRouter.js @@ -35,6 +35,22 @@ notificationAdapterRouter.post('/try', async (req, res) => { size: '666 2m', link: 'https://www.orange-coding.net', }, + { + price: '1500 €', + title: + 'This is a test listing with long and doubled title. This is a test listing with long and doubled title.', + address: 'some address', + size: '777 2m', + link: 'https://www.orange-coding.net', + }, + { + price: '2500 €', + title: 'This is a test listing with an image', + address: 'some address', + size: '555 2m', + link: 'https://www.orange-coding.net', + image: 'https://images.pexels.com/photos/106399/pexels-photo-106399.jpeg', + }, ], notificationConfig, jobKey: 'TestJob', diff --git a/lib/notification/adapter/apprise.js b/lib/notification/adapter/apprise.js index 77d019a..4004fbf 100644 --- a/lib/notification/adapter/apprise.js +++ b/lib/notification/adapter/apprise.js @@ -8,7 +8,7 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) = const jobName = job == null ? jobKey : job.name; const promises = newListings.map((newListing) => { const title = `${jobName} at ${serviceName}: ${newListing.title}`; - const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\Link: ${newListing.link}`; + const message = `Address: ${newListing.address}\nSize: ${newListing.size}\nPrice: ${newListing.price}\nLink: ${newListing.link}`; return fetch(server, { method: 'POST', headers: { 'Content-Type': 'application/json' }, diff --git a/lib/notification/adapter/telegram.js b/lib/notification/adapter/telegram.js index 03c489d..e95f646 100644 --- a/lib/notification/adapter/telegram.js +++ b/lib/notification/adapter/telegram.js @@ -15,10 +15,83 @@ const arrayChunks = (inputArray, perChunk) => all[ch] = [].concat(all[ch] || [], one); return all; }, []); -function shorten(str, len = 30) { + +function shorten(str, len = 45) { return str.length > len ? str.substring(0, len) + '...' : str; } -export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { + +function isValidURL(url) { + try { + new URL(url); + return true; + } catch (_) { + return false; + } +} + +const sendTelegramMessage = (token, chatId, message, isPhoto = false, photoUrl = '') => { + const url = isPhoto + ? `https://api.telegram.org/bot${token}/sendPhoto` + : `https://api.telegram.org/bot${token}/sendMessage`; + const body = isPhoto + ? JSON.stringify({ + chat_id: chatId, + photo: photoUrl, + caption: message, + parse_mode: 'HTML', + }) + : JSON.stringify({ + chat_id: chatId, + text: message, + parse_mode: 'HTML', + disable_web_page_preview: !isPhoto, + }); + + /** + * This is to not break the rate limit. It is to only send 1 message per second + */ + return new Promise((resolve, reject) => { + setTimeout(() => { + fetch(url, { + method: 'post', + body: body, + headers: { 'Content-Type': 'application/json' }, + }) + .then(() => { + resolve(); + }) + .catch(() => { + reject(); + }); + }, RATE_LIMIT_INTERVAL); + }); +}; + +const sendRichMessage = ({ serviceName, newListings, notificationConfig, jobKey }) => { + const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields; + const job = getJob(jobKey); + const jobName = job == null ? jobKey : job.name; + + const promises = newListings.map((o, index) => { + let message = ''; + /** + * Only send the job information once + */ + if (index === 0) { + message += `${jobName} (${serviceName}) found ${newListings.length} new listings:\n\n`; + } + message += + `${shorten(o.title.replace(/\*/g, '')).trim()}\n` + + `🏠 Address: ${o.address}\n` + + `💰 Price: ${o.price}\n` + + `📐 Size: ${o.size}\n`; + const imageURL = o.lazyImage || o.image; + return sendTelegramMessage(token, chatId, message, imageURL && isValidURL(imageURL), imageURL); + }); + return Promise.all(promises); +}; + +const sendPlainMessage = ({ serviceName, newListings, notificationConfig, jobKey }) => { const { token, chatId } = notificationConfig.find((adapter) => adapter.id === config.id).fields; const job = getJob(jobKey); const jobName = job == null ? jobKey : job.name; @@ -26,38 +99,27 @@ export const send = ({ serviceName, newListings, notificationConfig, jobKey }) = const chunks = arrayChunks(newListings, MAX_ENTITIES_PER_CHUNK); const promises = chunks.map((chunk) => { let message = `${jobName} (${serviceName}) found ${newListings.length} new listings:\n\n`; - message += chunk.map( - (o) => - `${shorten(o.title.replace(/\*/g, ''), 45).trim()}\n` + - [o.address, o.price, o.size].join(' | ') + - '\n\n', - ); - /** - * This is to not break the rate limit. It is to only send 1 message per second - */ - return new Promise((resolve, reject) => { - setTimeout(() => { - fetch(`https://api.telegram.org/bot${token}/sendMessage`, { - method: 'post', - body: JSON.stringify({ - chat_id: chatId, - text: message, - parse_mode: 'HTML', - disable_web_page_preview: true, - }), - headers: { 'Content-Type': 'application/json' }, - }) - .then(() => { - resolve(); - }) - .catch(() => { - reject(); - }); - }, RATE_LIMIT_INTERVAL); - }); + message += chunk + .map( + (o) => + `${shorten(o.title.replace(/\*/g, '')).trim()}\n` + + [o.address, o.price, o.size].join(' | '), + ) + .join('\n\n'); + return sendTelegramMessage(token, chatId, message); }); return Promise.all(promises); }; + +export const send = ({ serviceName, newListings, notificationConfig, jobKey }) => { + const { richMessage } = notificationConfig.find((adapter) => adapter.id === config.id).fields; + if (richMessage) { + return sendRichMessage({ serviceName, newListings, notificationConfig, jobKey }); + } else { + return sendPlainMessage({ serviceName, newListings, notificationConfig, jobKey }); + } +}; + export const config = { id: 'telegram', name: 'Telegram', @@ -74,5 +136,11 @@ export const config = { label: 'Chat Id', description: 'The chat id to send messages to you.', }, + richMessage: { + type: 'boolean', + label: 'Send rich message', + description: 'When selected sends a rich message with image.', + value: false, + }, }, }; diff --git a/lib/provider/immoscout.js b/lib/provider/immoscout.js index 7174227..506647d 100644 --- a/lib/provider/immoscout.js +++ b/lib/provider/immoscout.js @@ -24,6 +24,8 @@ const config = { title: '.result-list-entry .result-list-entry__brand-title-container h2 | removeNewline | trim', link: '.result-list-entry .result-list-entry__brand-title-container@href', address: '.result-list-entry .result-list-entry__map-link', + image: '.result-list-entry .gallery-container .slick-list .gallery__image@src', + lazyImage: '.result-list-entry .gallery-container .gallery__image@data-lazy-src', }, paginate: '#pager .align-right a@href', normalize: normalize, diff --git a/lib/provider/kleinanzeigen.js b/lib/provider/kleinanzeigen.js index 36fe56d..b3e913d 100755 --- a/lib/provider/kleinanzeigen.js +++ b/lib/provider/kleinanzeigen.js @@ -30,6 +30,7 @@ const config = { link: '.aditem-main .text-module-begin a@href | removeNewline | trim', description: '.aditem-main p:not(.text-module-end) | removeNewline | trim', address: '.aditem-main--top--left | trim | removeNewline', + image: '.aditem-image .imagebox img@src', }, paginate: '#srchrslt-pagination .pagination-next@href', normalize: normalize, diff --git a/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx b/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx index 70db99b..07b9056 100644 --- a/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx +++ b/ui/src/views/jobs/mutation/components/notificationAdapter/NotificationAdapterMutator.jsx @@ -142,8 +142,9 @@ export default function NotificationAdapterMutator({ return (
{uiElement.type === 'boolean' ? ( - { setValue(selectedAdapter, uiElement, key, checked); }} From f60eb0f9b5cb7026711193d0284a150cc78273f9 Mon Sep 17 00:00:00 2001 From: Vladislav Ivanov Date: Wed, 14 Aug 2024 22:41:37 +0200 Subject: [PATCH 2/3] add image to 1a-immobilien --- lib/provider/einsAImmobilien.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/provider/einsAImmobilien.js b/lib/provider/einsAImmobilien.js index 7cd9a1b..bcb21b6 100755 --- a/lib/provider/einsAImmobilien.js +++ b/lib/provider/einsAImmobilien.js @@ -28,6 +28,7 @@ const config = { rooms: '.tabelle .inner_object_data .data_boxes div:nth-child(2)', title: '.tabelle .inner_object_data .tabelle_inhalt_titel_black | removeNewline | trim', description: '.tabelle .inner_object_data .objekt_beschreibung | removeNewline | trim', + image: '.tabelle .inner_object_pic img@src', }, normalize: normalize, filter: applyBlacklist, From 70e3632f01085b5dcea24566e21895f7d73cf6c2 Mon Sep 17 00:00:00 2001 From: Vladislav Ivanov Date: Mon, 16 Sep 2024 23:29:40 +0200 Subject: [PATCH 3/3] PR fixup: replace ternary operator --- lib/notification/adapter/telegram.js | 36 +++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/notification/adapter/telegram.js b/lib/notification/adapter/telegram.js index e95f646..b6dd6dd 100644 --- a/lib/notification/adapter/telegram.js +++ b/lib/notification/adapter/telegram.js @@ -30,23 +30,27 @@ function isValidURL(url) { } const sendTelegramMessage = (token, chatId, message, isPhoto = false, photoUrl = '') => { - const url = isPhoto - ? `https://api.telegram.org/bot${token}/sendPhoto` - : `https://api.telegram.org/bot${token}/sendMessage`; - const body = isPhoto - ? JSON.stringify({ - chat_id: chatId, - photo: photoUrl, - caption: message, - parse_mode: 'HTML', - }) - : JSON.stringify({ - chat_id: chatId, - text: message, - parse_mode: 'HTML', - disable_web_page_preview: !isPhoto, - }); + let url = '' + let body = '' + if (isPhoto) { + url = `https://api.telegram.org/bot${token}/sendPhoto`; + body = JSON.stringify({ + chat_id: chatId, + photo: photoUrl, + caption: message, + parse_mode: 'HTML', + }); + } else { + url = `https://api.telegram.org/bot${token}/sendMessage`; + body = JSON.stringify({ + chat_id: chatId, + text: message, + parse_mode: 'HTML', + disable_web_page_preview: !isPhoto, + }); + } + /** * This is to not break the rate limit. It is to only send 1 message per second */