From 3c2d8c6872808ad1b000eb12b1188c6c3312fca0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 22:08:21 +0000 Subject: [PATCH 1/6] chore(deps-dev): bump @types/require-all from 3.0.5 to 3.0.6 Bumps [@types/require-all](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/require-all) from 3.0.5 to 3.0.6. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/require-all) --- updated-dependencies: - dependency-name: "@types/require-all" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- package.json | 2 +- pnpm-lock.yaml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 1c344241afa0d1..0af7eb9fbdf944 100644 --- a/package.json +++ b/package.json @@ -175,7 +175,7 @@ "@types/pidusage": "2.0.4", "@types/plist": "3.0.4", "@types/request-promise-native": "1.0.20", - "@types/require-all": "3.0.5", + "@types/require-all": "3.0.6", "@types/showdown": "2.0.3", "@types/supertest": "2.0.15", "@types/tiny-async-pool": "2.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b330f7c587cf42..246f54e4da745c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -278,8 +278,8 @@ devDependencies: specifier: 1.0.20 version: 1.0.20 '@types/require-all': - specifier: 3.0.5 - version: 3.0.5 + specifier: 3.0.6 + version: 3.0.6 '@types/showdown': specifier: 2.0.3 version: 2.0.3 @@ -1176,7 +1176,7 @@ packages: '@babel/runtime-corejs2': 7.23.2 '@postlight/ci-failed-test-reporter': 1.0.26 cheerio: 0.22.0 - difflib: github.com/postlight/difflib.js/32e8e38c7fcd935241b9baab71bb432fd9b166ed + difflib: git/github.com+postlight/difflib.js/32e8e38c7fcd935241b9baab71bb432fd9b166ed ellipsize: 0.1.0 iconv-lite: 0.5.0 moment: 2.29.4 @@ -1721,8 +1721,8 @@ packages: '@types/tough-cookie': 4.0.4 form-data: 2.5.1 - /@types/require-all@3.0.5: - resolution: {integrity: sha512-s5UgvpbPfoNGtHb0sCER3Xdi47xrkzT9qNr+uhozmfw/FKnjW5NstnkCgOTbHOhykzzDNceQN9XB3pRNdqyTEA==} + /@types/require-all@3.0.6: + resolution: {integrity: sha512-93iiG8N8kovFL08oenE76F7adIE00uzTdGV7FlFE9uPDtZ2f3pSuOS0ICWjG9OMhsn38eJjhf5CKFp9s7fzz/A==} dev: true /@types/responselike@1.0.0: @@ -8172,8 +8172,8 @@ packages: resolution: {integrity: sha512-V50KMwwzqJV0NpZIZFwfOD5/lyny3WlSzRiXgA0G7VUnRlqttta1L6UQIHzd6EuBY/cHGfwTIck7w1yH6Q5zUw==} dev: true - github.com/postlight/difflib.js/32e8e38c7fcd935241b9baab71bb432fd9b166ed: - resolution: {tarball: https://codeload.github.com/postlight/difflib.js/tar.gz/32e8e38c7fcd935241b9baab71bb432fd9b166ed} + git/github.com+postlight/difflib.js/32e8e38c7fcd935241b9baab71bb432fd9b166ed: + resolution: {commit: 32e8e38c7fcd935241b9baab71bb432fd9b166ed, repo: git@github.com:postlight/difflib.js.git, type: git} name: difflib version: 0.2.6 dependencies: From 94975463f385fed7a6e9c30dae2b620cc4debcde Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 7 Nov 2023 18:19:00 +0000 Subject: [PATCH 2/6] feat: support RSS3 UMS output also optimized middleware/template --- lib/middleware/template.js | 159 ++++++++++++++++++------------------- lib/utils/render.js | 2 + lib/views/rss3-ums.js | 58 ++++++++++++++ 3 files changed, 136 insertions(+), 83 deletions(-) create mode 100644 lib/views/rss3-ums.js diff --git a/lib/middleware/template.js b/lib/middleware/template.js index 1538d9813dd571..ad0b41abb15a99 100644 --- a/lib/middleware/template.js +++ b/lib/middleware/template.js @@ -1,7 +1,7 @@ -const { art, json } = require('@/utils/render'); +const { art, json, rss3_ums } = require('@/utils/render'); const path = require('path'); const config = require('@/config').value; -const typeRegex = /\.(atom|rss|debug\.json|json|\d+\.debug\.html)$/; +const typeRegex = /\.(atom|rss|ums|debug\.json|json|\d+\.debug\.html)$/; const { collapseWhitespace, convertDateToISO8601 } = require('@/utils/common-utils'); module.exports = async (ctx, next) => { @@ -16,111 +16,104 @@ module.exports = async (ctx, next) => { const outputType = ctx.state.type[1] || 'rss'; - if (outputType === 'debug.json' && config.debugInfo) { - ctx.set({ - 'Content-Type': 'application/json; charset=UTF-8', - }); - if (ctx.state.json) { - ctx.body = JSON.stringify(ctx.state.json, null, 4); - } else { - ctx.body = JSON.stringify({ message: 'plugin does not set debug json' }); - } - } - if (outputType.endsWith('.debug.html') && config.debugInfo) { - ctx.set({ - 'Content-Type': 'text/html; charset=UTF-8', - }); - - const index = parseInt(outputType.match(/(\d+)\.debug\.html$/)[1]); - if (!(ctx.state.data && ctx.state.data.item && ctx.state.data.item[index])) { - ctx.body = `ctx.state.data.item[${index}] not found`; - } else { - ctx.body = ctx.state.data.item[index].description; + // only enable when debugInfo=true + if (config.debugInfo) { + if (outputType === 'debug.json') { + ctx.set({ + 'Content-Type': 'application/json; charset=UTF-8', + }); + if (ctx.state.json) { + ctx.body = JSON.stringify(ctx.state.json, null, 4); + } else { + ctx.body = JSON.stringify({ message: 'plugin does not set debug json' }); + } } - } - if (outputType === 'json') { - ctx.set({ 'Content-Type': 'application/feed+json; charset=UTF-8' }); + if (outputType.endsWith('.debug.html')) { + ctx.set({ + 'Content-Type': 'text/html; charset=UTF-8', + }); + + const index = parseInt(outputType.match(/(\d+)\.debug\.html$/)[1]); + if (!(ctx.state.data && ctx.state.data.item && ctx.state.data.item[index])) { + ctx.body = `ctx.state.data.item[${index}] not found`; + } else { + ctx.body = ctx.state.data.item[index].description; + } + } } if (!ctx.body) { - let template; - - switch (outputType) { - case 'atom': - template = path.resolve(__dirname, '../views/atom.art'); - break; - case 'rss': - template = path.resolve(__dirname, '../views/rss.art'); - break; - default: - template = path.resolve(__dirname, '../views/rss.art'); - break; - } + const templateName = outputType === 'atom' ? 'atom.art' : 'rss.art'; + const template = path.resolve(__dirname, `../views/${templateName}`); if (ctx.state.data) { - ctx.state.data.title = collapseWhitespace(ctx.state.data.title); - ctx.state.data.subtitle = collapseWhitespace(ctx.state.data.subtitle); - ctx.state.data.author = collapseWhitespace(ctx.state.data.author); - - ctx.state.data.item && - ctx.state.data.item.forEach((item) => { - if (item.title) { - item.title = collapseWhitespace(item.title); - // trim title length - for (let length = 0, i = 0; i < item.title.length; i++) { - length += Buffer.from(item.title[i]).length !== 1 ? 2 : 1; - if (length > config.titleLengthLimit) { - item.title = `${item.title.slice(0, i)}...`; - break; - } - } + const collapseWhitespaceForProperties = (properties, obj) => { + properties.forEach((prop) => { + if (obj[prop]) { + obj[prop] = collapseWhitespace(obj[prop]); } - - if (typeof item.author === 'string') { - item.author = collapseWhitespace(item.author); - } else if (typeof item.author === 'object' && item.author !== null) { - for (const a of item.author) { - a.name = collapseWhitespace(a.name); - } - if (outputType !== 'json') { - item.author = item.author.map((a) => a.name).join(', '); + }); + }; + + collapseWhitespaceForProperties(['title', 'subtitle', 'author'], ctx.state.data); + + ctx.state.data.item?.forEach((item) => { + if (item.title) { + item.title = collapseWhitespace(item.title); + // trim title length + for (let length = 0, i = 0; i < item.title.length; i++) { + length += Buffer.from(item.title[i]).length !== 1 ? 2 : 1; + if (length > config.titleLengthLimit) { + item.title = `${item.title.slice(0, i)}...`; + break; } } - - if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && item.itunes_duration.indexOf(':') === -1) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) { - item.itunes_duration = +item.itunes_duration; - item.itunes_duration = - Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2); - } - - if (outputType !== 'rss') { - item.pubDate = convertDateToISO8601(item.pubDate); - item.updated = convertDateToISO8601(item.updated); + } + + if (typeof item.author === 'string') { + item.author = collapseWhitespace(item.author); + } else if (typeof item.author === 'object' && item.author !== null) { + item.author.forEach((a) => (a.name = collapseWhitespace(a.name))); + if (outputType !== 'json') { + item.author = item.author.map((a) => a.name).join(', '); } - }); + } + + if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && item.itunes_duration.indexOf(':') === -1) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) { + item.itunes_duration = +item.itunes_duration; + item.itunes_duration = + Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2); + } + + if (outputType !== 'rss') { + item.pubDate = convertDateToISO8601(item.pubDate); + item.updated = convertDateToISO8601(item.updated); + } + }); } - const routeTtl = (config.cache.routeExpire / 60) | 0; - const data = { lastBuildDate: new Date().toUTCString(), updated: new Date().toISOString(), - ttl: routeTtl, + ttl: (config.cache.routeExpire / 60) | 0, atomlink: ctx.request.href, ...ctx.state.data, }; + if (config.isPackage) { ctx.body = data; return; } - if (!template) { - return; - } - if (outputType !== 'json') { + + if (outputType === 'ums') { + ctx.set({ 'Content-Type': 'application/json; charset=UTF-8' }); + ctx.body = rss3_ums(data); + } else if (outputType === 'json') { + ctx.set({ 'Content-Type': 'application/feed+json; charset=UTF-8' }); + ctx.body = json(data); + } else { ctx.body = art(template, data); - return; } - ctx.body = json(data); } }; diff --git a/lib/utils/render.js b/lib/utils/render.js index 813f631c2475dd..5310ba5e3ec947 100644 --- a/lib/utils/render.js +++ b/lib/utils/render.js @@ -1,9 +1,11 @@ const art = require('art-template'); const json = require('@/views/json'); +const ums = require('@/views/rss3-ums'); // We may add more control over it later module.exports = { art, json, // This should be used by RSSHub middleware only. + ums, }; diff --git a/lib/views/rss3-ums.js b/lib/views/rss3-ums.js new file mode 100644 index 00000000000000..5f54a9aaa9d475 --- /dev/null +++ b/lib/views/rss3-ums.js @@ -0,0 +1,58 @@ +/** + * This function should be used by RSSHub middleware only. + * @param {object} data ctx.state.data + * @returns `JSON.stringify`-ed [UMS Result](https://docs.rss3.io/docs/unified-metadata-schemas) + */ + +const rss3_ums = (data) => { + const network = 'RSS'; + const tag = 'RSS'; + const type = 'article'; + const umsResult = { + data: data.item.map((item) => { + const owner = getOwnershipFieldFromURL(item); + return { + owner, + id: item.link, + network, + from: owner, + to: owner, + tag, + type, + direction: 'out', + feeValue: '0', + actions: [ + { + tag, + type, + platform: owner, + from: owner, + to: owner, + metadata: { + title: item.title, + description: item.description, + authors: [], + tags: typeof item.category === 'string' ? [item.category] : item.category, + pubDate: item.pubDate, + }, + related_urls: [item.link], + }, + ], + timestamp: item.updated, + }; + }), + }; + return JSON.stringify(umsResult, null, 4); +}; + +// we treat the domain as the owner of the content +function getOwnershipFieldFromURL(item) { + try { + const urlObj = new URL(item.link); + return urlObj.hostname; + } catch (e) { + return item.link; + } +} + +module.exports = rss3_ums; From 428d5e7cecd1b4751f1392a83751ee4a8947a849 Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 7 Nov 2023 21:54:58 +0000 Subject: [PATCH 3/6] fix: rss3_ums function name --- lib/utils/render.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/utils/render.js b/lib/utils/render.js index 5310ba5e3ec947..0bd5ad9e82b78b 100644 --- a/lib/utils/render.js +++ b/lib/utils/render.js @@ -1,11 +1,11 @@ const art = require('art-template'); const json = require('@/views/json'); -const ums = require('@/views/rss3-ums'); +const rss3_ums = require('@/views/rss3-ums'); // We may add more control over it later module.exports = { art, json, // This should be used by RSSHub middleware only. - ums, + rss3_ums, }; From 288f99e80963a8ed08b5d43af7bb07d897849a0e Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 7 Nov 2023 22:08:17 +0000 Subject: [PATCH 4/6] feat: handle current timestamp in UMS --- lib/middleware/template.js | 5 +++-- lib/views/rss3-ums.js | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/middleware/template.js b/lib/middleware/template.js index ad0b41abb15a99..6d102cd8c2d108 100644 --- a/lib/middleware/template.js +++ b/lib/middleware/template.js @@ -93,9 +93,10 @@ module.exports = async (ctx, next) => { }); } + const currentDate = new Date(); const data = { - lastBuildDate: new Date().toUTCString(), - updated: new Date().toISOString(), + lastBuildDate: currentDate.toUTCString(), + updated: currentDate.toISOString(), ttl: (config.cache.routeExpire / 60) | 0, atomlink: ctx.request.href, ...ctx.state.data, diff --git a/lib/views/rss3-ums.js b/lib/views/rss3-ums.js index 5f54a9aaa9d475..9716b5adac21de 100644 --- a/lib/views/rss3-ums.js +++ b/lib/views/rss3-ums.js @@ -1,3 +1,5 @@ +const dayjs = require('dayjs'); + /** * This function should be used by RSSHub middleware only. * @param {object} data ctx.state.data @@ -8,6 +10,7 @@ const rss3_ums = (data) => { const network = 'RSS'; const tag = 'RSS'; const type = 'article'; + const currentUnixTsp = dayjs().unix(); const umsResult = { data: data.item.map((item) => { const owner = getOwnershipFieldFromURL(item); @@ -38,7 +41,7 @@ const rss3_ums = (data) => { related_urls: [item.link], }, ], - timestamp: item.updated, + timestamp: dayjs(item.updated).unix() || currentUnixTsp, }; }), }; From 83f4d48747f08135dcc97232590632c68d6da340 Mon Sep 17 00:00:00 2001 From: Henry Date: Tue, 7 Nov 2023 22:10:17 +0000 Subject: [PATCH 5/6] feat: add authors --- lib/views/rss3-ums.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/views/rss3-ums.js b/lib/views/rss3-ums.js index 9716b5adac21de..33a951e1e423d7 100644 --- a/lib/views/rss3-ums.js +++ b/lib/views/rss3-ums.js @@ -32,11 +32,11 @@ const rss3_ums = (data) => { from: owner, to: owner, metadata: { - title: item.title, + authors: typeof item.author === 'string' ? [{ name: item.author }] : item.author, description: item.description, - authors: [], - tags: typeof item.category === 'string' ? [item.category] : item.category, pubDate: item.pubDate, + tags: typeof item.category === 'string' ? [item.category] : item.category, + title: item.title, }, related_urls: [item.link], }, From 48c175ef12410ad8e3ac00bc23236fe8c8c4407f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 22:12:57 +0000 Subject: [PATCH 6/6] chore(deps-dev): bump @types/showdown from 2.0.3 to 2.0.4 Bumps [@types/showdown](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/showdown) from 2.0.3 to 2.0.4. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/showdown) --- updated-dependencies: - dependency-name: "@types/showdown" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- package.json | 2 +- pnpm-lock.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 0af7eb9fbdf944..fdfa5e9c026f91 100644 --- a/package.json +++ b/package.json @@ -176,7 +176,7 @@ "@types/plist": "3.0.4", "@types/request-promise-native": "1.0.20", "@types/require-all": "3.0.6", - "@types/showdown": "2.0.3", + "@types/showdown": "2.0.4", "@types/supertest": "2.0.15", "@types/tiny-async-pool": "2.0.1", "@types/tough-cookie": "4.0.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 246f54e4da745c..64fc3f6aed5559 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -281,8 +281,8 @@ devDependencies: specifier: 3.0.6 version: 3.0.6 '@types/showdown': - specifier: 2.0.3 - version: 2.0.3 + specifier: 2.0.4 + version: 2.0.4 '@types/supertest': specifier: 2.0.15 version: 2.0.15 @@ -1746,8 +1746,8 @@ packages: '@types/node': 20.5.6 dev: true - /@types/showdown@2.0.3: - resolution: {integrity: sha512-cFuAcA3p2YPq8HR8KxvDXnOdccOZ74ypANB3kb3AL5Srji0QnteVw6vf4o7GJ8hMyz+uZ+nSQHVgXSgjYD1a5g==} + /@types/showdown@2.0.4: + resolution: {integrity: sha512-cSXSKOpTSr2HTdlGq8WskyZwNyxKhM7M/zJeLVdWjlUQmQ4d8TdtPrwz4JejglZdzIzSgU5loi5QUaEJF9JD8w==} dev: true /@types/stack-utils@2.0.1: