Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix/published date #2808

Merged
merged 6 commits into from
Sep 28, 2023
Merged

fix/published date #2808

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d37cb7f
fix published date in chinese not parsed correctly
sywhb Sep 26, 2023
60b7d50
fix long published date not parsed correctly
sywhb Sep 26, 2023
3399213
add test cases from economist and caixin
sywhb Sep 27, 2023
0ccf332
Update generated html
sywhb Sep 27, 2023
55e274a
better match of published date and avoid removing date string which i…
sywhb Sep 28, 2023
f0abdd6
Update generated html
sywhb Sep 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 40 additions & 15 deletions packages/readabilityjs/Readability.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,29 @@ const extractPublishedDateFromAuthor = (author)=> {
return [null, null];
}
const authorName = author.replace(/^by\s+/i, '');
const regex = /(January|February|March|April|May|June|July|August|September|Octrober|November|December)\s\d{1,2},\s\d{2,4}/;
if (!regex.test(author)) {
return [authorName, null];
}
const regex = /(January|February|March|April|May|June|July|August|September|October|November|December)\s\d{1,2},\s\d{2,4}/i;
const chineseDateRegex = /(\d{2,4})年(\d{1,2})月(\d{1,2})日/;

// English date
if (regex.test(author)) {
const match = author.match(regex) || [];
return [authorName.replace(regex, ''), match[0]];
}

const matchedDates = author.match(regex) || [];
const publishedAt = matchedDates[0];
// Chinese date
if (chineseDateRegex.test(author)) {
const match = author.match(chineseDateRegex);
if (match) {
const year = parseInt(match[1], 10);
const month = parseInt(match[2], 10) - 1; // January is 0 in JavaScript Date
const day = parseInt(match[3], 10);

const publishedAt = new Date(year, month, day);
return [authorName.replace(chineseDateRegex, ''), publishedAt];
}
}

return [authorName.replace(regex, ''), publishedAt];
return [authorName, null];
};

/**
Expand Down Expand Up @@ -204,8 +217,10 @@ Readability.prototype = {
DATES_REGEXPS: [
/([0-9]{4}[-\/]?((0[13-9]|1[012])[-\/]?(0[1-9]|[12][0-9]|30)|(0[13578]|1[02])[-\/]?31|02[-\/]?(0[1-9]|1[0-9]|2[0-8]))|([0-9]{2}(([2468][048]|[02468][48])|[13579][26])|([13579][26]|[02468][048]|0[0-9]|1[0-6])00)[-\/]?02[-\/]?29)/i,
/(((0[13-9]|1[012])[-/]?(0[1-9]|[12][0-9]|30)|(0[13578]|1[02])[-/]?31|02[-/]?(0[1-9]|1[0-9]|2[0-8]))[-/]?[0-9]{4}|02[-/]?29[-/]?([0-9]{2}(([2468][048]|[02468][48])|[13579][26])|([13579][26]|[02468][048]|0[0-9]|1[0-6])00))/i,
/(((0[1-9]|[12][0-9]|30)[-/]?(0[13-9]|1[012])|31[-/]?(0[13578]|1[02])|(0[1-9]|1[0-9]|2[0-8])[-/]?02)[-/]?[0-9]{4}|29[-/]?02[-/]?([0-9]{2}(([2468][048]|[02468][48])|[13579][26])|([13579][26]|[02468][048]|0[0-9]|1[0-6])00))/i
]
/(((0[1-9]|[12][0-9]|30)[-/]?(0[13-9]|1[012])|31[-/]?(0[13578]|1[02])|(0[1-9]|1[0-9]|2[0-8])[-/]?02)[-/]?[0-9]{4}|29[-/]?02[-/]?([0-9]{2}(([2468][048]|[02468][48])|[13579][26])|([13579][26]|[02468][048]|0[0-9]|1[0-6])00))/i,
],
LONG_DATE_REGEXP: /^(Jan(uary)?|Feb(ruary)?|Mar(ch)?|Apr(il)?|May|Jun(e)?|Jul(y)?|Aug(ust)?|Sep(tember)?|Oct(ober)?|Nov(ember)?|Dec(ember)?)\s\d{1,2}(?:st|nd|rd|th)?(,)?\s\d{2,4}$/i,
CHINESE_DATE_REGEXP: /^\d{2,4}年\d{1,2}月\d{1,2}日$/,
},

UNLIKELY_ROLES: ["menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog"],
Expand Down Expand Up @@ -1067,27 +1082,37 @@ Readability.prototype = {
// we don't want to check for dates in the URL's
if (node.tagName.toLowerCase() === 'a') return
// Searching for the real date in the text content
let dateRegExpFound = this.REGEXPS.DATES_REGEXPS.find(regexp => regexp.test(node.textContent.trim()))
dateRegExpFound && (dateRegExpFound = dateRegExpFound.exec(node.textContent.trim()))
const content = node.textContent.trim()
let dateFound
const dateRegExpFound = this.REGEXPS.DATES_REGEXPS.find(regexp => regexp.test(content))
if (dateRegExpFound) {
dateFound = dateRegExpFound.exec(content)[0]
} else if (this.REGEXPS.LONG_DATE_REGEXP.test(content)) {
dateFound = this.REGEXPS.LONG_DATE_REGEXP.exec(content)[0].replace(/st|nd|rd|th/i, '')
} else if (this.REGEXPS.CHINESE_DATE_REGEXP.test(content)) {
dateFound = this.REGEXPS.CHINESE_DATE_REGEXP.exec(content)[0].replace(/年|月/g, '-').replace(/日/g, '')
}

let publishedDateParsed
try {
// Trying to parse the Date from the content itself
publishedDateParsed = new Date(node.textContent.trim())
publishedDateParsed = new Date(content)
} catch (error) { }

if (
// Checking for the dates keywords through the attributes except URL's
((this._someNodeAttribute(node, ({ value, name }) => {
if (/href|uri|url/i.test(name)) return false;
return this.REGEXPS.publishedDate.test(value)
}) || dateRegExpFound) || (/date/i.test(matchString) && !isNaN(publishedDateParsed)))
}) || dateFound) || (/date/i.test(matchString) && !isNaN(publishedDateParsed)))
&& this._isValidPublishedDate(node.textContent)
) {
try {
if (isNaN(publishedDateParsed))
if (isNaN(publishedDateParsed)) {
// Trying to parse the Date from the found by REGEXP string
publishedDateParsed = new Date(dateRegExpFound[0])
publishedDateParsed = new Date(dateFound)
}

if (!isNaN(publishedDateParsed) && !this._articlePublishedDate)
this._articlePublishedDate = publishedDateParsed
}
Expand Down
Loading