Skip to content

Commit

Permalink
Add doc link checker
Browse files Browse the repository at this point in the history
  • Loading branch information
nwmac committed Sep 27, 2024
1 parent f1dce33 commit e069375
Show file tree
Hide file tree
Showing 3 changed files with 360 additions and 4 deletions.
356 changes: 356 additions & 0 deletions scripts/check-i18n-links
Original file line number Diff line number Diff line change
@@ -0,0 +1,356 @@
#!/usr/bin/env node

/**
* This script reads and parses all of the source code files, reads the US English
* translation files and compares the two to identify referenced strings that
* do not have a corresponding translation.
*
* This script is used in the PR gate to ensure code is not added without the
* corresponding translations.
*
*/

const fs = require('fs');
const path = require('path');
const yaml = require('js-yaml');
const request = require('request-promise-native');

const base = path.resolve(__dirname, '..');
const srcFolder = path.resolve(base, 'shell');
const pkgFolder = path.resolve(base, 'pkg');

// Simple shell colors
const reset = "\x1b[0m";
const cyan = `\x1b[96m`;
const yellow = `\x1b[33m`;
const white = `\x1b[97m`;
const bold = `\x1b[1m`;
const bg_red = `\x1b[41m`;

const DOCS_BASE_REGEX = /export const DOCS_BASE = '([^']*)';/;

const CATEGORIES = [
{
name: 'Rancher Manager Documentation',
regex: /^https:\/\/.*rancher\.com\/.*/
},
{
name: 'RKE2 Documentation',
regex: /^https:\/\/.*rke2\.io\/.*/
},
{
name: 'K3S Documentation',
regex: /^https:\/\/.*k3s\.io\/.*/
}
];

let docsBaseUrl = '';

// -s flag will show the details of all of the unused strings
// -x flag will cause script to return 0, even if there are errors
let showUnused = false;
let doNotReturnError = false;

// Simple arg parsing
if (process.argv.length > 2) {
process.argv.shift();
process.argv.shift();

process.argv.forEach((arg) => {
if (arg === '-s') {
showUnused = true;
} else if (arg === '-x') {
doNotReturnError = true;
}
});
}

// Read file to parse out the docs base
const docsBaseFile = fs.readFileSync(path.join(srcFolder, 'config', 'private-label.js'), 'utf8');
const docsBaseFileMatches = docsBaseFile.match(DOCS_BASE_REGEX);

if (docsBaseFileMatches && docsBaseFileMatches.length === 2) {
docsBaseUrl = docsBaseFileMatches[1];
}

function parseReference(ref) {
if (!ref.includes('$') || ref.startsWith('${')) {
return ref;
}

let out = '';
let inVar = false;
let variable = '';
let vars = 0;

for (let i=0;i<ref.length;i++) {
if (inVar) {
if (ref[i] === '}') {
inVar = false;
} else {
variable += ref[i];
}
} else if (ref[i] === '{') {
inVar = true;
vars++;
variable = '';
} else {
out += ref[i];
}
}

let p = out.replaceAll('"', '').split('.');

// Check for patterns like this: resourceDetail.detailTop.${annotationsVisible? 'hideAnnotations' : 'showAnnotations'}
if (vars === 1 && variable.includes('?')) {
const options = variable.substr(variable.indexOf('?') + 1);
const opts = options.split(':').map((o) => o.trim().replaceAll('\'', ''));

if (opts.length === 2) {
const a = p.map((o) => o === '$' ? opts[0] : o).join('.');
const b = p.map((o) => o === '$' ? opts[1] : o).join('.');

return [a, b];
}
}

p = p.map((a) => a.startsWith('$') ? `.*${ a.substr(1) }` : a);

return new RegExp(p.join('\\.'), 'g');
}

function readAndParseTranslations(filePath) {
const data = fs.readFileSync(filePath, 'utf8');

try {
const i18n = yaml.load(fs.readFileSync(filePath), 'utf8');

return parseTranslations(i18n);
} catch (e) {
console.log('Can not read i18n file'); // eslint-disable-line no-console
console.log(e); // eslint-disable-line no-console
process.exit(1);
}
}

function parseTranslations(obj, parent) {
let res = {};
Object.keys(obj).forEach((key) => {
const v = obj[key];
const pKey = parent ? `${parent}.${key}` : key;

if (v === null) {
// ignore
} else if (typeof v === 'object') {
res = {
...res,
...parseTranslations(v, pKey)
}
} else {
// Ensure empty strings work
res[pKey] = v.length === 0 ? '[empty]' : v;
}
});

return res;
}

const LINK_REGEX = /<[aA]\s[^>]*>/g;
const ATTR_REGEX = /(([a-zA-Z]*)=["']([^"']*))/g;

function parseLinks(str) {
const a = [...str.matchAll(LINK_REGEX)];

const links = [];

if (a && a.length) {
a.forEach((m) => {
const attrs = [...m[0].matchAll(ATTR_REGEX)];

attrs.forEach((attr) => {
if (attr.length === 4 && attr[2] === 'href') {
const link = attr[3].replace('{docsBase}', docsBaseUrl);

if (link.startsWith('http')) {
links.push(link);
}
}
});
});
}

return links;
}

function makeRegex(str) {
if (str.startsWith('/') && str.endsWith('/')) {
return new RegExp(str.substr(1, str.length - 2), 'g');
} else {
// String
// Support .* for a simple wildcard match
if (str.includes('*')) {
const parts = str.split('.');
const p = parts.map((s, i) => s === '*' ? (i === parts.length -1 ? '.*' : '[^\.]*') : s);

return new RegExp(`^${ p.join('\\.') }$`);
} else {
return str;
}
}
}

function doesMatch(stringOrRegex, str) {
if (typeof stringOrRegex === 'string') {
return str === stringOrRegex;
} else {
return str.match(stringOrRegex);
}
}
function loadI18nFiles(folder) {
let res = {};

// Find all of the test files
fs.readdirSync(folder).forEach((file) => {
const filePath = path.resolve(folder, file)
const isFolder = fs.lstatSync(filePath).isDirectory();

if (isFolder) {
res = {
...res,
...loadI18nFiles(filePath)
};
} else if (file === 'en-us.yaml') {
console.log(` ... ${ path.relative(base, filePath) }`);

const translations = readAndParseTranslations(filePath);

res = {
...res,
...translations
};
}
});

return res;
}

console.log('======================================'); // eslint-disable-line no-console
console.log(`${cyan}Checking source files for i18n strings${reset}`); // eslint-disable-line no-console
console.log('======================================'); // eslint-disable-line no-console

console.log(''); // eslint-disable-line no-console
console.log('Reading translation files:'); // eslint-disable-line no-console

let i18n = loadI18nFiles(srcFolder);

i18n = { ...i18n, ...loadI18nFiles(pkgFolder) };

console.log(`Read ${cyan}${ Object.keys(i18n).length }${reset} translations`); // eslint-disable-line no-console

//..console.log(`Found ${cyan}${ Object.keys(references).length }${reset} i18n references in code`); // eslint-disable-line no-console

let unused = 0;
const unusedKeys = [];

const links = [];

// Look for translations that are not used
Object.keys(i18n).forEach((str) => {
const link = parseLinks(i18n[str]);

links.push(...link);
});

console.log(`Discoverd ${cyan}${links.length}${reset} links`); // eslint-disable-line no-console

console.log(`${cyan}Links:${reset}`); // eslint-disable-line no-console
console.log(`${cyan}======${reset}`); // eslint-disable-line no-console

showByCategory(links, 'Links in', 'Other Links', cyan);

console.log(''); // eslint-disable-line no-console

function showByCategory(linksToShow, prefixLabel, otherLabel, color) {
const others = [];
const byCategory = {};

linksToShow.forEach((link) => {
let found = false;
CATEGORIES.forEach((category) => {
byCategory[category.name] = byCategory[category.name] || [];

if (!found && category.regex.test(link)) {
byCategory[category.name].push(link);
found = true;
}
});

if (!found) {
others.push(link);
}
});

CATEGORIES.forEach((category) => {
if (byCategory[category.name].length) {
console.log(`${color}${prefixLabel} ${category.name}${reset}`); // eslint-disable-line no-console
byCategory[category.name].forEach((link) => console.log(` ${link}`)); // eslint-disable-line no-console
}
});

if (others.length) {
console.log(`${color}${otherLabel}${reset}`); // eslint-disable-line no-console
others.forEach((link) => console.log(` ${link}`));
}
}

async function check(links) {
const badLinks = [];

for(let i =0; i<links.length; i++) {
const link = links[i];
let statusCode;
let statusMessage;

try {
const headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
'Accept': 'text/html',
'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8',
};

const r = await request(link, { resolveWithFullResponse: true, headers });

statusCode = r.statusCode;
statusMessage = r.statusMessage;
} catch (e) {
statusCode = e.statusCode;
statusMessage = e.statusMessage;
}

if (statusCode !== 200) {
const sc = `${ statusCode }`.padEnd(5);

console.log(` ${ link } : ${ sc } ${ statusMessage ? statusMessage : '' }`); // eslint-disable-line no-console

badLinks.push(link);
}
};

console.log(''); // eslint-disable-line no-console

if (badLinks.length === 0) {
console.log(`${cyan}${bold}Links Checked - all links could be fetched okay${reset}`); // eslint-disable-line no-console
} else {
console.log(`${yellow}${bold}Found ${badLinks.length} link(s) that could not be fetched${reset}`); // eslint-disable-line no-console
}

console.log(''); // eslint-disable-line no-console

showByCategory(badLinks, 'Broken links in', 'Other Broken links', yellow);

console.log(''); // eslint-disable-line no-console
}

console.log(`${cyan}Checking doc links ...${reset}`); // eslint-disable-line no-console

check(links);
4 changes: 2 additions & 2 deletions shell/assets/translations/en-us.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1885,7 +1885,7 @@ cluster:
header: Registry for Rancher System Container Images
label: Enable cluster scoped container registry for Rancher system container images
description: "If enabled, Rancher will pull container images from this registry during cluster provisioning. By default, Rancher will also use this registry when installing Rancher's official Helm chart apps. If the cluster scoped registry is disabled, system images are pulled from the System Default Registry in the global settings."
docsLinkRke2: "For help configuring private registry mirrors, see the RKE2 <a href=\"https://docs.rke2.io/install/containerd_registry_configuration/\" target=\"_blank\">documentation.</a>"
docsLinkRke2: "For help configuring private registry mirrors, see the RKE2 <a href=\"https://docs.rke2.io/install/containerd_registry_configuration\" target=\"_blank\">documentation.</a>"
docsLinkK3s: "For help configuring private registry mirrors, see the K3s <a href=\"https://docs.k3s.io/installation/private-registry\" target=\"_blank\">documentation.</a>"
provider:
aliyunecs: Aliyun ECS
Expand Down Expand Up @@ -5245,7 +5245,7 @@ setup:
skip: Skip
tip: What URL should be used for this {vendor} installation? All the nodes in your clusters will need to be able to reach this.
setPassword: The first order of business is to set a strong password for the default <code>{username}</code> user. We suggest using this random one generated just for you, but enter your own if you like.
telemetry: Allow collection of <a href="{docsBase}/faq/telemetry/" target="_blank" rel="noopener noreferrer nofollow">anonymous statistics</a> to help us improve {name}
telemetry: Allow collection of <a href="{docsBase}/faq/telemetry" target="_blank" rel="noopener noreferrer nofollow">anonymous statistics</a> to help us improve {name}
useManual: Set a specific password to use
useRandom: Use a randomly generated password
welcome: Welcome to {vendor}!
Expand Down
4 changes: 2 additions & 2 deletions shell/assets/translations/zh-hans.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1752,7 +1752,7 @@ cluster:
privateRegistry:
label: 为 Rancher 系统容器镜像启用集群级别的容器镜像仓库
description: "如果启用,Rancher 将在集群配置期间从该镜像仓库中拉取容器镜像。默认情况下,Rancher 在安装 Rancher 的官方 Helm Chart 应用程序时也会使用此镜像仓库。如果集群级别的镜像仓库被禁用,将从全局设置中的系统默认镜像仓库中拉取系统镜像。"
docsLinkRke2: "如需配置私有镜像仓库 mirror 的帮助,请参阅 RKE2 <a href=\"https://docs.rke2.io/install/containerd_registry_configuration/\" target=\"_blank\">文档</a>。"
docsLinkRke2: "如需配置私有镜像仓库 mirror 的帮助,请参阅 RKE2 <a href=\"https://docs.rke2.io/install/containerd_registry_configuration\" target=\"_blank\">文档</a>。"
docsLinkK3s: "如需配置私有镜像仓库 mirror 的帮助,请参阅 K3s <a href=\"https://docs.k3s.io/installation/private-registry\" target=\"_blank\">文档</a>。"
provider:
aliyunecs: Aliyun ECS
Expand Down Expand Up @@ -4967,7 +4967,7 @@ setup:
skip: 跳过
tip: 此 {vendor} 安装应使用什么 URL?集群中的所有节点都需要能访问该 URL。
setPassword: 请为默认用户 <code>{username}</code>设置强密码。建议使用生成的随机密码。你也可以自行设置。
telemetry: 允许收集<a href="{docsBase}/faq/telemetry/" target="_blank" rel="noopener noreferrer nofollow">匿名统计数据</a>,以帮我们改进 {name}。
telemetry: 允许收集<a href="{docsBase}/faq/telemetry" target="_blank" rel="noopener noreferrer nofollow">匿名统计数据</a>,以帮我们改进 {name}。
useManual: 设置密码
useRandom: 使用随机生成的密码
welcome: 欢迎使用 {vendor}!
Expand Down

0 comments on commit e069375

Please sign in to comment.