diff --git a/scanner/main.ts b/scanner/main.ts index 74b31a9..b649e6e 100644 --- a/scanner/main.ts +++ b/scanner/main.ts @@ -1,6 +1,7 @@ import Bottleneck from "bottleneck"; import specs from 'browser-specs' assert { type: "json" }; import fs from 'node:fs/promises'; +import { mean, quantile } from 'simple-statistics'; import octokit from './third_party/octokit-cache.js'; const ghLimiter = new Bottleneck({ @@ -20,54 +21,104 @@ interface IssueSummary { pull_request?: { draft: boolean }; milestone?: string; labels: string[]; + ageAtCloseMs?: number; +}; + +interface AgeStats { + count: number; + mean: number; + [percentile: string]: number; }; interface RepoSummary { - cached_at: number; + cachedAt: number; org: string; repo: string; issues: IssueSummary[]; labelsPresent: boolean; + ageAtCloseMs?: AgeStats; + openAgeMs?: AgeStats; +} + +interface GlobalStatsInput { + closeAgesMs: number[]; + openAgesMs: number[]; } -async function analyzeRepo(org: string, repo: string): Promise { +function ageStats(arr: number[]): AgeStats | undefined { + if (arr.length === 0) { + return undefined; + } + const result = { + count: arr.length, + mean: mean(arr), + } + const percentiles = [10, 25, 50, 75, 90]; + quantile(arr, percentiles.map(p => p / 100)).forEach((q, i) => { + result[percentiles[i]] = q; + }); + return result; +} + +async function analyzeRepo(org: string, repo: string, globalStats: GlobalStatsInput): Promise { + let result: RepoSummary | null = null; try { - const cachedRepo = JSON.parse(await fs.readFile(`summaries/${org}/${repo}.json`, { encoding: 'utf8' })); - if (Date.now() - cachedRepo.cached_at < 60 * 60 * 1000) { - return cachedRepo; - } + result = JSON.parse(await fs.readFile(`summaries/${org}/${repo}.json`, { encoding: 'utf8' }), + (key, value) => { + if (['created_at', 'closed_at'].includes(key)) { + return new Date(value); + } + return value; + }); } catch { // On error, fetch the body. } - const result: RepoSummary = { - cached_at: Date.now(), - org, repo, - issues: [], - labelsPresent: false - }; - - for (const issue of await getIssues(octokit, org, repo)) { - const info: IssueSummary = { - number: issue.number, - url: issue.html_url, - title: issue.title, - created_at: new Date(issue.created_at), - labels: issue.labels?.map(label => label.name) ?? [], + if (!result || Date.now() - result.cachedAt > 60 * 60 * 1000) { + result = { + cachedAt: Date.now(), + org, repo, + issues: [], + labelsPresent: false }; - if (issue.closed_at) { - info.closed_at = new Date(issue.closed_at); - } - if (issue.pull_request) { - info.pull_request = { draft: issue.pull_request.draft } - } - if (issue.milestone) { - info.milestone = issue.milestone.title; + + for (const issue of await getIssues(octokit, org, repo)) { + const info: IssueSummary = { + number: issue.number, + url: issue.html_url, + title: issue.title, + created_at: new Date(issue.created_at), + labels: issue.labels?.map(label => label.name) ?? [], + }; + if (issue.closed_at) { + info.closed_at = new Date(issue.closed_at); + } + if (issue.pull_request) { + info.pull_request = { draft: issue.pull_request.draft } + } + if (issue.milestone) { + info.milestone = issue.milestone.title; + } + result.issues.push(info); } - result.issues.push(info); + } - await fs.mkdir(`summaries/${org}`, { recursive: true }); - await fs.writeFile(`summaries/${org}/${repo}.json`, JSON.stringify(result, undefined, 2)); + const closeAgesMs: number[] = []; + const openAgesMs: number[] = []; + for (const issue of result.issues) { + if (issue.closed_at) { + closeAgesMs.push(issue.closed_at.valueOf() - issue.created_at.valueOf()); + } else { + openAgesMs.push(result.cachedAt - issue.created_at.valueOf()); + } } + result.ageAtCloseMs = ageStats(closeAgesMs); + result.openAgeMs = ageStats(openAgesMs); + globalStats.closeAgesMs.push(...closeAgesMs); + globalStats.openAgesMs.push(...openAgesMs); + + await fs.mkdir(`summaries/${org}`, { recursive: true }); + await fs.writeFile(`summaries/${org}/${repo}.json`, JSON.stringify(result, undefined, 2)); + return result; } @@ -93,7 +144,14 @@ async function main() { const [org, repo] = parts; githubRepos.push({ org, repo }); } - await Promise.all(githubRepos.map(({ org, repo }) => analyzeRepo(org, repo))); + + const globalStats: GlobalStatsInput = { openAgesMs: [], closeAgesMs: [] }; + await Promise.all(githubRepos.map(({ org, repo }) => analyzeRepo(org, repo, globalStats))); + + await fs.writeFile("summaries/global.json", JSON.stringify({ + ageAtCloseMs: ageStats(globalStats.closeAgesMs), + openAgeMs: ageStats(globalStats.openAgesMs), + }, undefined, 2)); } await main(); diff --git a/scanner/package.json b/scanner/package.json index cb949c8..dfe8bdd 100644 --- a/scanner/package.json +++ b/scanner/package.json @@ -16,7 +16,8 @@ "@octokit/plugin-throttling": "^8.0.0", "bottleneck": "^2.19.5", "browser-specs": "^3.62.0", - "node-fetch": "^3.3.2" + "node-fetch": "^3.3.2", + "simple-statistics": "^7.8.3" }, "devDependencies": { "@tsconfig/node20": "^20.1.2", diff --git a/scanner/pnpm-lock.yaml b/scanner/pnpm-lock.yaml index e7c0850..e5c3d13 100644 --- a/scanner/pnpm-lock.yaml +++ b/scanner/pnpm-lock.yaml @@ -20,6 +20,9 @@ dependencies: node-fetch: specifier: ^3.3.2 version: 3.3.2 + simple-statistics: + specifier: ^7.8.3 + version: 7.8.3 devDependencies: '@tsconfig/node20': @@ -182,6 +185,10 @@ packages: wrappy: 1.0.2 dev: false + /simple-statistics@7.8.3: + resolution: {integrity: sha512-JFvMY00t6SBGtwMuJ+nqgsx9ylkMiJ5JlK9bkj8AdvniIe5615wWQYkKHXe84XtSuc40G/tlrPu0A5/NlJvv8A==} + dev: false + /typescript@5.2.2: resolution: {integrity: sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==} engines: {node: '>=14.17'}