Skip to content

Commit

Permalink
Add types to graphql query results.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jyasskin
jyasskin committed Jan 17, 2024
1 parent e36d610 commit 4ed63db
Show file tree
Hide file tree
Showing 2 changed files with 180 additions and 68 deletions.
236 changes: 173 additions & 63 deletions scanner/github.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { Octokit as OctokitCore } from "@octokit/core";
import { paginateGraphql } from "@octokit/plugin-paginate-graphql";
import { throttling } from "@octokit/plugin-throttling";
import { RequestError } from "@octokit/request-error";
import { z } from "zod";
import config from "./third_party/config.cjs";

const Octokit = OctokitCore.plugin(throttling, paginateGraphql);
Expand Down Expand Up @@ -31,6 +32,10 @@ const octokit = new Octokit({
/** Enables syntax highlighting from the GraphQL editor extension. */
function gql(str: string) { return str; }

const instant = z.string().transform(val => Temporal.Instant.from(val));
const CreatedNode = z.object({
createdAt: instant,
});
const labeledFragment = gql(`fragment labeledFragment on LabeledEvent {
createdAt
label {
Expand All @@ -43,6 +48,13 @@ const unlabeledFragment = gql(`fragment unlabeledFragment on UnlabeledEvent {
name
}
}`);
const LabelingEvent = CreatedNode.extend({
label: z.object({
name: z.string(),
}),
});
type LabelingEvent = z.infer<typeof LabelingEvent>;

const issueFragment = gql(`fragment issueFragment on IssueConnection {
totalCount
pageInfo {
Expand Down Expand Up @@ -89,6 +101,69 @@ const issueFragment = gql(`fragment issueFragment on IssueConnection {
}
}
}`);
const PageInfo = z.object({
endCursor: z.string().nullable(),
hasNextPage: z.boolean(),
});
const ConnectionNode = z.object({
totalCount: z.number(),
pageInfo: PageInfo,
});
const Labels = z.object({
totalCount: z.number(),
nodes: z.array(z.object({
name: z.string(),
})),
});
const CommentFields = CreatedNode.extend({
author: z.object({
login: z.string(),
}).nullable(),
});
const TimelineItems = ConnectionNode.extend({
nodes: z.array(z.discriminatedUnion("__typename", [
z.object({ __typename: z.literal("ReadyForReviewEvent") }).merge(CreatedNode),
z.object({ __typename: z.literal("ConvertToDraftEvent") }).merge(CreatedNode),
z.object({ __typename: z.literal("ClosedEvent") }).merge(CreatedNode),
z.object({ __typename: z.literal("ReopenedEvent") }).merge(CreatedNode),
z.object({ __typename: z.literal("LabeledEvent") }).merge(LabelingEvent),
z.object({ __typename: z.literal("UnlabeledEvent") }).merge(LabelingEvent),
z.object({ __typename: z.literal("IssueComment") }).merge(CommentFields),
z.object({ __typename: z.literal("PullRequestReview") }).merge(CommentFields),
z.object({ __typename: z.literal("PullRequestReviewThread") }).extend({
comments: z.object({
nodes: z.array(CommentFields),
}),
}).merge(
// Added by fetchAllComments.
CommentFields.partial()),
])),
// Added by fetchAllComments.
totalComments: z.number().optional(),
commentPageInfo: PageInfo.optional(),
});
const IssueOrPr = z.object({
__typename: z.string(),
id: z.string(),
number: z.number(),
title: z.string(),
url: z.string(),
createdAt: instant,
isDraft: z.boolean().optional(),
author: z.object({
login: z.string(),
}).nullable(),
labels: Labels,
milestone: z.object({
url: z.string(),
title: z.string(),
}).nullable(),
timelineItems: TimelineItems,
});
export type IssueOrPr = z.infer<typeof IssueOrPr>;
const IssueConnection = ConnectionNode.extend({
nodes: z.array(IssueOrPr),
});
const prFragment = gql(`fragment prFragment on PullRequestConnection {
totalCount
pageInfo {
Expand Down Expand Up @@ -143,76 +218,88 @@ const prFragment = gql(`fragment prFragment on PullRequestConnection {
}
}`);

export async function fetchAllComments(needAllComments: any[], needEarlyComments: any[]) {
// First, fetch the early comments from every issue, and add them into the issue. Then we'll page
// through the comments on the issues that need a complete set.
needEarlyComments = needEarlyComments.concat(needAllComments);
const issueById = new Map<string, any>();
for (const issue of needEarlyComments) {
issueById.set(issue.id, issue);
const commentQuery = gql(`query ($ids: [ID!]!, $itemCount: Int!, $cursor: String) {
rateLimit {
cost
remaining
}
const query = gql(`query ($ids: [ID!]!, $itemCount: Int!, $cursor: String) {
rateLimit {
cost
remaining
}
nodes(ids: $ids) {
__typename
id
... on Issue {
timelineItems(first: $itemCount, after: $cursor, itemTypes: [ISSUE_COMMENT]) {
totalCount
pageInfo {
endCursor
hasNextPage
}
nodes {
__typename
...commentFields
}
nodes(ids: $ids) {
__typename
id
... on Issue {
timelineItems(first: $itemCount, after: $cursor, itemTypes: [ISSUE_COMMENT]) {
totalCount
pageInfo {
endCursor
hasNextPage
}
nodes {
__typename
...commentFields
}
}
... on PullRequest {
timelineItems(first: $itemCount, after: $cursor, itemTypes: [ISSUE_COMMENT, PULL_REQUEST_REVIEW, PULL_REQUEST_REVIEW_THREAD]) {
totalCount
pageInfo {
endCursor
hasNextPage
}
nodes {
__typename
...commentFields
... on PullRequestReviewThread {
comments(first: 1) {
nodes {
...commentFields
}
}
... on PullRequest {
timelineItems(first: $itemCount, after: $cursor, itemTypes: [ISSUE_COMMENT, PULL_REQUEST_REVIEW, PULL_REQUEST_REVIEW_THREAD]) {
totalCount
pageInfo {
endCursor
hasNextPage
}
nodes {
__typename
...commentFields
... on PullRequestReviewThread {
comments(first: 1) {
nodes {
...commentFields
}
}
}
}
}
}
}
fragment commentFields on Comment {
createdAt
author {
login
}
}`);
}
fragment commentFields on Comment {
createdAt
author {
login
}
}`);
const CommentQueryResult = z.object({
rateLimit: z.object({
cost: z.number(),
remaining: z.number(),
}),
nodes: z.array(z.object({
__typename: z.string(),
id: z.string(),
timelineItems: TimelineItems,
})),
});

export async function fetchAllComments(needAllComments: IssueOrPr[], needEarlyComments: IssueOrPr[]) {
// First, fetch the early comments from every issue, and add them into the issue. Then we'll page
// through the comments on the issues that need a complete set.
needEarlyComments = needEarlyComments.concat(needAllComments);
const issueById = new Map<string, IssueOrPr>();
for (const issue of needEarlyComments) {
issueById.set(issue.id, issue);
}
let rateLimit: { cost: number; remaining: number | null; } = { cost: 0, remaining: null };
let fetchAtOnce = 100;
const numEarlyComments = needEarlyComments.length;
while (needEarlyComments.length > 0) {
const initial = needEarlyComments.splice(0, fetchAtOnce);
const result: any = await octokit.graphql(query, {
const result = CommentQueryResult.parse(await octokit.graphql(commentQuery, {
ids: initial.map(issue => issue.id),
itemCount: 5,
});
}));
rateLimit.cost += result.rateLimit.cost;
rateLimit.remaining = result.rateLimit.remaining;
for (const issue of result.nodes) {
const fullIssue = issueById.get(issue.id);
const fullIssue = issueById.get(issue.id)!;
fullIssue.timelineItems.totalComments = issue.timelineItems.totalCount;
fullIssue.timelineItems.commentPageInfo = issue.timelineItems.pageInfo;
fullIssue.timelineItems.nodes.push(...issue.timelineItems.nodes);
Expand All @@ -221,29 +308,42 @@ export async function fetchAllComments(needAllComments: any[], needEarlyComments
console.log(`Fetched early comments for ${numEarlyComments} issues, with rate limit ${JSON.stringify(rateLimit)}.`);

for (const issue of needAllComments) {
if (!issue.timelineItems.commentPageInfo.hasNextPage) {
if (!issue.timelineItems.commentPageInfo?.hasNextPage) {
continue;
}
console.log(`Paging through comments on issue ${issue.number}; id ${issue.id}.`);
const result: any = await octokit.graphql.paginate(query, {
const result = CommentQueryResult.parse(await octokit.graphql.paginate(commentQuery, {
ids: [issue.id],
itemCount: 100,
cursor: issue.timelineItems.commentPageInfo.endCursor,
});
issue.timelineItems.nodes.push(...result.nodes.timelineItems.nodes);
}));
issue.timelineItems.nodes.push(...result.nodes[0].timelineItems.nodes);
for (const timelineItem of issue.timelineItems.nodes) {
if (timelineItem.__typename === 'PullRequestReviewThread') {
timelineItem.createdAt = timelineItem.comments.nodes[0]?.createdAt;
timelineItem.author = timelineItem.comments.nodes[0]?.author;
}
}
issue.timelineItems.nodes.sort((a, b) => Temporal.Instant.compare(a.createdAt, b.createdAt));
issue.timelineItems.nodes.sort((a, b) => Temporal.Instant.compare(a.createdAt!, b.createdAt!));
}
}

export async function getRepo(org, repo): Promise<any> {
const Repository = z.object({
labels: z.object({
totalCount: z.number(),
nodes: z.array(z.object({
name: z.string(),
})),
}),
issues: IssueConnection,
pullRequests: IssueConnection,
});
export type Repository = z.infer<typeof Repository>;
const RepositoryQueryResult = z.object({ repository: Repository });

export async function getRepo(org, repo): Promise<Repository> {
console.log(`Fetching ${org}/${repo}.`);
const result: any = await octokit.graphql(
const result = RepositoryQueryResult.parse(await octokit.graphql(
gql(`query ($owner: String!, $repoName: String!) {
repository(owner: $owner, name: $repoName) {
labels(first: 100, query: "Priority") {
Expand All @@ -267,7 +367,7 @@ export async function getRepo(org, repo): Promise<any> {
{
owner: org,
repoName: repo,
});
}));

if (result.repository.issues.pageInfo.hasNextPage) {
console.log(`Paging through issues on ${org}/${repo}.`);
Expand All @@ -287,9 +387,14 @@ export async function getRepo(org, repo): Promise<any> {
cursor: result.repository.issues.pageInfo.endCursor,
pageSize: 99, // 100 leads to a rate limit cost of 2.
};
let remainingIssues: any = null;
const IssueQueryResult = z.object({
repository: z.object({
issues: IssueConnection,
}),
});
let remainingIssues: null | z.infer<typeof IssueQueryResult> = null;
try {
remainingIssues = await octokit.graphql.paginate(query, vars);
remainingIssues = IssueQueryResult.parse(await octokit.graphql.paginate(query, vars));
} catch (e: unknown) {
if (e instanceof RequestError) {
let data: any = e.response?.data;
Expand All @@ -298,7 +403,7 @@ export async function getRepo(org, repo): Promise<any> {
"Something went wrong while executing your query. This may be the result of a timeout"))) {
vars.pageSize = Math.ceil(vars.pageSize / 4);
console.warn(`${JSON.stringify(data.errors)}\nScaling back to ${vars.pageSize} issues per page.`);
remainingIssues = await octokit.graphql.paginate(query, vars);
remainingIssues = IssueQueryResult.parse(await octokit.graphql.paginate(query, vars));
} else {
console.error(JSON.stringify(data));
throw e;
Expand All @@ -311,7 +416,12 @@ export async function getRepo(org, repo): Promise<any> {
}
if (result.repository.pullRequests.pageInfo.hasNextPage) {
console.log(`Paging through PRs on ${org}/${repo}.`);
const remainingPRs = await octokit.graphql.paginate(
const PrQueryResult = z.object({
repository: z.object({
pullRequests: IssueConnection,
}),
});
const remainingPRs = PrQueryResult.parse(await octokit.graphql.paginate(
gql(`query ($owner: String!, $repoName: String!, $cursor: String) {
repository(owner: $owner, name: $repoName) {
pullRequests(first: 50, states: [OPEN], after: $cursor) {
Expand All @@ -325,7 +435,7 @@ export async function getRepo(org, repo): Promise<any> {
owner: org,
repoName: repo,
cursor: result.repository.pullRequests.pageInfo.endCursor
});
}));
result.repository.pullRequests.nodes.push(...remainingPRs.repository.pullRequests.nodes);
}
return result.repository;
Expand Down
12 changes: 7 additions & 5 deletions scanner/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Temporal } from "@js-temporal/polyfill";
import { IssueSummary, RepoSummary } from '@lib/repo-summaries.js';
import fs from 'node:fs/promises';
import { browserSpecs } from "./browser-specs.js";
import { fetchAllComments, getRepo, logRateLimit } from "./github.js";
import { IssueOrPr, fetchAllComments, getRepo, logRateLimit } from "./github.js";
import { NEEDS_REPORTER_FEEDBACK, countSloTime, hasLabels, whichSlo } from "./slo.js";
import config from './third_party/config.cjs';

Expand Down Expand Up @@ -41,8 +41,8 @@ async function analyzeRepo(org: string, repoName: string, globalStats: GlobalSta
};

const allIssues = repo.issues.nodes.concat(repo.pullRequests.nodes);
const needEarlyComments: any[] = [];
const needAllComments: any[] = [];
const needEarlyComments: IssueOrPr[] = [];
const needAllComments: IssueOrPr[] = [];
for (const issue of allIssues) {
// Fetch comments for the issues whose SLO calculation needs their comments.
if (issue.timelineItems.nodes.some(item =>
Expand Down Expand Up @@ -74,7 +74,7 @@ async function analyzeRepo(org: string, repoName: string, globalStats: GlobalSta
}
};
if (issue.__typename === 'PullRequest') {
info.pull_request = { draft: issue.isDraft };
info.pull_request = { draft: issue.isDraft! };
}
if (issue.milestone) {
info.milestone = {
Expand All @@ -85,7 +85,9 @@ async function analyzeRepo(org: string, repoName: string, globalStats: GlobalSta
if (!result.labelsPresent && (
issue.milestone ||
issue.timelineItems.nodes.some(timelineItem =>
['IssueComment', 'PullRequestReview', 'PullRequestReviewThread'].includes(timelineItem.__typename)
(timelineItem.__typename === 'IssueComment' ||
timelineItem.__typename === 'PullRequestReview' ||
timelineItem.__typename === 'PullRequestReviewThread')
&& info.author !== timelineItem.author?.login
))) {
// If the repository doesn't have the triage labels, and an issue or PR has a comment from
Expand Down

0 comments on commit 4ed63db

Please sign in to comment.