Skip to content

Commit

Permalink
fix: github run api page<35 limit (#6876) (#6877)
Browse files Browse the repository at this point in the history 
* fix: github run api page<35 limit

* fix: some tips

* fix: update note

Co-authored-by: abeizn <zikuan.an@merico.dev>
  • Loading branch information
@github-actions @abeizn
github-actions[bot] and abeizn authored Jan 26, 2024
1 parent 19c6ec0 commit c645236
Showing 1 changed file with 262 additions and 8 deletions.
270 changes: 262 additions & 8 deletions backend/plugins/github/tasks/cicd_run_collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,6 @@ const RAW_RUN_TABLE = "github_api_runs"
// https://github.com/apache/incubator-devlake/issues/3199
const PAGE_SIZE = 30

type GithubRawRunsResult struct {
TotalCount int64 `json:"total_count"`
GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"`
}

type SimpleGithubApiJob struct {
ID int64
CreatedAt common.Iso8601Time `json:"created_at"`
Expand Down Expand Up @@ -80,7 +75,9 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
UrlTemplate: "repos/{{ .Params.Name }}/actions/runs",
Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
query := url.Values{}
query.Set("status", "completed")
// GitHub API returns only the first 34 pages (with a size of 30) when specifying status=compleleted, try the following API request to verify the problem.
// https://api.github.com/repos/apache/incubator-devlake/actions/runs?per_page=30&page=35&status=completed
// query.Set("status", "completed")
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
return query, nil
Expand All @@ -91,10 +88,21 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
if err != nil {
return nil, err
}
if len(body.GithubWorkflowRuns) == 0 {
if len(body.WorkflowRuns) == 0 {
return nil, nil
}
return body.GithubWorkflowRuns, nil
// filter out the runs that are not completed
filteredRuns := make([]json.RawMessage, 0)
for _, run := range body.WorkflowRuns {
if run.Status == "completed" {
runJSON, err := json.Marshal(run)
if err != nil {
return nil, errors.Convert(err)
}
filteredRuns = append(filteredRuns, json.RawMessage(runJSON))
}
}
return filteredRuns, nil
},
},
GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
Expand All @@ -115,3 +123,249 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
return collector.Execute()

}

type GithubRawRunsResult struct {
TotalCount int `json:"total_count"`
WorkflowRuns []struct {
ID int64 `json:"id"`
Name string `json:"name"`
NodeID string `json:"node_id"`
HeadBranch string `json:"head_branch"`
HeadSha string `json:"head_sha"`
Path string `json:"path"`
DisplayTitle string `json:"display_title"`
RunNumber int `json:"run_number"`
Event string `json:"event"`
Status string `json:"status"`
Conclusion string `json:"conclusion"`
WorkflowID int `json:"workflow_id"`
CheckSuiteID int64 `json:"check_suite_id"`
CheckSuiteNodeID string `json:"check_suite_node_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
PullRequests []struct {
URL string `json:"url"`
ID int `json:"id"`
Number int `json:"number"`
Head struct {
Ref string `json:"ref"`
Sha string `json:"sha"`
Repo struct {
ID int `json:"id"`
URL string `json:"url"`
Name string `json:"name"`
} `json:"repo"`
} `json:"head"`
Base struct {
Ref string `json:"ref"`
Sha string `json:"sha"`
Repo struct {
ID int `json:"id"`
URL string `json:"url"`
Name string `json:"name"`
} `json:"repo"`
} `json:"base"`
} `json:"pull_requests"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
Actor struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"actor"`
RunAttempt int `json:"run_attempt"`
ReferencedWorkflows []any `json:"referenced_workflows"`
RunStartedAt time.Time `json:"run_started_at"`
TriggeringActor struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"triggering_actor"`
JobsURL string `json:"jobs_url"`
LogsURL string `json:"logs_url"`
CheckSuiteURL string `json:"check_suite_url"`
ArtifactsURL string `json:"artifacts_url"`
CancelURL string `json:"cancel_url"`
RerunURL string `json:"rerun_url"`
PreviousAttemptURL any `json:"previous_attempt_url"`
WorkflowURL string `json:"workflow_url"`
HeadCommit struct {
ID string `json:"id"`
TreeID string `json:"tree_id"`
Message string `json:"message"`
Timestamp time.Time `json:"timestamp"`
Author struct {
Name string `json:"name"`
Email string `json:"email"`
} `json:"author"`
Committer struct {
Name string `json:"name"`
Email string `json:"email"`
} `json:"committer"`
} `json:"head_commit"`
Repository struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
URL string `json:"url"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
} `json:"repository"`
HeadRepository struct {
ID int `json:"id"`
NodeID string `json:"node_id"`
Name string `json:"name"`
FullName string `json:"full_name"`
Private bool `json:"private"`
Owner struct {
Login string `json:"login"`
ID int `json:"id"`
NodeID string `json:"node_id"`
AvatarURL string `json:"avatar_url"`
GravatarID string `json:"gravatar_id"`
URL string `json:"url"`
HTMLURL string `json:"html_url"`
FollowersURL string `json:"followers_url"`
FollowingURL string `json:"following_url"`
GistsURL string `json:"gists_url"`
StarredURL string `json:"starred_url"`
SubscriptionsURL string `json:"subscriptions_url"`
OrganizationsURL string `json:"organizations_url"`
ReposURL string `json:"repos_url"`
EventsURL string `json:"events_url"`
ReceivedEventsURL string `json:"received_events_url"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
} `json:"owner"`
HTMLURL string `json:"html_url"`
Description string `json:"description"`
Fork bool `json:"fork"`
URL string `json:"url"`
ForksURL string `json:"forks_url"`
KeysURL string `json:"keys_url"`
CollaboratorsURL string `json:"collaborators_url"`
TeamsURL string `json:"teams_url"`
HooksURL string `json:"hooks_url"`
IssueEventsURL string `json:"issue_events_url"`
EventsURL string `json:"events_url"`
AssigneesURL string `json:"assignees_url"`
BranchesURL string `json:"branches_url"`
TagsURL string `json:"tags_url"`
BlobsURL string `json:"blobs_url"`
GitTagsURL string `json:"git_tags_url"`
GitRefsURL string `json:"git_refs_url"`
TreesURL string `json:"trees_url"`
StatusesURL string `json:"statuses_url"`
LanguagesURL string `json:"languages_url"`
StargazersURL string `json:"stargazers_url"`
ContributorsURL string `json:"contributors_url"`
SubscribersURL string `json:"subscribers_url"`
SubscriptionURL string `json:"subscription_url"`
CommitsURL string `json:"commits_url"`
GitCommitsURL string `json:"git_commits_url"`
CommentsURL string `json:"comments_url"`
IssueCommentURL string `json:"issue_comment_url"`
ContentsURL string `json:"contents_url"`
CompareURL string `json:"compare_url"`
MergesURL string `json:"merges_url"`
ArchiveURL string `json:"archive_url"`
DownloadsURL string `json:"downloads_url"`
IssuesURL string `json:"issues_url"`
PullsURL string `json:"pulls_url"`
MilestonesURL string `json:"milestones_url"`
NotificationsURL string `json:"notifications_url"`
LabelsURL string `json:"labels_url"`
ReleasesURL string `json:"releases_url"`
DeploymentsURL string `json:"deployments_url"`
} `json:"head_repository"`
} `json:"workflow_runs"`
}

0 comments on commit c645236

Please sign in to comment.