diff --git a/backend/plugins/github/tasks/cicd_run_collector.go b/backend/plugins/github/tasks/cicd_run_collector.go index c94f194cac9..3cd1bd5d540 100644 --- a/backend/plugins/github/tasks/cicd_run_collector.go +++ b/backend/plugins/github/tasks/cicd_run_collector.go @@ -41,11 +41,6 @@ const RAW_RUN_TABLE = "github_api_runs" // https://github.com/apache/incubator-devlake/issues/3199 const PAGE_SIZE = 30 -type GithubRawRunsResult struct { - TotalCount int64 `json:"total_count"` - GithubWorkflowRuns []json.RawMessage `json:"workflow_runs"` -} - type SimpleGithubApiJob struct { ID int64 CreatedAt common.Iso8601Time `json:"created_at"` @@ -80,7 +75,9 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error { UrlTemplate: "repos/{{ .Params.Name }}/actions/runs", Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) { query := url.Values{} - query.Set("status", "completed") + // GitHub API returns only the first 34 pages (with a size of 30) when specifying status=compleleted, try the following API request to verify the problem. + // https://api.github.com/repos/apache/incubator-devlake/actions/runs?per_page=30&page=35&status=completed + // query.Set("status", "completed") query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) return query, nil @@ -91,10 +88,21 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return nil, err } - if len(body.GithubWorkflowRuns) == 0 { + if len(body.WorkflowRuns) == 0 { return nil, nil } - return body.GithubWorkflowRuns, nil + // filter out the runs that are not completed + filteredRuns := make([]json.RawMessage, 0) + for _, run := range body.WorkflowRuns { + if run.Status == "completed" { + runJSON, err := json.Marshal(run) + if err != nil { + return nil, errors.Convert(err) + } + filteredRuns = append(filteredRuns, json.RawMessage(runJSON)) + } + } + return filteredRuns, nil }, }, GetCreated: func(item json.RawMessage) (time.Time, errors.Error) { @@ -115,3 +123,249 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error { return collector.Execute() } + +type GithubRawRunsResult struct { + TotalCount int `json:"total_count"` + WorkflowRuns []struct { + ID int64 `json:"id"` + Name string `json:"name"` + NodeID string `json:"node_id"` + HeadBranch string `json:"head_branch"` + HeadSha string `json:"head_sha"` + Path string `json:"path"` + DisplayTitle string `json:"display_title"` + RunNumber int `json:"run_number"` + Event string `json:"event"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + WorkflowID int `json:"workflow_id"` + CheckSuiteID int64 `json:"check_suite_id"` + CheckSuiteNodeID string `json:"check_suite_node_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + PullRequests []struct { + URL string `json:"url"` + ID int `json:"id"` + Number int `json:"number"` + Head struct { + Ref string `json:"ref"` + Sha string `json:"sha"` + Repo struct { + ID int `json:"id"` + URL string `json:"url"` + Name string `json:"name"` + } `json:"repo"` + } `json:"head"` + Base struct { + Ref string `json:"ref"` + Sha string `json:"sha"` + Repo struct { + ID int `json:"id"` + URL string `json:"url"` + Name string `json:"name"` + } `json:"repo"` + } `json:"base"` + } `json:"pull_requests"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Actor struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"actor"` + RunAttempt int `json:"run_attempt"` + ReferencedWorkflows []any `json:"referenced_workflows"` + RunStartedAt time.Time `json:"run_started_at"` + TriggeringActor struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"triggering_actor"` + JobsURL string `json:"jobs_url"` + LogsURL string `json:"logs_url"` + CheckSuiteURL string `json:"check_suite_url"` + ArtifactsURL string `json:"artifacts_url"` + CancelURL string `json:"cancel_url"` + RerunURL string `json:"rerun_url"` + PreviousAttemptURL any `json:"previous_attempt_url"` + WorkflowURL string `json:"workflow_url"` + HeadCommit struct { + ID string `json:"id"` + TreeID string `json:"tree_id"` + Message string `json:"message"` + Timestamp time.Time `json:"timestamp"` + Author struct { + Name string `json:"name"` + Email string `json:"email"` + } `json:"author"` + Committer struct { + Name string `json:"name"` + Email string `json:"email"` + } `json:"committer"` + } `json:"head_commit"` + Repository struct { + ID int `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Private bool `json:"private"` + Owner struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"owner"` + HTMLURL string `json:"html_url"` + Description string `json:"description"` + Fork bool `json:"fork"` + URL string `json:"url"` + ForksURL string `json:"forks_url"` + KeysURL string `json:"keys_url"` + CollaboratorsURL string `json:"collaborators_url"` + TeamsURL string `json:"teams_url"` + HooksURL string `json:"hooks_url"` + IssueEventsURL string `json:"issue_events_url"` + EventsURL string `json:"events_url"` + AssigneesURL string `json:"assignees_url"` + BranchesURL string `json:"branches_url"` + TagsURL string `json:"tags_url"` + BlobsURL string `json:"blobs_url"` + GitTagsURL string `json:"git_tags_url"` + GitRefsURL string `json:"git_refs_url"` + TreesURL string `json:"trees_url"` + StatusesURL string `json:"statuses_url"` + LanguagesURL string `json:"languages_url"` + StargazersURL string `json:"stargazers_url"` + ContributorsURL string `json:"contributors_url"` + SubscribersURL string `json:"subscribers_url"` + SubscriptionURL string `json:"subscription_url"` + CommitsURL string `json:"commits_url"` + GitCommitsURL string `json:"git_commits_url"` + CommentsURL string `json:"comments_url"` + IssueCommentURL string `json:"issue_comment_url"` + ContentsURL string `json:"contents_url"` + CompareURL string `json:"compare_url"` + MergesURL string `json:"merges_url"` + ArchiveURL string `json:"archive_url"` + DownloadsURL string `json:"downloads_url"` + IssuesURL string `json:"issues_url"` + PullsURL string `json:"pulls_url"` + MilestonesURL string `json:"milestones_url"` + NotificationsURL string `json:"notifications_url"` + LabelsURL string `json:"labels_url"` + ReleasesURL string `json:"releases_url"` + DeploymentsURL string `json:"deployments_url"` + } `json:"repository"` + HeadRepository struct { + ID int `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + FullName string `json:"full_name"` + Private bool `json:"private"` + Owner struct { + Login string `json:"login"` + ID int `json:"id"` + NodeID string `json:"node_id"` + AvatarURL string `json:"avatar_url"` + GravatarID string `json:"gravatar_id"` + URL string `json:"url"` + HTMLURL string `json:"html_url"` + FollowersURL string `json:"followers_url"` + FollowingURL string `json:"following_url"` + GistsURL string `json:"gists_url"` + StarredURL string `json:"starred_url"` + SubscriptionsURL string `json:"subscriptions_url"` + OrganizationsURL string `json:"organizations_url"` + ReposURL string `json:"repos_url"` + EventsURL string `json:"events_url"` + ReceivedEventsURL string `json:"received_events_url"` + Type string `json:"type"` + SiteAdmin bool `json:"site_admin"` + } `json:"owner"` + HTMLURL string `json:"html_url"` + Description string `json:"description"` + Fork bool `json:"fork"` + URL string `json:"url"` + ForksURL string `json:"forks_url"` + KeysURL string `json:"keys_url"` + CollaboratorsURL string `json:"collaborators_url"` + TeamsURL string `json:"teams_url"` + HooksURL string `json:"hooks_url"` + IssueEventsURL string `json:"issue_events_url"` + EventsURL string `json:"events_url"` + AssigneesURL string `json:"assignees_url"` + BranchesURL string `json:"branches_url"` + TagsURL string `json:"tags_url"` + BlobsURL string `json:"blobs_url"` + GitTagsURL string `json:"git_tags_url"` + GitRefsURL string `json:"git_refs_url"` + TreesURL string `json:"trees_url"` + StatusesURL string `json:"statuses_url"` + LanguagesURL string `json:"languages_url"` + StargazersURL string `json:"stargazers_url"` + ContributorsURL string `json:"contributors_url"` + SubscribersURL string `json:"subscribers_url"` + SubscriptionURL string `json:"subscription_url"` + CommitsURL string `json:"commits_url"` + GitCommitsURL string `json:"git_commits_url"` + CommentsURL string `json:"comments_url"` + IssueCommentURL string `json:"issue_comment_url"` + ContentsURL string `json:"contents_url"` + CompareURL string `json:"compare_url"` + MergesURL string `json:"merges_url"` + ArchiveURL string `json:"archive_url"` + DownloadsURL string `json:"downloads_url"` + IssuesURL string `json:"issues_url"` + PullsURL string `json:"pulls_url"` + MilestonesURL string `json:"milestones_url"` + NotificationsURL string `json:"notifications_url"` + LabelsURL string `json:"labels_url"` + ReleasesURL string `json:"releases_url"` + DeploymentsURL string `json:"deployments_url"` + } `json:"head_repository"` + } `json:"workflow_runs"` +}