Skip to content

Commit

Permalink
fix: add collect sonarqube mode by file (#6507) (#6531)
Browse files Browse the repository at this point in the history 
* fix: add collect sonarqube mode by file

* fix: update some collector logic

* fix: some params

* fix: update some params

* fix: still need to push issues when the count exceed MAXISSUECOUNT

* fix: update log format and variable name

---------

Co-authored-by: abeizn <zikuan.an@merico.dev>
Co-authored-by: Lynwee <linwei.hou@merico.dev>
  • Loading branch information
@github-actions @abeizn @d4x1
3 people authored Nov 29, 2023
1 parent 9f48f30 commit e61a8c8
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 18 deletions.
8 changes: 4 additions & 4 deletions backend/plugins/sonarqube/impl/impl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,14 +87,14 @@ func (p Sonarqube) GetTablesInfo() []dal.Tabler {

func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
tasks.CollectIssuesMeta,
tasks.ExtractIssuesMeta,
tasks.CollectHotspotsMeta,
tasks.ExtractHotspotsMeta,
tasks.CollectAdditionalFilemetricsMeta,
tasks.ExtractAdditionalFileMetricsMeta,
tasks.CollectFilemetricsMeta,
tasks.ExtractFilemetricsMeta,
tasks.CollectIssuesMeta,
tasks.ExtractIssuesMeta,
tasks.CollectHotspotsMeta,
tasks.ExtractHotspotsMeta,
tasks.CollectAccountsMeta,
tasks.ExtractAccountsMeta,
tasks.ConvertProjectsMeta,
Expand Down
109 changes: 95 additions & 14 deletions backend/plugins/sonarqube/tasks/issues_collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ import (

const RAW_ISSUES_TABLE = "sonarqube_api_issues"

const MAXPAGES = 100
const MAXISSUECOUNT = 10000
const MININTERVAL = 10

var _ plugin.SubTaskEntryPoint = CollectIssues

type SonarqubeIssueIteratorNode struct {
Expand All @@ -40,6 +44,7 @@ type SonarqubeIssueIteratorNode struct {
Type string
CreatedAfter *time.Time
CreatedBefore *time.Time
FilePath string
}

func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
Expand All @@ -60,6 +65,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
Type: typ,
CreatedAfter: nil,
CreatedBefore: nil,
FilePath: "",
},
)
}
Expand All @@ -74,14 +80,20 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
Input: iterator,
Query: func(reqData *helper.RequestData) (url.Values, errors.Error) {
query := url.Values{}
query.Set("componentKeys", fmt.Sprintf("%v", data.Options.ProjectKey))
input, ok := reqData.Input.(*SonarqubeIssueIteratorNode)
if !ok {
return nil, errors.Default.New(fmt.Sprintf("Input to SonarqubeIssueIteratorNode failed:%+v", reqData.Input))
}
query.Set("severities", reqData.Input.(*SonarqubeIssueIteratorNode).Severity)
query.Set("statuses", reqData.Input.(*SonarqubeIssueIteratorNode).Status)
query.Set("types", reqData.Input.(*SonarqubeIssueIteratorNode).Type)
if input.FilePath != "" {
query.Del("facets")
query.Set("componentKeys", fmt.Sprintf("%v:%v", data.Options.ProjectKey, input.FilePath))
} else {
query.Set("componentKeys", fmt.Sprintf("%v", data.Options.ProjectKey))
query.Set("facets", "directories")
}
query.Set("severities", input.Severity)
query.Set("statuses", input.Status)
query.Set("types", input.Type)
if input.CreatedAfter != nil {
query.Set("createdAfter", GetFormatTime(input.CreatedAfter))
}
Expand All @@ -90,11 +102,12 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
}
query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size))

query.Encode()
return query, nil
},
GetTotalPages: func(res *http.Response, args *helper.ApiCollectorArgs) (int, errors.Error) {
body := &SonarqubePagination{}
body := &SonarqubePageInfo{}
err := helper.UnmarshalResponse(res, body)
if err != nil {
return 0, err
Expand All @@ -106,9 +119,11 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
}

query := res.Request.URL.Query()

// if get more than 10000 data, that need split it
if pages > 100 {
if pages > MAXPAGES {
severity := query.Get("severities")
status := query.Get("statuses")
typ := query.Get("types")
var createdAfterUnix int64
var createdBeforeUnix int64

Expand All @@ -133,17 +148,62 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
createdBeforeUnix = createdBefore.Unix()
}

// can not split it any more
if createdBeforeUnix-createdAfterUnix <= 10 {
return 100, nil
// can not split it by time
if createdBeforeUnix-createdAfterUnix <= MININTERVAL {
// split it by dir/fil
for _, facet := range body.Facets {
for _, value := range facet.Values {
if value.Count <= MAXISSUECOUNT {
iterator.Push(&SonarqubeIssueIteratorNode{
Severity: severity,
Status: status,
Type: typ,
CreatedAfter: createdAfter,
CreatedBefore: createdBefore,
FilePath: value.Val,
})
logger.Info("split by dir, and it's issue count:[%d] and file path:[%s]", value.Count, value.Val)
} else {
// split it by dir when it's issue count > 10000
resWithPath, err := data.ApiClient.Get("issues/search", url.Values{
"componentKeys": {fmt.Sprintf("%v", data.Options.ProjectKey)},
"directories": {value.Val},
"facets": {"files"},
"ps": {"1"},
}, nil)
if err != nil {
return 0, err
}
bodyWithPath := &SonarqubePageInfo{}
err = helper.UnmarshalResponse(resWithPath, bodyWithPath)
if err != nil {
return 0, err
}
if len(bodyWithPath.Facets) != 1 {
return 0, errors.Default.New(fmt.Sprintf("the facets count [%d] is not 1", len(bodyWithPath.Facets)))
}
for _, value2 := range bodyWithPath.Facets[0].Values {
if value2.Count > MAXISSUECOUNT {
logger.Warn(fmt.Errorf("the issue count [%d] exceeds the maximum page size", value2.Count), "")
}
iterator.Push(&SonarqubeIssueIteratorNode{
Severity: severity,
Status: status,
Type: typ,
CreatedAfter: createdAfter,
CreatedBefore: createdBefore,
FilePath: value2.Val,
})
logger.Info("split by fil, and it's issue count:[%d] and file path:[%s]", value2.Count, value2.Val)
}
}
}
}
return 0, nil
}

// split it
MidTime := time.Unix((createdAfterUnix+createdBeforeUnix)/2+1, 0)

severity := query.Get("severities")
status := query.Get("statuses")
typ := query.Get("types")
// left part
iterator.Push(&SonarqubeIssueIteratorNode{
Severity: severity,
Expand Down Expand Up @@ -239,3 +299,24 @@ func getTimeFromFormatTime(formatTime string) (*time.Time, errors.Error) {

return &t, nil
}

type SonarqubePageInfo struct {
Total int `json:"total"`
P int `json:"p"`
Ps int `json:"ps"`
Paging struct {
PageIndex int `json:"pageIndex"`
PageSize int `json:"pageSize"`
Total int `json:"total"`
} `json:"paging"`
EffortTotal int `json:"effortTotal"`
Issues []any `json:"issues"`
Components []any `json:"components"`
Facets []struct {
Property string `json:"property"`
Values []struct {
Val string `json:"val"`
Count int `json:"count"`
} `json:"values"`
} `json:"facets"`
}

0 comments on commit e61a8c8

Please sign in to comment.