feat: append prometheus sources to promi_src label on deduplication

internal/prometheus/alert.go

@@ -37,7 +37,7 @@ func (a Alert) Row() []string {
 		col = color.New(color.FgYellow).SprintFunc()
 	}
 
-	return []string{string(a.Labels["scraper"]), a.Job(), a.Name(), time.Since(a.ActiveAt).String(), col(string(a.State))}
+	return []string{string(a.Labels[sourceLabelName]), a.Job(), a.Name(), time.Since(a.ActiveAt).String(), col(string(a.State))}
 }
 
 // Job returns the job label value.
@@ -85,7 +85,7 @@ func (a Alerts) Filter(filters ...AlertFilterFunc) Alerts {
 // AlertByServer filters Alerts by prometheus server.
 func AlertByServer(r *regexp.Regexp) AlertFilterFunc {
 	return func(a Alert) bool {
-		return r.MatchString(string(a.Labels["scraper"]))
+		return r.MatchString(string(a.Labels[sourceLabelName]))
 	}
 }
 
@@ -144,7 +144,7 @@ func (c Client) Alerts(ctx context.Context) (Alerts, error) {
 
 	for r := range results {
 		for _, a := range r.alert.Alerts {
-			a.Labels["scraper"] = model.LabelValue(r.server)
+			a.Labels[sourceLabelName] = model.LabelValue(r.server)
 
 			if err := a.Labels.Validate(); err != nil {
 				return nil, err

internal/prometheus/client_test.go

@@ -34,7 +34,7 @@ func TestTarget(t *testing.T) {
 		m := map[string]bool{}
 
 		for _, target := range targets {
-			m[string(target.Labels["scraper"])] = true
+			m[string(target.Labels[sourceLabelName])] = true
 		}
 
 		assert.Len(t, m, 2)
@@ -73,7 +73,7 @@ func TestAlert(t *testing.T) {
 		m := map[string]bool{}
 
 		for _, target := range alerts {
-			m[string(target.Labels["scraper"])] = true
+			m[string(target.Labels[sourceLabelName])] = true
 		}
 
 		assert.Len(t, m, 2)

internal/prometheus/target.go

@@ -15,7 +15,9 @@ import (
 )
 
 const (
-	jobLabelName = "job"
+	jobLabelName    = "job"
+	sourcesJobName  = "promi_scrape_sources"
+	sourceLabelName = "promi_scrape_src"
 )
 
 // Targets is a slice of prometheus targets.
@@ -54,7 +56,7 @@ func (t Target) Row() []string {
 		col = color.New(color.FgYellow).SprintFunc()
 	}
 
-	server := string(t.Labels["scraper"])
+	server := string(t.Labels[sourceLabelName])
 
 	return []string{server, t.Job(), t.ScrapeURL, time.Since(t.LastScrape).String(), t.Labels.String(), t.ActiveTarget.LastError, col(string(t.Health))}
 }
@@ -75,7 +77,7 @@ func (c Client) Targets(ctx context.Context, appendScraperAsTarget bool) (Target
 			if appendScraperAsTarget {
 				a := v1.ActiveTarget{
 					ScrapeURL:  server,
-					ScrapePool: "aa_scraper",
+					ScrapePool: sourcesJobName,
 					GlobalURL:  server,
 					Labels: model.LabelSet{
 						"instance": model.LabelValue(server),
@@ -120,8 +122,8 @@ func (c Client) Targets(ctx context.Context, appendScraperAsTarget bool) (Target
 	for r := range results {
 		for i := range r.target.Active {
 			activeTarget := r.target.Active[i]
-			if activeTarget.Labels["job"] != "scrapers" {
-				activeTarget.Labels["scraper"] = model.LabelValue(r.server)
+			if activeTarget.ScrapePool != sourcesJobName {
+				activeTarget.Labels[sourceLabelName] = model.LabelValue(r.server)
 			}
 
 			if err := activeTarget.Labels.Validate(); err != nil {
@@ -164,7 +166,7 @@ func (t Targets) Filter(filters ...TargetFilterFunc) Targets {
 // TargetByServer filters Targets by prometheus server.
 func TargetByServer(r *regexp.Regexp) TargetFilterFunc {
 	return func(t Target) bool {
-		return r.MatchString(string(t.Labels["scraper"]))
+		return r.MatchString(string(t.Labels[sourceLabelName]))
 	}
 }
 
@@ -205,7 +207,7 @@ func (t Targets) Sort() {
 		case 1:
 			return false
 		}
-		switch strings.Compare(string(t[i].Labels["scraper"]), string(t[j].Labels["scraper"])) {
+		switch strings.Compare(string(t[i].Labels[sourceLabelName]), string(t[j].Labels[sourceLabelName])) {
 		case -1:
 			return true
 		case 1:
@@ -236,13 +238,17 @@ func (t Targets) Active() []*v1.ActiveTarget {
 
 // Deduplicate deduplicates targets by scrape url. If identical targets
 // have different health status, then HealthBad status has highest priority.
+// Multiple sources are appended in the sourceLabelName label.
 func (t Targets) Deduplicate() Targets {
 	m := map[string]Target{}
 
 	for i := range t {
 		target := t[i]
 		if tg, ok := m[target.ScrapeURL]; ok {
+			tg.addSource(target.getSource())
+
 			if target.Health == tg.Health || tg.Health == v1.HealthBad {
+				target.addSource(tg.getSource())
 				continue
 			}
 		}
@@ -284,3 +290,26 @@ func (l k8sLabels) Has(key string) bool {
 func (l k8sLabels) Get(key string) string {
 	return string(l.LabelSet[model.LabelName(key)])
 }
+
+func (t Target) addSource(src string) {
+	s := string(t.Labels[sourceLabelName])
+
+	if s == "" || src == "" || strings.Contains(s, src) {
+		return
+	}
+
+	l := []string{s}
+	if strings.Contains(s, ",") {
+		l = strings.Split(s, ",")
+	}
+
+	l = append(l, src)
+
+	sort.Strings(l)
+
+	t.Labels[sourceLabelName] = model.LabelValue(strings.Join(l, ","))
+}
+
+func (t Target) getSource() string {
+	return string(t.Labels[sourceLabelName])
+}

internal/prometheus/target_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -13,43 +14,62 @@ func TestDeduplicate(t *testing.T) {
 			ActiveTarget: v1.ActiveTarget{
 				ScrapeURL: "http://url1.example.com",
 				Health:    v1.HealthGood,
+				Labels: model.LabelSet{
+					sourceLabelName: "src1",
+				},
 			},
 		},
 		Target{
 			ActiveTarget: v1.ActiveTarget{
 				ScrapeURL: "http://url1.example.com",
 				Health:    v1.HealthGood,
+				Labels: model.LabelSet{
+					sourceLabelName: "src2",
+				},
 			},
 		},
 		Target{
 			ActiveTarget: v1.ActiveTarget{
 				ScrapeURL: "http://url2.example.com",
 				Health:    v1.HealthBad,
 				LastError: "scrapeErr",
+				Labels: model.LabelSet{
+					sourceLabelName: "src1",
+				},
 			},
 		},
 		Target{
 			ActiveTarget: v1.ActiveTarget{
 				ScrapeURL: "http://url2.example.com",
 				Health:    v1.HealthGood,
+				Labels: model.LabelSet{
+					sourceLabelName: "src2",
+				},
 			},
 		},
 		Target{
 			ActiveTarget: v1.ActiveTarget{
 				ScrapeURL: "http://url3.example.com",
 				Health:    v1.HealthBad,
 				LastError: "scrapeErr",
+				Labels: model.LabelSet{
+					sourceLabelName: "src1",
+				},
 			},
 		},
 		Target{
 			ActiveTarget: v1.ActiveTarget{
 				ScrapeURL: "http://url3.example.com",
 				Health:    v1.HealthGood,
+				Labels: model.LabelSet{
+					sourceLabelName: "src2",
+				},
 			},
 		},
 	}
 
 	dedup := targets.Deduplicate()
+	assert.NoError(t, dedup[0].Labels.Validate())
 
 	dedup.Sort()
 
@@ -58,20 +78,29 @@ func TestDeduplicate(t *testing.T) {
 			ActiveTarget: v1.ActiveTarget{
 				ScrapeURL: "http://url1.example.com",
 				Health:    v1.HealthGood,
+				Labels: model.LabelSet{
+					sourceLabelName: "src1,src2",
+				},
 			},
 		},
 		Target{
 			ActiveTarget: v1.ActiveTarget{
 				ScrapeURL: "http://url2.example.com",
 				Health:    v1.HealthBad,
 				LastError: "scrapeErr",
+				Labels: model.LabelSet{
+					sourceLabelName: "src1,src2",
+				},
 			},
 		},
 		Target{
 			ActiveTarget: v1.ActiveTarget{
 				ScrapeURL: "http://url3.example.com",
 				Health:    v1.HealthBad,
 				LastError: "scrapeErr",
+				Labels: model.LabelSet{
+					sourceLabelName: "src1,src2",
+				},
 			},
 		},
 	}