From 7c57d3957529aec3a439739325f4f0771d195f4e Mon Sep 17 00:00:00 2001
From: Piotr <17101802+thampiotr@users.noreply.github.com>
Date: Thu, 26 Sep 2024 18:17:37 +0100
Subject: [PATCH 1/5] support extra labels on mimir rules queries

---
 .../mimir/rules/kubernetes/events.go          | 83 +++++++++++++++++++
 .../mimir/rules/kubernetes/events_test.go     | 16 +++-
 2 files changed, 97 insertions(+), 2 deletions(-)

diff --git a/internal/component/mimir/rules/kubernetes/events.go b/internal/component/mimir/rules/kubernetes/events.go
index 9d16d77618..fc8c24de51 100644
--- a/internal/component/mimir/rules/kubernetes/events.go
+++ b/internal/component/mimir/rules/kubernetes/events.go
@@ -3,6 +3,7 @@ package rules
 import (
 	"context"
 	"fmt"
+	"github.com/prometheus/prometheus/promql/parser"
 	"maps"
 	"regexp"
 	"sync"
@@ -15,7 +16,9 @@ import (
 	"github.com/hashicorp/go-multierror"
 	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	promListers "github.com/prometheus-operator/prometheus-operator/pkg/client/listers/monitoring/v1"
+	promlabels "github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/rulefmt"
+	yamlv3 "gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/labels"
 	coreListers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/util/workqueue"
@@ -38,6 +41,8 @@ type eventProcessor struct {
 	ruleSelector      labels.Selector
 	namespacePrefix   string
 	externalLabels    map[string]string
+	//TODO: this needs to still be exposed properly to users
+	extraQueryLabels []queryLabel
 
 	metrics *metrics
 	logger  log.Logger
@@ -195,6 +200,22 @@ func (e *eventProcessor) desiredStateFromKubernetes() (kubernetes.RuleGroupsByNa
 				}
 			}
 
+			if len(e.extraQueryLabels) > 0 {
+				for _, rg := range groups {
+					for i, r := range rg.Rules {
+						query := r.Expr.Value
+						newQuery, err := addLabelsToQuery(query, e.extraQueryLabels)
+						if err != nil {
+							level.Error(e.logger).Log("msg", "failed to add labels to query", "query", query, "err", err)
+						}
+						rg.Rules[i].Expr = yamlv3.Node{
+							Kind:  yamlv3.ScalarNode,
+							Value: newQuery,
+						}
+					}
+				}
+			}
+
 			desiredState[mimirNs] = groups
 		}
 	}
@@ -202,6 +223,68 @@ func (e *eventProcessor) desiredStateFromKubernetes() (kubernetes.RuleGroupsByNa
 	return desiredState, nil
 }
 
+type queryLabel struct {
+	name      string
+	value     string
+	matchType string
+}
+
+func addLabelsToQuery(query string, labelsToAdd []queryLabel) (string, error) {
+	var err error
+	for _, label := range labelsToAdd {
+		query, err = labelsSetPromQL(query, label.matchType, label.name, label.value)
+		if err != nil {
+			return "", err
+		}
+	}
+	return query, nil
+}
+
+// Lifted from: https://github.com/prometheus/prometheus/blob/79a6238e195ecc1c20937036c1e3b4e3bdaddc49/cmd/promtool/main.go#L1242
+func labelsSetPromQL(query, labelMatchType, name, value string) (string, error) {
+	expr, err := parser.ParseExpr(query)
+	if err != nil {
+		return query, err
+	}
+
+	var matchType promlabels.MatchType
+	switch labelMatchType {
+	case parser.ItemType(parser.EQL).String():
+		matchType = promlabels.MatchEqual
+	case parser.ItemType(parser.NEQ).String():
+		matchType = promlabels.MatchNotEqual
+	case parser.ItemType(parser.EQL_REGEX).String():
+		matchType = promlabels.MatchRegexp
+	case parser.ItemType(parser.NEQ_REGEX).String():
+		matchType = promlabels.MatchNotRegexp
+	default:
+		return query, fmt.Errorf("invalid label match type: %s", labelMatchType)
+	}
+
+	parser.Inspect(expr, func(node parser.Node, path []parser.Node) error {
+		if n, ok := node.(*parser.VectorSelector); ok {
+			var found bool
+			for i, l := range n.LabelMatchers {
+				if l.Name == name {
+					n.LabelMatchers[i].Type = matchType
+					n.LabelMatchers[i].Value = value
+					found = true
+				}
+			}
+			if !found {
+				n.LabelMatchers = append(n.LabelMatchers, &promlabels.Matcher{
+					Type:  matchType,
+					Name:  name,
+					Value: value,
+				})
+			}
+		}
+		return nil
+	})
+
+	return expr.Pretty(0), nil
+}
+
 func convertCRDRuleGroupToRuleGroup(crd promv1.PrometheusRuleSpec) ([]rulefmt.RuleGroup, error) {
 	buf, err := yaml.Marshal(crd)
 	if err != nil {
diff --git a/internal/component/mimir/rules/kubernetes/events_test.go b/internal/component/mimir/rules/kubernetes/events_test.go
index 8bd09bd6b8..a211083375 100644
--- a/internal/component/mimir/rules/kubernetes/events_test.go
+++ b/internal/component/mimir/rules/kubernetes/events_test.go
@@ -251,6 +251,18 @@ func TestAdditionalLabels(t *testing.T) {
 		metrics:           newMetrics(),
 		logger:            log.With(log.NewLogfmtLogger(os.Stdout), "ts", log.DefaultTimestampUTC),
 		externalLabels:    map[string]string{"foo": "bar"},
+		extraQueryLabels: []queryLabel{
+			{
+				name:      "cluster",
+				matchType: "=",
+				value:     "big",
+			},
+			{
+				name:      "job",
+				matchType: "=",
+				value:     "good",
+			},
+		},
 	}
 
 	ctx := context.Background()
@@ -287,11 +299,11 @@ func TestAdditionalLabels(t *testing.T) {
 		expectedRule := `- name: group1
   rules:
   - alert: alert1
-    expr: expr1
+    expr: expr1{cluster="big",job="good"}
     labels:
       foo: bar
   - alert: alert2
-    expr: expr2
+    expr: expr2{cluster="big",job="good"}
     labels:
       foo: bar
 `

From c3b0d96ec28fce413449b0c7e9ae22e492249c76 Mon Sep 17 00:00:00 2001
From: Piotr <17101802+thampiotr@users.noreply.github.com>
Date: Fri, 27 Sep 2024 16:43:37 +0100
Subject: [PATCH 2/5] Add tests and expose query matchers to users.

---
 .../mimir/rules/kubernetes/events.go          |  68 +++----
 .../mimir/rules/kubernetes/events_test.go     | 177 +++++++++++++-----
 .../component/mimir/rules/kubernetes/rules.go |  25 +--
 .../mimir/rules/kubernetes/rules_test.go      |  94 ++++++++--
 .../component/mimir/rules/kubernetes/types.go |  46 +++++
 5 files changed, 298 insertions(+), 112 deletions(-)

diff --git a/internal/component/mimir/rules/kubernetes/events.go b/internal/component/mimir/rules/kubernetes/events.go
index fc8c24de51..800d86bdd4 100644
--- a/internal/component/mimir/rules/kubernetes/events.go
+++ b/internal/component/mimir/rules/kubernetes/events.go
@@ -3,26 +3,26 @@ package rules
 import (
 	"context"
 	"fmt"
-	"github.com/prometheus/prometheus/promql/parser"
 	"maps"
 	"regexp"
 	"sync"
 	"time"
 
 	"github.com/go-kit/log"
-	"github.com/grafana/alloy/internal/component/common/kubernetes"
-	"github.com/grafana/alloy/internal/mimir/client"
-	"github.com/grafana/alloy/internal/runtime/logging/level"
 	"github.com/hashicorp/go-multierror"
 	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	promListers "github.com/prometheus-operator/prometheus-operator/pkg/client/listers/monitoring/v1"
 	promlabels "github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/rulefmt"
-	yamlv3 "gopkg.in/yaml.v3"
+	"github.com/prometheus/prometheus/promql/parser"
 	"k8s.io/apimachinery/pkg/labels"
 	coreListers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/util/workqueue"
 	"sigs.k8s.io/yaml" // Used for CRD compatibility instead of gopkg.in/yaml.v2
+
+	"github.com/grafana/alloy/internal/component/common/kubernetes"
+	"github.com/grafana/alloy/internal/mimir/client"
+	"github.com/grafana/alloy/internal/runtime/logging/level"
 )
 
 const (
@@ -34,15 +34,14 @@ type eventProcessor struct {
 	stopChan chan struct{}
 	health   healthReporter
 
-	mimirClient       client.Interface
-	namespaceLister   coreListers.NamespaceLister
-	ruleLister        promListers.PrometheusRuleLister
-	namespaceSelector labels.Selector
-	ruleSelector      labels.Selector
-	namespacePrefix   string
-	externalLabels    map[string]string
-	//TODO: this needs to still be exposed properly to users
-	extraQueryLabels []queryLabel
+	mimirClient        client.Interface
+	namespaceLister    coreListers.NamespaceLister
+	ruleLister         promListers.PrometheusRuleLister
+	namespaceSelector  labels.Selector
+	ruleSelector       labels.Selector
+	namespacePrefix    string
+	externalLabels     map[string]string
+	extraQueryMatchers *ExtraQueryMatchers
 
 	metrics *metrics
 	logger  log.Logger
@@ -188,30 +187,27 @@ func (e *eventProcessor) desiredStateFromKubernetes() (kubernetes.RuleGroupsByNa
 			}
 
 			if len(e.externalLabels) > 0 {
-				for _, rule_group := range groups {
+				for _, ruleGroup := range groups {
 					// Refer to the slice element via its index,
 					// to make sure we mutate on the original and not a copy.
-					for i := range rule_group.Rules {
-						if rule_group.Rules[i].Labels == nil {
-							rule_group.Rules[i].Labels = make(map[string]string, len(e.externalLabels))
+					for i := range ruleGroup.Rules {
+						if ruleGroup.Rules[i].Labels == nil {
+							ruleGroup.Rules[i].Labels = make(map[string]string, len(e.externalLabels))
 						}
-						maps.Copy(rule_group.Rules[i].Labels, e.externalLabels)
+						maps.Copy(ruleGroup.Rules[i].Labels, e.externalLabels)
 					}
 				}
 			}
 
-			if len(e.extraQueryLabels) > 0 {
-				for _, rg := range groups {
-					for i, r := range rg.Rules {
-						query := r.Expr.Value
-						newQuery, err := addLabelsToQuery(query, e.extraQueryLabels)
+			if e.extraQueryMatchers != nil {
+				for _, ruleGroup := range groups {
+					for i := range ruleGroup.Rules {
+						query := ruleGroup.Rules[i].Expr.Value
+						newQuery, err := addMatchersToQuery(query, e.extraQueryMatchers.Matchers)
 						if err != nil {
-							level.Error(e.logger).Log("msg", "failed to add labels to query", "query", query, "err", err)
-						}
-						rg.Rules[i].Expr = yamlv3.Node{
-							Kind:  yamlv3.ScalarNode,
-							Value: newQuery,
+							level.Error(e.logger).Log("msg", "failed to add labels to PrometheusRule query", "query", query, "err", err)
 						}
+						ruleGroup.Rules[i].Expr.Value = newQuery
 					}
 				}
 			}
@@ -223,16 +219,10 @@ func (e *eventProcessor) desiredStateFromKubernetes() (kubernetes.RuleGroupsByNa
 	return desiredState, nil
 }
 
-type queryLabel struct {
-	name      string
-	value     string
-	matchType string
-}
-
-func addLabelsToQuery(query string, labelsToAdd []queryLabel) (string, error) {
+func addMatchersToQuery(query string, matchers []Matcher) (string, error) {
 	var err error
-	for _, label := range labelsToAdd {
-		query, err = labelsSetPromQL(query, label.matchType, label.name, label.value)
+	for _, s := range matchers {
+		query, err = labelsSetPromQL(query, s.MatchType, s.Name, s.Value)
 		if err != nil {
 			return "", err
 		}
@@ -282,7 +272,7 @@ func labelsSetPromQL(query, labelMatchType, name, value string) (string, error)
 		return nil
 	})
 
-	return expr.Pretty(0), nil
+	return expr.String(), nil
 }
 
 func convertCRDRuleGroupToRuleGroup(crd promv1.PrometheusRuleSpec) ([]rulefmt.RuleGroup, error) {
diff --git a/internal/component/mimir/rules/kubernetes/events_test.go b/internal/component/mimir/rules/kubernetes/events_test.go
index a211083375..1b5cf82c03 100644
--- a/internal/component/mimir/rules/kubernetes/events_test.go
+++ b/internal/component/mimir/rules/kubernetes/events_test.go
@@ -8,8 +8,6 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
-	"github.com/grafana/alloy/internal/component/common/kubernetes"
-	mimirClient "github.com/grafana/alloy/internal/mimir/client"
 	v1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	promListers "github.com/prometheus-operator/prometheus-operator/pkg/client/listers/monitoring/v1"
 	"github.com/prometheus/prometheus/model/rulefmt"
@@ -23,6 +21,9 @@ import (
 	coreListers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
+
+	"github.com/grafana/alloy/internal/component/common/kubernetes"
+	mimirClient "github.com/grafana/alloy/internal/mimir/client"
 )
 
 type fakeMimirClient struct {
@@ -86,17 +87,8 @@ func (m *fakeMimirClient) ListRules(_ context.Context, namespace string) (map[st
 }
 
 func TestEventLoop(t *testing.T) {
-	nsIndexer := cache.NewIndexer(
-		cache.DeletionHandlingMetaNamespaceKeyFunc,
-		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
-	)
-	nsLister := coreListers.NewNamespaceLister(nsIndexer)
-
-	ruleIndexer := cache.NewIndexer(
-		cache.DeletionHandlingMetaNamespaceKeyFunc,
-		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
-	)
-	ruleLister := promListers.NewPrometheusRuleLister(ruleIndexer)
+	nsIndexer := testNamespaceIndexer()
+	ruleIndexer := testRuleIndexer()
 
 	ns := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
@@ -131,8 +123,8 @@ func TestEventLoop(t *testing.T) {
 		stopChan:          make(chan struct{}),
 		health:            &fakeHealthReporter{},
 		mimirClient:       newFakeMimirClient(),
-		namespaceLister:   nsLister,
-		ruleLister:        ruleLister,
+		namespaceLister:   coreListers.NewNamespaceLister(nsIndexer),
+		ruleLister:        promListers.NewPrometheusRuleLister(ruleIndexer),
 		namespaceSelector: labels.Everything(),
 		ruleSelector:      labels.Everything(),
 		namespacePrefix:   "alloy",
@@ -190,17 +182,8 @@ func TestEventLoop(t *testing.T) {
 }
 
 func TestAdditionalLabels(t *testing.T) {
-	nsIndexer := cache.NewIndexer(
-		cache.DeletionHandlingMetaNamespaceKeyFunc,
-		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
-	)
-	nsLister := coreListers.NewNamespaceLister(nsIndexer)
-
-	ruleIndexer := cache.NewIndexer(
-		cache.DeletionHandlingMetaNamespaceKeyFunc,
-		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
-	)
-	ruleLister := promListers.NewPrometheusRuleLister(ruleIndexer)
+	nsIndexer := testNamespaceIndexer()
+	ruleIndexer := testRuleIndexer()
 
 	ns := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
@@ -228,7 +211,7 @@ func TestAdditionalLabels(t *testing.T) {
 							Alert: "alert2",
 							Expr:  intstr.FromString("expr2"),
 							Labels: map[string]string{
-								//This label should get overridden.
+								// This label should get overridden.
 								"foo": "lalalala",
 							},
 						},
@@ -243,26 +226,14 @@ func TestAdditionalLabels(t *testing.T) {
 		stopChan:          make(chan struct{}),
 		health:            &fakeHealthReporter{},
 		mimirClient:       newFakeMimirClient(),
-		namespaceLister:   nsLister,
-		ruleLister:        ruleLister,
+		namespaceLister:   coreListers.NewNamespaceLister(nsIndexer),
+		ruleLister:        promListers.NewPrometheusRuleLister(ruleIndexer),
 		namespaceSelector: labels.Everything(),
 		ruleSelector:      labels.Everything(),
 		namespacePrefix:   "alloy",
 		metrics:           newMetrics(),
 		logger:            log.With(log.NewLogfmtLogger(os.Stdout), "ts", log.DefaultTimestampUTC),
 		externalLabels:    map[string]string{"foo": "bar"},
-		extraQueryLabels: []queryLabel{
-			{
-				name:      "cluster",
-				matchType: "=",
-				value:     "big",
-			},
-			{
-				name:      "job",
-				matchType: "=",
-				value:     "good",
-			},
-		},
 	}
 
 	ctx := context.Background()
@@ -299,14 +270,134 @@ func TestAdditionalLabels(t *testing.T) {
 		expectedRule := `- name: group1
   rules:
   - alert: alert1
-    expr: expr1{cluster="big",job="good"}
+    expr: expr1
     labels:
       foo: bar
   - alert: alert2
-    expr: expr2{cluster="big",job="good"}
+    expr: expr2
     labels:
       foo: bar
 `
 		require.YAMLEq(t, expectedRule, string(ruleBuf))
 	}
 }
+
+func TestExtraQueryMatchers(t *testing.T) {
+	nsIndexer := testNamespaceIndexer()
+	ruleIndexer := testRuleIndexer()
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "namespace",
+			UID:  types.UID("33f8860c-bd06-4c0d-a0b1-a114d6b9937b"),
+		},
+	}
+
+	rule := &v1.PrometheusRule{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "name",
+			Namespace: "namespace",
+			UID:       types.UID("64aab764-c95e-4ee9-a932-cd63ba57e6cf"),
+		},
+		Spec: v1.PrometheusRuleSpec{
+			Groups: []v1.RuleGroup{
+				{
+					Name: "group1",
+					Rules: []v1.Rule{
+						{
+							Record: "record_rule_1",
+							Expr:   intstr.FromString("sum by (namespace) (rate(success{\"job\"=\"bad\"}[10m]) / rate(total{}[10m]))"),
+						},
+						{
+							Alert: "alert_1",
+							Expr:  intstr.FromString("sum by (namespace) (rate(success{\"foo\"=\"bar\"}[10m]) / (rate(success{\"job\"!~\"bad\"}[10m]) + rate(failure[10m]))) < 0.995"),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	processor := &eventProcessor{
+		queue:             workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
+		stopChan:          make(chan struct{}),
+		health:            &fakeHealthReporter{},
+		mimirClient:       newFakeMimirClient(),
+		namespaceLister:   coreListers.NewNamespaceLister(nsIndexer),
+		ruleLister:        promListers.NewPrometheusRuleLister(ruleIndexer),
+		namespaceSelector: labels.Everything(),
+		ruleSelector:      labels.Everything(),
+		namespacePrefix:   "alloy",
+		metrics:           newMetrics(),
+		logger:            log.With(log.NewLogfmtLogger(os.Stdout), "ts", log.DefaultTimestampUTC),
+		extraQueryMatchers: &ExtraQueryMatchers{Matchers: []Matcher{
+			{
+				Name:      "cluster",
+				MatchType: "=~",
+				Value:     "prod-.*",
+			},
+			{
+				Name:      "job",
+				MatchType: "=",
+				Value:     "good",
+			},
+		}},
+	}
+
+	ctx := context.Background()
+
+	// Do an initial sync of the Mimir ruler state before starting the event processing loop.
+	require.NoError(t, processor.syncMimir(ctx))
+	go processor.run(ctx)
+	defer processor.stop()
+
+	eventHandler := kubernetes.NewQueuedEventHandler(processor.logger, processor.queue)
+
+	// Add a namespace and rule to kubernetes
+	require.NoError(t, nsIndexer.Add(ns))
+	require.NoError(t, ruleIndexer.Add(rule))
+	eventHandler.OnAdd(rule, false)
+
+	// Wait for the rule to be added to mimir
+	rules := map[string][]rulefmt.RuleGroup{}
+	require.Eventually(t, func() bool {
+		var err error
+		rules, err = processor.mimirClient.ListRules(ctx, "")
+		require.NoError(t, err)
+		require.Equal(t, 1, len(rules))
+		return len(rules) == 1
+	}, 3*time.Second, 10*time.Millisecond)
+
+	// The map of rules has only one element.
+	for ruleName, rule := range rules {
+		require.Equal(t, "alloy/namespace/name/64aab764-c95e-4ee9-a932-cd63ba57e6cf", ruleName)
+
+		ruleBuf, err := yaml.Marshal(rule)
+		require.NoError(t, err)
+
+		expectedRule := `- name: group1
+  rules:
+    - expr: "sum by (namespace) (rate(success{cluster=~\"prod-.*\",job=\"good\"}[10m]) / rate(total{cluster=~\"prod-.*\",job=\"good\"}[10m]))"
+      record: record_rule_1
+    - alert: alert_1
+      expr: "sum by (namespace) (rate(success{cluster=~\"prod-.*\",foo=\"bar\",job=\"good\"}[10m]) / (rate(success{cluster=~\"prod-.*\",job=\"good\"}[10m]) + rate(failure{cluster=~\"prod-.*\",job=\"good\"}[10m]))) < 0.995"
+`
+		require.YAMLEq(t, expectedRule, string(ruleBuf))
+	}
+}
+
+func testRuleIndexer() cache.Indexer {
+	ruleIndexer := cache.NewIndexer(
+		cache.DeletionHandlingMetaNamespaceKeyFunc,
+		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+	)
+	return ruleIndexer
+}
+
+func testNamespaceIndexer() cache.Indexer {
+	nsIndexer := cache.NewIndexer(
+		cache.DeletionHandlingMetaNamespaceKeyFunc,
+		cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+	)
+	return nsIndexer
+}
diff --git a/internal/component/mimir/rules/kubernetes/rules.go b/internal/component/mimir/rules/kubernetes/rules.go
index bade96a8bc..d90263746b 100644
--- a/internal/component/mimir/rules/kubernetes/rules.go
+++ b/internal/component/mimir/rules/kubernetes/rules.go
@@ -445,18 +445,19 @@ func (c *Component) newEventProcessor(queue workqueue.RateLimitingInterface, sto
 	maps.Copy(externalLabels, c.args.ExternalLabels)
 
 	return &eventProcessor{
-		queue:             queue,
-		stopChan:          stopChan,
-		health:            c,
-		mimirClient:       c.mimirClient,
-		namespaceLister:   namespaceLister,
-		ruleLister:        ruleLister,
-		namespaceSelector: c.namespaceSelector,
-		ruleSelector:      c.ruleSelector,
-		namespacePrefix:   c.args.MimirNameSpacePrefix,
-		metrics:           c.metrics,
-		logger:            c.log,
-		externalLabels:    externalLabels,
+		queue:              queue,
+		stopChan:           stopChan,
+		health:             c,
+		mimirClient:        c.mimirClient,
+		namespaceLister:    namespaceLister,
+		ruleLister:         ruleLister,
+		namespaceSelector:  c.namespaceSelector,
+		ruleSelector:       c.ruleSelector,
+		namespacePrefix:    c.args.MimirNameSpacePrefix,
+		metrics:            c.metrics,
+		logger:             c.log,
+		externalLabels:     externalLabels,
+		extraQueryMatchers: c.args.ExtraQueryMatchers,
 	}
 }
 
diff --git a/internal/component/mimir/rules/kubernetes/rules_test.go b/internal/component/mimir/rules/kubernetes/rules_test.go
index 132497ae08..602ce025ff 100644
--- a/internal/component/mimir/rules/kubernetes/rules_test.go
+++ b/internal/component/mimir/rules/kubernetes/rules_test.go
@@ -20,32 +20,90 @@ import (
 	"github.com/grafana/alloy/syntax"
 )
 
-func TestAlloyConfig(t *testing.T) {
-	var exampleAlloyConfig = `
+func TestAlloyConfigs(t *testing.T) {
+	var testCases = []struct {
+		name                  string
+		config                string
+		expectedErrorContains string
+	}{
+		{
+			name: "basic working config",
+			config: `
 	address = "GRAFANA_CLOUD_METRICS_URL"
 	basic_auth {
 		username = "GRAFANA_CLOUD_USER"
 		password = "GRAFANA_CLOUD_API_KEY"
 	}
-	external_labels = {"label1" = "value1"}
-`
-
-	var args Arguments
-	err := syntax.Unmarshal([]byte(exampleAlloyConfig), &args)
-	require.NoError(t, err)
-}
-
-func TestBadAlloyConfig(t *testing.T) {
-	var exampleAlloyConfig = `
+	external_labels = {"label1" = "value1"}`,
+		},
+		{
+			name: "invalid http config",
+			config: `
 	address = "GRAFANA_CLOUD_METRICS_URL"
 	bearer_token = "token"
-	bearer_token_file = "/path/to/file.token"
-`
+	bearer_token_file = "/path/to/file.token"`,
+			expectedErrorContains: `at most one of basic_auth, authorization, oauth2, bearer_token & bearer_token_file must be configured`,
+		},
+		{
+			name: "query matchers valid",
+			config: `
+	address = "GRAFANA_CLOUD_METRICS_URL"
+	extra_query_matchers {
+		matcher {
+			name = "job"
+			match_type = "!="
+			value = "bar"
+		}
+		matcher {
+			name = "namespace"
+			match_type = "="
+			value = "all"
+		}
+		matcher {
+			name = "namespace"
+			match_type = "!~"
+			value = ".+"
+		}
+		matcher {
+			name = "cluster"
+			match_type = "=~"
+			value = "prod-.*"
+		}
+	}
+`,
+		},
+		{
+			name: "query matchers empty",
+			config: `
+	address = "GRAFANA_CLOUD_METRICS_URL"
+	extra_query_matchers {}`,
+		},
+		{
+			name: "query matchers invalid",
+			config: `
+	address = "GRAFANA_CLOUD_METRICS_URL"
+	extra_query_matchers {
+		matcher {
+			name = "job"
+			match_type = "!!"
+			value = "bar"
+		}
+	}`,
+			expectedErrorContains: `invalid match type`,
+		},
+	}
 
-	// Make sure the squashed HTTPClientConfig Validate function is being utilized correctly
-	var args Arguments
-	err := syntax.Unmarshal([]byte(exampleAlloyConfig), &args)
-	require.ErrorContains(t, err, "at most one of basic_auth, authorization, oauth2, bearer_token & bearer_token_file must be configured")
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			var args Arguments
+			err := syntax.Unmarshal([]byte(tc.config), &args)
+			if tc.expectedErrorContains == "" {
+				require.NoError(t, err)
+			} else {
+				require.ErrorContains(t, err, tc.expectedErrorContains)
+			}
+		})
+	}
 }
 
 type fakeCluster struct{}
diff --git a/internal/component/mimir/rules/kubernetes/types.go b/internal/component/mimir/rules/kubernetes/types.go
index ecbc8fbb15..c872a0d8f1 100644
--- a/internal/component/mimir/rules/kubernetes/types.go
+++ b/internal/component/mimir/rules/kubernetes/types.go
@@ -1,13 +1,27 @@
 package rules
 
 import (
+	"errors"
 	"fmt"
+	"slices"
 	"time"
 
+	promlabels "github.com/prometheus/prometheus/model/labels"
+
 	"github.com/grafana/alloy/internal/component/common/config"
 	"github.com/grafana/alloy/internal/component/common/kubernetes"
 )
 
+var (
+	// This should contain all valid match types for extra query matchers.
+	validMatchTypes = []string{
+		promlabels.MatchEqual.String(),
+		promlabels.MatchNotEqual.String(),
+		promlabels.MatchRegexp.String(),
+		promlabels.MatchNotRegexp.String(),
+	}
+)
+
 type Arguments struct {
 	Address              string                  `alloy:"address,attr"`
 	TenantID             string                  `alloy:"tenant_id,attr,optional"`
@@ -17,6 +31,7 @@ type Arguments struct {
 	SyncInterval         time.Duration           `alloy:"sync_interval,attr,optional"`
 	MimirNameSpacePrefix string                  `alloy:"mimir_namespace_prefix,attr,optional"`
 	ExternalLabels       map[string]string       `alloy:"external_labels,attr,optional"`
+	ExtraQueryMatchers   *ExtraQueryMatchers     `alloy:"extra_query_matchers,block,optional"`
 
 	RuleSelector          kubernetes.LabelSelector `alloy:"rule_selector,block,optional"`
 	RuleNamespaceSelector kubernetes.LabelSelector `alloy:"rule_namespace_selector,block,optional"`
@@ -42,7 +57,38 @@ func (args *Arguments) Validate() error {
 	if args.MimirNameSpacePrefix == "" {
 		return fmt.Errorf("mimir_namespace_prefix must not be empty")
 	}
+	if err := args.ExtraQueryMatchers.Validate(); err != nil {
+		return err
+	}
 
 	// We must explicitly Validate because HTTPClientConfig is squashed and it won't run otherwise
 	return args.HTTPClientConfig.Validate()
 }
+
+type ExtraQueryMatchers struct {
+	Matchers []Matcher `alloy:"matcher,block,optional"`
+}
+
+func (e *ExtraQueryMatchers) Validate() error {
+	if e == nil {
+		return nil
+	}
+	var errs error
+	for _, matcher := range e.Matchers {
+		errs = errors.Join(errs, matcher.Validate())
+	}
+	return errs
+}
+
+type Matcher struct {
+	Name      string `alloy:"name,attr"`
+	Value     string `alloy:"value,attr"`
+	MatchType string `alloy:"match_type,attr"`
+}
+
+func (m Matcher) Validate() error {
+	if !slices.Contains(validMatchTypes, m.MatchType) {
+		return fmt.Errorf("invalid match type: %q", m.MatchType)
+	}
+	return nil
+}

From 04d63034d50d554559209c0e209bc2063e044906 Mon Sep 17 00:00:00 2001
From: Piotr <17101802+thampiotr@users.noreply.github.com>
Date: Fri, 27 Sep 2024 17:13:13 +0100
Subject: [PATCH 3/5] Docs

---
 .../mimir/mimir.rules.kubernetes.md           | 132 ++++++++++++------
 1 file changed, 87 insertions(+), 45 deletions(-)

diff --git a/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md b/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md
index 07e87a2fca..f8af042bf3 100644
--- a/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md
+++ b/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md
@@ -50,25 +50,25 @@ mimir.rules.kubernetes "LABEL" {
 
 `mimir.rules.kubernetes` supports the following arguments:
 
-Name                     | Type                | Description                                                                                      | Default       | Required
--------------------------|---------------------|--------------------------------------------------------------------------------------------------|---------------|---------
-`address`                | `string`            | URL of the Mimir ruler.                                                                          |               | yes
-`tenant_id`              | `string`            | Mimir tenant ID.                                                                                 |               | no
-`use_legacy_routes`      | `bool`              | Whether to use [deprecated][gem-2_2] ruler API endpoints.                                        | false         | no
-`prometheus_http_prefix` | `string`            | Path prefix for [Mimir's Prometheus endpoint][gem-path-prefix].                                  | `/prometheus` | no
-`sync_interval`          | `duration`          | Amount of time between reconciliations with Mimir.                                               | "5m"          | no
-`mimir_namespace_prefix` | `string`            | Prefix used to differentiate multiple {{< param "PRODUCT_NAME" >}} deployments.                  | "alloy"       | no
-`bearer_token_file`      | `string`            | File containing a bearer token to authenticate with.                                             |               | no
-`bearer_token`           | `secret`            | Bearer token to authenticate with.                                                               |               | no
-`enable_http2`           | `bool`              | Whether HTTP2 is supported for requests.                                                         | `true`        | no
-`follow_redirects`       | `bool`              | Whether redirects returned by the server should be followed.                                     | `true`        | no
-`proxy_url`              | `string`            | HTTP proxy to send requests through.                                                             |               | no
-`no_proxy`               | `string`            | Comma-separated list of IP addresses, CIDR notations, and domain names to exclude from proxying. |               | no
-`proxy_from_environment` | `bool`              | Use the proxy URL indicated by environment variables.                                            | `false`       | no
-`proxy_connect_header`   | `map(list(secret))` | Specifies headers to send to proxies during CONNECT requests.                                    |               | no
-`external_labels`        | `map(string)`       | Labels to add to each rule.                                                                      | `{}` | no
-
- At most, one of the following can be provided:
+| Name                     | Type                | Description                                                                                      | Default       | Required |
+|--------------------------|---------------------|--------------------------------------------------------------------------------------------------|---------------|----------|
+| `address`                | `string`            | URL of the Mimir ruler.                                                                          |               | yes      |
+| `tenant_id`              | `string`            | Mimir tenant ID.                                                                                 |               | no       |
+| `use_legacy_routes`      | `bool`              | Whether to use [deprecated][gem-2_2] ruler API endpoints.                                        | false         | no       |
+| `prometheus_http_prefix` | `string`            | Path prefix for [Mimir's Prometheus endpoint][gem-path-prefix].                                  | `/prometheus` | no       |
+| `sync_interval`          | `duration`          | Amount of time between reconciliations with Mimir.                                               | "5m"          | no       |
+| `mimir_namespace_prefix` | `string`            | Prefix used to differentiate multiple {{< param "PRODUCT_NAME" >}} deployments.                  | "alloy"       | no       |
+| `bearer_token_file`      | `string`            | File containing a bearer token to authenticate with.                                             |               | no       |
+| `bearer_token`           | `secret`            | Bearer token to authenticate with.                                                               |               | no       |
+| `enable_http2`           | `bool`              | Whether HTTP2 is supported for requests.                                                         | `true`        | no       |
+| `follow_redirects`       | `bool`              | Whether redirects returned by the server should be followed.                                     | `true`        | no       |
+| `proxy_url`              | `string`            | HTTP proxy to send requests through.                                                             |               | no       |
+| `no_proxy`               | `string`            | Comma-separated list of IP addresses, CIDR notations, and domain names to exclude from proxying. |               | no       |
+| `proxy_from_environment` | `bool`              | Use the proxy URL indicated by environment variables.                                            | `false`       | no       |
+| `proxy_connect_header`   | `map(list(secret))` | Specifies headers to send to proxies during CONNECT requests.                                    |               | no       |
+| `external_labels`        | `map(string)`       | Labels to add to each rule.                                                                      | `{}`          | no       |
+
+At most, one of the following can be provided:
  - [`bearer_token` argument](#arguments).
  - [`bearer_token_file` argument](#arguments).
  - [`basic_auth` block][basic_auth].
@@ -105,17 +105,19 @@ This is useful if you configure Mimir to use a different [prefix][gem-path-prefi
 The following blocks are supported inside the definition of
 `mimir.rules.kubernetes`:
 
-Hierarchy                                  | Block                  | Description                                              | Required
--------------------------------------------|------------------------|----------------------------------------------------------|---------
-rule_namespace_selector                    | [label_selector][]     | Label selector for `Namespace` resources.                | no
-rule_namespace_selector > match_expression | [match_expression][]   | Label match expression for `Namespace` resources.        | no
-rule_selector                              | [label_selector][]     | Label selector for `PrometheusRule` resources.           | no
-rule_selector > match_expression           | [match_expression][]   | Label match expression for `PrometheusRule` resources.   | no
-basic_auth                                 | [basic_auth][]         | Configure basic_auth for authenticating to the endpoint. | no
-authorization                              | [authorization][]      | Configure generic authorization to the endpoint.         | no
-oauth2                                     | [oauth2][]             | Configure OAuth2 for authenticating to the endpoint.     | no
-oauth2 > tls_config                        | [tls_config][]         | Configure TLS settings for connecting to the endpoint.   | no
-tls_config                                 | [tls_config][]         | Configure TLS settings for connecting to the endpoint.   | no
+| Hierarchy                                  | Block                    | Description                                              | Required |
+|--------------------------------------------|--------------------------|----------------------------------------------------------|----------|
+| extra_query_matchers                       | [extra_query_matchers][] | Additional label matchers to add to each query.          | no       |
+| extra_query_matchers > matcher             | [matcher][]              | A label matcher to add to query.                         | no       |
+| rule_namespace_selector                    | [label_selector][]       | Label selector for `Namespace` resources.                | no       |
+| rule_namespace_selector > match_expression | [match_expression][]     | Label match expression for `Namespace` resources.        | no       |
+| rule_selector                              | [label_selector][]       | Label selector for `PrometheusRule` resources.           | no       |
+| rule_selector > match_expression           | [match_expression][]     | Label match expression for `PrometheusRule` resources.   | no       |
+| basic_auth                                 | [basic_auth][]           | Configure basic_auth for authenticating to the endpoint. | no       |
+| authorization                              | [authorization][]        | Configure generic authorization to the endpoint.         | no       |
+| oauth2                                     | [oauth2][]               | Configure OAuth2 for authenticating to the endpoint.     | no       |
+| oauth2 > tls_config                        | [tls_config][]           | Configure TLS settings for connecting to the endpoint.   | no       |
+| tls_config                                 | [tls_config][]           | Configure TLS settings for connecting to the endpoint.   | no       |
 
 The `>` symbol indicates deeper levels of nesting. For example,
 `oauth2 > tls_config` refers to a `tls_config` block defined inside
@@ -127,6 +129,28 @@ an `oauth2` block.
 [tls_config]: #tls_config-block
 [label_selector]: #label_selector-block
 [match_expression]: #match_expression-block
+[extra_query_matchers]: #extra_query_matchers-block
+[matcher]: #matcher-block
+
+### extra_query_matchers block
+
+The `extra_query_matchers` block has no attributes. It contains zero or more [matcher][] blocks.
+These blocks allow to add extra label matchers to all queries that are discovered by `mimir.rules.kubernetes`
+component. The algorithm of adding the label matchers to queries is the same as the one provided by 
+`promtool promql label-matchers set` command in [promtool](https://github.com/prometheus/prometheus/tree/main/cmd/promtool).
+
+### matcher block
+
+The `matcher` block describes a label matcher that will be added to each query found in `PrometheusRule` CRDs.
+
+The following arguments are supported:
+
+| Name         | Type     | Description                                        | Default | Required |
+|--------------|----------|----------------------------------------------------|---------|----------|
+| `name`       | `string` | Name of the label to match.                        |         | yes      |
+| `match_type` | `string` | The type of match. One of `=`, `!=`, `=~` and `!~` |         | yes      |
+| `value`      | `string` | Value of the label to match.                       |         | yes      |
+
 
 ### label_selector block
 
@@ -134,9 +158,9 @@ The `label_selector` block describes a Kubernetes label selector for rule or nam
 
 The following arguments are supported:
 
-Name           | Type          | Description                                       | Default                     | Required
----------------|---------------|---------------------------------------------------|-----------------------------|---------
-`match_labels` | `map(string)` | Label keys and values used to discover resources. | `{}` | yes
+| Name           | Type          | Description                                       | Default | Required |
+|----------------|---------------|---------------------------------------------------|---------|----------|
+| `match_labels` | `map(string)` | Label keys and values used to discover resources. | `{}`    | yes      |
 
 When the `match_labels` argument is empty, all resources will be matched.
 
@@ -146,11 +170,11 @@ The `match_expression` block describes a Kubernetes label match expression for r
 
 The following arguments are supported:
 
-Name       | Type           | Description                                        | Default | Required
------------|----------------|----------------------------------------------------|---------|---------
-`key`      | `string`       | The label name to match against.                   |         | yes
-`operator` | `string`       | The operator to use when matching.                 |         | yes
-`values`   | `list(string)` | The values used when matching.                     |         | no
+| Name       | Type           | Description                        | Default | Required |
+|------------|----------------|------------------------------------|---------|----------|
+| `key`      | `string`       | The label name to match against.   |         | yes      |
+| `operator` | `string`       | The operator to use when matching. |         | yes      |
+| `values`   | `list(string)` | The values used when matching.     |         | no       |
 
 The `operator` argument should be one of the following strings:
 
@@ -204,13 +228,13 @@ actually exist.
 
 ## Debug metrics
 
-Metric Name                                   | Type        | Description
-----------------------------------------------|-------------|-------------------------------------------------------------------------
-`mimir_rules_config_updates_total`            | `counter`   | Number of times the configuration has been updated.
-`mimir_rules_events_total`                    | `counter`   | Number of events processed, partitioned by event type.
-`mimir_rules_events_failed_total`             | `counter`   | Number of events that failed to be processed, partitioned by event type.
-`mimir_rules_events_retried_total`            | `counter`   | Number of events that were retried, partitioned by event type.
-`mimir_rules_client_request_duration_seconds` | `histogram` | Duration of requests to the Mimir API.
+| Metric Name                                   | Type        | Description                                                              |
+|-----------------------------------------------|-------------|--------------------------------------------------------------------------|
+| `mimir_rules_config_updates_total`            | `counter`   | Number of times the configuration has been updated.                      |
+| `mimir_rules_events_total`                    | `counter`   | Number of events processed, partitioned by event type.                   |
+| `mimir_rules_events_failed_total`             | `counter`   | Number of events that failed to be processed, partitioned by event type. |
+| `mimir_rules_events_retried_total`            | `counter`   | Number of events that were retried, partitioned by event type.           |
+| `mimir_rules_client_request_duration_seconds` | `histogram` | Duration of requests to the Mimir API.                                   |
 
 ## Example
 
@@ -253,6 +277,24 @@ mimir.rules.kubernetes "default" {
 }
 ```
 
+This example adds label matcher `{cluster=~"prod-.*"}` to all the queries discovered by `mimir.rules.kubernetes`.
+
+```alloy
+mimir.rules.kubernetes "default" {
+    address = "GRAFANA_CLOUD_METRICS_URL"
+    extra_query_matchers {
+        matcher {
+            name = "cluster"
+            match_type = "=~"
+            value = "prod-.*"
+        }
+    }
+}
+```
+
+If a query in the form of `up != 1` is found in `PrometheusRule` CRDs, 
+it will be modified to `up{cluster=~"prod-.*"} != 1` before sending it to Mimir.
+
 The following example is an RBAC configuration for Kubernetes. It authorizes {{< param "PRODUCT_NAME" >}} to query the Kubernetes REST API:
 
 ```yaml

From 453e253075f1a450a26240b8e6a94f4590e49c60 Mon Sep 17 00:00:00 2001
From: Piotr <17101802+thampiotr@users.noreply.github.com>
Date: Fri, 27 Sep 2024 17:16:17 +0100
Subject: [PATCH 4/5] Changelog

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 061ec26b47..bfa302c3e6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,11 @@ Main (unreleased)
 
 - Add the function `path_join` to the stdlib. (@wildum)
 
+### Enhancements
+
+- The `mimir.rules.kubernetes` component now supports adding extra label matchers
+  to all queries discovered via `PrometheusRule` CRDs. (@thampiotr)
+
 ### Bugfixes
 
 - Windows installer: Don't quote alloy binary path in registy (@jkroepke)

From a13c540ef31b90b4a0031e858d9062ca4099ce81 Mon Sep 17 00:00:00 2001
From: Piotr <17101802+thampiotr@users.noreply.github.com>
Date: Mon, 30 Sep 2024 11:02:42 +0100
Subject: [PATCH 5/5] Apply suggestions from code review

Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>
---
 .../reference/components/mimir/mimir.rules.kubernetes.md      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md b/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md
index f8af042bf3..8652a3b35c 100644
--- a/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md
+++ b/docs/sources/reference/components/mimir/mimir.rules.kubernetes.md
@@ -135,7 +135,7 @@ an `oauth2` block.
 ### extra_query_matchers block
 
 The `extra_query_matchers` block has no attributes. It contains zero or more [matcher][] blocks.
-These blocks allow to add extra label matchers to all queries that are discovered by `mimir.rules.kubernetes`
+These blocks allow you to add extra label matchers to all queries that are discovered by `mimir.rules.kubernetes`
 component. The algorithm of adding the label matchers to queries is the same as the one provided by 
 `promtool promql label-matchers set` command in [promtool](https://github.com/prometheus/prometheus/tree/main/cmd/promtool).
 
@@ -148,7 +148,7 @@ The following arguments are supported:
 | Name         | Type     | Description                                        | Default | Required |
 |--------------|----------|----------------------------------------------------|---------|----------|
 | `name`       | `string` | Name of the label to match.                        |         | yes      |
-| `match_type` | `string` | The type of match. One of `=`, `!=`, `=~` and `!~` |         | yes      |
+| `match_type` | `string` | The type of match. One of `=`, `!=`, `=~` and `!~`. |         | yes      |
 | `value`      | `string` | Value of the label to match.                       |         | yes      |