diff --git a/controllers/remote/cluster_cache_tracker.go b/controllers/remote/cluster_cache_tracker.go
index b0e85ebf9a53..f5ecb5ada259 100644
--- a/controllers/remote/cluster_cache_tracker.go
+++ b/controllers/remote/cluster_cache_tracker.go
@@ -69,6 +69,8 @@ var ErrClusterLocked = errors.New("cluster is locked already")
 type ClusterCacheTracker struct {
 	log logr.Logger
 
+	cacheByObject map[client.Object]cache.ByObject
+
 	clientUncachedObjects []client.Object
 	clientQPS             float32
 	clientBurst           int
@@ -115,6 +117,9 @@ type ClusterCacheTrackerOptions struct {
 	// Defaults to a no-op logger if it's not set.
 	Log *logr.Logger
 
+	// CacheByObject restricts the cache's ListWatch to the desired fields per GVK at the specified object.
+	CacheByObject map[client.Object]cache.ByObject
+
 	// ClientUncachedObjects instructs the Client to never cache the following objects,
 	// it'll instead query the API server directly.
 	// Defaults to never caching ConfigMap and Secret if not set.
@@ -191,6 +196,7 @@ func NewClusterCacheTracker(manager ctrl.Manager, options ClusterCacheTrackerOpt
 		controllerPodMetadata: controllerPodMetadata,
 		log:                   *options.Log,
 		clientUncachedObjects: options.ClientUncachedObjects,
+		cacheByObject:         options.CacheByObject,
 		clientQPS:             options.ClientQPS,
 		clientBurst:           options.ClientBurst,
 		client:                manager.GetClient(),
@@ -485,6 +491,7 @@ func (t *ClusterCacheTracker) createCachedClient(ctx context.Context, config *re
 		HTTPClient: httpClient,
 		Scheme:     t.scheme,
 		Mapper:     mapper,
+		ByObject:   t.cacheByObject,
 	}
 	remoteCache, err := cache.New(config, cacheOptions)
 	if err != nil {