From a17e12d2caabb919d2e82650352c761b9be07ebe Mon Sep 17 00:00:00 2001
From: Niall D <4562759+driev@users.noreply.github.com>
Date: Mon, 4 Nov 2024 09:22:17 +0000
Subject: [PATCH 1/4] feat(scheduler): exposing the deleted resource ttl as a
 CLI param (#5994)

* exposing the deleted resource ttl as a CLI param

* switching to seconds and adding a little test for the cleanup functions

* merge v2

* Revert "merge v2"

This reverts commit 4cf864426a0b2d58bf3f67062f11600ffbd1904c.
---
 scheduler/cmd/scheduler/main.go              | 10 +++--
 scheduler/pkg/server/server_test.go          | 44 +++++++++++++-------
 scheduler/pkg/store/experiment/db.go         | 19 +++++----
 scheduler/pkg/store/experiment/db_test.go    | 18 ++++----
 scheduler/pkg/store/experiment/store.go      |  6 +--
 scheduler/pkg/store/experiment/store_test.go | 12 ++++--
 scheduler/pkg/store/pipeline/db.go           | 30 +++++++------
 scheduler/pkg/store/pipeline/db_test.go      | 30 ++++++-------
 scheduler/pkg/store/pipeline/store.go        |  6 +--
 scheduler/pkg/store/pipeline/store_test.go   |  8 +++-
 scheduler/pkg/store/utils/utils.go           |  5 +--
 11 files changed, 107 insertions(+), 81 deletions(-)

diff --git a/scheduler/cmd/scheduler/main.go b/scheduler/cmd/scheduler/main.go
index 132025bcea..e5f2e46948 100644
--- a/scheduler/cmd/scheduler/main.go
+++ b/scheduler/cmd/scheduler/main.go
@@ -55,10 +55,11 @@ var (
 	tracingConfigPath            string
 	dbPath                       string
 	nodeID                       string
-	allowPlaintxt                bool //scheduler server
+	allowPlaintxt                bool // scheduler server
 	autoscalingDisabled          bool
 	kafkaConfigPath              string
 	schedulerReadyTimeoutSeconds uint
+	deletedResourceTTLSeconds    uint
 )
 
 const (
@@ -115,6 +116,9 @@ func init() {
 
 	// Timeout for scheduler to be ready
 	flag.UintVar(&schedulerReadyTimeoutSeconds, "scheduler-ready-timeout-seconds", 300, "Timeout for scheduler to be ready")
+
+	// This TTL is set in badger DB
+	flag.UintVar(&deletedResourceTTLSeconds, "deleted-resource-ttl-seconds", 86400, "TTL for deleted experiments and pipelines (in seconds)")
 }
 
 func getNamespace() string {
@@ -211,11 +215,11 @@ func main() {
 	// Do here after other services created so eventHub events will be handled on pipeline/experiment load
 	// If we start earlier events will be sent but not received by services that start listening "late" to eventHub
 	if dbPath != "" {
-		err := ps.InitialiseOrRestoreDB(dbPath)
+		err := ps.InitialiseOrRestoreDB(dbPath, deletedResourceTTLSeconds)
 		if err != nil {
 			log.WithError(err).Fatalf("Failed to initialise pipeline db at %s", dbPath)
 		}
-		err = es.InitialiseOrRestoreDB(dbPath)
+		err = es.InitialiseOrRestoreDB(dbPath, deletedResourceTTLSeconds)
 		if err != nil {
 			log.WithError(err).Fatalf("Failed to initialise experiment db at %s", dbPath)
 		}
diff --git a/scheduler/pkg/server/server_test.go b/scheduler/pkg/server/server_test.go
index 26f0623c54..e5e05915cb 100644
--- a/scheduler/pkg/server/server_test.go
+++ b/scheduler/pkg/server/server_test.go
@@ -127,7 +127,8 @@ func TestLoadModel(t *testing.T) {
 					},
 				},
 			},
-			model: &pb.Model{Meta: &pb.MetaData{Name: "model1"},
+			model: &pb.Model{
+				Meta: &pb.MetaData{Name: "model1"},
 				ModelSpec: &pb.ModelSpec{
 					Uri:          "gs://model",
 					Requirements: []string{"sklearn"},
@@ -384,8 +385,11 @@ func TestUnloadModel(t *testing.T) {
 		{
 			name: "Simple",
 			req: []*pba.AgentSubscribeRequest{
-				{ServerName: "server1", ReplicaIdx: 0, Shared: true, AvailableMemoryBytes: 1000,
-					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn"}}}},
+				{
+					ServerName: "server1", ReplicaIdx: 0, Shared: true, AvailableMemoryBytes: 1000,
+					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn"}},
+				},
+			},
 			model:      &pb.Model{Meta: &pb.MetaData{Name: "model1"}, ModelSpec: &pb.ModelSpec{Uri: "gs://model", Requirements: []string{"sklearn"}, MemoryBytes: &smallMemory}, DeploymentSpec: &pb.DeploymentSpec{Replicas: 1}},
 			code:       codes.OK,
 			modelState: store.ModelTerminated,
@@ -393,8 +397,11 @@ func TestUnloadModel(t *testing.T) {
 		{
 			name: "Multiple",
 			req: []*pba.AgentSubscribeRequest{
-				{ServerName: "server1", ReplicaIdx: 0, Shared: true, AvailableMemoryBytes: 1000,
-					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn", "xgboost"}}}},
+				{
+					ServerName: "server1", ReplicaIdx: 0, Shared: true, AvailableMemoryBytes: 1000,
+					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn", "xgboost"}},
+				},
+			},
 			model:      &pb.Model{Meta: &pb.MetaData{Name: "model1"}, ModelSpec: &pb.ModelSpec{Uri: "gs://model", Requirements: []string{"sklearn", "xgboost"}, MemoryBytes: &smallMemory}, DeploymentSpec: &pb.DeploymentSpec{Replicas: 1}},
 			code:       codes.OK,
 			modelState: store.ModelTerminated,
@@ -402,10 +409,15 @@ func TestUnloadModel(t *testing.T) {
 		{
 			name: "TwoReplicas",
 			req: []*pba.AgentSubscribeRequest{
-				{ServerName: "server1", ReplicaIdx: 0, Shared: true, AvailableMemoryBytes: 1000,
-					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn"}}},
-				{ServerName: "server1", ReplicaIdx: 1, Shared: true, AvailableMemoryBytes: 1000,
-					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn"}}}},
+				{
+					ServerName: "server1", ReplicaIdx: 0, Shared: true, AvailableMemoryBytes: 1000,
+					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn"}},
+				},
+				{
+					ServerName: "server1", ReplicaIdx: 1, Shared: true, AvailableMemoryBytes: 1000,
+					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn"}},
+				},
+			},
 			model:      &pb.Model{Meta: &pb.MetaData{Name: "model1"}, ModelSpec: &pb.ModelSpec{Uri: "gs://model", Requirements: []string{"sklearn"}, MemoryBytes: &smallMemory}, DeploymentSpec: &pb.DeploymentSpec{Replicas: 2}},
 			code:       codes.OK,
 			modelState: store.ModelTerminated,
@@ -413,10 +425,14 @@ func TestUnloadModel(t *testing.T) {
 		{
 			name: "NotExist",
 			req: []*pba.AgentSubscribeRequest{
-				{ServerName: "server1", ReplicaIdx: 0, Shared: true, AvailableMemoryBytes: 1000,
-					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn"}}}},
+				{
+					ServerName: "server1", ReplicaIdx: 0, Shared: true, AvailableMemoryBytes: 1000,
+					ReplicaConfig: &pba.ReplicaConfig{InferenceSvc: "server1", InferenceHttpPort: 1, Capabilities: []string{"sklearn"}},
+				},
+			},
 			model: nil,
-			code:  codes.FailedPrecondition},
+			code:  codes.FailedPrecondition,
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
@@ -518,7 +534,6 @@ func TestLoadPipeline(t *testing.T) {
 			}
 		})
 	}
-
 }
 
 func TestUnloadPipeline(t *testing.T) {
@@ -562,7 +577,7 @@ func TestUnloadPipeline(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			path := fmt.Sprintf("%s/db", t.TempDir())
-			_ = test.server.pipelineHandler.(*pipeline.PipelineStore).InitialiseOrRestoreDB(path)
+			_ = test.server.pipelineHandler.(*pipeline.PipelineStore).InitialiseOrRestoreDB(path, 10)
 			if test.loadReq != nil {
 				err := test.server.pipelineHandler.AddPipeline(test.loadReq.Pipeline)
 				g.Expect(err).To(BeNil())
@@ -575,7 +590,6 @@ func TestUnloadPipeline(t *testing.T) {
 			}
 		})
 	}
-
 }
 
 func TestPipelineStatus(t *testing.T) {
diff --git a/scheduler/pkg/store/experiment/db.go b/scheduler/pkg/store/experiment/db.go
index e41f3bd87f..391a6859f6 100644
--- a/scheduler/pkg/store/experiment/db.go
+++ b/scheduler/pkg/store/experiment/db.go
@@ -10,6 +10,8 @@ the Change License after the Change Date as each is defined in accordance with t
 package experiment
 
 import (
+	"time"
+
 	"github.com/dgraph-io/badger/v3"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/proto"
@@ -25,19 +27,21 @@ const (
 )
 
 type ExperimentDBManager struct {
-	db     *badger.DB
-	logger logrus.FieldLogger
+	db                 *badger.DB
+	logger             logrus.FieldLogger
+	deletedResourceTTL time.Duration
 }
 
-func newExperimentDbManager(path string, logger logrus.FieldLogger) (*ExperimentDBManager, error) {
+func newExperimentDbManager(path string, logger logrus.FieldLogger, deletedResourceTTL uint) (*ExperimentDBManager, error) {
 	db, err := utils.Open(path, logger, "experimentDb")
 	if err != nil {
 		return nil, err
 	}
 
 	edb := &ExperimentDBManager{
-		db:     db,
-		logger: logger,
+		db:                 db,
+		logger:             logger,
+		deletedResourceTTL: time.Duration(deletedResourceTTL * uint(time.Second)),
 	}
 
 	version, err := edb.getVersion()
@@ -73,9 +77,8 @@ func (edb *ExperimentDBManager) save(experiment *Experiment) error {
 			return err
 		})
 	} else {
-		ttl := utils.DeletedResourceTTL
 		return edb.db.Update(func(txn *badger.Txn) error {
-			e := badger.NewEntry([]byte(experiment.Name), experimentBytes).WithTTL(ttl)
+			e := badger.NewEntry([]byte(experiment.Name), experimentBytes).WithTTL(edb.deletedResourceTTL)
 			err = txn.SetEntry(e)
 			return err
 		})
@@ -119,7 +122,7 @@ func (edb *ExperimentDBManager) restore(
 				}
 				experiment := CreateExperimentFromSnapshot(&snapshot)
 				if experiment.Deleted {
-					experiment.DeletedAt = utils.GetDeletedAt(item)
+					experiment.DeletedAt = utils.GetDeletedAt(item, edb.deletedResourceTTL)
 					err = stopExperimentCb(experiment)
 				} else {
 					// otherwise attempt to start the experiment
diff --git a/scheduler/pkg/store/experiment/db_test.go b/scheduler/pkg/store/experiment/db_test.go
index 4356a9a753..b8085a1e46 100644
--- a/scheduler/pkg/store/experiment/db_test.go
+++ b/scheduler/pkg/store/experiment/db_test.go
@@ -102,7 +102,7 @@ func TestSaveWithTTL(t *testing.T) {
 
 	path := fmt.Sprintf("%s/db", t.TempDir())
 	logger := log.New()
-	db, err := newExperimentDbManager(getExperimentDbFolder(path), logger)
+	db, err := newExperimentDbManager(getExperimentDbFolder(path), logger, 10)
 	g.Expect(err).To(BeNil())
 	err = db.save(experiment)
 	g.Expect(err).To(BeNil())
@@ -307,7 +307,7 @@ func TestSaveAndRestore(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			path := fmt.Sprintf("%s/db", t.TempDir())
 			logger := log.New()
-			db, err := newExperimentDbManager(getExperimentDbFolder(path), logger)
+			db, err := newExperimentDbManager(getExperimentDbFolder(path), logger, 10)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.experiments {
 				err := db.save(p)
@@ -317,7 +317,7 @@ func TestSaveAndRestore(t *testing.T) {
 			g.Expect(err).To(BeNil())
 
 			es := NewExperimentServer(log.New(), nil, nil, nil)
-			err = es.InitialiseOrRestoreDB(path)
+			err = es.InitialiseOrRestoreDB(path, 10)
 			g.Expect(err).To(BeNil())
 			for idx, p := range test.experiments {
 				if !test.errors[idx] {
@@ -379,7 +379,7 @@ func TestSaveAndRestoreDeletedExperiments(t *testing.T) {
 			g.Expect(test.experiment.Deleted).To(BeTrue(), "this is a test for deleted experiments")
 			path := fmt.Sprintf("%s/db", t.TempDir())
 			logger := log.New()
-			edb, err := newExperimentDbManager(getExperimentDbFolder(path), logger)
+			edb, err := newExperimentDbManager(getExperimentDbFolder(path), logger, 10)
 			g.Expect(err).To(BeNil())
 			if !test.withTTL {
 				err = saveWithOutTTL(&test.experiment, edb.db)
@@ -392,7 +392,7 @@ func TestSaveAndRestoreDeletedExperiments(t *testing.T) {
 			g.Expect(err).To(BeNil())
 
 			es := NewExperimentServer(log.New(), nil, nil, nil)
-			err = es.InitialiseOrRestoreDB(path)
+			err = es.InitialiseOrRestoreDB(path, 10)
 			g.Expect(err).To(BeNil())
 
 			if !test.withTTL {
@@ -538,7 +538,7 @@ func TestGetExperimentFromDB(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			path := fmt.Sprintf("%s/db", t.TempDir())
 			logger := log.New()
-			db, err := newExperimentDbManager(getExperimentDbFolder(path), logger)
+			db, err := newExperimentDbManager(getExperimentDbFolder(path), logger, 10)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.experiments {
 				err := db.save(p)
@@ -678,7 +678,7 @@ func TestDeleteExperimentFromDB(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			path := fmt.Sprintf("%s/db", t.TempDir())
 			logger := log.New()
-			db, err := newExperimentDbManager(getExperimentDbFolder(path), logger)
+			db, err := newExperimentDbManager(getExperimentDbFolder(path), logger, 10)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.experiments {
 				err := db.save(p)
@@ -844,7 +844,7 @@ func TestMigrateFromV1ToV2(t *testing.T) {
 			_ = db.Close()
 
 			// migrate
-			edb, err := newExperimentDbManager(getExperimentDbFolder(path), logger)
+			edb, err := newExperimentDbManager(getExperimentDbFolder(path), logger, 10)
 			g.Expect(err).To(BeNil())
 
 			g.Expect(err).To(BeNil())
@@ -856,7 +856,7 @@ func TestMigrateFromV1ToV2(t *testing.T) {
 
 			// check that we have no experiments in the db format
 			es := NewExperimentServer(log.New(), nil, nil, nil)
-			err = es.InitialiseOrRestoreDB(path)
+			err = es.InitialiseOrRestoreDB(path, 10)
 			g.Expect(err).To(BeNil())
 			g.Expect(len(es.experiments)).To(Equal(0))
 		})
diff --git a/scheduler/pkg/store/experiment/store.go b/scheduler/pkg/store/experiment/store.go
index 30b9b22593..d4ba9ed918 100644
--- a/scheduler/pkg/store/experiment/store.go
+++ b/scheduler/pkg/store/experiment/store.go
@@ -108,7 +108,7 @@ func (es *ExperimentStore) addExperimentInMap(experiment *Experiment) error {
 	}
 }
 
-func (es *ExperimentStore) InitialiseOrRestoreDB(path string) error {
+func (es *ExperimentStore) InitialiseOrRestoreDB(path string, deletedResourceTTL uint) error {
 	logger := es.logger.WithField("func", "initialiseDB")
 	experimentDbPath := getExperimentDbFolder(path)
 	logger.Infof("Initialise DB at %s", experimentDbPath)
@@ -116,7 +116,7 @@ func (es *ExperimentStore) InitialiseOrRestoreDB(path string) error {
 	if err != nil {
 		return err
 	}
-	db, err := newExperimentDbManager(experimentDbPath, es.logger)
+	db, err := newExperimentDbManager(experimentDbPath, es.logger, deletedResourceTTL)
 	if err != nil {
 		return err
 	}
@@ -484,7 +484,7 @@ func (es *ExperimentStore) cleanupDeletedExperiments() {
 						es.logger.Warnf("could not update DB TTL for experiment: %s", experiment.Name)
 					}
 				}
-			} else if experiment.DeletedAt.Add(utils.DeletedResourceTTL).Before(time.Now()) {
+			} else if experiment.DeletedAt.Add(es.db.deletedResourceTTL).Before(time.Now()) {
 				delete(es.experiments, experiment.Name)
 				es.logger.Info("cleaning up deleted experiment: %s", experiment.Name)
 			}
diff --git a/scheduler/pkg/store/experiment/store_test.go b/scheduler/pkg/store/experiment/store_test.go
index b6fc5a27b8..870dfd579b 100644
--- a/scheduler/pkg/store/experiment/store_test.go
+++ b/scheduler/pkg/store/experiment/store_test.go
@@ -195,7 +195,7 @@ func TestStartExperiment(t *testing.T) {
 			g.Expect(err).To(BeNil())
 			server := NewExperimentServer(logger, eventHub, fakeModelStore{}, fakePipelineStore{})
 			// init db
-			_ = server.InitialiseOrRestoreDB(path)
+			_ = server.InitialiseOrRestoreDB(path, 10)
 			for _, ea := range test.experiments {
 				err := server.StartExperiment(ea.experiment)
 				if ea.fail {
@@ -262,7 +262,7 @@ func TestStopExperiment(t *testing.T) {
 			path := fmt.Sprintf("%s/db", t.TempDir())
 
 			// init db
-			err := test.store.InitialiseOrRestoreDB(path)
+			err := test.store.InitialiseOrRestoreDB(path, 1)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.store.experiments {
 				err := test.store.db.save(p)
@@ -288,6 +288,10 @@ func TestStopExperiment(t *testing.T) {
 				// check db
 				experimentFromDB, _ := test.store.db.get(test.experimentName)
 				g.Expect(experimentFromDB.Deleted).To(BeTrue())
+
+				time.Sleep(1 * time.Second)
+				test.store.cleanupDeletedExperiments()
+				g.Expect(test.store.experiments[test.experimentName]).To(BeNil())
 			}
 		})
 	}
@@ -413,7 +417,7 @@ func TestRestoreExperiments(t *testing.T) {
 				experiments:     make(map[string]*Experiment),
 			}
 			// init db
-			err := store.InitialiseOrRestoreDB(path)
+			err := store.InitialiseOrRestoreDB(path, 10)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.experiments {
 				err := store.db.save(p)
@@ -422,7 +426,7 @@ func TestRestoreExperiments(t *testing.T) {
 			_ = store.db.Stop()
 
 			// restore from db now that we have state on disk
-			_ = store.InitialiseOrRestoreDB(path)
+			_ = store.InitialiseOrRestoreDB(path, 10)
 
 			for _, p := range test.experiments {
 				experimentFromDB, _ := store.db.get(p.Name)
diff --git a/scheduler/pkg/store/pipeline/db.go b/scheduler/pkg/store/pipeline/db.go
index f778105796..0bdbaa8f9f 100644
--- a/scheduler/pkg/store/pipeline/db.go
+++ b/scheduler/pkg/store/pipeline/db.go
@@ -10,6 +10,8 @@ the Change License after the Change Date as each is defined in accordance with t
 package pipeline
 
 import (
+	"time"
+
 	"github.com/dgraph-io/badger/v3"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/proto"
@@ -25,19 +27,21 @@ const (
 )
 
 type PipelineDBManager struct {
-	db     *badger.DB
-	logger logrus.FieldLogger
+	db                 *badger.DB
+	logger             logrus.FieldLogger
+	deletedResourceTTL time.Duration
 }
 
-func newPipelineDbManager(path string, logger logrus.FieldLogger) (*PipelineDBManager, error) {
+func newPipelineDbManager(path string, logger logrus.FieldLogger, deletedResourceTTL uint) (*PipelineDBManager, error) {
 	db, err := utils.Open(path, logger, "pipelineDb")
 	if err != nil {
 		return nil, err
 	}
 
 	pdb := &PipelineDBManager{
-		db:     db,
-		logger: logger,
+		db:                 db,
+		logger:             logger,
+		deletedResourceTTL: time.Duration(deletedResourceTTL * uint(time.Second)),
 	}
 
 	version, err := pdb.getVersion()
@@ -57,22 +61,20 @@ func newPipelineDbManager(path string, logger logrus.FieldLogger) (*PipelineDBMa
 }
 
 // a nil ttl will save the pipeline indefinitely
-func save(pipeline *Pipeline, db *badger.DB) error {
+func (pdb *PipelineDBManager) save(pipeline *Pipeline) error {
 	pipelineProto := CreatePipelineSnapshotFromPipeline(pipeline)
 	pipelineBytes, err := proto.Marshal(pipelineProto)
 	if err != nil {
 		return err
 	}
 	if !pipeline.Deleted {
-		return db.Update(func(txn *badger.Txn) error {
+		return pdb.db.Update(func(txn *badger.Txn) error {
 			err = txn.Set([]byte(pipeline.Name), pipelineBytes)
 			return err
 		})
 	} else {
-		// useful for testing
-		ttl := utils.DeletedResourceTTL
-		return db.Update(func(txn *badger.Txn) error {
-			e := badger.NewEntry([]byte(pipeline.Name), pipelineBytes).WithTTL(ttl)
+		return pdb.db.Update(func(txn *badger.Txn) error {
+			e := badger.NewEntry([]byte(pipeline.Name), pipelineBytes).WithTTL(pdb.deletedResourceTTL)
 			err = txn.SetEntry(e)
 			return err
 		})
@@ -83,10 +85,6 @@ func (pdb *PipelineDBManager) Stop() error {
 	return utils.Stop(pdb.db)
 }
 
-func (pdb *PipelineDBManager) save(pipeline *Pipeline) error {
-	return save(pipeline, pdb.db)
-}
-
 // TODO: delete unused pipelines from the store as for now it increases indefinitely
 func (pdb *PipelineDBManager) delete(name string) error {
 	return utils.Delete(pdb.db, name)
@@ -125,7 +123,7 @@ func (pdb *PipelineDBManager) restore(createPipelineCb func(pipeline *Pipeline))
 				}
 
 				if pipeline.Deleted {
-					pipeline.DeletedAt = utils.GetDeletedAt(item)
+					pipeline.DeletedAt = utils.GetDeletedAt(item, pdb.deletedResourceTTL)
 				}
 
 				createPipelineCb(pipeline)
diff --git a/scheduler/pkg/store/pipeline/db_test.go b/scheduler/pkg/store/pipeline/db_test.go
index f3e87375f6..414cf90638 100644
--- a/scheduler/pkg/store/pipeline/db_test.go
+++ b/scheduler/pkg/store/pipeline/db_test.go
@@ -21,7 +21,6 @@ import (
 	"google.golang.org/protobuf/proto"
 
 	"github.com/seldonio/seldon-core/scheduler/v2/pkg/store"
-	"github.com/seldonio/seldon-core/scheduler/v2/pkg/store/utils"
 )
 
 func TestSaveWithTTL(t *testing.T) {
@@ -50,10 +49,12 @@ func TestSaveWithTTL(t *testing.T) {
 		},
 		Deleted: true,
 	}
+	ttl := time.Duration(time.Second)
+	pipeline.DeletedAt = time.Now().Add(ttl)
 
 	path := fmt.Sprintf("%s/db", t.TempDir())
 	logger := log.New()
-	db, err := newPipelineDbManager(getPipelineDbFolder(path), logger)
+	db, err := newPipelineDbManager(getPipelineDbFolder(path), logger, 10)
 	g.Expect(err).To(BeNil())
 	err = db.save(pipeline)
 	g.Expect(err).To(BeNil())
@@ -181,7 +182,7 @@ func TestSaveAndRestore(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			path := fmt.Sprintf("%s/db", t.TempDir())
 			logger := log.New()
-			db, err := newPipelineDbManager(getPipelineDbFolder(path), logger)
+			db, err := newPipelineDbManager(getPipelineDbFolder(path), logger, 10)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.pipelines {
 				err := db.save(p)
@@ -191,7 +192,7 @@ func TestSaveAndRestore(t *testing.T) {
 			g.Expect(err).To(BeNil())
 
 			ps := NewPipelineStore(log.New(), nil, fakeModelStore{status: map[string]store.ModelState{}})
-			err = ps.InitialiseOrRestoreDB(path)
+			err = ps.InitialiseOrRestoreDB(path, 10)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.pipelines {
 				g.Expect(cmp.Equal(p, ps.pipelines[p.Name])).To(BeTrue())
@@ -252,12 +253,12 @@ func TestSaveAndRestoreDeletedPipelines(t *testing.T) {
 			g.Expect(test.pipeline.Deleted).To(BeTrue(), "this is a test for deleted pipelines")
 			path := fmt.Sprintf("%s/db", t.TempDir())
 			logger := log.New()
-			pdb, err := newPipelineDbManager(getPipelineDbFolder(path), logger)
+			pdb, err := newPipelineDbManager(getPipelineDbFolder(path), logger, 10)
 			g.Expect(err).To(BeNil())
 			if !test.withTTL {
 				err = saveWithOutTTL(&test.pipeline, pdb.db)
 			} else {
-				test.pipeline.DeletedAt = time.Now().Add(-utils.DeletedResourceTTL)
+				test.pipeline.DeletedAt = time.Now()
 				err = pdb.save(&test.pipeline)
 			}
 			g.Expect(err).To(BeNil())
@@ -265,7 +266,7 @@ func TestSaveAndRestoreDeletedPipelines(t *testing.T) {
 			g.Expect(err).To(BeNil())
 
 			ps := NewPipelineStore(log.New(), nil, fakeModelStore{status: map[string]store.ModelState{}})
-			err = ps.InitialiseOrRestoreDB(path)
+			err = ps.InitialiseOrRestoreDB(path, 10)
 			g.Expect(err).To(BeNil())
 
 			if !test.withTTL {
@@ -344,7 +345,7 @@ func TestGetPipelineFromDB(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			path := fmt.Sprintf("%s/db", t.TempDir())
 			logger := log.New()
-			db, err := newPipelineDbManager(getPipelineDbFolder(path), logger)
+			db, err := newPipelineDbManager(getPipelineDbFolder(path), logger, 10)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.pipelines {
 				err := db.save(p)
@@ -419,7 +420,7 @@ func TestDeletePipelineFromDB(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			path := fmt.Sprintf("%s/db", t.TempDir())
 			logger := log.New()
-			db, err := newPipelineDbManager(getPipelineDbFolder(path), logger)
+			db, err := newPipelineDbManager(getPipelineDbFolder(path), logger, 10)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.pipelines {
 				err := db.save(p)
@@ -537,18 +538,17 @@ func TestMigrateFromV1ToV2(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			path := fmt.Sprintf("%s/db", t.TempDir())
-			logger := log.New()
-			db, err := utils.Open(getPipelineDbFolder(path), logger, "pipelineDb")
+			ps := NewPipelineStore(log.New(), nil, fakeModelStore{status: map[string]store.ModelState{}})
+			err := ps.InitialiseOrRestoreDB(path, 10)
 			g.Expect(err).To(BeNil())
 			for _, p := range test.pipelines {
-				err := save(p, db)
+				err := ps.db.save(p)
 				g.Expect(err).To(BeNil())
 			}
-			err = db.Close()
+			err = ps.db.db.Close()
 			g.Expect(err).To(BeNil())
 
-			ps := NewPipelineStore(log.New(), nil, fakeModelStore{status: map[string]store.ModelState{}})
-			err = ps.InitialiseOrRestoreDB(path)
+			err = ps.InitialiseOrRestoreDB(path, 10)
 			g.Expect(err).To(BeNil())
 
 			// make sure we still have the pipelines
diff --git a/scheduler/pkg/store/pipeline/store.go b/scheduler/pkg/store/pipeline/store.go
index d6e2c0d979..36ddc16405 100644
--- a/scheduler/pkg/store/pipeline/store.go
+++ b/scheduler/pkg/store/pipeline/store.go
@@ -81,7 +81,7 @@ func getPipelineDbFolder(basePath string) string {
 	return filepath.Join(basePath, pipelineDbFolder)
 }
 
-func (ps *PipelineStore) InitialiseOrRestoreDB(path string) error {
+func (ps *PipelineStore) InitialiseOrRestoreDB(path string, deletedResourceTTL uint) error {
 	logger := ps.logger.WithField("func", "initialiseDB")
 	pipelineDbPath := getPipelineDbFolder(path)
 	logger.Infof("Initialise DB at %s", pipelineDbPath)
@@ -89,7 +89,7 @@ func (ps *PipelineStore) InitialiseOrRestoreDB(path string) error {
 	if err != nil {
 		return err
 	}
-	db, err := newPipelineDbManager(pipelineDbPath, ps.logger)
+	db, err := newPipelineDbManager(pipelineDbPath, ps.logger, deletedResourceTTL)
 	if err != nil {
 		return err
 	}
@@ -458,7 +458,7 @@ func (ps *PipelineStore) cleanupDeletedPipelines() {
 						ps.logger.Warnf("could not update DB TTL for pipeline: %s", pipeline.Name)
 					}
 				}
-			} else if pipeline.DeletedAt.Add(utils.DeletedResourceTTL).Before(time.Now()) {
+			} else if pipeline.DeletedAt.Add(ps.db.deletedResourceTTL).Before(time.Now()) {
 				delete(ps.pipelines, pipeline.Name)
 				ps.logger.Info("cleaning up deleted pipeline: %s", pipeline.Name)
 			}
diff --git a/scheduler/pkg/store/pipeline/store_test.go b/scheduler/pkg/store/pipeline/store_test.go
index 51e53bf6e5..3d0b1d2870 100644
--- a/scheduler/pkg/store/pipeline/store_test.go
+++ b/scheduler/pkg/store/pipeline/store_test.go
@@ -12,6 +12,7 @@ package pipeline
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	. "github.com/onsi/gomega"
 	"github.com/sirupsen/logrus"
@@ -234,7 +235,7 @@ func TestAddPipeline(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			logger := logrus.New()
 			path := fmt.Sprintf("%s/db", t.TempDir())
-			db, _ := newPipelineDbManager(getPipelineDbFolder(path), logger)
+			db, _ := newPipelineDbManager(getPipelineDbFolder(path), logger, 10)
 			test.store.db = db
 			err := test.store.AddPipeline(test.proto)
 			if test.err == nil {
@@ -385,7 +386,7 @@ func TestRemovePipeline(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			logger := logrus.New()
 			path := fmt.Sprintf("%s/db", t.TempDir())
-			db, _ := newPipelineDbManager(getPipelineDbFolder(path), logger)
+			db, _ := newPipelineDbManager(getPipelineDbFolder(path), logger, 1)
 			test.store.db = db
 			err := test.store.RemovePipeline(test.pipelineName)
 			if test.err == nil {
@@ -408,6 +409,9 @@ func TestRemovePipeline(t *testing.T) {
 				g.Expect(actualPipeline.LastVersion).To(Equal(expectedPipeline.LastVersion))
 				g.Expect(len(actualPipeline.Versions)).To(Equal(len(expectedPipeline.Versions)))
 				g.Expect(len(actualPipeline.Versions)).To(Equal(len(expectedPipeline.Versions)))
+				time.Sleep(1 * time.Second)
+				test.store.cleanupDeletedPipelines()
+				g.Expect(test.store.pipelines[test.pipelineName]).To(BeNil())
 			} else {
 				g.Expect(err.Error()).To(Equal(test.err.Error()))
 			}
diff --git a/scheduler/pkg/store/utils/utils.go b/scheduler/pkg/store/utils/utils.go
index 5d0b3bd0bc..489de3d497 100644
--- a/scheduler/pkg/store/utils/utils.go
+++ b/scheduler/pkg/store/utils/utils.go
@@ -18,7 +18,6 @@ import (
 
 const (
 	VersionKey                                    = "__version_key__"
-	DeletedResourceTTL              time.Duration = time.Duration(24 * time.Hour)
 	DeletedResourceCleanupFrequency time.Duration = time.Duration(10 * time.Minute)
 )
 
@@ -48,11 +47,11 @@ func SaveVersion(db *badger.DB, version string) error {
 	})
 }
 
-func GetDeletedAt(item *badger.Item) time.Time {
+func GetDeletedAt(item *badger.Item, ttl time.Duration) time.Time {
 	if item.ExpiresAt() == 0 {
 		return time.Time{}
 	} else {
-		return time.Unix(int64(item.ExpiresAt()), 0).Add(-DeletedResourceTTL)
+		return time.Unix(int64(item.ExpiresAt()), 0).Add(-ttl)
 	}
 }
 

From ac839c363930202bb7165601e2ceb0f6362a731b Mon Sep 17 00:00:00 2001
From: Niall D <4562759+driev@users.noreply.github.com>
Date: Mon, 4 Nov 2024 14:32:21 +0000
Subject: [PATCH 2/4] fix(scheduler): always send model events for deleted
 models (#5992)

* always send model events for deleted models

* trying to make it easier to read

* fix broken logic

* simplify loop

* replace recursive call with a loop

* add more logs and update one condition for clarity
---
 scheduler/pkg/store/memory.go        | 92 +++++++++++++---------------
 scheduler/pkg/store/memory_status.go |  2 +-
 scheduler/pkg/store/memory_test.go   |  4 +-
 3 files changed, 46 insertions(+), 52 deletions(-)

diff --git a/scheduler/pkg/store/memory.go b/scheduler/pkg/store/memory.go
index e2a3b6e611..6b18917671 100644
--- a/scheduler/pkg/store/memory.go
+++ b/scheduler/pkg/store/memory.go
@@ -293,6 +293,7 @@ func (m *MemoryStore) updateLoadedModelsImpl(
 	if !ok {
 		return nil, fmt.Errorf("failed to find model %s", modelKey)
 	}
+
 	modelVersion := model.Latest()
 	if version != modelVersion.GetVersion() {
 		return nil, fmt.Errorf(
@@ -310,83 +311,74 @@ func (m *MemoryStore) updateLoadedModelsImpl(
 	if !ok {
 		return nil, fmt.Errorf("failed to find server %s", serverKey)
 	}
+
+	assignedReplicaIds := make(map[int]struct{})
 	for _, replica := range replicas {
-		_, ok := server.replicas[replica.GetReplicaIdx()]
-		if !ok {
+		if _, ok := server.replicas[replica.replicaIdx]; !ok {
 			return nil, fmt.Errorf(
-				"Failed to reserve replica %d as it does not exist on server %s",
-				replica.GetReplicaIdx(), serverKey,
+				"failed to reserve replica %d as it does not exist on server %s",
+				replica.replicaIdx, serverKey,
 			)
 		}
+		assignedReplicaIds[replica.replicaIdx] = struct{}{}
 	}
 
-	if modelVersion.HasServer() && modelVersion.Server() != serverKey {
+	for modelVersion.HasServer() && modelVersion.Server() != serverKey {
 		logger.Debugf("Adding new version as server changed to %s from %s", modelVersion.Server(), serverKey)
 		m.addNextModelVersion(model, model.Latest().modelDefn)
-		return m.updateLoadedModelsImpl(modelKey, model.Latest().GetVersion(), serverKey, replicas)
+		modelVersion = model.Latest()
 	}
 
-	// Update model that need to be placed on a replica to request loading
-	updatedReplicas := make(map[int]bool)
-	updated := false
-	for _, replica := range replicas {
-		existingState := modelVersion.replicas[replica.GetReplicaIdx()]
-		if !existingState.State.AlreadyLoadingOrLoaded() {
+	//  resevere memory for existing replicas that are not already loading or loaded
+	replicaStateUpdated := false
+	for replicaIdx := range assignedReplicaIds {
+		if existingState, ok := modelVersion.replicas[replicaIdx]; !ok {
 			logger.Debugf(
-				"Setting model %s version %d on server %s replica %d to LoadRequested",
-				modelKey, modelVersion.version, serverKey, replica.GetReplicaIdx(),
+				"Model %s version %d state %s on server %s replica %d does not exist yet and should be loaded",
+				modelKey, modelVersion.version, existingState.State.String(), serverKey, replicaIdx,
 			)
-			modelVersion.SetReplicaState(replica.GetReplicaIdx(), LoadRequested, "")
-			m.updateReservedMemory(LoadRequested, serverKey, replica.GetReplicaIdx(), modelVersion.GetRequiredMemory())
-			updated = true
+			modelVersion.SetReplicaState(replicaIdx, LoadRequested, "")
+			m.updateReservedMemory(LoadRequested, serverKey, replicaIdx, modelVersion.GetRequiredMemory())
+			replicaStateUpdated = true
 		} else {
 			logger.Debugf(
-				"model %s on server %s replica %d already loaded",
-				modelKey, serverKey, replica.GetReplicaIdx(),
+				"Checking if model %s version %d state %s on server %s replica %d should be loaded",
+				modelKey, modelVersion.version, existingState.State.String(), serverKey, replicaIdx,
 			)
+			if !existingState.State.AlreadyLoadingOrLoaded() {
+				modelVersion.SetReplicaState(replicaIdx, LoadRequested, "")
+				m.updateReservedMemory(LoadRequested, serverKey, replicaIdx, modelVersion.GetRequiredMemory())
+				replicaStateUpdated = true
+			}
 		}
-		updatedReplicas[replica.GetReplicaIdx()] = true
 	}
-	// Unload any existing model replicas assignments no longer needed
-	for replicaIdx, existingState := range modelVersion.ReplicaState() {
-		logger.Debugf(
-			"Looking at replicaidx %d with state %s but ignoring processed %v",
-			replicaIdx, existingState.State.String(), updatedReplicas,
-		)
 
-		if _, ok := updatedReplicas[replicaIdx]; !ok {
-			if !existingState.State.UnloadingOrUnloaded() {
-				if existingState.State == Draining {
-					logger.Debugf(
-						"model %s version %d on server %s replica %d is Draining",
-						modelKey, modelVersion.version, serverKey, replicaIdx,
-					)
-				} else {
-					logger.Debugf(
-						"Setting model %s version %d on server %s replica %d to UnloadEnvoyRequested",
-						modelKey, modelVersion.version, serverKey, replicaIdx,
-					)
-					modelVersion.SetReplicaState(replicaIdx, UnloadEnvoyRequested, "")
-					updated = true
-				}
-			} else {
-				logger.Debugf(
-					"model %s on server %s replica %d already unloading or can't be unloaded",
-					modelKey, serverKey, replicaIdx,
-				)
+	// Unload any existing model replicas assignments that are no longer part of the replica set
+	for replicaIdx, existingState := range modelVersion.ReplicaState() {
+		if _, ok := assignedReplicaIds[replicaIdx]; !ok {
+			logger.Debugf(
+				"Checking if replicaidx %d with state %s should be unloaded",
+				replicaIdx, existingState.State.String(),
+			)
+			if !existingState.State.UnloadingOrUnloaded() && existingState.State != Draining {
+				modelVersion.SetReplicaState(replicaIdx, UnloadEnvoyRequested, "")
+				replicaStateUpdated = true
 			}
 		}
 	}
 
 	// in cases where we did have a previous ScheduleFailed, we need to reflect the change here
 	// this could be in the cases where we are scaling down a model and the new replica count can be all deployed
-	if updated || modelVersion.state.State == ScheduleFailed {
+	// and always send an update for deleted models, so the operator will remove them from k8s
+	if replicaStateUpdated || modelVersion.state.State == ScheduleFailed || model.IsDeleted() {
 		logger.Debugf("Updating model status for model %s server %s", modelKey, serverKey)
 		modelVersion.server = serverKey
 		m.updateModelStatus(true, model.IsDeleted(), modelVersion, model.GetLastAvailableModelVersion())
 		return &coordinator.ModelEventMsg{ModelName: modelVersion.GetMeta().GetName(), ModelVersion: modelVersion.GetVersion()}, nil
+	} else {
+		logger.Debugf("Model status update not required for model %s server %s as no replicas were updated", modelKey, serverKey)
+		return nil, nil
 	}
-	return nil, nil
 }
 
 func (m *MemoryStore) UnloadVersionModels(modelKey string, version uint32) (bool, error) {
@@ -415,7 +407,7 @@ func (m *MemoryStore) unloadVersionModelsImpl(modelKey string, version uint32) (
 	}
 	modelVersion := model.GetVersion(version)
 	if modelVersion == nil {
-		return nil, false, fmt.Errorf("Version not found for model %s, version %d", modelKey, version)
+		return nil, false, fmt.Errorf("version not found for model %s, version %d", modelKey, version)
 	}
 
 	updated := false
@@ -482,7 +474,7 @@ func (m *MemoryStore) updateModelStateImpl(
 	desiredState ModelReplicaState,
 	reason string,
 ) (*coordinator.ModelEventMsg, error) {
-	logger := m.logger.WithField("func", "UpdateModelState")
+	logger := m.logger.WithField("func", "updateModelStateImpl")
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
diff --git a/scheduler/pkg/store/memory_status.go b/scheduler/pkg/store/memory_status.go
index 9a2dfc14de..d7e3dc6bfb 100644
--- a/scheduler/pkg/store/memory_status.go
+++ b/scheduler/pkg/store/memory_status.go
@@ -88,7 +88,7 @@ func updateModelState(isLatest bool, modelVersion *ModelVersion, prevModelVersio
 			modelReason = stats.lastFailedReason
 			modelTimestamp = stats.lastFailedStateTime
 		} else if (modelVersion.GetDeploymentSpec() != nil && stats.replicasAvailable == modelVersion.GetDeploymentSpec().Replicas) || // equal to desired replicas
-			(stats.replicasAvailable > 0 && prevModelVersion != nil && modelVersion != prevModelVersion && prevModelVersion.state.State == ModelAvailable) { //TODO In future check if available replicas is > minReplicas
+			(stats.replicasAvailable > 0 && prevModelVersion != nil && modelVersion != prevModelVersion && prevModelVersion.state.State == ModelAvailable) { // TODO In future check if available replicas is > minReplicas
 			modelState = ModelAvailable
 		} else {
 			modelState = ModelProgressing
diff --git a/scheduler/pkg/store/memory_test.go b/scheduler/pkg/store/memory_test.go
index 80c3f499ef..0f16b5e1b7 100644
--- a/scheduler/pkg/store/memory_test.go
+++ b/scheduler/pkg/store/memory_test.go
@@ -684,9 +684,10 @@ func TestUpdateLoadedModels(t *testing.T) {
 				test.store.models[test.modelKey].SetDeleted()
 			}
 			ms := NewMemoryStore(logger, test.store, eventHub)
-			err = ms.UpdateLoadedModels(test.modelKey, test.version, test.serverKey, test.replicas)
+			msg, err := ms.updateLoadedModelsImpl(test.modelKey, test.version, test.serverKey, test.replicas)
 			if !test.err {
 				g.Expect(err).To(BeNil())
+				g.Expect(msg).ToNot(BeNil())
 				for replicaIdx, state := range test.expectedStates {
 					mv := test.store.models[test.modelKey].Latest()
 					g.Expect(mv).ToNot(BeNil())
@@ -700,6 +701,7 @@ func TestUpdateLoadedModels(t *testing.T) {
 				}
 			} else {
 				g.Expect(err).ToNot(BeNil())
+				g.Expect(msg).To(BeNil())
 			}
 		})
 	}

From 1ecc543f5e82c39540b2771937bfe1a634cbb82b Mon Sep 17 00:00:00 2001
From: Lucian Carata <lc525@users.noreply.github.com>
Date: Wed, 6 Nov 2024 12:49:58 +0000
Subject: [PATCH 3/4] feat(docs): add more configuration details to the HPA
 docs (#6019)

* improve overall docs clarity
* explain prometheus-adapter ConfigMap customisation options
* mention that `spec.replicas` needs to be present in the Model CR for HPA to work
* change `apply` to `replace` in kubectl command updating the prometheus-adapter ConfigMap
* clarify in what namespaces we should load the various manifests in the example
* add cluster operation guidelines

Co-authored-by: Rajakavitha Kodhandapani <krajakavitha@gmail.com>
* review with doc improvements suggestions & fixes

---------

Co-authored-by: paulb-seldon <141156400+paulb-seldon@users.noreply.github.com>
Co-authored-by: Rajakavitha Kodhandapani <krajakavitha@gmail.com>
---
 docs-gb/kubernetes/hpa-rps-autoscaling.md | 464 ++++++++++++++++------
 1 file changed, 348 insertions(+), 116 deletions(-)

diff --git a/docs-gb/kubernetes/hpa-rps-autoscaling.md b/docs-gb/kubernetes/hpa-rps-autoscaling.md
index 8e815f187d..9dc9b910e8 100644
--- a/docs-gb/kubernetes/hpa-rps-autoscaling.md
+++ b/docs-gb/kubernetes/hpa-rps-autoscaling.md
@@ -1,7 +1,7 @@
 
 # HPA Autoscaling in single-model serving
 
-Learn about how to jointly autoscale model and server replicas based on a metric of inference
+Learn how to jointly autoscale model and server replicas based on a metric of inference
 requests per second (RPS) using HPA, when there is a one-to-one correspondence between models
 and servers (single-model serving). This will require:
 
@@ -13,9 +13,9 @@ and servers (single-model serving). This will require:
 * Configuring HPA manifests to scale Models and the corresponding Server replicas based on the
   custom metrics
 
-### Installing and configuring the Prometheus Adapter
+## Installing and configuring the Prometheus Adapter
 
-The role of the Prometheus Adapter is to expose queries on metrics in prometheus as k8s custom
+The role of the Prometheus Adapter is to expose queries on metrics in Prometheus as k8s custom
 or external metrics. Those can then be accessed by HPA in order to take scaling decisions.
 
 To install through helm:
@@ -26,10 +26,13 @@ helm repo update
 helm install --set prometheus.url='http://seldon-monitoring-prometheus' hpa-metrics prometheus-community/prometheus-adapter -n seldon-monitoring
 ```
 
-In the commands above, we install `prometheus-adapter` as a helm release named `hpa-metrics` in
-the same namespace as our prometheus install, and point to its service URL (without the port).
+These commands install `prometheus-adapter` as a helm release named `hpa-metrics` in
+the same namespace where Prometheus is installed, and point to its service URL (without the port).
 
-If running prometheus on a different port than the default 9090, you can also pass `--set
+The URL is not fully qualified as it references a Prometheus instance running in the same namespace.
+If you are using a separately-managed Prometheus instance, please update the URL accordingly.
+
+If you are running Prometheus on a different port than the default 9090, you can also pass `--set
 prometheus.port=[custom_port]` You may inspect all the options available as helm values by
 running `helm show values prometheus-community/prometheus-adapter`
 
@@ -38,18 +41,14 @@ per-model RPS values. On install, the adapter has created a `ConfigMap` in the s
 itself, named `[helm_release_name]-prometheus-adapter`. In our case, it will be
 `hpa-metrics-prometheus-adapter`.
 
-We want to overwrite this ConfigMap with the content below (please change the name if your helm
-release has a different one). The manifest contains embedded documentation, highlighting how we
-match the `seldon_model_infer_total` metric in Prometheus, compute a rate via a `metricsQuery`
-and expose this to k8s as the `infer_rps` metric, on a per (model, namespace) basis.
-
-Other aggregations on per (server, namespace) and (pod, namespace) are also exposed and may be
-used in HPA, but we will focus on the (model, namespace) aggregation in the examples below.
+Overwrite the ConfigMap as shown in the following manifest, after applying any required customizations.
 
-You may want to modify some of the settings to match the prometheus query that you typically use
-for RPS metrics. For example, the `metricsQuery` below computes the RPS by calling [`rate()`]
-(https://prometheus.io/docs/prometheus/latest/querying/functions/#rate) with a 1 minute window.
+{% hint style="warning" %}
+Change the `name` if you've chosen a different value for the `prometheus-adapter` helm release name.
+Change the `namespace` to match the namespace where `prometheus-adapter` is installed.
+{% endhint %}
 
+{% code title="prometheus-adapter.config.yaml" lineNumbers="true" %}
 ````yaml
 apiVersion: v1
 kind: ConfigMap
@@ -59,83 +58,21 @@ metadata:
 data:
   config.yaml: |-
     "rules":
-    # Rule matching Seldon inference requests-per-second metrics and exposing aggregations for
-    # specific k8s models, servers, pods and namespaces
-    #
-    # Uses the prometheus-side `seldon_model_(.*)_total` inference request count metrics to
-    # compute and expose k8s custom metrics on inference RPS `${1}_rps`. A prometheus metric named
-    # `seldon_model_infer_total` will be exposed as multiple `[group-by-k8s-resource]/infer_rps`
-    # k8s metrics, for consumption by HPA.
-    #
-    # One k8s metric is generated for each k8s resource associated with a prometheus metric, as
-    # defined in the "Association" section below. Because this association is defined based on
-    # labels present in the prometheus metric, the number of generated k8s metrics will vary
-    # depending on what labels are available in each discovered prometheus metric.
-    #
-    # The resources associated through this rule (when available as labels for each of the
-    # discovered prometheus metrics) are:
-    # - models
-    # - servers
-    # - pods (inference server pods)
-    # - namespaces
-    #
-    # For example, you will get aggregated metrics for `models.mlops.seldon.io/iris0/infer_rps`,
-    # `servers.mlops.seldon.io/mlserver/infer_rps`, `pods/mlserver-0/infer_rps`,
-    # `namespaces/seldon-mesh/infer_rps`
-    #
-    # Metrics associated with any resource except the namespace one (models, servers and pods)
-    # need to be requested in the context of a particular namespace.
-    #
-    # To fetch those k8s metrics manually once the prometheus-adapter is running, you can run:
-    #
-    # For "namespaced" resources, i.e. models, servers and pods (replace values in brackets):
-    # ```
-    # kubectl get --raw
-    # "/apis/custom.metrics.k8s.io/v1beta1/namespaces/[NAMESPACE]/[RESOURCE_NAME]/[CR_NAME]/infer_rps"
-    # ```
-    #
-    # For example:
-    # ```
-    # kubectl get --raw
-    # "/apis/custom.metrics.k8s.io/v1beta1/namespaces/seldon-mesh/models.mlops.seldon.io/iris0/infer_rps"
-    # ```
-    #
-    # For the namespace resource, you can get the namespace-level aggregation of the metric with:
-    # ```
-    # kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/namespaces/*/metrics/infer_rps"
-    # ```
     -
-      # Metric discovery: selects subset of metrics exposed in Prometheus, based on name and
-      # filters
       "seriesQuery": |
          {__name__=~"^seldon_model.*_total",namespace!=""}
       "seriesFilters":
         - "isNot": "^seldon_.*_seconds_total"
         - "isNot": "^seldon_.*_aggregate_.*"
-      # Association: maps label values in the Prometheus metric to K8s resources (native or CRs)
-      # Below, we associate the "model" prometheus metric label to the corresponding Seldon Model
-      # CR, the "server" label to the Seldon Server CR, etc.
       "resources":
         "overrides":
           "model": {group: "mlops.seldon.io", resource: "model"}
           "server": {group: "mlops.seldon.io", resource: "server"}
           "pod": {resource: "pod"}
           "namespace": {resource: "namespace"}
-      # Rename prometheus metrics to get k8s metric names that reflect the processing done via
-      # the query applied to those metrics (actual query below under the "metricsQuery" key)
       "name":
         "matches": "^seldon_model_(.*)_total"
         "as": "${1}_rps"
-      # The actual query to be executed against Prometheus to retrieve the metric value
-      # Here:
-      #   - .Series is replaced by the discovered prometheus metric name (e.g.
-      #     `seldon_model_infer_total`)
-      #   - .LabelMatchers, when requesting a metric for a namespaced resource X with name x in
-      #     namespace n, is replaced by `X=~"x",namespace="n"`. For example, `model=~"iris0",
-      #     namespace="seldon-mesh"`. When requesting the namespace resource itself, only the
-      #     `namespace="n"` is kept.
-      #   - .GroupBy is replaced by the resource type of the requested metric (e.g. `model`,
-      #     `server`, `pod` or `namespace`).
       "metricsQuery": |
         sum by (<<.GroupBy>>) (
           rate (
@@ -143,70 +80,154 @@ data:
           )
         )
 ````
+{% endcode %}
+
+In this example, a single rule is defined to fetch the `seldon_model_infer_total` metric
+from Prometheus, compute its rate over a 1 minute window, and expose this to k8s as the `infer_rps`
+metric, with aggregations at model, server, inference server pod and namespace level.
+
+A list of all the Prometheus metrics exposed by Seldon Core 2 in relation to Models, Servers and Pipelines is available [here](../metrics/operational.md),
+and those may be used when customizing the configuration.
+
+### Understanding prometheus-adapter rule definitions
+
+The rule definition can be broken down in four parts:
+
+* _Discovery_ (the `seriesQuery` and `seriesFilters` keys) controls what Prometheus
+    metrics are considered for exposure via the k8s custom metrics API.
+
+  In the example, all the Seldon Prometheus metrics of the form `seldon_model_*_total` are
+  considered, excluding metrics pre-aggregated across all models (`.*_aggregate_.*`) as well as
+  the cummulative infer time per model (`.*_seconds_total`). For RPS, we are only interested in
+  the model inference count (`seldon_model_infer_total`)
+
+* _Association_ (the `resources` key) controls the Kubernetes resources that a particular
+    metric can be attached to or aggregated over.
+
+  The resources key defines an association between certain labels from the Prometheus metric and
+  k8s resources. For example, on line 17, `"model": {group: "mlops.seldon.io", resource: "model"}`
+  lets `prometheus-adapter` know that, for the selected Prometheus metrics, the value of the
+  "model" label represents the name of a k8s `model.mlops.seldon.io` CR.
+
+  One k8s custom metric is generated for each k8s resource associated with a prometheus metric.
+  In this way, it becomes possible to request the k8s custom metric values for
+  `models.mlops.seldon.io/iris` or for `servers.mlops.seldon.io/mlserver`.
+
+  The labels that *do not* refer to a `namespace` resource generate "namespaced" custom
+  metrics (the label values refer to resources which are part of a namespace) -- this
+  distinction becomes important when needing to fetch the metrics via kubectl, and in
+  understanding how certain Prometheus query template placeholders are replaced.
+
+
+* _Naming_ (the `name` key) configures the naming of the k8s custom metric.
+
+  In the example ConfigMap, this is configured to take the Prometheus metric named
+  `seldon_model_infer_total` and expose custom metric endpoints named `infer_rps`, which when
+  called return the result of a query over the Prometheus metric.
+
+  The matching over the Prometheus metric name uses regex group capture expressions (line 22),
+  which are then be referenced in the custom metric name (line 23).
 
-Apply the config, and restart the prometheus adapter deployment (this restart is required so
-that prometheus-adapter picks up the new config):
+* _Querying_ (the `metricsQuery` key) defines how a request for a specific k8s custom metric gets
+    converted into a Prometheus query.
+
+  The query can make use of the following placeholders:
+
+    - .Series is replaced by the discovered prometheus metric name (e.g. `seldon_model_infer_total`)
+    - .LabelMatchers, when requesting a namespaced metric for resource `X` with name `x` in
+    namespace `n`, is replaced by `X=~"x",namespace="n"`. For example, `model=~"iris0",
+    namespace="seldon-mesh"`. When requesting the namespace resource itself, only the
+    `namespace="n"` is kept.
+    - .GroupBy is replaced by the resource type of the requested metric (e.g. `model`,
+    `server`, `pod` or `namespace`).
+
+  You may want to modify the query in the example to match the one that you typically use in
+  your monitoring setup for RPS metrics. The example calls [`rate()`](https://prometheus.io/docs/prometheus/latest/querying/functions/#rate)
+  with a 1 minute window.
+
+
+For a complete reference for how `prometheus-adapter` can be configured via the `ConfigMap`, please
+consult the docs [here](https://github.com/kubernetes-sigs/prometheus-adapter/blob/master/docs/config.md).
+
+
+
+Once you have applied any necessary customizations, replace the default prometheus-adapter config
+with the new one, and restart the deployment (this restart is required so that prometheus-adapter
+picks up the new config):
 
 ```sh
-# Apply prometheus adapter config
-kubectl apply -f prometheus-adapter.config.yaml
-# Restart prom-adapter pods
+# Replace default prometheus adapter config
+kubectl replace -f prometheus-adapter.config.yaml
+# Restart prometheus-adapter pods
 kubectl rollout restart deployment hpa-metrics-prometheus-adapter -n seldon-monitoring
 ```
 
+### Testing the install using the custom metrics API
+
 In order to test that the prometheus adapter config works and everything is set up correctly,
-you can issue raw kubectl requests against the custom metrics API, as described below.
+you can issue raw kubectl requests against the custom metrics API
 
 {% hint style="info" %}
-If no inference requests were issued towards any model in the Seldon install, the metrics
-configured above will not be available in prometheus, and thus will also not appear when
-checking via the commands below. Therefore, please first run some inference requests towards a
-sample model to ensure that the metrics are available — this is only required for the testing of
-the install.
+If no inference requests were issued towards any model in the Core 2 install, the corresponding
+metrics will not be available in prometheus, and thus will also not appear when checking via
+kubectl. Therefore, please first run some inference requests towards a sample model to ensure
+that the metrics are available — this is only required for the testing of the install.
 {% endhint %}
 
-**Testing the prometheus-adapter install using the custom metrics API**
-
-List available metrics
+Listing the available metrics:
 
 ```sh
 kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/ | jq .
 ```
 
-Fetching model RPS metric for specific (namespace, model) pair:
+For namespaced metrics, the general template for fetching is:
 
 ```sh
-kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespaces/seldon-mesh/models.mlops.seldon.io/irisa0/infer_rps
+kubectl get --raw "/apis/custom.metrics.k8s.io/v1beta1/namespaces/[NAMESPACE]/[API_RESOURCE_NAME]/[CR_NAME]/[METRIC_NAME]"
 ```
 
-Fetching model RPS metric aggregated at the (namespace, server) level:
+For example:
 
-```sh
-kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespaces/seldon-mesh/servers.mlops.seldon.io/mlserver/infer_rps
-```
+* Fetching model RPS metric for a specific `(namespace, model)` pair `(seldon-mesh, irisa0)`:
 
-Fetching model RPS metric aggregated at the (namespace, pod) level:
+    ```sh
+    kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespaces/seldon-mesh/models.mlops.seldon.io/irisa0/infer_rps
+    ```
 
-```sh
-kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespaces/seldon-mesh/pods/mlserver-0/infer_rps
-```
+* Fetching model RPS metric aggregated at the `(namespace, server)` level `(seldon-mesh, mlserver)`:
+
+    ```sh
+    kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespaces/seldon-mesh/servers.mlops.seldon.io/mlserver/infer_rps
+    ```
 
-Fetching the same metric aggregated at namespace level:
+* Fetching model RPS metric aggregated at the `(namespace, pod)` level `(seldon-mesh, mlserver-0)`:
 
-```sh
-kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespaces/*/metrics/infer_rps
-```
+    ```sh
+    kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespaces/seldon-mesh/pods/mlserver-0/infer_rps
+    ```
 
-### Configuring HPA manifests
+* Fetching the same metric aggregated at `namespace` level `(seldon-mesh)`:
+
+    ```sh
+    kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1/namespaces/*/metrics/infer_rps
+    ```
+
+## Configuring HPA manifests
 
 For every (Model, Server) pair you want to autoscale, you need to apply 2 HPA manifests based on
 the same metric: one scaling the Model, the other the Server. The example below only works if
 the mapping between Models and Servers is 1-to-1 (i.e no multi-model serving).
 
 Consider a model named `irisa0` with the following manifest. Please note we don’t set
-`minReplicas/maxReplicas` this is in order to disable the seldon-specific autoscaling so that it
-doesn’t interact with HPA.
+`minReplicas/maxReplicas`. This disables the seldon lag-based autoscaling so that it
+doesn’t interact with HPA (separate `minReplicas/maxReplicas` configs will be set on the HPA
+side)
 
+You must also explicitly define a value for `spec.replicas`. This is the key modified by HPA
+to increase the number of replicas, and if not present in the manifest it will result in HPA not
+working until the Model CR is modified to have `spec.replicas` defined.
+
+{% code title="irisa0.yaml" lineNumbers="false" %}
 ```yaml
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Model
@@ -220,10 +241,12 @@ spec:
   - sklearn
   storageUri: gs://seldon-models/testing/iris1
 ```
+{% endcode %}
 
 Let’s scale this model when it is deployed on a server named `mlserver`, with a target RPS **per
 replica** of 3 RPS (higher RPS would trigger scale-up, lower would trigger scale-down):
 
+{% code title="irisa0-hpa.yaml" lineNumbers="true" %}
 ```yaml
 apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
@@ -275,9 +298,10 @@ spec:
         type: AverageValue
         averageValue: 3
 ```
+{% endcode %}
 
 In the preceding HPA manifests, the scaling metric is exactly the same, and uses the exact same
-parameters: this is to ensure that both the Models and the Servers are scaled up/down at
+parameters. This is to ensure that both the Models and the Servers are scaled up/down at
 approximately the same time. Small variations in the scale-up time are expected because each HPA
 samples the metrics independently, at regular intervals.
 
@@ -292,7 +316,7 @@ In order to ensure similar scaling behaviour between Models and Servers, the num
 `minReplicas` and `maxReplicas`, as well as any other configured scaling policies should be kept
 in sync across the HPA for the model and the server.
 
-#### Details on custom metrics of type Object
+### Details on custom metrics of type Object
 
 The HPA manifests use metrics of type "Object" that fetch the data used in scaling
 decisions by querying k8s metrics associated with a particular k8s object.  The endpoints that
@@ -355,13 +379,50 @@ by the Model HPA. Once a pod is owned by a given HPA it is not available for oth
 so we would no longer be able to scale the Server CRs using HPA.
 {% endhint %}
 
-#### Advanced settings
+### HPA sampling of custom metrics
+
+Each HPA CR has it's own timer on which it samples the specified custom metrics. This timer
+starts when the CR is created, with sampling of the metric being done at regular intervals (by
+default, 15 seconds).
 
-*   Filtering metrics by additional labels on the prometheus metric
+As a side effect of this, creating the Model HPA and the Server HPA (for a given model) at
+different times will mean that the scaling decisions on the two are taken at different times.
+Even when creating the two CRs together as part of the same manifest, there will usually be a
+small delay between the point where the Model and Server `spec.replicas` values are changed.
+
+Despite this delay, the two will converge to the same number when the decisions are taken based
+on the same metric (as in the previous examples).
+
+When showing the HPA CR information via `kubectl get`, a column of the output will display the
+current metric value per replica and the target average value in the format `[per replica metric value]/[target]`.
+This information is updated in accordance to the sampling rate of each HPA resource. It is
+therefore expected to sometimes see different metric values for the Model and it's corresponding
+Server.
+
+{% hint style="info" %}
+Some versions of k8s will display `[per pod metric value]` instead of `[per replica metric value]`,
+with the number of pods being computed based on a label selector present in the target resource
+CR (the `status.selector` value for the Model or Server in the Core 2 case).
+
+HPA is designed so that multiple HPA CRs cannot target the same underlying pod with this selector
+(with HPA stopping when such a condition is detected). This means that in Core 2, the Model and
+Server selector cannot be the same. A design choice was made to assign the Model a unique
+selector that does not match any pods.
+
+As a result, for the k8s versions displaying `[per pod metric value]`, the information shown for
+the Model HPA CR will be an overflow caused by division by zero. This is only a display artefact,
+with the Model HPA continuing to work normally. The actual value of the metric can be seen by
+inspecting the corresponding Server HPA CR, or by fetching the metric directly via `kubectl get
+--raw`
+{% endhint %}
+
+### Advanced settings
+
+*   Filtering metrics by additional labels on the prometheus metric:
 
     The prometheus metric from which the model RPS is computed has the following labels:
 
-    ```yaml
+    ```c-like
     seldon_model_infer_total{
         code="200", 
         container="agent", 
@@ -396,11 +457,182 @@ so we would no longer be able to scale the Server CRs using HPA.
               matchLabels:
                 method_type: rest
           target:
-    	type: AverageValue
+    	    type: AverageValue
             averageValue: "3"
     ```
-*   Customise scale-up / scale-down rate & properties by using scaling policies as described in
+*   Customize scale-up / scale-down rate & properties by using scaling policies as described in
     the [HPA scaling policies docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior)
 
 *   For more resources, please consult the [HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/)
     and the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/)
+
+
+## Cluster operation guidelines when using HPA-based scaling
+
+When deploying HPA-based scaling for Seldon Core 2 models and servers as part of a production deployment,
+it is important to understand the exact interactions between HPA-triggered actions and Seldon Core 2
+scheduling, as well as potential pitfalls in choosing particular HPA configurations.
+
+Using the default scaling policy, HPA is relatively aggressive on scale-up (responding quickly
+to increases in load), with a maximum replicas increase of either 4 every 15 seconds or 100% of
+existing replicas within the same period (**whichever is highest**). In contrast, scaling-down
+is more gradual, with HPA only scaling down to the maximum number of recommended replicas in the
+most recent 5 minute rolling window, in order to avoid flapping. Those parameters can be
+customized via [scaling policies](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#configurable-scaling-behavior).
+
+When using custom metrics such as RPS, the actual number of replicas added during scale-up or
+reduced during scale-down will entirely depend, alongside the maximums imposed by the policy, on
+the configured target (`averageValue` RPS per replica) and on how quickly the inferencing load
+varies in your cluster. All three need to be considered jointly in order to deliver both an
+efficient use of resources and meeting SLAs.
+
+### Customizing per-replica RPS targets and replica limits
+
+Naturally, the first thing to consider is an estimated peak inference load (including some
+margins) for each of the models in the cluster. If the minimum number of model
+replicas needed to serve that load without breaching latency SLAs is known, it should be set as
+`spec.maxReplicas`, with the HPA `target.averageValue` set to `peak_infer_RPS`/`maxReplicas`.
+
+If `maxReplicas` is not already known, an open-loop load test with a slowly ramping up request
+rate should be done on the target model (one replica, no scaling). This would allow you to
+determine the RPS (inference request throughput) when latency SLAs are breached or (depending on
+the desired operation point) when latency starts increasing. You would then set the HPA
+`target.averageValue` taking some margin below this saturation RPS, and compute
+`spec.maxReplicas` as `peak_infer_RPS`/`target.averageValue`. The margin taken below the
+saturation point is very important, because scaling-up cannot be instant (it requires spinning
+up new pods, downloading model artifacts, etc.). In the period until the new replicas become
+available, any load increases will still need to be absorbed by the existing replicas.
+
+If there are multiple models which typically experience peak load in a correlated manner, you
+need to ensure that sufficient cluster resources are available for k8s to concurrently schedule
+the maximum number of server pods, with each pod holding one model replica. This can be ensured
+by using either [Cluster Autoscaler](https://kubernetes.io/docs/concepts/cluster-administration/cluster-autoscaling/)
+or, when running workloads in the cloud, any provider-specific cluster autoscaling services.
+
+{% hint style="warning" %}
+It is important for the cluster to have sufficient resources for creating the total number of
+desired server replicas set by the HPA CRs across all the models at a given time.
+
+Not having sufficient cluster resources to serve the number of replicas configured by HPA at a
+given moment, in particular under aggressive scale-up HPA policies, may result in breaches of
+SLAs. This is discussed in more detail in the following section.
+{% endhint %}
+
+A similar approach should be taken for setting `minReplicas`, in relation to estimated RPS in
+the low-load regime. However, it's useful to balance lower resource usage to immediate
+availability of replicas for inference rate increases from that lowest load point. If low-load
+regimes only occur for small periods of time, and especially combined with a high rate of increase
+in RPS when moving out of the low-load regime, it might be worth to set the `minReplicas` floor
+higher in order to ensure SLAs are met at all times.
+
+
+### Customizing HPA policy settings for ensuring correct scaling behaviour
+
+Each `spec.replica` value change for a Model or Server triggers a rescheduling event for the
+Seldon Core 2 scheduler, which considers any updates that are needed in mapping Model replicas to
+Server replicas  such as rescheduling failed Model replicas, loading new ones, unloading in the case of the number of replicas going down, etc.
+
+Two characteristics in the current implementation are important in terms of
+autoscaling and configuring the HPA scale-up policy:
+
+- The scheduler does not create new Server replicas when the existing replicas are not
+    sufficient for loading a Model's replicas (one Model replica per Server replica). Whenever
+    a Model requests more replicas than available on any of the available Servers, its `ModelReady`
+    condition transitions to `Status: False` with a `ScheduleFailed` message. However, any
+    replicas of that Model that are already loaded at that point remain available for servicing
+    inference load.
+
+- There is no _partial_ scheduling of replicas. For example, consider a model with 2 replicas,
+    currently loaded on a server with 3 replicas (two of those server replicas will have the
+    model loaded). If you update the model replicas to 4, the scheduler will transition the
+    model to `ScheduleFailed`, seeing that it cannot satisfy the requested number of replicas.
+    The existing 2 model replicas will continue to serve traffic, *but a third replica will
+    not be loaded* onto the remaining server replica.
+
+    In other words, the scheduler either schedules all the requested replicas, or,
+    if unable to do so, leaves the state of the cluster unchanged.
+
+    Introducing partial scheduling would make the overall results of assigning models to servers
+    significantly less predictable and ephemeral. This is because models may end up moved
+    back-and forth between servers depending on the speed with which various server replicas
+    become available. Network partitions or other transient errors may also trigger large changes
+    to the model-to-server assignments, making it challenging to sustain consistent data plane
+    load during those periods.
+
+Taken together, the two Core 2 scheduling characteristics, combined with a very aggressive HPA
+scale-up policy and a continuously increasing RPS may lead to the following pathological case:
+
+- Based on RPS, HPA decides to increase both the Model and Server replicas from 2 (an example
+start stable state) to 8. While the 6 new Server pods get scheduled and get the Model loaded
+onto them, the scheduler will transition the Model into the `ScheduleFailed` state, because it
+cannot fulfill the requested replicas requirement. During this period, the initial 2 Model
+replicas continue to serve load, but are using their RPS margins and getting closer to the
+saturation point.
+- At the same time, load continues to increase, so HPA further increases the number of
+required Model and Server replicas from 8 to 12, before all of the 6 new Server pods had a chance
+to become available. The new replica target for the scheduler also becomes 12, and this would
+not be satisfied until all the 12 Server replicas are available. The 2 Model replicas that are
+available may by now be saturated and the infer latency spikes up, breaching set SLAs.
+- The process may continue until load stabilizes.
+- If at any point the number of requested replicas (<=`maxReplicas`) exceeds the resource
+capacity of the cluster, the requested server replica count will never be reached and thus the
+Model will remain permanently in the `ScheduleFailed` state.
+
+While most likely encountered during continuous ramp-up RPS load tests with autoscaling enabled,
+the pathological case example is a good showcase for the elements that need to be taken
+into account when setting the HPA policies.
+
+- The speed with which new Server replicas can become available versus how many new replicas may
+  HPA request in a given time:
+    - The HPA scale-up policy should not be configured to request more replicas than can
+      become available in the specified time. The following example reflects a confidence that 5
+      Server pods will become available within 90 seconds, with some safety margin. The default
+      scale-up config, that also adds a percentage based policy (double the existing replicas
+      within the set `periodSeconds`) is not recommended because of this.
+    - Perhaps more importantly, there is no reason to scale faster than the time it takes for
+      replicas to become available - this is the true maximum rate with which scaling up can
+      happen anyway. Because the underlying Server replica pods are part of a stateful set, they
+      are created sequentially by k8s.
+
+{% code title="hpa-custom-policy.yaml" lineNumbers="true" %}
+```yaml
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: irisa0-model-hpa
+  namespace: seldon-mesh
+spec:
+  scaleTargetRef:
+    ...
+  minReplicas: 1
+  maxReplicas: 3
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 60
+      policies:
+      - type: Pods
+        value: 5
+        periodSeconds: 90
+  metrics:
+    ...
+```
+{% endcode %}
+
+- The duration of transient load spikes which you might want to absorb within the existing
+  per-replica RPS margins.
+    - The previous example, at line 13, configures a scale-up stabilization window of one minute.
+      It means that for all of the HPA recommended replicas in the last 60 second window (4
+      samples of the custom metric considering the default sampling rate), only the *smallest*
+      will be applied.
+    - Such stabilization windows should be set depending on typical load patterns in your
+        cluster: not being too aggressive in reacting to increased load will allow you to
+        achieve cost savings, but has the disadvantage of a delayed reaction if the load spike turns
+        out to be sustained.
+
+- The duration of any typical/expected sustained ramp-up period, and the RPS increase rate
+    during this period.
+    - It is useful to consider whether the replica scale-up rate configured via the policy (line
+        15 in the example) is able to keep-up with this RPS increase rate.
+    - Such a scenario may appear, for example, if you are planning for a smooth traffic ramp-up
+        in a blue-green deployment as you are draining the "blue" deployment and transitioning
+        to the "green" one

From 7fc340255a6a0e5a36e1d80ea844ba78bc103493 Mon Sep 17 00:00:00 2001
From: Sherif Akoush <sa@seldon.io>
Date: Wed, 6 Nov 2024 14:10:45 +0000
Subject: [PATCH 4/4] fix: Use generation id to bootstrap version id (#6029)

* use generation id to bootstrap versions

* add event type for sync operation

* move event to response

* introduce a second stage of sync process after the scheduler is ready

* send the right resources based on the event type for the control plane

* update control plane test

* add test coverage

* update control plane test

* allow model progressing status update

* add test for generation id

* Update scheduler/pkg/store/memory.go

Co-authored-by: Lucian Carata <lc525@users.noreply.github.com>

---------

Co-authored-by: Lucian Carata <lc525@users.noreply.github.com>
---
 apis/go/mlops/scheduler/scheduler.pb.go    | 613 ++++++++++++---------
 apis/mlops/scheduler/scheduler.proto       |   7 +-
 operator/scheduler/client.go               |  47 +-
 operator/scheduler/control_plane.go        |   2 +-
 operator/scheduler/utils_test.go           |  12 +-
 scheduler/pkg/server/control_plane.go      |  20 +-
 scheduler/pkg/server/control_plane_test.go |  91 +++
 scheduler/pkg/store/memory.go              |  11 +-
 scheduler/pkg/store/memory_test.go         |  75 ++-
 9 files changed, 563 insertions(+), 315 deletions(-)

diff --git a/apis/go/mlops/scheduler/scheduler.pb.go b/apis/go/mlops/scheduler/scheduler.pb.go
index 54bc9aa90d..ee133d0b31 100644
--- a/apis/go/mlops/scheduler/scheduler.pb.go
+++ b/apis/go/mlops/scheduler/scheduler.pb.go
@@ -430,6 +430,55 @@ func (PipelineVersionState_PipelineStatus) EnumDescriptor() ([]byte, []int) {
 	return file_mlops_scheduler_scheduler_proto_rawDescGZIP(), []int{51, 0}
 }
 
+type ControlPlaneResponse_Event int32
+
+const (
+	ControlPlaneResponse_UNKNOWN_EVENT  ControlPlaneResponse_Event = 0
+	ControlPlaneResponse_SEND_SERVERS   ControlPlaneResponse_Event = 1 // initial sync for the servers
+	ControlPlaneResponse_SEND_RESOURCES ControlPlaneResponse_Event = 2 // send models / pipelines / experiments
+)
+
+// Enum value maps for ControlPlaneResponse_Event.
+var (
+	ControlPlaneResponse_Event_name = map[int32]string{
+		0: "UNKNOWN_EVENT",
+		1: "SEND_SERVERS",
+		2: "SEND_RESOURCES",
+	}
+	ControlPlaneResponse_Event_value = map[string]int32{
+		"UNKNOWN_EVENT":  0,
+		"SEND_SERVERS":   1,
+		"SEND_RESOURCES": 2,
+	}
+)
+
+func (x ControlPlaneResponse_Event) Enum() *ControlPlaneResponse_Event {
+	p := new(ControlPlaneResponse_Event)
+	*p = x
+	return p
+}
+
+func (x ControlPlaneResponse_Event) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (ControlPlaneResponse_Event) Descriptor() protoreflect.EnumDescriptor {
+	return file_mlops_scheduler_scheduler_proto_enumTypes[7].Descriptor()
+}
+
+func (ControlPlaneResponse_Event) Type() protoreflect.EnumType {
+	return &file_mlops_scheduler_scheduler_proto_enumTypes[7]
+}
+
+func (x ControlPlaneResponse_Event) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use ControlPlaneResponse_Event.Descriptor instead.
+func (ControlPlaneResponse_Event) EnumDescriptor() ([]byte, []int) {
+	return file_mlops_scheduler_scheduler_proto_rawDescGZIP(), []int{55, 0}
+}
+
 type LoadModelRequest struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -3763,6 +3812,8 @@ type ControlPlaneResponse struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
+
+	Event ControlPlaneResponse_Event `protobuf:"varint,1,opt,name=event,proto3,enum=seldon.mlops.scheduler.ControlPlaneResponse_Event" json:"event,omitempty"`
 }
 
 func (x *ControlPlaneResponse) Reset() {
@@ -3797,6 +3848,13 @@ func (*ControlPlaneResponse) Descriptor() ([]byte, []int) {
 	return file_mlops_scheduler_scheduler_proto_rawDescGZIP(), []int{55}
 }
 
+func (x *ControlPlaneResponse) GetEvent() ControlPlaneResponse_Event {
+	if x != nil {
+		return x.Event
+	}
+	return ControlPlaneResponse_UNKNOWN_EVENT
+}
+
 var File_mlops_scheduler_scheduler_proto protoreflect.FileDescriptor
 
 var file_mlops_scheduler_scheduler_proto_rawDesc = []byte{
@@ -4409,141 +4467,150 @@ var file_mlops_scheduler_scheduler_proto_rawDesc = []byte{
 	0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
 	0x26, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x4e, 0x61, 0x6d,
 	0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
-	0x62, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x16, 0x0a, 0x14, 0x43, 0x6f, 0x6e, 0x74, 0x72,
-	0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a,
-	0x27, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12,
-	0x09, 0x0a, 0x05, 0x4d, 0x4f, 0x44, 0x45, 0x4c, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x49,
-	0x50, 0x45, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x32, 0xde, 0x0f, 0x0a, 0x09, 0x53, 0x63, 0x68,
-	0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x12, 0x6b, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
-	0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e,
-	0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e,
-	0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f,
+	0x62, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xa2, 0x01, 0x0a, 0x14, 0x43, 0x6f, 0x6e, 0x74,
+	0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
+	0x12, 0x48, 0x0a, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32,
+	0x32, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73,
+	0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c,
+	0x50, 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x45, 0x76,
+	0x65, 0x6e, 0x74, 0x52, 0x05, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x22, 0x40, 0x0a, 0x05, 0x45, 0x76,
+	0x65, 0x6e, 0x74, 0x12, 0x11, 0x0a, 0x0d, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x45,
+	0x56, 0x45, 0x4e, 0x54, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x45, 0x4e, 0x44, 0x5f, 0x53,
+	0x45, 0x52, 0x56, 0x45, 0x52, 0x53, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x45, 0x4e, 0x44,
+	0x5f, 0x52, 0x45, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x53, 0x10, 0x02, 0x2a, 0x27, 0x0a, 0x0c,
+	0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05,
+	0x4d, 0x4f, 0x44, 0x45, 0x4c, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x50, 0x49, 0x50, 0x45, 0x4c,
+	0x49, 0x4e, 0x45, 0x10, 0x01, 0x32, 0xde, 0x0f, 0x0a, 0x09, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75,
+	0x6c, 0x65, 0x72, 0x12, 0x6b, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4e, 0x6f, 0x74,
+	0x69, 0x66, 0x79, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f,
 	0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72,
-	0x76, 0x65, 0x72, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x09, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c,
-	0x12, 0x28, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e,
-	0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f,
-	0x64, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x73, 0x65, 0x6c,
-	0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75,
-	0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x0b, 0x55, 0x6e, 0x6c, 0x6f, 0x61,
-	0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e,
-	0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e,
-	0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65,
-	0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70,
-	0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f,
-	0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x00, 0x12, 0x6b, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e,
-	0x65, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73,
-	0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50,
-	0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c,
+	0x76, 0x65, 0x72, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e,
+	0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72,
+	0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
+	0x12, 0x62, 0x0a, 0x09, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x12, 0x28, 0x2e,
+	0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68,
+	0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e,
+	0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72,
+	0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x0b, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4d, 0x6f,
+	0x64, 0x65, 0x6c, 0x12, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f,
+	0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c,
+	0x6f, 0x61, 0x64, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
+	0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73,
+	0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4d,
+	0x6f, 0x64, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6b,
+	0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x2b,
 	0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63,
 	0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65,
-	0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x71,
-	0x0a, 0x0e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65,
-	0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e,
-	0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64,
-	0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
-	0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73,
-	0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50,
-	0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x00, 0x12, 0x74, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x72, 0x74, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69,
-	0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c,
-	0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x74,
-	0x61, 0x72, 0x74, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c,
-	0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x74,
-	0x61, 0x72, 0x74, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x0e, 0x53, 0x74, 0x6f, 0x70, 0x45,
-	0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64,
-	0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c,
-	0x65, 0x72, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e,
-	0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f,
-	0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65,
-	0x72, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6d, 0x0a, 0x0c, 0x53, 0x65,
-	0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c,
-	0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75,
-	0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e,
-	0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72,
-	0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x6a, 0x0a, 0x0b, 0x4d, 0x6f, 0x64,
-	0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f,
-	0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65,
-	0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c,
-	0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f,
-	0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x73, 0x0a, 0x0e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e,
-	0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e,
-	0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72,
-	0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52,
-	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e,
+	0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65,
+	0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64,
+	0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e,
+	0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x0e, 0x55,
+	0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x2d, 0x2e,
+	0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68,
+	0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70,
+	0x65, 0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73,
+	0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65,
+	0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x69, 0x70, 0x65,
+	0x6c, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x74,
+	0x0a, 0x0f, 0x53, 0x74, 0x61, 0x72, 0x74, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e,
+	0x74, 0x12, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73,
+	0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74,
+	0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x2f, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73,
+	0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74,
+	0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x22, 0x00, 0x12, 0x71, 0x0a, 0x0e, 0x53, 0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65,
+	0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e,
 	0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e,
-	0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x10, 0x45, 0x78,
-	0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f,
-	0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63,
-	0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65,
-	0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
-	0x30, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73,
-	0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d,
-	0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
-	0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x74, 0x0a, 0x0f, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c,
-	0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f,
-	0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65,
-	0x72, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75,
-	0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f,
-	0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65,
-	0x72, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75,
-	0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7c, 0x0a, 0x15, 0x53,
-	0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74,
-	0x61, 0x74, 0x75, 0x73, 0x12, 0x31, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c,
-	0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65,
-	0x72, 0x76, 0x65, 0x72, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e,
+	0x53, 0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d,
+	0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53,
+	0x74, 0x6f, 0x70, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73,
+	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6d, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e,
 	0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72,
-	0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x14, 0x53, 0x75, 0x62,
-	0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75,
-	0x73, 0x12, 0x30, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73,
+	0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c,
+	0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x6a, 0x0a, 0x0b, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53,
+	0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d,
+	0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d,
+	0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+	0x74, 0x1a, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73,
 	0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c,
-	0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f,
-	0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64,
-	0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x22, 0x00, 0x30, 0x01, 0x12, 0x88, 0x01, 0x0a, 0x19, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
-	0x62, 0x65, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74,
-	0x75, 0x73, 0x12, 0x35, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70,
-	0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65,
-	0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69,
-	0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x73, 0x65, 0x6c, 0x64,
-	0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c,
-	0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61,
-	0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12,
-	0x82, 0x01, 0x0a, 0x17, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x50, 0x69, 0x70,
-	0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x33, 0x2e, 0x73, 0x65,
+	0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
+	0x30, 0x01, 0x12, 0x73, 0x0a, 0x0e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74,
+	0x61, 0x74, 0x75, 0x73, 0x12, 0x2d, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c,
+	0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69,
+	0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75,
+	0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f,
+	0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70,
+	0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x10, 0x45, 0x78, 0x70, 0x65, 0x72,
+	0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2f, 0x2e, 0x73, 0x65,
 	0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64,
-	0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x75, 0x62,
-	0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e,
-	0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69,
-	0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
-	0x22, 0x00, 0x30, 0x01, 0x12, 0x82, 0x01, 0x0a, 0x15, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69,
-	0x62, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x12, 0x37,
+	0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53,
+	0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x73,
+	0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65,
+	0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74,
+	0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
+	0x30, 0x01, 0x12, 0x74, 0x0a, 0x0f, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53,
+	0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2e, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d,
+	0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53,
+	0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2f, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d,
+	0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53,
+	0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65,
+	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x7c, 0x0a, 0x15, 0x53, 0x75, 0x62, 0x73,
+	0x63, 0x72, 0x69, 0x62, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75,
+	0x73, 0x12, 0x31, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73,
+	0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65,
+	0x72, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c,
+	0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x53, 0x65,
+	0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x79, 0x0a, 0x14, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72,
+	0x69, 0x62, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x30,
 	0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63,
-	0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50,
-	0x6c, 0x61, 0x6e, 0x65, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e,
-	0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72,
-	0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73,
-	0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74,
-	0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x69, 0x6f,
-	0x2f, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2d, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x61, 0x70, 0x69,
-	0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x32, 0x2f, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2f, 0x73, 0x63,
-	0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x75, 0x62,
+	0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x1a, 0x2b, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e,
+	0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53,
+	0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30,
+	0x01, 0x12, 0x88, 0x01, 0x0a, 0x19, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x45,
+	0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12,
+	0x35, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73,
+	0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d,
+	0x65, 0x6e, 0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e,
+	0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e,
+	0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6d, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73,
+	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x82, 0x01, 0x0a,
+	0x17, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69,
+	0x6e, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x33, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f,
+	0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65,
+	0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72,
+	0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e,
+	0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68,
+	0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x50, 0x69, 0x70, 0x65, 0x6c, 0x69, 0x6e, 0x65, 0x53,
+	0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30,
+	0x01, 0x12, 0x82, 0x01, 0x0a, 0x15, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x43,
+	0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x12, 0x37, 0x2e, 0x73, 0x65,
+	0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64,
+	0x75, 0x6c, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e,
+	0x65, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71,
+	0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x2e, 0x6d, 0x6c,
+	0x6f, 0x70, 0x73, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x43, 0x6f,
+	0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
+	0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x6c, 0x64, 0x6f, 0x6e, 0x69, 0x6f, 0x2f, 0x73, 0x65,
+	0x6c, 0x64, 0x6f, 0x6e, 0x2d, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67,
+	0x6f, 0x2f, 0x76, 0x32, 0x2f, 0x6d, 0x6c, 0x6f, 0x70, 0x73, 0x2f, 0x73, 0x63, 0x68, 0x65, 0x64,
+	0x75, 0x6c, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -4558,7 +4625,7 @@ func file_mlops_scheduler_scheduler_proto_rawDescGZIP() []byte {
 	return file_mlops_scheduler_scheduler_proto_rawDescData
 }
 
-var file_mlops_scheduler_scheduler_proto_enumTypes = make([]protoimpl.EnumInfo, 7)
+var file_mlops_scheduler_scheduler_proto_enumTypes = make([]protoimpl.EnumInfo, 8)
 var file_mlops_scheduler_scheduler_proto_msgTypes = make([]protoimpl.MessageInfo, 60)
 var file_mlops_scheduler_scheduler_proto_goTypes = []any{
 	(ResourceType)(0),                         // 0: seldon.mlops.scheduler.ResourceType
@@ -4568,160 +4635,162 @@ var file_mlops_scheduler_scheduler_proto_goTypes = []any{
 	(PipelineInput_JoinOp)(0),                 // 4: seldon.mlops.scheduler.PipelineInput.JoinOp
 	(PipelineOutput_JoinOp)(0),                // 5: seldon.mlops.scheduler.PipelineOutput.JoinOp
 	(PipelineVersionState_PipelineStatus)(0),  // 6: seldon.mlops.scheduler.PipelineVersionState.PipelineStatus
-	(*LoadModelRequest)(nil),                  // 7: seldon.mlops.scheduler.LoadModelRequest
-	(*Model)(nil),                             // 8: seldon.mlops.scheduler.Model
-	(*MetaData)(nil),                          // 9: seldon.mlops.scheduler.MetaData
-	(*DeploymentSpec)(nil),                    // 10: seldon.mlops.scheduler.DeploymentSpec
-	(*ModelSpec)(nil),                         // 11: seldon.mlops.scheduler.ModelSpec
-	(*ParameterSpec)(nil),                     // 12: seldon.mlops.scheduler.ParameterSpec
-	(*ExplainerSpec)(nil),                     // 13: seldon.mlops.scheduler.ExplainerSpec
-	(*KubernetesMeta)(nil),                    // 14: seldon.mlops.scheduler.KubernetesMeta
-	(*StreamSpec)(nil),                        // 15: seldon.mlops.scheduler.StreamSpec
-	(*StorageConfig)(nil),                     // 16: seldon.mlops.scheduler.StorageConfig
-	(*LoadModelResponse)(nil),                 // 17: seldon.mlops.scheduler.LoadModelResponse
-	(*ModelReference)(nil),                    // 18: seldon.mlops.scheduler.ModelReference
-	(*UnloadModelRequest)(nil),                // 19: seldon.mlops.scheduler.UnloadModelRequest
-	(*UnloadModelResponse)(nil),               // 20: seldon.mlops.scheduler.UnloadModelResponse
-	(*ModelStatusResponse)(nil),               // 21: seldon.mlops.scheduler.ModelStatusResponse
-	(*ModelVersionStatus)(nil),                // 22: seldon.mlops.scheduler.ModelVersionStatus
-	(*ModelStatus)(nil),                       // 23: seldon.mlops.scheduler.ModelStatus
-	(*ModelReplicaStatus)(nil),                // 24: seldon.mlops.scheduler.ModelReplicaStatus
-	(*ServerStatusRequest)(nil),               // 25: seldon.mlops.scheduler.ServerStatusRequest
-	(*ServerStatusResponse)(nil),              // 26: seldon.mlops.scheduler.ServerStatusResponse
-	(*ServerReplicaResources)(nil),            // 27: seldon.mlops.scheduler.ServerReplicaResources
-	(*ModelSubscriptionRequest)(nil),          // 28: seldon.mlops.scheduler.ModelSubscriptionRequest
-	(*ModelStatusRequest)(nil),                // 29: seldon.mlops.scheduler.ModelStatusRequest
-	(*ServerNotifyRequest)(nil),               // 30: seldon.mlops.scheduler.ServerNotifyRequest
-	(*ServerNotify)(nil),                      // 31: seldon.mlops.scheduler.ServerNotify
-	(*ServerNotifyResponse)(nil),              // 32: seldon.mlops.scheduler.ServerNotifyResponse
-	(*ServerSubscriptionRequest)(nil),         // 33: seldon.mlops.scheduler.ServerSubscriptionRequest
-	(*StartExperimentRequest)(nil),            // 34: seldon.mlops.scheduler.StartExperimentRequest
-	(*Experiment)(nil),                        // 35: seldon.mlops.scheduler.Experiment
-	(*ExperimentConfig)(nil),                  // 36: seldon.mlops.scheduler.ExperimentConfig
-	(*ExperimentCandidate)(nil),               // 37: seldon.mlops.scheduler.ExperimentCandidate
-	(*ExperimentMirror)(nil),                  // 38: seldon.mlops.scheduler.ExperimentMirror
-	(*StartExperimentResponse)(nil),           // 39: seldon.mlops.scheduler.StartExperimentResponse
-	(*StopExperimentRequest)(nil),             // 40: seldon.mlops.scheduler.StopExperimentRequest
-	(*StopExperimentResponse)(nil),            // 41: seldon.mlops.scheduler.StopExperimentResponse
-	(*ExperimentSubscriptionRequest)(nil),     // 42: seldon.mlops.scheduler.ExperimentSubscriptionRequest
-	(*ExperimentStatusResponse)(nil),          // 43: seldon.mlops.scheduler.ExperimentStatusResponse
-	(*LoadPipelineRequest)(nil),               // 44: seldon.mlops.scheduler.LoadPipelineRequest
-	(*ExperimentStatusRequest)(nil),           // 45: seldon.mlops.scheduler.ExperimentStatusRequest
-	(*Pipeline)(nil),                          // 46: seldon.mlops.scheduler.Pipeline
-	(*PipelineStep)(nil),                      // 47: seldon.mlops.scheduler.PipelineStep
-	(*Batch)(nil),                             // 48: seldon.mlops.scheduler.Batch
-	(*PipelineInput)(nil),                     // 49: seldon.mlops.scheduler.PipelineInput
-	(*PipelineOutput)(nil),                    // 50: seldon.mlops.scheduler.PipelineOutput
-	(*LoadPipelineResponse)(nil),              // 51: seldon.mlops.scheduler.LoadPipelineResponse
-	(*UnloadPipelineRequest)(nil),             // 52: seldon.mlops.scheduler.UnloadPipelineRequest
-	(*UnloadPipelineResponse)(nil),            // 53: seldon.mlops.scheduler.UnloadPipelineResponse
-	(*PipelineStatusRequest)(nil),             // 54: seldon.mlops.scheduler.PipelineStatusRequest
-	(*PipelineSubscriptionRequest)(nil),       // 55: seldon.mlops.scheduler.PipelineSubscriptionRequest
-	(*PipelineStatusResponse)(nil),            // 56: seldon.mlops.scheduler.PipelineStatusResponse
-	(*PipelineWithState)(nil),                 // 57: seldon.mlops.scheduler.PipelineWithState
-	(*PipelineVersionState)(nil),              // 58: seldon.mlops.scheduler.PipelineVersionState
-	(*SchedulerStatusRequest)(nil),            // 59: seldon.mlops.scheduler.SchedulerStatusRequest
-	(*SchedulerStatusResponse)(nil),           // 60: seldon.mlops.scheduler.SchedulerStatusResponse
-	(*ControlPlaneSubscriptionRequest)(nil),   // 61: seldon.mlops.scheduler.ControlPlaneSubscriptionRequest
-	(*ControlPlaneResponse)(nil),              // 62: seldon.mlops.scheduler.ControlPlaneResponse
-	nil,                                       // 63: seldon.mlops.scheduler.ModelVersionStatus.ModelReplicaStateEntry
-	nil,                                       // 64: seldon.mlops.scheduler.PipelineStep.TensorMapEntry
-	nil,                                       // 65: seldon.mlops.scheduler.PipelineInput.TensorMapEntry
-	nil,                                       // 66: seldon.mlops.scheduler.PipelineOutput.TensorMapEntry
-	(*timestamppb.Timestamp)(nil),             // 67: google.protobuf.Timestamp
+	(ControlPlaneResponse_Event)(0),           // 7: seldon.mlops.scheduler.ControlPlaneResponse.Event
+	(*LoadModelRequest)(nil),                  // 8: seldon.mlops.scheduler.LoadModelRequest
+	(*Model)(nil),                             // 9: seldon.mlops.scheduler.Model
+	(*MetaData)(nil),                          // 10: seldon.mlops.scheduler.MetaData
+	(*DeploymentSpec)(nil),                    // 11: seldon.mlops.scheduler.DeploymentSpec
+	(*ModelSpec)(nil),                         // 12: seldon.mlops.scheduler.ModelSpec
+	(*ParameterSpec)(nil),                     // 13: seldon.mlops.scheduler.ParameterSpec
+	(*ExplainerSpec)(nil),                     // 14: seldon.mlops.scheduler.ExplainerSpec
+	(*KubernetesMeta)(nil),                    // 15: seldon.mlops.scheduler.KubernetesMeta
+	(*StreamSpec)(nil),                        // 16: seldon.mlops.scheduler.StreamSpec
+	(*StorageConfig)(nil),                     // 17: seldon.mlops.scheduler.StorageConfig
+	(*LoadModelResponse)(nil),                 // 18: seldon.mlops.scheduler.LoadModelResponse
+	(*ModelReference)(nil),                    // 19: seldon.mlops.scheduler.ModelReference
+	(*UnloadModelRequest)(nil),                // 20: seldon.mlops.scheduler.UnloadModelRequest
+	(*UnloadModelResponse)(nil),               // 21: seldon.mlops.scheduler.UnloadModelResponse
+	(*ModelStatusResponse)(nil),               // 22: seldon.mlops.scheduler.ModelStatusResponse
+	(*ModelVersionStatus)(nil),                // 23: seldon.mlops.scheduler.ModelVersionStatus
+	(*ModelStatus)(nil),                       // 24: seldon.mlops.scheduler.ModelStatus
+	(*ModelReplicaStatus)(nil),                // 25: seldon.mlops.scheduler.ModelReplicaStatus
+	(*ServerStatusRequest)(nil),               // 26: seldon.mlops.scheduler.ServerStatusRequest
+	(*ServerStatusResponse)(nil),              // 27: seldon.mlops.scheduler.ServerStatusResponse
+	(*ServerReplicaResources)(nil),            // 28: seldon.mlops.scheduler.ServerReplicaResources
+	(*ModelSubscriptionRequest)(nil),          // 29: seldon.mlops.scheduler.ModelSubscriptionRequest
+	(*ModelStatusRequest)(nil),                // 30: seldon.mlops.scheduler.ModelStatusRequest
+	(*ServerNotifyRequest)(nil),               // 31: seldon.mlops.scheduler.ServerNotifyRequest
+	(*ServerNotify)(nil),                      // 32: seldon.mlops.scheduler.ServerNotify
+	(*ServerNotifyResponse)(nil),              // 33: seldon.mlops.scheduler.ServerNotifyResponse
+	(*ServerSubscriptionRequest)(nil),         // 34: seldon.mlops.scheduler.ServerSubscriptionRequest
+	(*StartExperimentRequest)(nil),            // 35: seldon.mlops.scheduler.StartExperimentRequest
+	(*Experiment)(nil),                        // 36: seldon.mlops.scheduler.Experiment
+	(*ExperimentConfig)(nil),                  // 37: seldon.mlops.scheduler.ExperimentConfig
+	(*ExperimentCandidate)(nil),               // 38: seldon.mlops.scheduler.ExperimentCandidate
+	(*ExperimentMirror)(nil),                  // 39: seldon.mlops.scheduler.ExperimentMirror
+	(*StartExperimentResponse)(nil),           // 40: seldon.mlops.scheduler.StartExperimentResponse
+	(*StopExperimentRequest)(nil),             // 41: seldon.mlops.scheduler.StopExperimentRequest
+	(*StopExperimentResponse)(nil),            // 42: seldon.mlops.scheduler.StopExperimentResponse
+	(*ExperimentSubscriptionRequest)(nil),     // 43: seldon.mlops.scheduler.ExperimentSubscriptionRequest
+	(*ExperimentStatusResponse)(nil),          // 44: seldon.mlops.scheduler.ExperimentStatusResponse
+	(*LoadPipelineRequest)(nil),               // 45: seldon.mlops.scheduler.LoadPipelineRequest
+	(*ExperimentStatusRequest)(nil),           // 46: seldon.mlops.scheduler.ExperimentStatusRequest
+	(*Pipeline)(nil),                          // 47: seldon.mlops.scheduler.Pipeline
+	(*PipelineStep)(nil),                      // 48: seldon.mlops.scheduler.PipelineStep
+	(*Batch)(nil),                             // 49: seldon.mlops.scheduler.Batch
+	(*PipelineInput)(nil),                     // 50: seldon.mlops.scheduler.PipelineInput
+	(*PipelineOutput)(nil),                    // 51: seldon.mlops.scheduler.PipelineOutput
+	(*LoadPipelineResponse)(nil),              // 52: seldon.mlops.scheduler.LoadPipelineResponse
+	(*UnloadPipelineRequest)(nil),             // 53: seldon.mlops.scheduler.UnloadPipelineRequest
+	(*UnloadPipelineResponse)(nil),            // 54: seldon.mlops.scheduler.UnloadPipelineResponse
+	(*PipelineStatusRequest)(nil),             // 55: seldon.mlops.scheduler.PipelineStatusRequest
+	(*PipelineSubscriptionRequest)(nil),       // 56: seldon.mlops.scheduler.PipelineSubscriptionRequest
+	(*PipelineStatusResponse)(nil),            // 57: seldon.mlops.scheduler.PipelineStatusResponse
+	(*PipelineWithState)(nil),                 // 58: seldon.mlops.scheduler.PipelineWithState
+	(*PipelineVersionState)(nil),              // 59: seldon.mlops.scheduler.PipelineVersionState
+	(*SchedulerStatusRequest)(nil),            // 60: seldon.mlops.scheduler.SchedulerStatusRequest
+	(*SchedulerStatusResponse)(nil),           // 61: seldon.mlops.scheduler.SchedulerStatusResponse
+	(*ControlPlaneSubscriptionRequest)(nil),   // 62: seldon.mlops.scheduler.ControlPlaneSubscriptionRequest
+	(*ControlPlaneResponse)(nil),              // 63: seldon.mlops.scheduler.ControlPlaneResponse
+	nil,                                       // 64: seldon.mlops.scheduler.ModelVersionStatus.ModelReplicaStateEntry
+	nil,                                       // 65: seldon.mlops.scheduler.PipelineStep.TensorMapEntry
+	nil,                                       // 66: seldon.mlops.scheduler.PipelineInput.TensorMapEntry
+	nil,                                       // 67: seldon.mlops.scheduler.PipelineOutput.TensorMapEntry
+	(*timestamppb.Timestamp)(nil),             // 68: google.protobuf.Timestamp
 }
 var file_mlops_scheduler_scheduler_proto_depIdxs = []int32{
-	8,  // 0: seldon.mlops.scheduler.LoadModelRequest.model:type_name -> seldon.mlops.scheduler.Model
-	9,  // 1: seldon.mlops.scheduler.Model.meta:type_name -> seldon.mlops.scheduler.MetaData
-	11, // 2: seldon.mlops.scheduler.Model.modelSpec:type_name -> seldon.mlops.scheduler.ModelSpec
-	10, // 3: seldon.mlops.scheduler.Model.deploymentSpec:type_name -> seldon.mlops.scheduler.DeploymentSpec
-	15, // 4: seldon.mlops.scheduler.Model.streamSpec:type_name -> seldon.mlops.scheduler.StreamSpec
-	14, // 5: seldon.mlops.scheduler.MetaData.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
-	16, // 6: seldon.mlops.scheduler.ModelSpec.storageConfig:type_name -> seldon.mlops.scheduler.StorageConfig
-	13, // 7: seldon.mlops.scheduler.ModelSpec.explainer:type_name -> seldon.mlops.scheduler.ExplainerSpec
-	12, // 8: seldon.mlops.scheduler.ModelSpec.parameters:type_name -> seldon.mlops.scheduler.ParameterSpec
-	18, // 9: seldon.mlops.scheduler.UnloadModelRequest.model:type_name -> seldon.mlops.scheduler.ModelReference
-	14, // 10: seldon.mlops.scheduler.UnloadModelRequest.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
-	22, // 11: seldon.mlops.scheduler.ModelStatusResponse.versions:type_name -> seldon.mlops.scheduler.ModelVersionStatus
-	14, // 12: seldon.mlops.scheduler.ModelVersionStatus.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
-	63, // 13: seldon.mlops.scheduler.ModelVersionStatus.modelReplicaState:type_name -> seldon.mlops.scheduler.ModelVersionStatus.ModelReplicaStateEntry
-	23, // 14: seldon.mlops.scheduler.ModelVersionStatus.state:type_name -> seldon.mlops.scheduler.ModelStatus
-	8,  // 15: seldon.mlops.scheduler.ModelVersionStatus.modelDefn:type_name -> seldon.mlops.scheduler.Model
+	9,  // 0: seldon.mlops.scheduler.LoadModelRequest.model:type_name -> seldon.mlops.scheduler.Model
+	10, // 1: seldon.mlops.scheduler.Model.meta:type_name -> seldon.mlops.scheduler.MetaData
+	12, // 2: seldon.mlops.scheduler.Model.modelSpec:type_name -> seldon.mlops.scheduler.ModelSpec
+	11, // 3: seldon.mlops.scheduler.Model.deploymentSpec:type_name -> seldon.mlops.scheduler.DeploymentSpec
+	16, // 4: seldon.mlops.scheduler.Model.streamSpec:type_name -> seldon.mlops.scheduler.StreamSpec
+	15, // 5: seldon.mlops.scheduler.MetaData.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
+	17, // 6: seldon.mlops.scheduler.ModelSpec.storageConfig:type_name -> seldon.mlops.scheduler.StorageConfig
+	14, // 7: seldon.mlops.scheduler.ModelSpec.explainer:type_name -> seldon.mlops.scheduler.ExplainerSpec
+	13, // 8: seldon.mlops.scheduler.ModelSpec.parameters:type_name -> seldon.mlops.scheduler.ParameterSpec
+	19, // 9: seldon.mlops.scheduler.UnloadModelRequest.model:type_name -> seldon.mlops.scheduler.ModelReference
+	15, // 10: seldon.mlops.scheduler.UnloadModelRequest.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
+	23, // 11: seldon.mlops.scheduler.ModelStatusResponse.versions:type_name -> seldon.mlops.scheduler.ModelVersionStatus
+	15, // 12: seldon.mlops.scheduler.ModelVersionStatus.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
+	64, // 13: seldon.mlops.scheduler.ModelVersionStatus.modelReplicaState:type_name -> seldon.mlops.scheduler.ModelVersionStatus.ModelReplicaStateEntry
+	24, // 14: seldon.mlops.scheduler.ModelVersionStatus.state:type_name -> seldon.mlops.scheduler.ModelStatus
+	9,  // 15: seldon.mlops.scheduler.ModelVersionStatus.modelDefn:type_name -> seldon.mlops.scheduler.Model
 	1,  // 16: seldon.mlops.scheduler.ModelStatus.state:type_name -> seldon.mlops.scheduler.ModelStatus.ModelState
-	67, // 17: seldon.mlops.scheduler.ModelStatus.lastChangeTimestamp:type_name -> google.protobuf.Timestamp
+	68, // 17: seldon.mlops.scheduler.ModelStatus.lastChangeTimestamp:type_name -> google.protobuf.Timestamp
 	2,  // 18: seldon.mlops.scheduler.ModelReplicaStatus.state:type_name -> seldon.mlops.scheduler.ModelReplicaStatus.ModelReplicaState
-	67, // 19: seldon.mlops.scheduler.ModelReplicaStatus.lastChangeTimestamp:type_name -> google.protobuf.Timestamp
-	27, // 20: seldon.mlops.scheduler.ServerStatusResponse.resources:type_name -> seldon.mlops.scheduler.ServerReplicaResources
-	14, // 21: seldon.mlops.scheduler.ServerStatusResponse.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
-	18, // 22: seldon.mlops.scheduler.ModelStatusRequest.model:type_name -> seldon.mlops.scheduler.ModelReference
-	31, // 23: seldon.mlops.scheduler.ServerNotifyRequest.servers:type_name -> seldon.mlops.scheduler.ServerNotify
-	14, // 24: seldon.mlops.scheduler.ServerNotify.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
-	35, // 25: seldon.mlops.scheduler.StartExperimentRequest.experiment:type_name -> seldon.mlops.scheduler.Experiment
-	37, // 26: seldon.mlops.scheduler.Experiment.candidates:type_name -> seldon.mlops.scheduler.ExperimentCandidate
-	38, // 27: seldon.mlops.scheduler.Experiment.mirror:type_name -> seldon.mlops.scheduler.ExperimentMirror
-	36, // 28: seldon.mlops.scheduler.Experiment.config:type_name -> seldon.mlops.scheduler.ExperimentConfig
-	14, // 29: seldon.mlops.scheduler.Experiment.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
+	68, // 19: seldon.mlops.scheduler.ModelReplicaStatus.lastChangeTimestamp:type_name -> google.protobuf.Timestamp
+	28, // 20: seldon.mlops.scheduler.ServerStatusResponse.resources:type_name -> seldon.mlops.scheduler.ServerReplicaResources
+	15, // 21: seldon.mlops.scheduler.ServerStatusResponse.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
+	19, // 22: seldon.mlops.scheduler.ModelStatusRequest.model:type_name -> seldon.mlops.scheduler.ModelReference
+	32, // 23: seldon.mlops.scheduler.ServerNotifyRequest.servers:type_name -> seldon.mlops.scheduler.ServerNotify
+	15, // 24: seldon.mlops.scheduler.ServerNotify.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
+	36, // 25: seldon.mlops.scheduler.StartExperimentRequest.experiment:type_name -> seldon.mlops.scheduler.Experiment
+	38, // 26: seldon.mlops.scheduler.Experiment.candidates:type_name -> seldon.mlops.scheduler.ExperimentCandidate
+	39, // 27: seldon.mlops.scheduler.Experiment.mirror:type_name -> seldon.mlops.scheduler.ExperimentMirror
+	37, // 28: seldon.mlops.scheduler.Experiment.config:type_name -> seldon.mlops.scheduler.ExperimentConfig
+	15, // 29: seldon.mlops.scheduler.Experiment.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
 	0,  // 30: seldon.mlops.scheduler.Experiment.resourceType:type_name -> seldon.mlops.scheduler.ResourceType
-	14, // 31: seldon.mlops.scheduler.ExperimentStatusResponse.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
-	46, // 32: seldon.mlops.scheduler.LoadPipelineRequest.pipeline:type_name -> seldon.mlops.scheduler.Pipeline
-	47, // 33: seldon.mlops.scheduler.Pipeline.steps:type_name -> seldon.mlops.scheduler.PipelineStep
-	50, // 34: seldon.mlops.scheduler.Pipeline.output:type_name -> seldon.mlops.scheduler.PipelineOutput
-	14, // 35: seldon.mlops.scheduler.Pipeline.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
-	49, // 36: seldon.mlops.scheduler.Pipeline.input:type_name -> seldon.mlops.scheduler.PipelineInput
-	64, // 37: seldon.mlops.scheduler.PipelineStep.tensorMap:type_name -> seldon.mlops.scheduler.PipelineStep.TensorMapEntry
+	15, // 31: seldon.mlops.scheduler.ExperimentStatusResponse.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
+	47, // 32: seldon.mlops.scheduler.LoadPipelineRequest.pipeline:type_name -> seldon.mlops.scheduler.Pipeline
+	48, // 33: seldon.mlops.scheduler.Pipeline.steps:type_name -> seldon.mlops.scheduler.PipelineStep
+	51, // 34: seldon.mlops.scheduler.Pipeline.output:type_name -> seldon.mlops.scheduler.PipelineOutput
+	15, // 35: seldon.mlops.scheduler.Pipeline.kubernetesMeta:type_name -> seldon.mlops.scheduler.KubernetesMeta
+	50, // 36: seldon.mlops.scheduler.Pipeline.input:type_name -> seldon.mlops.scheduler.PipelineInput
+	65, // 37: seldon.mlops.scheduler.PipelineStep.tensorMap:type_name -> seldon.mlops.scheduler.PipelineStep.TensorMapEntry
 	3,  // 38: seldon.mlops.scheduler.PipelineStep.inputsJoin:type_name -> seldon.mlops.scheduler.PipelineStep.JoinOp
 	3,  // 39: seldon.mlops.scheduler.PipelineStep.triggersJoin:type_name -> seldon.mlops.scheduler.PipelineStep.JoinOp
-	48, // 40: seldon.mlops.scheduler.PipelineStep.batch:type_name -> seldon.mlops.scheduler.Batch
+	49, // 40: seldon.mlops.scheduler.PipelineStep.batch:type_name -> seldon.mlops.scheduler.Batch
 	4,  // 41: seldon.mlops.scheduler.PipelineInput.joinType:type_name -> seldon.mlops.scheduler.PipelineInput.JoinOp
 	4,  // 42: seldon.mlops.scheduler.PipelineInput.triggersJoin:type_name -> seldon.mlops.scheduler.PipelineInput.JoinOp
-	65, // 43: seldon.mlops.scheduler.PipelineInput.tensorMap:type_name -> seldon.mlops.scheduler.PipelineInput.TensorMapEntry
+	66, // 43: seldon.mlops.scheduler.PipelineInput.tensorMap:type_name -> seldon.mlops.scheduler.PipelineInput.TensorMapEntry
 	5,  // 44: seldon.mlops.scheduler.PipelineOutput.stepsJoin:type_name -> seldon.mlops.scheduler.PipelineOutput.JoinOp
-	66, // 45: seldon.mlops.scheduler.PipelineOutput.tensorMap:type_name -> seldon.mlops.scheduler.PipelineOutput.TensorMapEntry
-	57, // 46: seldon.mlops.scheduler.PipelineStatusResponse.versions:type_name -> seldon.mlops.scheduler.PipelineWithState
-	46, // 47: seldon.mlops.scheduler.PipelineWithState.pipeline:type_name -> seldon.mlops.scheduler.Pipeline
-	58, // 48: seldon.mlops.scheduler.PipelineWithState.state:type_name -> seldon.mlops.scheduler.PipelineVersionState
+	67, // 45: seldon.mlops.scheduler.PipelineOutput.tensorMap:type_name -> seldon.mlops.scheduler.PipelineOutput.TensorMapEntry
+	58, // 46: seldon.mlops.scheduler.PipelineStatusResponse.versions:type_name -> seldon.mlops.scheduler.PipelineWithState
+	47, // 47: seldon.mlops.scheduler.PipelineWithState.pipeline:type_name -> seldon.mlops.scheduler.Pipeline
+	59, // 48: seldon.mlops.scheduler.PipelineWithState.state:type_name -> seldon.mlops.scheduler.PipelineVersionState
 	6,  // 49: seldon.mlops.scheduler.PipelineVersionState.status:type_name -> seldon.mlops.scheduler.PipelineVersionState.PipelineStatus
-	67, // 50: seldon.mlops.scheduler.PipelineVersionState.lastChangeTimestamp:type_name -> google.protobuf.Timestamp
-	24, // 51: seldon.mlops.scheduler.ModelVersionStatus.ModelReplicaStateEntry.value:type_name -> seldon.mlops.scheduler.ModelReplicaStatus
-	30, // 52: seldon.mlops.scheduler.Scheduler.ServerNotify:input_type -> seldon.mlops.scheduler.ServerNotifyRequest
-	7,  // 53: seldon.mlops.scheduler.Scheduler.LoadModel:input_type -> seldon.mlops.scheduler.LoadModelRequest
-	19, // 54: seldon.mlops.scheduler.Scheduler.UnloadModel:input_type -> seldon.mlops.scheduler.UnloadModelRequest
-	44, // 55: seldon.mlops.scheduler.Scheduler.LoadPipeline:input_type -> seldon.mlops.scheduler.LoadPipelineRequest
-	52, // 56: seldon.mlops.scheduler.Scheduler.UnloadPipeline:input_type -> seldon.mlops.scheduler.UnloadPipelineRequest
-	34, // 57: seldon.mlops.scheduler.Scheduler.StartExperiment:input_type -> seldon.mlops.scheduler.StartExperimentRequest
-	40, // 58: seldon.mlops.scheduler.Scheduler.StopExperiment:input_type -> seldon.mlops.scheduler.StopExperimentRequest
-	25, // 59: seldon.mlops.scheduler.Scheduler.ServerStatus:input_type -> seldon.mlops.scheduler.ServerStatusRequest
-	29, // 60: seldon.mlops.scheduler.Scheduler.ModelStatus:input_type -> seldon.mlops.scheduler.ModelStatusRequest
-	54, // 61: seldon.mlops.scheduler.Scheduler.PipelineStatus:input_type -> seldon.mlops.scheduler.PipelineStatusRequest
-	45, // 62: seldon.mlops.scheduler.Scheduler.ExperimentStatus:input_type -> seldon.mlops.scheduler.ExperimentStatusRequest
-	59, // 63: seldon.mlops.scheduler.Scheduler.SchedulerStatus:input_type -> seldon.mlops.scheduler.SchedulerStatusRequest
-	33, // 64: seldon.mlops.scheduler.Scheduler.SubscribeServerStatus:input_type -> seldon.mlops.scheduler.ServerSubscriptionRequest
-	28, // 65: seldon.mlops.scheduler.Scheduler.SubscribeModelStatus:input_type -> seldon.mlops.scheduler.ModelSubscriptionRequest
-	42, // 66: seldon.mlops.scheduler.Scheduler.SubscribeExperimentStatus:input_type -> seldon.mlops.scheduler.ExperimentSubscriptionRequest
-	55, // 67: seldon.mlops.scheduler.Scheduler.SubscribePipelineStatus:input_type -> seldon.mlops.scheduler.PipelineSubscriptionRequest
-	61, // 68: seldon.mlops.scheduler.Scheduler.SubscribeControlPlane:input_type -> seldon.mlops.scheduler.ControlPlaneSubscriptionRequest
-	32, // 69: seldon.mlops.scheduler.Scheduler.ServerNotify:output_type -> seldon.mlops.scheduler.ServerNotifyResponse
-	17, // 70: seldon.mlops.scheduler.Scheduler.LoadModel:output_type -> seldon.mlops.scheduler.LoadModelResponse
-	20, // 71: seldon.mlops.scheduler.Scheduler.UnloadModel:output_type -> seldon.mlops.scheduler.UnloadModelResponse
-	51, // 72: seldon.mlops.scheduler.Scheduler.LoadPipeline:output_type -> seldon.mlops.scheduler.LoadPipelineResponse
-	53, // 73: seldon.mlops.scheduler.Scheduler.UnloadPipeline:output_type -> seldon.mlops.scheduler.UnloadPipelineResponse
-	39, // 74: seldon.mlops.scheduler.Scheduler.StartExperiment:output_type -> seldon.mlops.scheduler.StartExperimentResponse
-	41, // 75: seldon.mlops.scheduler.Scheduler.StopExperiment:output_type -> seldon.mlops.scheduler.StopExperimentResponse
-	26, // 76: seldon.mlops.scheduler.Scheduler.ServerStatus:output_type -> seldon.mlops.scheduler.ServerStatusResponse
-	21, // 77: seldon.mlops.scheduler.Scheduler.ModelStatus:output_type -> seldon.mlops.scheduler.ModelStatusResponse
-	56, // 78: seldon.mlops.scheduler.Scheduler.PipelineStatus:output_type -> seldon.mlops.scheduler.PipelineStatusResponse
-	43, // 79: seldon.mlops.scheduler.Scheduler.ExperimentStatus:output_type -> seldon.mlops.scheduler.ExperimentStatusResponse
-	60, // 80: seldon.mlops.scheduler.Scheduler.SchedulerStatus:output_type -> seldon.mlops.scheduler.SchedulerStatusResponse
-	26, // 81: seldon.mlops.scheduler.Scheduler.SubscribeServerStatus:output_type -> seldon.mlops.scheduler.ServerStatusResponse
-	21, // 82: seldon.mlops.scheduler.Scheduler.SubscribeModelStatus:output_type -> seldon.mlops.scheduler.ModelStatusResponse
-	43, // 83: seldon.mlops.scheduler.Scheduler.SubscribeExperimentStatus:output_type -> seldon.mlops.scheduler.ExperimentStatusResponse
-	56, // 84: seldon.mlops.scheduler.Scheduler.SubscribePipelineStatus:output_type -> seldon.mlops.scheduler.PipelineStatusResponse
-	62, // 85: seldon.mlops.scheduler.Scheduler.SubscribeControlPlane:output_type -> seldon.mlops.scheduler.ControlPlaneResponse
-	69, // [69:86] is the sub-list for method output_type
-	52, // [52:69] is the sub-list for method input_type
-	52, // [52:52] is the sub-list for extension type_name
-	52, // [52:52] is the sub-list for extension extendee
-	0,  // [0:52] is the sub-list for field type_name
+	68, // 50: seldon.mlops.scheduler.PipelineVersionState.lastChangeTimestamp:type_name -> google.protobuf.Timestamp
+	7,  // 51: seldon.mlops.scheduler.ControlPlaneResponse.event:type_name -> seldon.mlops.scheduler.ControlPlaneResponse.Event
+	25, // 52: seldon.mlops.scheduler.ModelVersionStatus.ModelReplicaStateEntry.value:type_name -> seldon.mlops.scheduler.ModelReplicaStatus
+	31, // 53: seldon.mlops.scheduler.Scheduler.ServerNotify:input_type -> seldon.mlops.scheduler.ServerNotifyRequest
+	8,  // 54: seldon.mlops.scheduler.Scheduler.LoadModel:input_type -> seldon.mlops.scheduler.LoadModelRequest
+	20, // 55: seldon.mlops.scheduler.Scheduler.UnloadModel:input_type -> seldon.mlops.scheduler.UnloadModelRequest
+	45, // 56: seldon.mlops.scheduler.Scheduler.LoadPipeline:input_type -> seldon.mlops.scheduler.LoadPipelineRequest
+	53, // 57: seldon.mlops.scheduler.Scheduler.UnloadPipeline:input_type -> seldon.mlops.scheduler.UnloadPipelineRequest
+	35, // 58: seldon.mlops.scheduler.Scheduler.StartExperiment:input_type -> seldon.mlops.scheduler.StartExperimentRequest
+	41, // 59: seldon.mlops.scheduler.Scheduler.StopExperiment:input_type -> seldon.mlops.scheduler.StopExperimentRequest
+	26, // 60: seldon.mlops.scheduler.Scheduler.ServerStatus:input_type -> seldon.mlops.scheduler.ServerStatusRequest
+	30, // 61: seldon.mlops.scheduler.Scheduler.ModelStatus:input_type -> seldon.mlops.scheduler.ModelStatusRequest
+	55, // 62: seldon.mlops.scheduler.Scheduler.PipelineStatus:input_type -> seldon.mlops.scheduler.PipelineStatusRequest
+	46, // 63: seldon.mlops.scheduler.Scheduler.ExperimentStatus:input_type -> seldon.mlops.scheduler.ExperimentStatusRequest
+	60, // 64: seldon.mlops.scheduler.Scheduler.SchedulerStatus:input_type -> seldon.mlops.scheduler.SchedulerStatusRequest
+	34, // 65: seldon.mlops.scheduler.Scheduler.SubscribeServerStatus:input_type -> seldon.mlops.scheduler.ServerSubscriptionRequest
+	29, // 66: seldon.mlops.scheduler.Scheduler.SubscribeModelStatus:input_type -> seldon.mlops.scheduler.ModelSubscriptionRequest
+	43, // 67: seldon.mlops.scheduler.Scheduler.SubscribeExperimentStatus:input_type -> seldon.mlops.scheduler.ExperimentSubscriptionRequest
+	56, // 68: seldon.mlops.scheduler.Scheduler.SubscribePipelineStatus:input_type -> seldon.mlops.scheduler.PipelineSubscriptionRequest
+	62, // 69: seldon.mlops.scheduler.Scheduler.SubscribeControlPlane:input_type -> seldon.mlops.scheduler.ControlPlaneSubscriptionRequest
+	33, // 70: seldon.mlops.scheduler.Scheduler.ServerNotify:output_type -> seldon.mlops.scheduler.ServerNotifyResponse
+	18, // 71: seldon.mlops.scheduler.Scheduler.LoadModel:output_type -> seldon.mlops.scheduler.LoadModelResponse
+	21, // 72: seldon.mlops.scheduler.Scheduler.UnloadModel:output_type -> seldon.mlops.scheduler.UnloadModelResponse
+	52, // 73: seldon.mlops.scheduler.Scheduler.LoadPipeline:output_type -> seldon.mlops.scheduler.LoadPipelineResponse
+	54, // 74: seldon.mlops.scheduler.Scheduler.UnloadPipeline:output_type -> seldon.mlops.scheduler.UnloadPipelineResponse
+	40, // 75: seldon.mlops.scheduler.Scheduler.StartExperiment:output_type -> seldon.mlops.scheduler.StartExperimentResponse
+	42, // 76: seldon.mlops.scheduler.Scheduler.StopExperiment:output_type -> seldon.mlops.scheduler.StopExperimentResponse
+	27, // 77: seldon.mlops.scheduler.Scheduler.ServerStatus:output_type -> seldon.mlops.scheduler.ServerStatusResponse
+	22, // 78: seldon.mlops.scheduler.Scheduler.ModelStatus:output_type -> seldon.mlops.scheduler.ModelStatusResponse
+	57, // 79: seldon.mlops.scheduler.Scheduler.PipelineStatus:output_type -> seldon.mlops.scheduler.PipelineStatusResponse
+	44, // 80: seldon.mlops.scheduler.Scheduler.ExperimentStatus:output_type -> seldon.mlops.scheduler.ExperimentStatusResponse
+	61, // 81: seldon.mlops.scheduler.Scheduler.SchedulerStatus:output_type -> seldon.mlops.scheduler.SchedulerStatusResponse
+	27, // 82: seldon.mlops.scheduler.Scheduler.SubscribeServerStatus:output_type -> seldon.mlops.scheduler.ServerStatusResponse
+	22, // 83: seldon.mlops.scheduler.Scheduler.SubscribeModelStatus:output_type -> seldon.mlops.scheduler.ModelStatusResponse
+	44, // 84: seldon.mlops.scheduler.Scheduler.SubscribeExperimentStatus:output_type -> seldon.mlops.scheduler.ExperimentStatusResponse
+	57, // 85: seldon.mlops.scheduler.Scheduler.SubscribePipelineStatus:output_type -> seldon.mlops.scheduler.PipelineStatusResponse
+	63, // 86: seldon.mlops.scheduler.Scheduler.SubscribeControlPlane:output_type -> seldon.mlops.scheduler.ControlPlaneResponse
+	70, // [70:87] is the sub-list for method output_type
+	53, // [53:70] is the sub-list for method input_type
+	53, // [53:53] is the sub-list for extension type_name
+	53, // [53:53] is the sub-list for extension extendee
+	0,  // [0:53] is the sub-list for field type_name
 }
 
 func init() { file_mlops_scheduler_scheduler_proto_init() }
@@ -5430,7 +5499,7 @@ func file_mlops_scheduler_scheduler_proto_init() {
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_mlops_scheduler_scheduler_proto_rawDesc,
-			NumEnums:      7,
+			NumEnums:      8,
 			NumMessages:   60,
 			NumExtensions: 0,
 			NumServices:   1,
diff --git a/apis/mlops/scheduler/scheduler.proto b/apis/mlops/scheduler/scheduler.proto
index 163afdfd5d..3263ebcc1b 100644
--- a/apis/mlops/scheduler/scheduler.proto
+++ b/apis/mlops/scheduler/scheduler.proto
@@ -390,7 +390,12 @@ message ControlPlaneSubscriptionRequest {
 }
 
 message ControlPlaneResponse {
-
+  enum Event {
+    UNKNOWN_EVENT = 0;
+    SEND_SERVERS = 1;  // initial sync for the servers
+    SEND_RESOURCES = 2;  // send models / pipelines / experiments
+  }
+  Event event = 1;
 }
 
 // [END Messages]
diff --git a/operator/scheduler/client.go b/operator/scheduler/client.go
index 98492d31c4..1500b7fb3a 100644
--- a/operator/scheduler/client.go
+++ b/operator/scheduler/client.go
@@ -142,34 +142,37 @@ func (s *SchedulerClient) startEventHanders(namespace string, conn *grpc.ClientC
 	}()
 }
 
-func (s *SchedulerClient) handleStateOnReconnect(context context.Context, grpcClient scheduler.SchedulerClient, namespace string) error {
-	// on new reconnects we send a list of servers to the schedule
-	err := s.handleRegisteredServers(context, grpcClient, namespace)
-	if err != nil {
-		s.logger.Error(err, "Failed to send registered server to scheduler")
-	}
-
-	if err == nil {
-		err = s.handleExperiments(context, grpcClient, namespace)
+func (s *SchedulerClient) handleStateOnReconnect(context context.Context, grpcClient scheduler.SchedulerClient, namespace string, operation scheduler.ControlPlaneResponse_Event) error {
+	switch operation {
+	case scheduler.ControlPlaneResponse_SEND_SERVERS:
+		// on new reconnects we send a list of servers to the schedule
+		err := s.handleRegisteredServers(context, grpcClient, namespace)
 		if err != nil {
-			s.logger.Error(err, "Failed to send experiments to scheduler")
+			s.logger.Error(err, "Failed to send registered server to scheduler")
 		}
-	}
-
-	if err == nil {
-		err = s.handlePipelines(context, grpcClient, namespace)
+		return err
+	case scheduler.ControlPlaneResponse_SEND_RESOURCES:
+		err := s.handleExperiments(context, grpcClient, namespace)
 		if err != nil {
-			s.logger.Error(err, "Failed to send pipelines to scheduler")
+			s.logger.Error(err, "Failed to send experiments to scheduler")
 		}
-	}
-
-	if err == nil {
-		err = s.handleModels(context, grpcClient, namespace)
-		if err != nil {
-			s.logger.Error(err, "Failed to send models to scheduler")
+		if err == nil {
+			err = s.handlePipelines(context, grpcClient, namespace)
+			if err != nil {
+				s.logger.Error(err, "Failed to send pipelines to scheduler")
+			}
 		}
+		if err == nil {
+			err = s.handleModels(context, grpcClient, namespace)
+			if err != nil {
+				s.logger.Error(err, "Failed to send models to scheduler")
+			}
+		}
+		return err
+	default:
+		s.logger.Info("Unknown operation", "operation", operation)
+		return fmt.Errorf("Unknown operation %v", operation)
 	}
-	return err
 }
 
 func (s *SchedulerClient) RemoveConnection(namespace string) {
diff --git a/operator/scheduler/control_plane.go b/operator/scheduler/control_plane.go
index 9834cfea98..9f0d1dfd8d 100644
--- a/operator/scheduler/control_plane.go
+++ b/operator/scheduler/control_plane.go
@@ -50,7 +50,7 @@ func (s *SchedulerClient) SubscribeControlPlaneEvents(ctx context.Context, grpcC
 		logger.Info("Received event to handle state", "event", event)
 
 		fn := func() error {
-			return s.handleStateOnReconnect(ctx, grpcClient, namespace)
+			return s.handleStateOnReconnect(ctx, grpcClient, namespace, event.GetEvent())
 		}
 		_, err = execWithTimeout(fn, execTimeOut)
 		if err != nil {
diff --git a/operator/scheduler/utils_test.go b/operator/scheduler/utils_test.go
index fc71c0b505..92b384d8e6 100644
--- a/operator/scheduler/utils_test.go
+++ b/operator/scheduler/utils_test.go
@@ -109,7 +109,7 @@ func (s *mockSchedulerServerSubscribeGrpcClient) Recv() (*scheduler.ServerStatus
 // Control Plane subscribe mock grpc client
 
 type mockControlPlaneSubscribeGrpcClient struct {
-	sent bool
+	sent int
 	grpc.ClientStream
 }
 
@@ -120,9 +120,13 @@ func newMockControlPlaneSubscribeGrpcClient() *mockControlPlaneSubscribeGrpcClie
 }
 
 func (s *mockControlPlaneSubscribeGrpcClient) Recv() (*scheduler.ControlPlaneResponse, error) {
-	if !s.sent {
-		s.sent = true
-		return &scheduler.ControlPlaneResponse{}, nil
+	if s.sent == 0 {
+		s.sent++
+		return &scheduler.ControlPlaneResponse{Event: scheduler.ControlPlaneResponse_SEND_SERVERS}, nil
+	}
+	if s.sent == 1 {
+		s.sent++
+		return &scheduler.ControlPlaneResponse{Event: scheduler.ControlPlaneResponse_SEND_RESOURCES}, nil
 	}
 	return nil, io.EOF
 }
diff --git a/scheduler/pkg/server/control_plane.go b/scheduler/pkg/server/control_plane.go
index a3031d6df8..03e2468d04 100644
--- a/scheduler/pkg/server/control_plane.go
+++ b/scheduler/pkg/server/control_plane.go
@@ -23,8 +23,15 @@ func (s *SchedulerServer) SubscribeControlPlane(req *pb.ControlPlaneSubscription
 		return err
 	}
 
-	fin := make(chan bool)
+	s.synchroniser.WaitReady()
+
+	err = s.sendResourcesMarker(stream)
+	if err != nil {
+		logger.WithError(err).Errorf("Failed to send resources marker to %s", req.GetSubscriberName())
+		return err
+	}
 
+	fin := make(chan bool)
 	s.controlPlaneStream.mu.Lock()
 	s.controlPlaneStream.streams[stream] = &ControlPlaneSubsription{
 		name:   req.GetSubscriberName(),
@@ -61,11 +68,20 @@ func (s *SchedulerServer) StopSendControlPlaneEvents() {
 // this is to mark the initial start of a new stream (at application level)
 // as otherwise the other side sometimes doesnt know if the scheduler has established a new stream explicitly
 func (s *SchedulerServer) sendStartServerStreamMarker(stream pb.Scheduler_SubscribeControlPlaneServer) error {
-	ssr := &pb.ControlPlaneResponse{}
+	ssr := &pb.ControlPlaneResponse{Event: pb.ControlPlaneResponse_SEND_SERVERS}
 	_, err := sendWithTimeout(func() error { return stream.Send(ssr) }, s.timeout)
 	if err != nil {
 		return err
 	}
+	return nil
+}
 
+// this is to mark a stage to send resources (models, pipelines, experiments) from the controller
+func (s *SchedulerServer) sendResourcesMarker(stream pb.Scheduler_SubscribeControlPlaneServer) error {
+	ssr := &pb.ControlPlaneResponse{Event: pb.ControlPlaneResponse_SEND_RESOURCES}
+	_, err := sendWithTimeout(func() error { return stream.Send(ssr) }, s.timeout)
+	if err != nil {
+		return err
+	}
 	return nil
 }
diff --git a/scheduler/pkg/server/control_plane_test.go b/scheduler/pkg/server/control_plane_test.go
index b7d0eb5dfe..ceecc8fd4e 100644
--- a/scheduler/pkg/server/control_plane_test.go
+++ b/scheduler/pkg/server/control_plane_test.go
@@ -10,15 +10,22 @@ the Change License after the Change Date as each is defined in accordance with t
 package server
 
 import (
+	"context"
+	"fmt"
 	"testing"
 	"time"
 
 	. "github.com/onsi/gomega"
 	log "github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 
 	pb "github.com/seldonio/seldon-core/apis/go/v2/mlops/scheduler"
 
+	"github.com/seldonio/seldon-core/scheduler/v2/pkg/coordinator"
+	"github.com/seldonio/seldon-core/scheduler/v2/pkg/internal/testing_utils"
 	"github.com/seldonio/seldon-core/scheduler/v2/pkg/store"
+	"github.com/seldonio/seldon-core/scheduler/v2/pkg/synchroniser"
 )
 
 func TestStartServerStream(t *testing.T) {
@@ -67,7 +74,91 @@ func TestStartServerStream(t *testing.T) {
 				}
 
 				g.Expect(msr).ToNot(BeNil())
+				g.Expect(msr.Event).To(Equal(pb.ControlPlaneResponse_SEND_SERVERS))
 			}
+
+			err = test.server.sendResourcesMarker(stream)
+			if test.err {
+				g.Expect(err).ToNot(BeNil())
+			} else {
+				g.Expect(err).To(BeNil())
+
+				var msr *pb.ControlPlaneResponse
+				select {
+				case next := <-stream.msgs:
+					msr = next
+				default:
+					t.Fail()
+				}
+
+				g.Expect(msr).ToNot(BeNil())
+				g.Expect(msr.Event).To(Equal(pb.ControlPlaneResponse_SEND_RESOURCES))
+			}
+		})
+	}
+}
+
+func TestSubscribeControlPlane(t *testing.T) {
+	log.SetLevel(log.DebugLevel)
+	g := NewGomegaWithT(t)
+
+	type test struct {
+		name string
+	}
+	tests := []test{
+		{
+			name: "simple",
+		},
+	}
+
+	getStream := func(context context.Context, port int) (*grpc.ClientConn, pb.Scheduler_SubscribeControlPlaneClient) {
+		conn, _ := grpc.NewClient(fmt.Sprintf(":%d", port), grpc.WithTransportCredentials(insecure.NewCredentials()))
+		grpcClient := pb.NewSchedulerClient(conn)
+		client, _ := grpcClient.SubscribeControlPlane(
+			context,
+			&pb.ControlPlaneSubscriptionRequest{SubscriberName: "dummy"},
+		)
+		return conn, client
+	}
+
+	createTestScheduler := func() *SchedulerServer {
+		logger := log.New()
+		logger.SetLevel(log.WarnLevel)
+
+		eventHub, err := coordinator.NewEventHub(logger)
+		g.Expect(err).To(BeNil())
+
+		sync := synchroniser.NewSimpleSynchroniser(time.Duration(10 * time.Millisecond))
+
+		s := NewSchedulerServer(logger, nil, nil, nil, nil, eventHub, sync)
+		sync.Signals(1)
+
+		return s
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			server := createTestScheduler()
+			port, err := testing_utils.GetFreePortForTest()
+			if err != nil {
+				t.Fatal(err)
+			}
+			err = server.startServer(uint(port), false)
+			if err != nil {
+				t.Fatal(err)
+			}
+			time.Sleep(100 * time.Millisecond)
+
+			conn, client := getStream(context.Background(), port)
+
+			msg, _ := client.Recv()
+			g.Expect(msg.GetEvent()).To(Equal(pb.ControlPlaneResponse_SEND_SERVERS))
+
+			msg, _ = client.Recv()
+			g.Expect(msg.Event).To(Equal(pb.ControlPlaneResponse_SEND_RESOURCES))
+
+			conn.Close()
+			server.StopSendControlPlaneEvents()
 		})
 	}
 }
diff --git a/scheduler/pkg/store/memory.go b/scheduler/pkg/store/memory.go
index 6b18917671..4d011ac987 100644
--- a/scheduler/pkg/store/memory.go
+++ b/scheduler/pkg/store/memory.go
@@ -89,7 +89,11 @@ func (m *MemoryStore) addModelVersionIfNotExists(req *agent.ModelVersion) (*Mode
 }
 
 func (m *MemoryStore) addNextModelVersion(model *Model, pbmodel *pb.Model) {
-	version := uint32(1)
+	// if we start from a clean state, lets use the generation id as the starting version
+	// this is to ensure that we have monotonic increasing version numbers
+	// and we never reset back to 1
+	generation := pbmodel.GetMeta().GetKubernetesMeta().GetGeneration()
+	version := max(uint32(1), uint32(generation))
 	if model.Latest() != nil {
 		version = model.Latest().GetVersion() + 1
 	}
@@ -329,7 +333,7 @@ func (m *MemoryStore) updateLoadedModelsImpl(
 		modelVersion = model.Latest()
 	}
 
-	//  resevere memory for existing replicas that are not already loading or loaded
+	//  reserve memory for existing replicas that are not already loading or loaded
 	replicaStateUpdated := false
 	for replicaIdx := range assignedReplicaIds {
 		if existingState, ok := modelVersion.replicas[replicaIdx]; !ok {
@@ -370,7 +374,8 @@ func (m *MemoryStore) updateLoadedModelsImpl(
 	// in cases where we did have a previous ScheduleFailed, we need to reflect the change here
 	// this could be in the cases where we are scaling down a model and the new replica count can be all deployed
 	// and always send an update for deleted models, so the operator will remove them from k8s
-	if replicaStateUpdated || modelVersion.state.State == ScheduleFailed || model.IsDeleted() {
+	// also send an update for progressing models so the operator can update the status in the case of a network glitch where the model generation has been updated
+	if replicaStateUpdated || modelVersion.state.State == ScheduleFailed || model.IsDeleted() || modelVersion.state.State == ModelProgressing {
 		logger.Debugf("Updating model status for model %s server %s", modelKey, serverKey)
 		modelVersion.server = serverKey
 		m.updateModelStatus(true, model.IsDeleted(), modelVersion, model.GetLastAvailableModelVersion())
diff --git a/scheduler/pkg/store/memory_test.go b/scheduler/pkg/store/memory_test.go
index 0f16b5e1b7..51e0890161 100644
--- a/scheduler/pkg/store/memory_test.go
+++ b/scheduler/pkg/store/memory_test.go
@@ -48,6 +48,21 @@ func TestUpdateModel(t *testing.T) {
 			},
 			expectedVersion: 1,
 		},
+		{
+			name:  "simple with generation",
+			store: NewLocalSchedulerStore(),
+			loadModelReq: &pb.LoadModelRequest{
+				Model: &pb.Model{
+					Meta: &pb.MetaData{
+						Name: "model",
+						KubernetesMeta: &pb.KubernetesMeta{
+							Generation: 100,
+						},
+					},
+				},
+			},
+			expectedVersion: 100,
+		},
 		{
 			name: "VersionAlreadyExists",
 			store: &LocalSchedulerStore{
@@ -337,15 +352,16 @@ func TestUpdateLoadedModels(t *testing.T) {
 	memBytes := uint64(1)
 
 	type test struct {
-		name           string
-		store          *LocalSchedulerStore
-		modelKey       string
-		version        uint32
-		serverKey      string
-		replicas       []*ServerReplica
-		expectedStates map[int]ReplicaStatus
-		err            bool
-		isModelDeleted bool
+		name               string
+		store              *LocalSchedulerStore
+		modelKey           string
+		version            uint32
+		serverKey          string
+		replicas           []*ServerReplica
+		expectedStates     map[int]ReplicaStatus
+		err                bool
+		isModelDeleted     bool
+		expectedModelState *ModelStatus
 	}
 
 	tests := []test{
@@ -673,6 +689,42 @@ func TestUpdateLoadedModels(t *testing.T) {
 			isModelDeleted: true,
 			expectedStates: map[int]ReplicaStatus{},
 		},
+		{
+			name: "ProgressModelLoading",
+			store: &LocalSchedulerStore{
+				models: map[string]*Model{"model": {
+					versions: []*ModelVersion{
+						{
+							modelDefn: &pb.Model{ModelSpec: &pb.ModelSpec{MemoryBytes: &memBytes}, DeploymentSpec: &pb.DeploymentSpec{Replicas: 1}},
+							server:    "server",
+							version:   1,
+							replicas: map[int]ReplicaStatus{
+								0: {State: Available},
+								1: {State: Unloaded},
+							},
+							state: ModelStatus{State: ModelProgressing},
+						},
+					},
+				}},
+				servers: map[string]*Server{
+					"server": {
+						name: "server",
+						replicas: map[int]*ServerReplica{
+							0: {},
+							1: {},
+						},
+					},
+				},
+			},
+			modelKey:  "model",
+			version:   1,
+			serverKey: "server",
+			replicas: []*ServerReplica{
+				{replicaIdx: 0},
+			},
+			expectedStates:     map[int]ReplicaStatus{0: {State: Available}, 1: {State: Unloaded}},
+			expectedModelState: &ModelStatus{State: ModelAvailable},
+		},
 	}
 
 	for _, test := range tests {
@@ -688,8 +740,8 @@ func TestUpdateLoadedModels(t *testing.T) {
 			if !test.err {
 				g.Expect(err).To(BeNil())
 				g.Expect(msg).ToNot(BeNil())
+				mv := test.store.models[test.modelKey].Latest()
 				for replicaIdx, state := range test.expectedStates {
-					mv := test.store.models[test.modelKey].Latest()
 					g.Expect(mv).ToNot(BeNil())
 					g.Expect(mv.GetModelReplicaState(replicaIdx)).To(Equal(state.State))
 					ss, _ := ms.GetServer(test.serverKey, false, true)
@@ -699,6 +751,9 @@ func TestUpdateLoadedModels(t *testing.T) {
 						g.Expect(ss.Replicas[replicaIdx].GetReservedMemory()).To(Equal(uint64(0)))
 					}
 				}
+				if test.expectedModelState != nil {
+					g.Expect(mv.state.State).To(Equal(test.expectedModelState.State))
+				}
 			} else {
 				g.Expect(err).ToNot(BeNil())
 				g.Expect(msg).To(BeNil())