From c5bdf3fd14417bbd84569a8b2f8fa604b09ca45b Mon Sep 17 00:00:00 2001 From: John Date: Tue, 31 Dec 2024 14:12:35 -0700 Subject: [PATCH 1/8] Fix: Add a TTL on build containers --- pkg/abstractions/image/build.go | 9 +++- pkg/abstractions/image/image.go | 74 +++++++++++++++++++++++++++++++-- pkg/common/config.default.yaml | 1 + pkg/gateway/gateway.go | 1 + pkg/types/config.go | 1 + 5 files changed, 82 insertions(+), 4 deletions(-) diff --git a/pkg/abstractions/image/build.go b/pkg/abstractions/image/build.go index f9390915c..3ca004a2c 100644 --- a/pkg/abstractions/image/build.go +++ b/pkg/abstractions/image/build.go @@ -43,6 +43,7 @@ type Builder struct { registry *common.ImageRegistry containerRepo repository.ContainerRepository tailscale *network.Tailscale + rdb *common.RedisClient } type BuildStep struct { @@ -130,13 +131,14 @@ func (o *BuildOpts) addPythonRequirements() { o.PythonPackages = append(filteredPythonPackages, baseRequirementsSlice...) } -func NewBuilder(config types.AppConfig, registry *common.ImageRegistry, scheduler *scheduler.Scheduler, tailscale *network.Tailscale, containerRepo repository.ContainerRepository) (*Builder, error) { +func NewBuilder(config types.AppConfig, registry *common.ImageRegistry, scheduler *scheduler.Scheduler, tailscale *network.Tailscale, containerRepo repository.ContainerRepository, rdb *common.RedisClient) (*Builder, error) { return &Builder{ config: config, scheduler: scheduler, tailscale: tailscale, registry: registry, containerRepo: containerRepo, + rdb: rdb, }, nil } @@ -283,6 +285,11 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co return err } + err = b.rdb.HSetNX(ctx, imageBuildContainersCreatedAtKey, containerId, time.Now().Unix()).Err() + if err != nil { + outputChan <- common.OutputMsg{Done: true, Success: false, Msg: fmt.Sprintf("Unknown error occurred.\n %s", err.Error())} + } + conn, err := network.ConnectToHost(ctx, hostname, time.Second*30, b.tailscale, b.config.Tailscale) if err != nil { outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Failed to connect to build container.\n"} diff --git a/pkg/abstractions/image/image.go b/pkg/abstractions/image/image.go index 6bd587ac0..7d048f98a 100644 --- a/pkg/abstractions/image/image.go +++ b/pkg/abstractions/image/image.go @@ -3,6 +3,8 @@ package image import ( "context" "fmt" + "strconv" + "time" "github.com/beam-cloud/beta9/pkg/auth" "github.com/beam-cloud/beta9/pkg/common" @@ -26,6 +28,7 @@ type RuncImageService struct { builder *Builder config types.AppConfig backendRepo repository.BackendRepository + rdb *common.RedisClient } type ImageServiceOpts struct { @@ -34,6 +37,11 @@ type ImageServiceOpts struct { BackendRepo repository.BackendRepository Scheduler *scheduler.Scheduler Tailscale *network.Tailscale + RedisClient *common.RedisClient +} + +type ImageBuildContainerState struct { + CreatedAt int64 `redis:"created_at"` } func NewRuncImageService( @@ -45,16 +53,21 @@ func NewRuncImageService( return nil, err } - builder, err := NewBuilder(opts.Config, registry, opts.Scheduler, opts.Tailscale, opts.ContainerRepo) + builder, err := NewBuilder(opts.Config, registry, opts.Scheduler, opts.Tailscale, opts.ContainerRepo, opts.RedisClient) if err != nil { return nil, err } - return &RuncImageService{ + is := RuncImageService{ builder: builder, config: opts.Config, backendRepo: opts.BackendRepo, - }, nil + rdb: opts.RedisClient, + } + + go is.monitorImageContainers(ctx) + + return &is, nil } func (is *RuncImageService) VerifyImageBuild(ctx context.Context, in *pb.VerifyImageBuildRequest) (*pb.VerifyImageBuildResponse, error) { @@ -184,6 +197,59 @@ func (is *RuncImageService) retrieveBuildSecrets(ctx context.Context, secrets [] return buildSecrets, nil } +func (is *RuncImageService) monitorImageContainers(ctx context.Context) { + ticker := time.NewTicker(5 * time.Second) + + for { + select { + case <-ticker.C: + containerKeys, err := is.rdb.HKeys(ctx, imageBuildContainersCreatedAtKey).Result() + if err != nil { + log.Error().Err(err).Msg("failed to get active image build containers") + return + } + + for _, key := range containerKeys { + createdAt := is.rdb.HGet(ctx, imageBuildContainersCreatedAtKey, key).Val() + if createdAt == "" { + log.Error().Msg("failed to get created at time for container") + continue + } + + createdAtInt, err := strconv.ParseInt(createdAt, 10, 64) + if err != nil { + log.Error().Err(err).Msg("failed to parse container created at time") + continue + } + + // See if the container still exists + _, err = is.builder.containerRepo.GetContainerState(key) + if err != nil { + if errors.Is(err, &types.ErrCheckpointNotFound{}) { + is.rdb.HDel(ctx, imageBuildContainersCreatedAtKey, key) + } + + log.Error().Err(err).Msg("failed to get container state") + continue + } + + if createdAtInt < time.Now().Unix()-is.config.ImageService.ImageBuildTimeoutS { + is.builder.scheduler.Stop( + &types.StopContainerArgs{ + ContainerId: key, + Force: true, + }, + ) + + is.rdb.HDel(ctx, imageBuildContainersCreatedAtKey, key) + } + } + case <-ctx.Done(): + return + } + } +} + func convertBuildSteps(buildSteps []*pb.BuildStep) []BuildStep { steps := make([]BuildStep, len(buildSteps)) for i, s := range buildSteps { @@ -194,3 +260,5 @@ func convertBuildSteps(buildSteps []*pb.BuildStep) []BuildStep { } return steps } + +var imageBuildContainersCreatedAtKey = "image:containers:created_at" diff --git a/pkg/common/config.default.yaml b/pkg/common/config.default.yaml index bcca17ef2..02fd9e08a 100644 --- a/pkg/common/config.default.yaml +++ b/pkg/common/config.default.yaml @@ -61,6 +61,7 @@ imageService: registryStore: local registryCredentialProvider: docker buildContainerPoolSelector: build + imageBuildTimeoutS: 3600 registries: docker: username: beamcloud diff --git a/pkg/gateway/gateway.go b/pkg/gateway/gateway.go index 1a2ff75bf..aa923ebb4 100644 --- a/pkg/gateway/gateway.go +++ b/pkg/gateway/gateway.go @@ -250,6 +250,7 @@ func (g *Gateway) registerServices() error { Scheduler: g.Scheduler, Tailscale: g.Tailscale, BackendRepo: g.BackendRepo, + RedisClient: g.RedisClient, }) if err != nil { return err diff --git a/pkg/types/config.go b/pkg/types/config.go index 744557d27..216093e75 100644 --- a/pkg/types/config.go +++ b/pkg/types/config.go @@ -132,6 +132,7 @@ type ImageServiceConfig struct { BuildContainerMemory int64 `key:"buildContainerMemory" json:"build_container_memory"` BuildContainerPoolSelector string `key:"buildContainerPoolSelector" json:"build_container_pool_selector"` Runner RunnerConfig `key:"runner" json:"runner"` + ImageBuildTimeoutS int64 `key:"imageBuildTimeoutS" json:"image_build_timeout_s"` } type ImageRegistriesConfig struct { From aabae1dfe556e6345d46a87b5d457633fe731f7f Mon Sep 17 00:00:00 2001 From: John Date: Tue, 31 Dec 2024 14:15:44 -0700 Subject: [PATCH 2/8] remove unused struct --- pkg/abstractions/image/image.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/abstractions/image/image.go b/pkg/abstractions/image/image.go index 7d048f98a..5ddce7ee9 100644 --- a/pkg/abstractions/image/image.go +++ b/pkg/abstractions/image/image.go @@ -40,10 +40,6 @@ type ImageServiceOpts struct { RedisClient *common.RedisClient } -type ImageBuildContainerState struct { - CreatedAt int64 `redis:"created_at"` -} - func NewRuncImageService( ctx context.Context, opts ImageServiceOpts, From f685877ade73df487f40fcfbcb6025cd614ec429 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 31 Dec 2024 14:16:25 -0700 Subject: [PATCH 3/8] Remove error string --- pkg/abstractions/image/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/abstractions/image/build.go b/pkg/abstractions/image/build.go index 3ca004a2c..7a2094567 100644 --- a/pkg/abstractions/image/build.go +++ b/pkg/abstractions/image/build.go @@ -287,7 +287,7 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co err = b.rdb.HSetNX(ctx, imageBuildContainersCreatedAtKey, containerId, time.Now().Unix()).Err() if err != nil { - outputChan <- common.OutputMsg{Done: true, Success: false, Msg: fmt.Sprintf("Unknown error occurred.\n %s", err.Error())} + outputChan <- common.OutputMsg{Done: true, Success: false, Msg: fmt.Sprintf("Unknown error occurred.\n")} } conn, err := network.ConnectToHost(ctx, hostname, time.Second*30, b.tailscale, b.config.Tailscale) From cab329fa46c8a948576c0a164d49b95f6528ae04 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 2 Jan 2025 16:12:57 -0700 Subject: [PATCH 4/8] Use key event manager --- pkg/abstractions/image/build.go | 23 ++++++- pkg/abstractions/image/image.go | 103 ++++++++++++++++---------------- pkg/common/key_events.go | 4 ++ pkg/types/scheduler.go | 2 +- 4 files changed, 77 insertions(+), 55 deletions(-) diff --git a/pkg/abstractions/image/build.go b/pkg/abstractions/image/build.go index 7a2094567..a98c921d7 100644 --- a/pkg/abstractions/image/build.go +++ b/pkg/abstractions/image/build.go @@ -285,11 +285,14 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co return err } - err = b.rdb.HSetNX(ctx, imageBuildContainersCreatedAtKey, containerId, time.Now().Unix()).Err() + err = b.rdb.Set(ctx, Keys.imageContainerTTL(containerId), "1", time.Duration(imageContainerTtlS)*time.Second).Err() if err != nil { - outputChan <- common.OutputMsg{Done: true, Success: false, Msg: fmt.Sprintf("Unknown error occurred.\n")} + outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Failed to connect to build container.\n"} + return err } + go b.keepAlive(ctx, containerId, ctx.Done()) + conn, err := network.ConnectToHost(ctx, hostname, time.Second*30, b.tailscale, b.config.Tailscale) if err != nil { outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Failed to connect to build container.\n"} @@ -444,6 +447,22 @@ func (b *Builder) Exists(ctx context.Context, imageId string) bool { return b.registry.Exists(ctx, imageId) } +func (b *Builder) keepAlive(ctx context.Context, containerId string, done <-chan struct{}) { + ticker := time.NewTicker(time.Duration(containerKeepAliveIntervalS) * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-done: + return + case <-ticker.C: + b.rdb.Set(ctx, Keys.imageContainerTTL(containerId), "1", time.Duration(imageContainerTtlS)*time.Second).Err() + } + } +} + var imageNamePattern = regexp.MustCompile( `^` + // Assert position at the start of the string `(?:(?P(?:(?:localhost|[\w.-]+(?:\.[\w.-]+)+)(?::\d+)?)|[\w]+:\d+)\/)?` + // Optional registry, which can be localhost, a domain with optional port, or a simple registry with port diff --git a/pkg/abstractions/image/image.go b/pkg/abstractions/image/image.go index 5ddce7ee9..03c96bf97 100644 --- a/pkg/abstractions/image/image.go +++ b/pkg/abstractions/image/image.go @@ -3,8 +3,7 @@ package image import ( "context" "fmt" - "strconv" - "time" + "strings" "github.com/beam-cloud/beta9/pkg/auth" "github.com/beam-cloud/beta9/pkg/common" @@ -25,10 +24,12 @@ type ImageService interface { type RuncImageService struct { pb.UnimplementedImageServiceServer - builder *Builder - config types.AppConfig - backendRepo repository.BackendRepository - rdb *common.RedisClient + builder *Builder + config types.AppConfig + backendRepo repository.BackendRepository + rdb *common.RedisClient + keyEventChan chan common.KeyEvent + keyEventManager *common.KeyEventManager } type ImageServiceOpts struct { @@ -40,6 +41,9 @@ type ImageServiceOpts struct { RedisClient *common.RedisClient } +const containerKeepAliveIntervalS int = 30 +const imageContainerTtlS int = 30 + func NewRuncImageService( ctx context.Context, opts ImageServiceOpts, @@ -54,14 +58,23 @@ func NewRuncImageService( return nil, err } + keyEventManager, err := common.NewKeyEventManager(opts.RedisClient) + if err != nil { + return nil, err + } + is := RuncImageService{ - builder: builder, - config: opts.Config, - backendRepo: opts.BackendRepo, - rdb: opts.RedisClient, + builder: builder, + config: opts.Config, + backendRepo: opts.BackendRepo, + keyEventChan: make(chan common.KeyEvent), + keyEventManager: keyEventManager, + rdb: opts.RedisClient, } go is.monitorImageContainers(ctx) + go is.keyEventManager.ListenForPattern(ctx, Keys.imageContainerTTL("*"), is.keyEventChan) + go is.keyEventManager.ListenForPattern(ctx, common.RedisKeys.SchedulerContainerState("*"), is.keyEventChan) return &is, nil } @@ -194,51 +207,27 @@ func (is *RuncImageService) retrieveBuildSecrets(ctx context.Context, secrets [] } func (is *RuncImageService) monitorImageContainers(ctx context.Context) { - ticker := time.NewTicker(5 * time.Second) - for { select { - case <-ticker.C: - containerKeys, err := is.rdb.HKeys(ctx, imageBuildContainersCreatedAtKey).Result() - if err != nil { - log.Error().Err(err).Msg("failed to get active image build containers") - return - } - - for _, key := range containerKeys { - createdAt := is.rdb.HGet(ctx, imageBuildContainersCreatedAtKey, key).Val() - if createdAt == "" { - log.Error().Msg("failed to get created at time for container") - continue - } - - createdAtInt, err := strconv.ParseInt(createdAt, 10, 64) - if err != nil { - log.Error().Err(err).Msg("failed to parse container created at time") - continue - } - - // See if the container still exists - _, err = is.builder.containerRepo.GetContainerState(key) - if err != nil { - if errors.Is(err, &types.ErrCheckpointNotFound{}) { - is.rdb.HDel(ctx, imageBuildContainersCreatedAtKey, key) - } - - log.Error().Err(err).Msg("failed to get container state") - continue - } - - if createdAtInt < time.Now().Unix()-is.config.ImageService.ImageBuildTimeoutS { - is.builder.scheduler.Stop( - &types.StopContainerArgs{ - ContainerId: key, + case event := <-is.keyEventChan: + switch event.Operation { + case common.KeyOperationSet: + if strings.Contains(event.Key, common.RedisKeys.SchedulerContainerState("")) { + containerId := strings.TrimPrefix(is.keyEventManager.TrimKeyspacePrefix(event.Key), common.RedisKeys.SchedulerContainerState("")) + + if is.rdb.Exists(ctx, Keys.imageContainerTTL(containerId)).Val() == 0 { + is.builder.scheduler.Stop(&types.StopContainerArgs{ + ContainerId: containerId, Force: true, - }, - ) - - is.rdb.HDel(ctx, imageBuildContainersCreatedAtKey, key) + }) + } } + case common.KeyOperationExpired: + containerId := strings.TrimPrefix(is.keyEventManager.TrimKeyspacePrefix(event.Key), Keys.imageContainerTTL("")) + is.builder.scheduler.Stop(&types.StopContainerArgs{ + ContainerId: containerId, + Force: true, + }) } case <-ctx.Done(): return @@ -257,4 +246,14 @@ func convertBuildSteps(buildSteps []*pb.BuildStep) []BuildStep { return steps } -var imageBuildContainersCreatedAtKey = "image:containers:created_at" +var ( + imageContainerTTL string = "image:container_ttl:%s" +) + +var Keys = &keys{} + +type keys struct{} + +func (k *keys) imageContainerTTL(containerId string) string { + return fmt.Sprintf(imageContainerTTL, containerId) +} diff --git a/pkg/common/key_events.go b/pkg/common/key_events.go index cdd5c082b..800f16688 100644 --- a/pkg/common/key_events.go +++ b/pkg/common/key_events.go @@ -33,6 +33,10 @@ func NewKeyEventManager(rdb *RedisClient) (*KeyEventManager, error) { return &KeyEventManager{rdb: rdb}, nil } +func (kem *KeyEventManager) TrimKeyspacePrefix(key string) string { + return strings.TrimPrefix(key, keyspacePrefix) +} + func (kem *KeyEventManager) fetchExistingKeys(patternPrefix string) ([]string, error) { pattern := fmt.Sprintf("%s*", patternPrefix) diff --git a/pkg/types/scheduler.go b/pkg/types/scheduler.go index 09b6ac17c..fea0206e4 100644 --- a/pkg/types/scheduler.go +++ b/pkg/types/scheduler.go @@ -126,7 +126,7 @@ const ( ContainerResourceUsageEmissionInterval time.Duration = 3 * time.Second ) const ContainerStateTtlSWhilePending int = 600 -const ContainerStateTtlS int = 120 +const ContainerStateTtlS int = 30 const WorkspaceQuotaTtlS int = 600 type ErrContainerStateNotFound struct { From 1a09a18fa20cacd640debf635fce9b6674c93182 Mon Sep 17 00:00:00 2001 From: John Date: Thu, 2 Jan 2025 16:14:30 -0700 Subject: [PATCH 5/8] Cleanup --- pkg/abstractions/image/image.go | 2 +- pkg/common/config.default.yaml | 1 - pkg/types/config.go | 1 - pkg/types/scheduler.go | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/abstractions/image/image.go b/pkg/abstractions/image/image.go index 03c96bf97..c4f6d3154 100644 --- a/pkg/abstractions/image/image.go +++ b/pkg/abstractions/image/image.go @@ -42,7 +42,7 @@ type ImageServiceOpts struct { } const containerKeepAliveIntervalS int = 30 -const imageContainerTtlS int = 30 +const imageContainerTtlS int = 60 func NewRuncImageService( ctx context.Context, diff --git a/pkg/common/config.default.yaml b/pkg/common/config.default.yaml index 02fd9e08a..bcca17ef2 100644 --- a/pkg/common/config.default.yaml +++ b/pkg/common/config.default.yaml @@ -61,7 +61,6 @@ imageService: registryStore: local registryCredentialProvider: docker buildContainerPoolSelector: build - imageBuildTimeoutS: 3600 registries: docker: username: beamcloud diff --git a/pkg/types/config.go b/pkg/types/config.go index 216093e75..744557d27 100644 --- a/pkg/types/config.go +++ b/pkg/types/config.go @@ -132,7 +132,6 @@ type ImageServiceConfig struct { BuildContainerMemory int64 `key:"buildContainerMemory" json:"build_container_memory"` BuildContainerPoolSelector string `key:"buildContainerPoolSelector" json:"build_container_pool_selector"` Runner RunnerConfig `key:"runner" json:"runner"` - ImageBuildTimeoutS int64 `key:"imageBuildTimeoutS" json:"image_build_timeout_s"` } type ImageRegistriesConfig struct { diff --git a/pkg/types/scheduler.go b/pkg/types/scheduler.go index fea0206e4..09b6ac17c 100644 --- a/pkg/types/scheduler.go +++ b/pkg/types/scheduler.go @@ -126,7 +126,7 @@ const ( ContainerResourceUsageEmissionInterval time.Duration = 3 * time.Second ) const ContainerStateTtlSWhilePending int = 600 -const ContainerStateTtlS int = 30 +const ContainerStateTtlS int = 120 const WorkspaceQuotaTtlS int = 600 type ErrContainerStateNotFound struct { From 049a1d8afdf97480f8802c5f5bd8042a13c2be64 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 7 Jan 2025 15:15:35 -0700 Subject: [PATCH 6/8] Update naming --- pkg/abstractions/image/build.go | 6 +++--- pkg/abstractions/image/image.go | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/abstractions/image/build.go b/pkg/abstractions/image/build.go index c852ea000..0dc07a175 100644 --- a/pkg/abstractions/image/build.go +++ b/pkg/abstractions/image/build.go @@ -287,7 +287,7 @@ func (b *Builder) Build(ctx context.Context, opts *BuildOpts, outputChan chan co return err } - err = b.rdb.Set(ctx, Keys.imageContainerTTL(containerId), "1", time.Duration(imageContainerTtlS)*time.Second).Err() + err = b.rdb.Set(ctx, Keys.imageBuildContainerTTL(containerId), "1", time.Duration(imageContainerTtlS)*time.Second).Err() if err != nil { outputChan <- common.OutputMsg{Done: true, Success: false, Msg: "Failed to connect to build container.\n"} return err @@ -450,7 +450,7 @@ func (b *Builder) Exists(ctx context.Context, imageId string) bool { } func (b *Builder) keepAlive(ctx context.Context, containerId string, done <-chan struct{}) { - ticker := time.NewTicker(time.Duration(containerKeepAliveIntervalS) * time.Second) + ticker := time.NewTicker(time.Duration(buildContainerKeepAliveIntervalS) * time.Second) defer ticker.Stop() for { @@ -460,7 +460,7 @@ func (b *Builder) keepAlive(ctx context.Context, containerId string, done <-chan case <-done: return case <-ticker.C: - b.rdb.Set(ctx, Keys.imageContainerTTL(containerId), "1", time.Duration(imageContainerTtlS)*time.Second).Err() + b.rdb.Set(ctx, Keys.imageBuildContainerTTL(containerId), "1", time.Duration(imageContainerTtlS)*time.Second).Err() } } } diff --git a/pkg/abstractions/image/image.go b/pkg/abstractions/image/image.go index c4f6d3154..788fcff85 100644 --- a/pkg/abstractions/image/image.go +++ b/pkg/abstractions/image/image.go @@ -41,7 +41,7 @@ type ImageServiceOpts struct { RedisClient *common.RedisClient } -const containerKeepAliveIntervalS int = 30 +const buildContainerKeepAliveIntervalS int = 30 const imageContainerTtlS int = 60 func NewRuncImageService( @@ -73,7 +73,7 @@ func NewRuncImageService( } go is.monitorImageContainers(ctx) - go is.keyEventManager.ListenForPattern(ctx, Keys.imageContainerTTL("*"), is.keyEventChan) + go is.keyEventManager.ListenForPattern(ctx, Keys.imageBuildContainerTTL("*"), is.keyEventChan) go is.keyEventManager.ListenForPattern(ctx, common.RedisKeys.SchedulerContainerState("*"), is.keyEventChan) return &is, nil @@ -215,7 +215,7 @@ func (is *RuncImageService) monitorImageContainers(ctx context.Context) { if strings.Contains(event.Key, common.RedisKeys.SchedulerContainerState("")) { containerId := strings.TrimPrefix(is.keyEventManager.TrimKeyspacePrefix(event.Key), common.RedisKeys.SchedulerContainerState("")) - if is.rdb.Exists(ctx, Keys.imageContainerTTL(containerId)).Val() == 0 { + if is.rdb.Exists(ctx, Keys.imageBuildContainerTTL(containerId)).Val() == 0 { is.builder.scheduler.Stop(&types.StopContainerArgs{ ContainerId: containerId, Force: true, @@ -223,7 +223,7 @@ func (is *RuncImageService) monitorImageContainers(ctx context.Context) { } } case common.KeyOperationExpired: - containerId := strings.TrimPrefix(is.keyEventManager.TrimKeyspacePrefix(event.Key), Keys.imageContainerTTL("")) + containerId := strings.TrimPrefix(is.keyEventManager.TrimKeyspacePrefix(event.Key), Keys.imageBuildContainerTTL("")) is.builder.scheduler.Stop(&types.StopContainerArgs{ ContainerId: containerId, Force: true, @@ -247,13 +247,13 @@ func convertBuildSteps(buildSteps []*pb.BuildStep) []BuildStep { } var ( - imageContainerTTL string = "image:container_ttl:%s" + imageBuildContainerTTL string = "image:build_container_ttl:%s" ) var Keys = &keys{} type keys struct{} -func (k *keys) imageContainerTTL(containerId string) string { - return fmt.Sprintf(imageContainerTTL, containerId) +func (k *keys) imageBuildContainerTTL(containerId string) string { + return fmt.Sprintf(imageBuildContainerTTL, containerId) } From 824d2c9e95d31364ec4ca1a8ebfc825faea409b9 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 7 Jan 2025 15:23:28 -0700 Subject: [PATCH 7/8] remove 2nd trimkeyspaceprefix fn --- pkg/common/key_events.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/common/key_events.go b/pkg/common/key_events.go index 7b776b841..800f16688 100644 --- a/pkg/common/key_events.go +++ b/pkg/common/key_events.go @@ -53,10 +53,6 @@ func (kem *KeyEventManager) fetchExistingKeys(patternPrefix string) ([]string, e return trimmedKeys, nil } -func (kem *KeyEventManager) TrimKeyspacePrefix(key string) string { - return strings.TrimPrefix(key, keyspacePrefix) -} - func (kem *KeyEventManager) ListenForPattern(ctx context.Context, patternPrefix string, keyEventChan chan KeyEvent) error { existingKeys, err := kem.fetchExistingKeys(patternPrefix) if err != nil { From 573a7aba6ac5ffcb0a471f6578ff141a9bd41b62 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 7 Jan 2025 15:44:13 -0700 Subject: [PATCH 8/8] Lower keepalive interval --- pkg/abstractions/image/image.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/abstractions/image/image.go b/pkg/abstractions/image/image.go index 788fcff85..95d43e8ef 100644 --- a/pkg/abstractions/image/image.go +++ b/pkg/abstractions/image/image.go @@ -41,7 +41,7 @@ type ImageServiceOpts struct { RedisClient *common.RedisClient } -const buildContainerKeepAliveIntervalS int = 30 +const buildContainerKeepAliveIntervalS int = 10 const imageContainerTtlS int = 60 func NewRuncImageService(