Skip to content

Commit

Permalink
Adding timeout in waitRules
Browse files Browse the repository at this point in the history
Signed-off-by: Rohit Aggarwal <rohit.aggarwal@broadcom.com>
  • Loading branch information
rohitagg2020 committed Aug 5, 2024
1 parent 4992886 commit 0d11980
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 6 deletions.
1 change: 1 addition & 0 deletions pkg/kapp/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ type WaitRuleConditionMatcher struct {
Success bool
SupportsObservedGeneration bool
UnblockChanges bool
Timeout string
}

type WaitRuleYtt struct {
Expand Down
58 changes: 52 additions & 6 deletions pkg/kapp/resourcesmisc/custom_waiting_resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,16 @@ package resourcesmisc

import (
"fmt"
"sync"
"time"

ctlconf "carvel.dev/kapp/pkg/kapp/config"
ctlres "carvel.dev/kapp/pkg/kapp/resources"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var timeoutMap sync.Map

type CustomWaitingResource struct {
resource ctlres.Resource
waitRule ctlconf.WaitRule
Expand Down Expand Up @@ -81,16 +85,41 @@ func (s CustomWaitingResource) IsDoneApplying() DoneApplyState {
hasConditionWaitingForGeneration := false
// Check on failure conditions first
for _, condMatcher := range s.waitRule.ConditionMatchers {
for _, cond := range obj.Status.Conditions {
if cond.Type == condMatcher.Type && cond.Status == condMatcher.Status {
// Check whether timeout has occured
var isTimeOutConditionPresent bool
if len(condMatcher.Timeout) != 0 {
for _, cond := range obj.Status.Conditions {
if condMatcher.SupportsObservedGeneration && obj.Metadata.Generation != cond.ObservedGeneration {
hasConditionWaitingForGeneration = true
continue
}
if condMatcher.Failure {
return DoneApplyState{Done: true, Successful: false, Message: fmt.Sprintf(
"Encountered failure condition %s == %s: %s (message: %s)",
cond.Type, condMatcher.Status, cond.Reason, cond.Message)}
if condMatcher.Type == cond.Type && cond.Status == condMatcher.Status {
isTimeOutConditionPresent = true
if s.hasTimeoutOccurred(condMatcher.Timeout, fmt.Sprintf("%s.%s", s.resource.Namespace(), s.resource.Name())) {
return DoneApplyState{Done: true, Successful: false, Message: fmt.Sprintf(
"Encountered failure condition %s == %s: %s (message: %s) continuously for %s duration",
cond.Type, condMatcher.Status, cond.Reason, cond.Message, condMatcher.Timeout)}
}
return DoneApplyState{Done: false, Message: fmt.Sprintf(
"%s: %s (message: %s)",
cond.Type, cond.Reason, cond.Message)}
}
}
if !isTimeOutConditionPresent {
timeoutMap.Delete(fmt.Sprintf("%s.%s", s.resource.Namespace(), s.resource.Name()))
}
} else {
for _, cond := range obj.Status.Conditions {
if cond.Type == condMatcher.Type && cond.Status == condMatcher.Status {
if condMatcher.SupportsObservedGeneration && obj.Metadata.Generation != cond.ObservedGeneration {
hasConditionWaitingForGeneration = true
continue
}
if condMatcher.Failure {
return DoneApplyState{Done: true, Successful: false, Message: fmt.Sprintf(
"Encountered failure condition %s == %s: %s (message: %s)",
cond.Type, condMatcher.Status, cond.Reason, cond.Message)}
}
}
}
}
Expand Down Expand Up @@ -132,3 +161,20 @@ func (s CustomWaitingResource) IsDoneApplying() DoneApplyState {

return DoneApplyState{Done: false, Message: "No failing or successful conditions found"}
}

func (s CustomWaitingResource) hasTimeoutOccurred(timeout string, key string) bool {
expiryTime, found := timeoutMap.Load(key)
if found {
return time.Now().Sub(expiryTime.(time.Time)) > 0
}
timeoutMap.Store(key, time.Now().Add(parseDuration(timeout)))
return false
}

func parseDuration(str string) time.Duration {
dur, err := time.ParseDuration(str)
if err != nil {
dur = 15 * time.Minute
}
return dur
}
47 changes: 47 additions & 0 deletions test/e2e/wait_timeout_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package e2e

import (
"fmt"
"strings"
"testing"

Expand Down Expand Up @@ -32,6 +33,33 @@ func TestWaitTimeout(t *testing.T) {
restartPolicy: Never
`

yaml2 := `
apiVersion: v1
kind: Pod
metadata:
name: nginx
spec:
containers:
- name: nginx
image: %s
ports:
- containerPort: 80
---
apiVersion: kapp.k14s.io/v1alpha1
kind: Config
waitRules:
- supportsObservedGeneration: true
conditionMatchers:
- type: ContainersReady
status: "False"
timeout: 50s
- type: Ready
status: "True"
success: true
resourceMatchers:
- apiVersionKindMatcher: {apiVersion: v1, kind: Pod}
`

name := "test-wait-timeout"
cleanUp := func() {
kapp.Run([]string{"delete", "-a", name})
Expand Down Expand Up @@ -67,4 +95,23 @@ func TestWaitTimeout(t *testing.T) {

require.NoErrorf(t, err, "Expected to be successful without resource timeout")
})

cleanUp()

logger.Section("Deploy timeout after staying in a condition for certain time", func() {
_, err := kapp.RunWithOpts([]string{"deploy", "-f", "-", "-a", name, "--json"},
RunOpts{IntoNs: true, AllowError: true, StdinReader: strings.NewReader(fmt.Sprintf(yaml2, "nginx:200"))})

require.Error(t, err)
require.Contains(t, err.Error(), "Encountered failure condition")
})

cleanUp()

logger.Section("Deploy should be successful", func() {
_, err := kapp.RunWithOpts([]string{"deploy", "-f", "-", "-a", name, "--json"},
RunOpts{IntoNs: true, AllowError: true, StdinReader: strings.NewReader(fmt.Sprintf(yaml2, "nginx"))})

require.NoError(t, err)
})
}

0 comments on commit 0d11980

Please sign in to comment.