Merge pull request #755 from rancher-max/simple-retry

Add some simple retry to service rio commands

cli/cmd/weight/weight.go

@@ -72,7 +72,9 @@ func GenerateWeightAndRolloutConfig(ctx *clicontext.CLIContext, obj types.Resour
 		return 0, nil, errors.New("cannot perform rollout longer than 10 hours") // over 10 hours we go under increment of 1/10k, given 2 second. Also see safety valve below in increment.
 	}
 	svcs, err := util.ListAppServicesFromAppName(ctx, obj.Namespace, obj.App)
-	if err != nil {
+	if err != nil && err.Error() == "no services found" || len(svcs) == 0 { // todo: make this string check less brittle
+		return targetPercentage, &riov1.RolloutConfig{}, nil
+	} else if err != nil {
 		return 0, nil, err
 	}
 

tests/integration/endpoint_test.go

@@ -16,7 +16,7 @@ func endpointTests(t *testing.T, when spec.G, it spec.S) {
 	when("a service is running and another is staged", func() {
 
 		it.Before(func() {
-			service.Create(t, "ibuildthecloud/demo:v1")
+			service.Create(t, "--weight", "100", "ibuildthecloud/demo:v1")
 		})
 		it.After(func() {
 			service.Remove()
@@ -51,7 +51,7 @@ func endpointTests(t *testing.T, when spec.G, it spec.S) {
 
 	when("a staged service is promoted", func() {
 		it.Before(func() {
-			service.Create(t, "ibuildthecloud/demo:v1")
+			service.Create(t, "--weight", "100", "ibuildthecloud/demo:v1")
 			stagedService = service.Stage("ibuildthecloud/demo:v3", "v3")
 			stagedService.Promote()
 		})

tests/integration/route_test.go

@@ -17,7 +17,7 @@ func routeTests(t *testing.T, when spec.G, it spec.S) {
 	var routeB testutil.TestRoute
 
 	it.Before(func() {
-		service.Create(t, "ibuildthecloud/demo:v1")
+		service.Create(t, "--weight", "100", "ibuildthecloud/demo:v1")
 		stagedService = service.Stage("ibuildthecloud/demo:v3", "v3")
 	})
 

tests/integration/stage_test.go

@@ -15,7 +15,7 @@ func stageTests(t *testing.T, when spec.G, it spec.S) {
 	var stagedService testutil.TestService
 
 	it.Before(func() {
-		service.Create(t, "--label", "x=y", "--annotations", "a=b", "ibuildthecloud/demo:v1")
+		service.Create(t, "--weight", "100", "--label", "x=y", "--annotations", "a=b", "ibuildthecloud/demo:v1")
 	})
 
 	it.After(func() {

tests/integration/weight_test.go

@@ -14,7 +14,7 @@ func weightTests(t *testing.T, when spec.G, it spec.S) {
 	var stagedService testutil.TestService
 
 	it.Before(func() {
-		service.Create(t, "ibuildthecloud/demo:v1")
+		service.Create(t, "--weight", "100", "ibuildthecloud/demo:v1")
 		stagedService = service.Stage("ibuildthecloud/demo:v3", "v3")
 	})
 

tests/testutil/service.go

@@ -32,7 +32,7 @@ type TestService struct {
 // returns a new TestService with it attached. Guarantees ready state but not live endpoint
 func (ts *TestService) Create(t *testing.T, source ...string) {
 	args, envs := ts.createArgs(t, source...)
-	_, err := RioCmd(args, envs...)
+	_, err := RioCmdWithRetry(args, envs...)
 	if err != nil {
 		ts.T.Fatalf("Failed to create service %s: %v", ts.Name, err.Error())
 	}
@@ -119,7 +119,7 @@ func (ts *TestService) Remove() {
 		}
 	}
 	if ts.Service.Status.DeploymentReady {
-		_, err := RioCmd([]string{"rm", ts.Name})
+		_, err := RioCmdWithRetry([]string{"rm", ts.Name})
 		if err != nil {
 			ts.T.Log(err.Error())
 		}
@@ -137,7 +137,7 @@ func (ts *TestService) Attach() []string {
 
 // Call "rio scale ns/service={scaleTo}"
 func (ts *TestService) Scale(scaleTo int) {
-	_, err := RioCmd([]string{"scale", fmt.Sprintf("%s=%d", ts.Name, scaleTo)})
+	_, err := RioCmdWithRetry([]string{"scale", fmt.Sprintf("%s=%d", ts.Name, scaleTo)})
 	if err != nil {
 		ts.T.Fatalf("scale command failed:  %v", err.Error())
 	}
@@ -158,7 +158,7 @@ func (ts *TestService) WeightWithoutWaiting(weightSpec int, args ...string) {
 	args = append(
 		[]string{"weight"},
 		append(args, fmt.Sprintf("%s=%d", ts.Name, weightSpec))...)
-	_, err := RioCmd(args)
+	_, err := RioCmdWithRetry(args)
 	if err != nil {
 		ts.T.Fatalf("weight command failed:  %v", err.Error())
 	}
@@ -183,7 +183,7 @@ func (ts *TestService) Weight(weightSpec int, args ...string) {
 
 // Call "rio stage --image={source} ns/name:{version}", this will return a new TestService
 func (ts *TestService) Stage(source, version string) TestService {
-	_, err := RioCmd([]string{"stage", "--image", source, ts.Name, version})
+	_, err := RioCmdWithRetry([]string{"stage", "--image", source, ts.Name, version})
 	if err != nil {
 		ts.T.Fatalf("stage command failed:  %v", err.Error())
 	}
@@ -205,7 +205,7 @@ func (ts *TestService) Promote(args ...string) {
 	args = append(
 		[]string{"promote", "--pause=false", "--duration=0s"},
 		append(args, ts.Name)...)
-	_, err := RioCmd(args)
+	_, err := RioCmdWithRetry(args)
 	if err != nil {
 		ts.T.Fatalf("promote command failed:  %v", err.Error())
 	}
@@ -229,7 +229,7 @@ func (ts *TestService) Logs(args ...string) []string {
 
 // Exec calls "rio exec ns/service {command}" on this service
 func (ts *TestService) Exec(command ...string) string {
-	out, err := RioCmd(append([]string{"exec", ts.Name}, command...))
+	out, err := RioCmdWithRetry(append([]string{"exec", ts.Name}, command...))
 	if err != nil {
 		ts.T.Fatalf("exec command failed:  %v", err.Error())
 	}
@@ -309,7 +309,7 @@ func (ts *TestService) GetSpecWeight() int {
 
 // Return service's computed (actual) weight, not the spec (end-goal) weight
 func (ts *TestService) GetCurrentWeight() int {
-	out, err := RioCmd([]string{"ps", "--format", "{{.Obj | id}}::{{.Data.Weight}}"})
+	out, err := RioCmdWithRetry([]string{"ps", "--format", "{{.Obj | id}}::{{.Data.Weight}}"})
 	if err != nil {
 		ts.T.Fatal(err)
 	}
@@ -564,7 +564,7 @@ func (ts *TestService) WaitForScaleDown() error {
 
 // reload calls inspect on the service and uses that to reload our object
 func (ts *TestService) reload() error {
-	out, err := RioCmd([]string{"inspect", "--format", "json", ts.Name})
+	out, err := RioCmdWithRetry([]string{"inspect", "--format", "json", ts.Name})
 	if err != nil {
 		return err
 	}
@@ -594,7 +594,7 @@ func (ts *TestService) reloadBuild() error {
 
 // load a "rio export..." response into a new TestService obj
 func (ts *TestService) loadExport(args []string) (TestService, error) {
-	out, err := RioCmd(args)
+	out, err := RioCmdWithRetry(args)
 	if err != nil {
 		return TestService{}, fmt.Errorf("export command failed: %v", err.Error())
 	}

tests/testutil/testutil.go

@@ -74,6 +74,16 @@ func RioCmd(args []string, envs ...string) (string, error) {
 	return string(stdOutErr), nil
 }
 
+// RioCmdWithRetry executes rio CLI commands with your arguments in testing namespace
+// Example: args=["run", "-n", "test", "nginx"] would run: "rio --namespace testing-namespace run -n test nginx"
+func RioCmdWithRetry(args []string, envs ...string) (string, error) {
+	out, err := retry(5, 1, RioCmd, args, envs...)
+	if err != nil {
+		return "", fmt.Errorf("%s: %s", err.Error(), out)
+	}
+	return string(out), nil
+}
+
 // RioCmdWithTail executes rio CLI commands that tail output with your arguments in testing namespace.
 // Example: args=["attach", "nginx"] would run: "rio --namespace testing-namespace attach nginx"
 func RioCmdWithTail(timeoutSeconds int, args []string, envs ...string) ([]string, error) {
@@ -346,3 +356,21 @@ func GetCNAMEInfo() string {
 func getZoneIDInfo() string {
 	return os.Getenv("RIO_ROUTE53_ZONEID")
 }
+
+// retry function is intended for retrying command line commands invocations that collisioned while updating kubernetes objects
+func retry(attempts int, sleep time.Duration, f func(args []string, envs ...string) (string, error), args []string, envs ...string) (string, error) {
+	strOut, err := f(args, envs...)
+	if err != nil {
+		if s, ok := err.(stop); ok {
+			return strOut, s.error
+		}
+		if attempts--; attempts > 0 {
+			jitter := time.Duration(rand.Int63n(int64(sleep)))
+			sleep = sleep + jitter/2
+			time.Sleep(sleep)
+			return retry(attempts, 2*sleep, f, args, envs...)
+		}
+		return strOut, err
+	}
+	return strOut, nil
+}