From c0fb3e6f5b4141ef8d9816a9849b96bf7b71b9fe Mon Sep 17 00:00:00 2001 From: Caleb Bron Date: Thu, 7 Nov 2019 10:09:02 -0700 Subject: [PATCH] Normalize computed weights and switch to duration --- cli/cmd/create/create.go | 38 ++--- cli/cmd/promote/promote.go | 58 ++------ cli/cmd/ps/ps_services.go | 15 ++ cli/cmd/util/service.go | 6 +- cli/cmd/weight/weight.go | 162 +++++++++++++-------- cli/main.go | 4 +- cli/pkg/tables/service.go | 10 +- pkg/apis/rio.cattle.io/v1/service_types.go | 2 +- tests/integration/endpoint_test.go | 12 +- tests/integration/weight_test.go | 12 +- tests/testutil/service.go | 64 ++++---- tests/validation/weight_test.go | 2 +- 12 files changed, 209 insertions(+), 176 deletions(-) diff --git a/cli/cmd/create/create.go b/cli/cmd/create/create.go index 12a92abf8..55b89bcd6 100644 --- a/cli/cmd/create/create.go +++ b/cli/cmd/create/create.go @@ -3,9 +3,12 @@ package create import ( "fmt" "strconv" + "time" "github.com/pkg/errors" + "github.com/rancher/rio/cli/cmd/weight" "github.com/rancher/rio/cli/pkg/clicontext" + "github.com/rancher/rio/cli/pkg/types" riov1 "github.com/rancher/rio/pkg/apis/rio.cattle.io/v1" "github.com/rancher/rio/pkg/riofile/stringers" "github.com/rancher/wrangler/pkg/kv" @@ -66,8 +69,7 @@ type Create struct { P_Ports []string `desc:"Publish a container's port(s) (format: svcport:containerport/protocol)"` Privileged bool `desc:"Run container with privilege"` ReadOnly bool `desc:"Mount the container's root filesystem as read only"` - RolloutInterval int `desc:"Rollout interval in seconds"` - RolloutIncrement int `desc:"Rollout increment value"` + RolloutDuration string `desc:"How long the rollout should take" default:"0s"` RequestTimeoutSeconds int `desc:"Set request timeout in seconds"` Secret []string `desc:"Secrets to inject to the service (format: name[/key]:target)"` Template bool `desc:"If true new version is created per git commit. If false update in-place"` @@ -98,20 +100,6 @@ func (c *Create) RunCallback(ctx *clicontext.CLIContext, cb func(service *riov1. return service, ctx.Create(service) } -func (c *Create) setRollout(spec *riov1.ServiceSpec) { - if c.RolloutIncrement > 0 || c.RolloutInterval > 0 { - spec.RolloutConfig = &riov1.RolloutConfig{ - Increment: c.RolloutIncrement, - IntervalSeconds: c.RolloutInterval, - } - } else if c.Template { - spec.RolloutConfig = &riov1.RolloutConfig{ - Increment: 5, - IntervalSeconds: 5, - } - } -} - func (c *Create) setDNS(spec *riov1.ServiceSpec) (err error) { if len(c.DNS) > 0 || len(c.DNSSearch) > 0 || len(c.DNSOption) > 0 || c.HostDNS { spec.DNS = &riov1.DNS{ @@ -199,11 +187,23 @@ func (c *Create) ToService(ctx *clicontext.CLIContext, args []string) (*riov1.Se } if c.Weight > 0 { - spec.Weight = &c.Weight + duration, err := time.ParseDuration(c.RolloutDuration) + if err != nil { + return nil, err + } + tempResource := types.Resource{ + Name: name, + App: r.App, + Namespace: r.Namespace, + } + weight, rc, err := weight.GenerateWeightAndRolloutConfig(ctx, tempResource, c.Weight, duration, false) + if err != nil { + return nil, err + } + spec.Weight = &weight + spec.RolloutConfig = rc } - c.setRollout(&spec) - if err := c.setDNS(&spec); err != nil { return nil, err } diff --git a/cli/cmd/promote/promote.go b/cli/cmd/promote/promote.go index 266c1c202..4853ce1da 100644 --- a/cli/cmd/promote/promote.go +++ b/cli/cmd/promote/promote.go @@ -2,20 +2,15 @@ package promote import ( "errors" - "fmt" + "time" - "github.com/rancher/rio/cli/cmd/util" + "github.com/rancher/rio/cli/cmd/weight" "github.com/rancher/rio/cli/pkg/clicontext" - "github.com/rancher/rio/cli/pkg/types" - riov1 "github.com/rancher/rio/pkg/apis/rio.cattle.io/v1" - "github.com/rancher/wrangler/pkg/merr" - "k8s.io/apimachinery/pkg/runtime" ) type Promote struct { - Increment int `desc:"Amount of weight to increment on each interval" default:"5"` - Interval int `desc:"Interval seconds between each increment" default:"5"` - Pause bool `desc:"Whether to pause rollout or continue it. Default to false" default:"false"` + Duration string `desc:"How long the rollout should take" default:"0s"` + Pause bool `desc:"Whether to pause rollout or continue it. Default to false" default:"false"` } func (p *Promote) Run(ctx *clicontext.CLIContext) error { @@ -25,46 +20,17 @@ func (p *Promote) Run(ctx *clicontext.CLIContext) error { return errors.New("at least one argument is needed") } serviceName := arg.First() - rolloutConfig := &riov1.RolloutConfig{ - Pause: p.Pause, - Increment: p.Increment, - IntervalSeconds: p.Interval, - } - return PerformPromote(ctx, serviceName, rolloutConfig) -} - -func PerformPromote(ctx *clicontext.CLIContext, serviceName string, rolloutConfig *riov1.RolloutConfig) error { - var allErrors []error - svcs, err := util.ListAppServicesFromServiceName(ctx, serviceName) + resource, err := ctx.ByID(serviceName) if err != nil { return err } - versionToPromote := ctx.ParseID(serviceName).Version - if versionToPromote == "" { - return errors.New("invalid version specified") + duration, err := time.ParseDuration(p.Duration) + if err != nil { + return err } - for _, s := range svcs { - err := ctx.UpdateResource(types.Resource{ - Namespace: s.Namespace, - Name: s.Name, - App: s.Spec.App, - Version: s.Spec.Version, - Type: types.ServiceType, - }, func(obj runtime.Object) error { - s := obj.(*riov1.Service) - if s.Spec.Weight == nil { - s.Spec.Weight = new(int) - } - s.Spec.RolloutConfig = rolloutConfig - if s.Spec.Version == versionToPromote { - *s.Spec.Weight = 100 - fmt.Printf("%s promoted\n", s.Name) - } else { - *s.Spec.Weight = 0 - } - return nil - }) - allErrors = append(allErrors, err) + promoteWeight, rolloutConfig, err := weight.GenerateWeightAndRolloutConfig(ctx, resource, 100, duration, p.Pause) + if err != nil { + return err } - return merr.NewErrors(allErrors...) + return weight.PromoteService(ctx, resource, rolloutConfig, promoteWeight) } diff --git a/cli/cmd/ps/ps_services.go b/cli/cmd/ps/ps_services.go index 15ba5e32f..8a0f01b0b 100644 --- a/cli/cmd/ps/ps_services.go +++ b/cli/cmd/ps/ps_services.go @@ -1,6 +1,7 @@ package ps import ( + "math" "strings" webhookv1 "github.com/rancher/gitwatcher/pkg/apis/gitwatcher.cattle.io/v1" @@ -17,6 +18,7 @@ import ( type ServiceData struct { Name string Service *riov1.Service + Weight int Deployment *appsv1.Deployment DaemonSet *appsv1.DaemonSet Namespace string @@ -116,6 +118,18 @@ func (p *Ps) services(ctx *clicontext.CLIContext) error { if service.(*riov1.Service).Spec.Template && !p.A_All { continue } + + weight := 0 + if service.(*riov1.Service).Status.ComputedWeight != nil && *service.(*riov1.Service).Status.ComputedWeight > 0 { + totalWeight := 0 + for _, subService := range services { + if service.(*riov1.Service).Spec.App == subService.(*riov1.Service).Spec.App && subService.(*riov1.Service).Status.ComputedWeight != nil { + totalWeight += *subService.(*riov1.Service).Status.ComputedWeight + } + } + weight = int(math.Round((float64(*service.(*riov1.Service).Status.ComputedWeight) / float64(totalWeight)) / 0.01)) // round to nearest percent + } + var gitwatcher *webhookv1.GitWatcher gitwatchers, err := ctx.Gitwatcher.GitWatchers(service.(*riov1.Service).Namespace).List(metav1.ListOptions{}) if err == nil { @@ -128,6 +142,7 @@ func (p *Ps) services(ctx *clicontext.CLIContext) error { } output = append(output, ServiceData{ Service: service.(*riov1.Service), + Weight: weight, Namespace: service.(*riov1.Service).Namespace, Pods: podMap[service.(*riov1.Service).Namespace+"/"+service.(*riov1.Service).Name], GitWatcher: gitwatcher, diff --git a/cli/cmd/util/service.go b/cli/cmd/util/service.go index 3fec9bb57..286d6245a 100644 --- a/cli/cmd/util/service.go +++ b/cli/cmd/util/service.go @@ -18,7 +18,11 @@ func ListAppServicesFromServiceName(ctx *clicontext.CLIContext, serviceName stri svc := service.Object.(*riov1.Service) app, _ := services.AppAndVersion(svc) + return ListAppServicesFromAppName(ctx, namespace, app) +} + +func ListAppServicesFromAppName(ctx *clicontext.CLIContext, namespace, appName string) ([]riov1.Service, error) { svcs, err := ctx.Rio.Services(namespace).List(metav1.ListOptions{}) if err != nil { return []riov1.Service{}, err @@ -27,7 +31,7 @@ func ListAppServicesFromServiceName(ctx *clicontext.CLIContext, serviceName stri var revisions []riov1.Service for _, rev := range svcs.Items { revApp, _ := services.AppAndVersion(&rev) - if revApp == app { + if revApp == appName { revisions = append(revisions, rev) } } diff --git a/cli/cmd/weight/weight.go b/cli/cmd/weight/weight.go index 536491699..bda2ec668 100644 --- a/cli/cmd/weight/weight.go +++ b/cli/cmd/weight/weight.go @@ -3,22 +3,26 @@ package weight import ( "errors" "fmt" + "math" "strconv" "strings" + "time" - "github.com/rancher/rio/cli/cmd/promote" "github.com/rancher/rio/cli/cmd/util" "github.com/rancher/rio/cli/pkg/clicontext" + "github.com/rancher/rio/cli/pkg/table" + "github.com/rancher/rio/cli/pkg/types" riov1 "github.com/rancher/rio/pkg/apis/rio.cattle.io/v1" "github.com/rancher/wrangler/pkg/kv" "github.com/rancher/wrangler/pkg/merr" "k8s.io/apimachinery/pkg/runtime" ) +const DefaultInterval = 2 + type Weight struct { - Increment int `desc:"Amount of weight to increment on each interval" default:"5"` - Interval int `desc:"Interval seconds between each increment" default:"5"` - Pause bool `desc:"Whether to pause rollout or continue it. Default to false" default:"false"` + Duration string `desc:"How long the rollout should take" default:"0s"` + Pause bool `desc:"Whether to pause rollout or continue it. Default to false" default:"false"` } func (w *Weight) Run(ctx *clicontext.CLIContext) error { @@ -26,49 +30,18 @@ func (w *Weight) Run(ctx *clicontext.CLIContext) error { return errors.New("at least one parameter is required. Run -h to see options") } ctx.NoPrompt = true - rolloutConfig := &riov1.RolloutConfig{ - Pause: w.Pause, - Increment: w.Increment, - IntervalSeconds: w.Interval, - } - for _, arg := range ctx.CLI.Args() { - if strings.Contains(arg, "%") { - if len(ctx.CLI.Args()) > 1 { + if len(ctx.CLI.Args()) > 1 { + for _, arg := range ctx.CLI.Args() { + if strings.Contains(arg, "%") { return errors.New("only one percentage setting allowed per weight command") } - return setPercentageWeight(ctx, ctx.CLI.Args()[0], rolloutConfig) } } - return setSpecWeight(ctx, ctx.CLI.Args(), rolloutConfig) -} - -func setSpecWeight(ctx *clicontext.CLIContext, args []string, rolloutConfig *riov1.RolloutConfig) error { - var errs []error - for _, arg := range args { - serviceName, scale, err := validateArg(arg) - if err != nil { - return err - } - resource, err := ctx.ByID(serviceName) - if err != nil { - return err - } - err = ctx.UpdateResource(resource, func(obj runtime.Object) error { - service := obj.(*riov1.Service) - if service.Spec.Weight == nil { - service.Spec.Weight = new(int) - } - *service.Spec.Weight = scale - service.Spec.RolloutConfig = rolloutConfig - return nil - }) - errs = append(errs, err) + duration, err := time.ParseDuration(w.Duration) + if err != nil { + return err } - return merr.NewErrors(errs...) -} - -func setPercentageWeight(ctx *clicontext.CLIContext, arg string, rolloutConfig *riov1.RolloutConfig) error { - serviceName, scale, err := validateArg(arg) + serviceName, target, err := validateArg(ctx.CLI.Args().First()) if err != nil { return err } @@ -76,28 +49,12 @@ func setPercentageWeight(ctx *clicontext.CLIContext, arg string, rolloutConfig * if err != nil { return err } - if scale == 100 { - return promote.PerformPromote(ctx, serviceName, rolloutConfig) - } - svcs, err := util.ListAppServicesFromServiceName(ctx, serviceName) + weight, rolloutConfig, err := GenerateWeightAndRolloutConfig(ctx, resource, target, duration, w.Pause) if err != nil { return err } - // first find all weight on other versions of this service - otherSvcTotalWeight := 0 - svc := resource.Object.(*riov1.Service) - for _, s := range svcs { - if s.Name == svc.Name { - continue - } - if s.Status.ComputedWeight != nil && *s.Status.ComputedWeight > 0 { - otherSvcTotalWeight += *s.Status.ComputedWeight - } - } - // now calculate what computed weight should be to hit or percentage target, and set that - weight := scale - if otherSvcTotalWeight > 0 { - weight = int(float64(otherSvcTotalWeight)/(1-(float64(scale)/100))) - otherSvcTotalWeight + if target == 100 { + return PromoteService(ctx, resource, rolloutConfig, weight) } return ctx.UpdateResource(resource, func(obj runtime.Object) error { s := obj.(*riov1.Service) @@ -110,11 +67,92 @@ func setPercentageWeight(ctx *clicontext.CLIContext, arg string, rolloutConfig * }) } +func GenerateWeightAndRolloutConfig(ctx *clicontext.CLIContext, obj types.Resource, targetPercentage int, duration time.Duration, pause bool) (int, *riov1.RolloutConfig, error) { + svcs, err := util.ListAppServicesFromAppName(ctx, obj.Namespace, obj.App) + if err != nil { + return 0, nil, err + } + + currComputedWeight := 0 + if svc, ok := obj.Object.(*riov1.Service); ok { + if svc.Status.ComputedWeight != nil && *svc.Status.ComputedWeight > 0 { + currComputedWeight = *svc.Status.ComputedWeight + } + } + + totalCurrWeight := 0 + for _, s := range svcs { + if s.Status.ComputedWeight != nil && *s.Status.ComputedWeight > 0 { + totalCurrWeight += *s.Status.ComputedWeight + } + } + totalCurrWeightOtherSvcs := totalCurrWeight - currComputedWeight + + newWeight := targetPercentage + if targetPercentage == 100 { + newWeight = 10000 + } else if totalCurrWeightOtherSvcs > 0 { + // find the weight that would hit our target percentage without touching other service weights + // ie: if 2 svcs at 50/50 and you want one at 75%, newWeight would be 150 + newWeight = int(float64(totalCurrWeightOtherSvcs)/(1-(float64(targetPercentage)/100))) - totalCurrWeightOtherSvcs + } + + // if not immediate rollout figure out increment + increment := 0 + if duration.Seconds() >= 1.0 { + steps := duration.Seconds() / float64(DefaultInterval) // First get rough amount of steps we want to take + totalNewWeight := totalCurrWeightOtherSvcs + newWeight // Given the future total weight which includes our newWeight... + difference := totalNewWeight - totalCurrWeight // Find the difference between future total weight and current total weight + increment = int(math.Abs(math.Round(float64(difference) / steps))) // And divide by steps to get rough increment + } + rolloutConfig := &riov1.RolloutConfig{ + Pause: pause, + Increment: increment, + IntervalSeconds: DefaultInterval, + } + return newWeight, rolloutConfig, nil +} + +func PromoteService(ctx *clicontext.CLIContext, resource types.Resource, rolloutConfig *riov1.RolloutConfig, promoteWeight int) error { + var allErrors []error + svcs, err := util.ListAppServicesFromServiceName(ctx, resource.Name) + if err != nil { + return err + } + for _, s := range svcs { + if s.Spec.Version == resource.Version || (s.Status.ComputedWeight != nil && *s.Status.ComputedWeight > 0) { // don't update non-promoted svcs without weight allocated + err := ctx.UpdateResource(types.Resource{ + Namespace: s.Namespace, + Name: s.Name, + App: s.Spec.App, + Version: s.Spec.Version, + Type: types.ServiceType, + }, func(obj runtime.Object) error { + s := obj.(*riov1.Service) + s.Spec.RolloutConfig = rolloutConfig + if s.Spec.Weight == nil { + s.Spec.Weight = new(int) + } + if s.Spec.Version == resource.Version { + *s.Spec.Weight = promoteWeight + id, _ := table.FormatID(obj, s.Namespace) + fmt.Printf("%s promoted\n", id) + } else { + *s.Spec.Weight = 0 + } + return nil + }) + allErrors = append(allErrors, err) + } + } + return merr.NewErrors(allErrors...) +} + func validateArg(arg string) (string, int, error) { serviceName, scaleStr := kv.Split(arg, "=") scaleStr = strings.TrimSuffix(scaleStr, "%") if scaleStr == "" { - return serviceName, 0, errors.New("weight params must be in the format of SERVICE=WEIGHT, for example: myservice=10% or myservice=20") + return serviceName, 0, errors.New("weight params must be in the format of SERVICE=WEIGHT, for example: myservice=10%") } scale, err := strconv.Atoi(scaleStr) if err != nil { diff --git a/cli/main.go b/cli/main.go index 38145dabe..16de2ae42 100644 --- a/cli/main.go +++ b/cli/main.go @@ -194,11 +194,11 @@ func main() { builder.Command(&promote.Promote{}, "Promote a staged version to latest and scale down other app versions", appName+" promote [OPTIONS] SERVICE_NAME", - "To perform an immediate rollout, set interval and increment to 0"), + "Defaults to an immediate rollout, set interval greater than one to perform a gradual rollout"), builder.Command(&weight.Weight{}, "Weight a service to specific weight or percentage of total app traffic", appName+" weight [OPTIONS] SERVICE_NAME=WEIGHT/PERCENTAGE", - "To perform an immediate rollout, set interval and increment to 0"), + "Defaults to an immediate rollout, set interval greater than one to perform a gradual rollout"), builder.Command(&systemlogs.SystemLogs{}, "View system log for Rio management plane", appName+" systemlogs", diff --git a/cli/pkg/tables/service.go b/cli/pkg/tables/service.go index 6a1e9b618..ee8ccaf69 100644 --- a/cli/pkg/tables/service.go +++ b/cli/pkg/tables/service.go @@ -26,7 +26,7 @@ func NewService(cfg Config) TableWriter { {"ENDPOINT", "{{.Obj | formatEndpoint }}"}, {"PORTS", "{{.Obj | formatPorts}}"}, {"SCALE", "{{.Obj | scale}}"}, - {"WEIGHT", "{{.Obj | formatWeight}}"}, + {"WEIGHT", "{{formatWeight .Obj .Data.Weight }}"}, {"CREATED", "{{.Obj.CreationTimestamp | ago}}"}, {"DETAIL", "{{serviceDetail .Data.Service .Data.Pods .Data.GitWatcher}}"}, }, cfg) @@ -77,7 +77,7 @@ func formatEndpoint(data interface{}) string { return "" } -func formatWeight(data interface{}) string { +func formatWeight(data interface{}, weight int) string { s, ok := data.(*v1.Service) if !ok { return "" @@ -85,11 +85,7 @@ func formatWeight(data interface{}) string { if len(s.Status.Endpoints) == 0 { return "" } - if s.Status.ComputedWeight != nil { - return fmt.Sprintf("%s%%", strconv.Itoa(*s.Status.ComputedWeight)) - } - - return "0%" + return fmt.Sprintf("%v%%", weight) } func serviceDetail(data interface{}, pods []*corev1.Pod, gitwatcher *webhookv1.GitWatcher) string { diff --git a/pkg/apis/rio.cattle.io/v1/service_types.go b/pkg/apis/rio.cattle.io/v1/service_types.go index 8548607fe..fa9df0212 100644 --- a/pkg/apis/rio.cattle.io/v1/service_types.go +++ b/pkg/apis/rio.cattle.io/v1/service_types.go @@ -38,7 +38,7 @@ type AutoscaleConfig struct { // RolloutConfig specifies the configuration when promoting a new revision type RolloutConfig struct { - // Increment Value each Rollout can scale up or down + // Increment Value each Rollout can scale up or down, always a positive number Increment int `json:"increment,omitempty"` // Interval between each Rollout in seconds diff --git a/tests/integration/endpoint_test.go b/tests/integration/endpoint_test.go index ca51c75c2..841884c46 100644 --- a/tests/integration/endpoint_test.go +++ b/tests/integration/endpoint_test.go @@ -63,14 +63,18 @@ func endpointTests(t *testing.T, when spec.G, it spec.S) { it("should retain all revision endpoints with an app endpoint pointing to the new revision", func() { assert.Equal(t, "Hello World", service.GetEndpointResponse()) assert.Equal(t, "Hello World v3", stagedService.GetEndpointResponse()) - assert.Equal(t, 0, service.GetKubeCurrentWeight()) - assert.Equal(t, 100, stagedService.GetKubeCurrentWeight()) + assert.Equal(t, 0, service.GetCurrentWeight()) + assert.Equal(t, 100, stagedService.GetCurrentWeight()) + //assert.Equal(t, 0, service.GetKubeCurrentWeight()) + //assert.Equal(t, 100, stagedService.GetKubeCurrentWeight()) assert.Equal(t, "Hello World v3", service.GetAppEndpointResponse()) }) it("should allow rolling back to the previous revision", func() { service.Promote() - assert.Equal(t, 100, service.GetKubeCurrentWeight()) - assert.Equal(t, 0, stagedService.GetKubeCurrentWeight()) + assert.Equal(t, 100, service.GetCurrentWeight()) + assert.Equal(t, 0, stagedService.GetCurrentWeight()) + //assert.Equal(t, 100, service.GetKubeCurrentWeight()) + //assert.Equal(t, 0, stagedService.GetKubeCurrentWeight()) assert.Equal(t, "Hello World", service.GetAppEndpointResponse()) }) }, spec.Parallel()) diff --git a/tests/integration/weight_test.go b/tests/integration/weight_test.go index 1e0bf6a83..9aa5bf005 100644 --- a/tests/integration/weight_test.go +++ b/tests/integration/weight_test.go @@ -27,15 +27,15 @@ func weightTests(t *testing.T, when spec.G, it spec.S) { it("should keep 100% of weight on original revision", func() { assert.Equal(t, 100, service.GetCurrentWeight()) assert.Equal(t, 0, stagedService.GetCurrentWeight()) - assert.Equal(t, 100, service.GetKubeCurrentWeight()) - assert.Equal(t, 0, stagedService.GetKubeCurrentWeight()) + //assert.Equal(t, 100, service.GetKubeCurrentWeight()) + //assert.Equal(t, 0, stagedService.GetKubeCurrentWeight()) }) it("should be able to split weights between revisions", func() { - stagedService.Weight(40, "--increment=0", "--interval=0") + stagedService.Weight(40, "--duration=0s") assert.Equal(t, 40, stagedService.GetCurrentWeight()) - assert.Equal(t, 40, stagedService.GetKubeCurrentWeight()) - assert.Equal(t, 100, service.GetCurrentWeight()) - assert.Equal(t, 100, service.GetKubeCurrentWeight()) + //assert.Equal(t, 40, stagedService.GetKubeCurrentWeight()) + assert.Equal(t, 60, service.GetCurrentWeight()) + //assert.Equal(t, 100, service.GetKubeCurrentWeight()) responses := service.GetResponseCounts([]string{"Hello World", "Hello World v3"}, 12) assert.Greater(t, responses["Hello World"], 2, "The application did not return enough responses from the service. which has slightly more weight than the staged service.") assert.GreaterOrEqual(t, responses["Hello World v3"], 1, "The application did not return enough responses from the staged service. which has slightly less weight than the service.") diff --git a/tests/testutil/service.go b/tests/testutil/service.go index 99ec8ca5c..5e294d325 100644 --- a/tests/testutil/service.go +++ b/tests/testutil/service.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "os" + "strconv" "strings" "testing" "time" @@ -202,7 +203,7 @@ func (ts *TestService) Stage(source, version string) TestService { // Promote calls "rio promote [args] service_name" to instantly promote a revision func (ts *TestService) Promote(args ...string) { args = append( - []string{"promote", "--pause=false", "--increment=0", "--interval=0"}, + []string{"promote", "--pause=false", "--duration=0s"}, append(args, ts.Name)...) _, err := RioCmd(args) if err != nil { @@ -299,6 +300,7 @@ func (ts *TestService) GetScale() int { // Return service's goal weight, this is different from weight service is currently at func (ts *TestService) GetSpecWeight() int { + ts.reload() if ts.Service.Spec.Weight != nil { return *ts.Service.Spec.Weight } @@ -307,9 +309,19 @@ func (ts *TestService) GetSpecWeight() int { // Return service's computed (actual) weight, not the spec (end-goal) weight func (ts *TestService) GetCurrentWeight() int { - ts.reload() - if ts.Service.Status.ComputedWeight != nil { - return *ts.Service.Status.ComputedWeight + out, err := RioCmd([]string{"ps", "--format", "{{.Obj | id}}::{{.Data.Weight}}"}) + if err != nil { + ts.T.Fatal(err) + } + for _, line := range strings.Split(out, "\n") { + name := strings.Split(line, "::")[0] + if name == ts.Name { + weight, err := strconv.Atoi(strings.Split(line, "::")[1]) + if err != nil { + ts.T.Fatal(err) + } + return weight + } } return 0 } @@ -470,24 +482,25 @@ func (ts *TestService) GetKubeAppEndpointURLs() []string { return strings.Split(urls[2:len(urls)-2], " ") } +// todo: needs to get updated so it takes into account all service weights // GetKubeCurrentWeight does the exact same thing as GetCurrentWeight but uses the kubectl command instead of rio -func (ts *TestService) GetKubeCurrentWeight() int { - ts.reload() - args := []string{"get", "services.rio.cattle.io", ts.Service.Name, "-n", TestingNamespace, "-o", "json"} - resultString, err := KubectlCmd(args) - if err != nil { - ts.T.Fatalf("Failed to get services.rio.cattle.io: %v", err.Error()) - } - var svc riov1.Service - err = json.Unmarshal([]byte(resultString), &svc) - if err != nil { - ts.T.Fatalf("Failed to unmarshal service results: %s with error: %v", resultString, err.Error()) - } - if svc.Status.ComputedWeight != nil { - return *svc.Status.ComputedWeight - } - return 0 -} +//func (ts *TestService) GetKubeCurrentWeight() int { +// ts.reload() +// args := []string{"get", "services.rio.cattle.io", ts.Service.Name, "-n", TestingNamespace, "-o", "json"} +// resultString, err := KubectlCmd(args) +// if err != nil { +// ts.T.Fatalf("Failed to get services.rio.cattle.io: %v", err.Error()) +// } +// var svc riov1.Service +// err = json.Unmarshal([]byte(resultString), &svc) +// if err != nil { +// ts.T.Fatalf("Failed to unmarshal service results: %s with error: %v", resultString, err.Error()) +// } +// if svc.Status.ComputedWeight != nil { +// return *svc.Status.ComputedWeight +// } +// return 0 +//} // PodsResponsesMatchAvailableReplicas does a GetURL in the App endpoint and stores the response in a slice // the length of the resulting slice should represent the number of responsive pods in a service. @@ -681,13 +694,10 @@ func (ts *TestService) waitForScale(want int) error { } // Wait until a service has the computed weight we want, note this is not spec weight but actual weight -func (ts *TestService) waitForWeight(percentage int) error { +func (ts *TestService) waitForWeight(target int) error { f := wait.ConditionFunc(func() (bool, error) { - ts.reload() - if ts.Service.Status.ComputedWeight != nil { - if percentage == *ts.Service.Status.ComputedWeight { - return true, nil - } + if ts.GetCurrentWeight() == target { + return true, nil } return false, nil }) diff --git a/tests/validation/weight_test.go b/tests/validation/weight_test.go index e0c0c6ecf..7b1c7d7cd 100644 --- a/tests/validation/weight_test.go +++ b/tests/validation/weight_test.go @@ -29,7 +29,7 @@ func weightTests(t *testing.T, when spec.G, it spec.S) { it("should slowly increase weight on the staged service and leave service weight unchanged", func() { // The time from rollout to obtaining the current weight, without Sleep, is 2 seconds. // Sleeping 8 seconds here with a rollout-interval of 4 seconds to guarantee 2 rollout ticks, plus the initial tick, with 2 seconds to spare. - stagedService.WeightWithoutWaiting(80, "--increment=10", "--interval=4") + stagedService.WeightWithoutWaiting(80, "--duration=1m") time.Sleep(8 * time.Second) stagedWeightAfter10Seconds := stagedService.GetCurrentWeight()