Add exponential backoff for requests that receive PV drivers missing …

…error (#262)

* Add exponential backoff for requests that receive PV drivers missing error

* Retry VM lacks feature errors as well. PV drivers aren't loaded when trying to reboot

* Add provider configuration settings for retry mode and max retry time

(cherry picked from commit 9f22000)

* Update provider docs with new configuration settings

client/client.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff/v3"
 	gorillawebsocket "github.com/gorilla/websocket"
 	"github.com/sourcegraph/jsonrpc2"
 	"github.com/sourcegraph/jsonrpc2/websocket"
@@ -118,16 +119,31 @@ type XOClient interface {
 }
 
 type Client struct {
-	rpc        jsonrpc2.JSONRPC2
-	httpClient http.Client
-	restApiURL *url.URL
+	RetryMode    RetryMode
+	RetryMaxTime time.Duration
+	rpc          jsonrpc2.JSONRPC2
+	httpClient   http.Client
+	restApiURL   *url.URL
 }
 
+type RetryMode int
+
+const (
+	None RetryMode = iota // specifies that no retries will be made
+	// Specifies that exponential backoff will be used for certain retryable errors. When
+	// a guest is booting there is the potential for a race condition if the given action
+	// relies on the existance of a PV driver (unplugging / plugging a device). This open
+	// allows the provider to retry these errors until the guest is initialized.
+	Backoff
+)
+
 type Config struct {
 	Url                string
 	Username           string
 	Password           string
 	InsecureSkipVerify bool
+	RetryMode          RetryMode
+	RetryMaxTime       time.Duration
 }
 
 var dialer = gorillawebsocket.Dialer{
@@ -222,39 +238,70 @@ func NewClient(config Config) (XOClient, error) {
 		},
 	}
 	return &Client{
-		rpc:        c,
-		httpClient: httpClient,
-		restApiURL: restApiURL,
+		RetryMode:    config.RetryMode,
+		RetryMaxTime: config.RetryMaxTime,
+		rpc:          c,
+		httpClient:   httpClient,
+		restApiURL:   restApiURL,
 	}, nil
 }
 
-func (c *Client) Call(method string, params, result interface{}, opt ...jsonrpc2.CallOption) error {
-	err := c.rpc.Call(context.Background(), method, params, result, opt...)
-	var callRes interface{}
-	t := reflect.TypeOf(result)
-	if t == nil || t.Kind() != reflect.Ptr {
-		callRes = result
-	} else {
-		callRes = reflect.ValueOf(result).Elem()
+func (c *Client) IsRetryableError(err jsonrpc2.Error) bool {
+
+	if c.RetryMode == None {
+		return false
 	}
-	log.Printf("[TRACE] Made rpc call `%s` with params: %v and received %+v: result with error: %v\n", method, params, callRes, err)
 
-	if err != nil {
-		rpcErr, ok := err.(*jsonrpc2.Error)
+	// Error code 11 corresponds to an error condition where a VM is missing PV drivers.
+	// https://github.com/vatesfr/xen-orchestra/blob/a3a2fda157fa30af4b93d34c99bac550f7c82bbc/packages/xo-common/api-errors.js#L95
 
-		if !ok {
-			return err
+	// During the boot process, there is a race condition where the PV drivers aren't available yet and
+	// making XO api calls during this time can return a VM_MISSING_PV_DRIVERS error. These errors can
+	// be treated as retryable since we want to wait until the VM has finished booting and its PV driver
+	// is initialized.
+	if err.Code == 11 || err.Code == 14 {
+		return true
+	}
+	return false
+}
+
+func (c *Client) Call(method string, params, result interface{}) error {
+	operation := func() error {
+		err := c.rpc.Call(context.Background(), method, params, result)
+		var callRes interface{}
+		t := reflect.TypeOf(result)
+		if t == nil || t.Kind() != reflect.Ptr {
+			callRes = result
+		} else {
+			callRes = reflect.ValueOf(result).Elem()
 		}
+		log.Printf("[TRACE] Made rpc call `%s` with params: %v and received %+v: result with error: %v\n", method, params, callRes, err)
+
+		if err != nil {
+			rpcErr, ok := err.(*jsonrpc2.Error)
 
-		data := rpcErr.Data
+			if !ok {
+				return backoff.Permanent(err)
+			}
 
-		if data == nil {
-			return err
-		}
+			if c.IsRetryableError(*rpcErr) {
+				return err
+			}
 
-		return errors.New(fmt.Sprintf("%s: %s", err, *data))
+			data := rpcErr.Data
+
+			if data == nil {
+				return backoff.Permanent(err)
+			}
+
+			return backoff.Permanent(errors.New(fmt.Sprintf("%s: %s", err, *data)))
+		}
+		return nil
 	}
-	return nil
+
+	bo := backoff.NewExponentialBackOff()
+	bo.MaxElapsedTime = c.RetryMaxTime
+	return backoff.Retry(operation, bo)
 }
 
 type RefreshComparison interface {

docs/index.md

@@ -50,5 +50,7 @@ provider "xenorchestra" {
 
 - `insecure` (Boolean) Whether SSL should be verified or not. Can be set via the XOA_INSECURE environment variable.
 - `password` (String) Password for xoa api. Can be set via the XOA_PASSWORD environment variable.
+- `retry_max_time` (String) If `retry_mode` is set, this specifies the duration for which the backoff method will continue retries. Can be set via the `XOA_RETRY_MAX_TIME` environment variable
+- `retry_mode` (String) Specifies if retries should be attempted for requests that require eventual . Can be set via the XOA_RETRY_MODE environment variable.
 - `url` (String) Hostname of the xoa router. Can be set via the XOA_URL environment variable.
 - `username` (String) User account for xoa api. Can be set via the XOA_USER environment variable.

go.mod

@@ -3,6 +3,7 @@ module github.com/ddelnano/terraform-provider-xenorchestra
 go 1.16
 
 require (
+	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
 	github.com/ddelnano/terraform-provider-xenorchestra/client v0.0.0-00010101000000-000000000000
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.4.3
 )

go.sum

@@ -56,6 +56,8 @@ github.com/aws/aws-sdk-go v1.25.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M=
+github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=

xoa/provider.go

@@ -1,8 +1,21 @@
 package xoa
 
 import (
+	"errors"
+	"fmt"
+	"regexp"
+	"time"
+
 	"github.com/ddelnano/terraform-provider-xenorchestra/client"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
+)
+
+var (
+	retryModeMap = map[string]client.RetryMode{
+		"none":    client.None,
+		"backoff": client.Backoff,
+	}
 )
 
 func Provider() *schema.Provider {
@@ -33,6 +46,20 @@ func Provider() *schema.Provider {
 				DefaultFunc: schema.EnvDefaultFunc("XOA_INSECURE", nil),
 				Description: "Whether SSL should be verified or not. Can be set via the XOA_INSECURE environment variable.",
 			},
+			"retry_mode": &schema.Schema{
+				Type:         schema.TypeString,
+				Optional:     true,
+				DefaultFunc:  schema.EnvDefaultFunc("XOA_RETRY_MODE", "backoff"),
+				Description:  "Specifies if retries should be attempted for requests that require eventual . Can be set via the XOA_RETRY_MODE environment variable.",
+				ValidateFunc: validation.StringInSlice([]string{"backoff", "none"}, false),
+			},
+			"retry_max_time": &schema.Schema{
+				Type:         schema.TypeString,
+				Optional:     true,
+				DefaultFunc:  schema.EnvDefaultFunc("XOA_RETRY_MAX_TIME", "5m"),
+				Description:  "If `retry_mode` is set, this specifies the duration for which the backoff method will continue retries. Can be set via the `XOA_RETRY_MAX_TIME` environment variable",
+				ValidateFunc: validation.StringMatch(regexp.MustCompile(`^[0-9]+(\.[0-9]+)?(ms|s|m|h)$`), "must be a number immediately followed by ms (milliseconds), s (seconds), m (minutes), or h (hours). For example, \"30s\" for 30 seconds."),
+			},
 		},
 		ResourcesMap: map[string]*schema.Resource{
 			"xenorchestra_acl":            resourceAcl(),
@@ -66,11 +93,26 @@ func xoaConfigure(d *schema.ResourceData) (interface{}, error) {
 	username := d.Get("username").(string)
 	password := d.Get("password").(string)
 	insecure := d.Get("insecure").(bool)
+	retryMode := d.Get("retry_mode").(string)
+	retryMaxTime := d.Get("retry_max_time").(string)
+
+	duration, err := time.ParseDuration(retryMaxTime)
+	if err != nil {
+		return client.Config{}, err
+	}
+
+	retry, ok := retryModeMap[retryMode]
+	if !ok {
+		return client.Config{}, errors.New(fmt.Sprintf("retry mode provided invalid: %s", retryMode))
+	}
+
 	config := client.Config{
 		Url:                url,
 		Username:           username,
 		Password:           password,
 		InsecureSkipVerify: insecure,
+		RetryMode:          retry,
+		RetryMaxTime:       duration,
 	}
 	return client.NewClient(config)
 }