Skip to content

Commit

Permalink
Use our own ChainedTokenCredential for easy debugging
Browse files Browse the repository at this point in the history
Use our own ChainedTokenCredential for easy debugging

Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>
  • Loading branch information
ywk253100 committed Apr 8, 2024
1 parent f85f877 commit 2b80813
Show file tree
Hide file tree
Showing 4 changed files with 193 additions and 4 deletions.
113 changes: 113 additions & 0 deletions pkg/util/azure/chained_credential.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package azure

import (
"context"
"errors"
"fmt"
"sync"

"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
)

// This file is a copy of https://github.com/Azure/azure-sdk-for-go/blob/sdk/azidentity/v1.5.1/sdk/azidentity/chained_token_credential.go with a change that
// removes the specific error checking logic.
// Velero chains the ConfigCredential, WorkloadIdentityCredential, ManagedIdentityCredential and uses them to do the auth in order.
// The original ChainedTokenCredential only reports the error got from the last credential and ignores the others in some cases, this causes confusion
// if the root cause is from the former credentials.
// For example, if users provide an invalid certificate as the credential, the original reports a managed identity credential error, this is hard to debug.
// With the change in this file, the new ChainedTokenCredential reports all errors of all of the credentials.

// ChainedTokenCredential links together multiple credentials and tries them sequentially when authenticating. By default,
// it tries all the credentials until one authenticates, after which it always uses that credential.
type ChainedTokenCredential struct {
cond *sync.Cond
iterating bool
name string
retrySources bool
sources []azcore.TokenCredential
successfulCredential azcore.TokenCredential
}

// NewChainedTokenCredential creates a ChainedTokenCredential. Pass nil for options to accept defaults.
func NewChainedTokenCredential(sources []azcore.TokenCredential, options *azidentity.ChainedTokenCredentialOptions) (*ChainedTokenCredential, error) {
if len(sources) == 0 {
return nil, errors.New("sources must contain at least one TokenCredential")
}
for _, source := range sources {
if source == nil { // cannot have a nil credential in the chain or else the application will panic when GetToken() is called on nil
return nil, errors.New("sources cannot contain nil")
}
}
cp := make([]azcore.TokenCredential, len(sources))
copy(cp, sources)
if options == nil {
options = &azidentity.ChainedTokenCredentialOptions{}
}
return &ChainedTokenCredential{
cond: sync.NewCond(&sync.Mutex{}),
name: "ChainedTokenCredential",
retrySources: options.RetrySources,
sources: cp,
}, nil
}

// GetToken calls GetToken on the chained credentials in turn, stopping when one returns a token.
// This method is called automatically by Azure SDK clients.
func (c *ChainedTokenCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) {
if !c.retrySources {
// ensure only one goroutine at a time iterates the sources and perhaps sets c.successfulCredential
c.cond.L.Lock()
for {
if c.successfulCredential != nil {
c.cond.L.Unlock()
return c.successfulCredential.GetToken(ctx, opts)
}
if !c.iterating {
c.iterating = true
// allow other goroutines to wait while this one iterates
c.cond.L.Unlock()
break
}
c.cond.Wait()
}
}

var (
err error
errs []error
successfulCredential azcore.TokenCredential
token azcore.AccessToken
)
for _, cred := range c.sources {
token, err = cred.GetToken(ctx, opts)
if err == nil {
successfulCredential = cred
break
}
errs = append(errs, err)
}
if c.iterating {
c.cond.L.Lock()
// this is nil when all credentials returned an error
c.successfulCredential = successfulCredential
c.iterating = false
c.cond.L.Unlock()
c.cond.Broadcast()
}
// err is the error returned by the last GetToken call. It will be nil when that call succeeds
if err != nil {
msg := createChainedErrorMessage(errs)
err = fmt.Errorf("%s", msg)
}
return token, err
}

func createChainedErrorMessage(errs []error) string {
msg := "failed to acquire a token.\nAttempted credentials:"
for _, err := range errs {
msg += fmt.Sprintf("\n\t%s", err.Error())
}
return msg
}
42 changes: 42 additions & 0 deletions pkg/util/azure/chained_credential_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package azure

import (
"context"
"errors"
"testing"

"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestNewChainedTokenCredential(t *testing.T) {
// empty source
var sources []azcore.TokenCredential
_, err := NewChainedTokenCredential(sources, nil)
require.NotNil(t, err)

// contain nil source
sources = append(sources, nil)
_, err = NewChainedTokenCredential(sources, nil)
require.NotNil(t, err)

// valid
sources = []azcore.TokenCredential{&credentialErrorReporter{}}
credential, err := NewChainedTokenCredential(sources, nil)
require.Nil(t, err)
assert.NotNil(t, credential)
}

func TestGetToken(t *testing.T) {
sources := []azcore.TokenCredential{&credentialErrorReporter{err: &credentialError{
credType: "fake",
err: errors.New("fake error"),
}}}
credential, err := NewChainedTokenCredential(sources, nil)
require.Nil(t, err)

_, err = credential.GetToken(context.Background(), policy.TokenRequestOptions{})
require.NotNil(t, err)
}
37 changes: 33 additions & 4 deletions pkg/util/azure/credential.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ limitations under the License.
package azure

import (
"context"
"fmt"
"os"
"strings"

Expand Down Expand Up @@ -46,7 +48,9 @@ func NewCredential(creds map[string]string, options policy.ClientOptions) (azcor
if err == nil {
credential = append(credential, cfgCred)
} else {
errMsgs = append(errMsgs, err.Error())
credentialErr := &credentialError{credType: "ConfigCredential", err: err}
errMsgs = append(errMsgs, credentialErr.Error())
credential = append(credential, &credentialErrorReporter{err: credentialErr})
}

// workload identity credential
Expand All @@ -57,7 +61,9 @@ func NewCredential(creds map[string]string, options policy.ClientOptions) (azcor
if err == nil {
credential = append(credential, wic)
} else {
errMsgs = append(errMsgs, err.Error())
credentialErr := &credentialError{credType: "WorkloadIdentityCredential", err: err}
errMsgs = append(errMsgs, credentialErr.Error())
credential = append(credential, &credentialErrorReporter{err: credentialErr})
}

//managed identity credential
Expand All @@ -66,14 +72,16 @@ func NewCredential(creds map[string]string, options policy.ClientOptions) (azcor
if err == nil {
credential = append(credential, msi)
} else {
errMsgs = append(errMsgs, err.Error())
credentialErr := &credentialError{credType: "ManagedIdentityCredential", err: err}
errMsgs = append(errMsgs, credentialErr.Error())
credential = append(credential, &credentialErrorReporter{err: credentialErr})
}

if len(credential) == 0 {
return nil, errors.Errorf("failed to create Azure credential: %s", strings.Join(errMsgs, "\n\t"))
}

return azidentity.NewChainedTokenCredential(credential, nil)
return NewChainedTokenCredential(credential, nil)
}

type configCredentialOptions struct {
Expand Down Expand Up @@ -145,3 +153,24 @@ func newConfigCredential(creds map[string]string, options configCredentialOption

return nil, errors.New("incomplete credential configuration. Only AZURE_TENANT_ID and AZURE_CLIENT_ID are set")
}

type credentialError struct {
credType string
err error
}

func (c *credentialError) Error() string {
return fmt.Sprintf("%s: %s", c.credType, c.err.Error())
}

// credentialErrorReporter is a substitute for credentials that couldn't be constructed.
// Its GetToken method always returns an error having the same message as
// the error that prevented constructing the credential. This ensures the message is present
// in the error returned by ChainedTokenCredential.GetToken()
type credentialErrorReporter struct {
err *credentialError
}

func (c *credentialErrorReporter) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) {
return azcore.AccessToken{}, c.err
}
5 changes: 5 additions & 0 deletions pkg/util/kube/periodical_enqueue_source.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ type PeriodicalEnqueueSourceOption struct {
func (p *PeriodicalEnqueueSource) Start(ctx context.Context, h handler.EventHandler, q workqueue.RateLimitingInterface, predicates ...predicate.Predicate) error {
go wait.Until(func() {
p.logger.Debug("enqueueing resources ...")
// TODO add comment to explain why
if err := meta.SetList(p.objList, nil); err != nil {
p.logger.WithError(err).Error("error reset resource list")
return
}
if err := p.List(ctx, p.objList); err != nil {
p.logger.WithError(err).Error("error listing resources")
return
Expand Down

0 comments on commit 2b80813

Please sign in to comment.