Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: Put app to the operation queue after refresh queue processing to avoid race condition [#18500] #18694

Merged
merged 1 commit into from
Jul 31, 2024
Merged

fix: Put app to the operation queue after refresh queue processing to avoid race condition [#18500] #18694

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions controller/appcontroller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -910,10 +910,8 @@ func (ctrl *ApplicationController) requestAppRefresh(appName string, compareWith
}
if after != nil {
ctrl.appRefreshQueue.AddAfter(key, *after)
ctrl.appOperationQueue.AddAfter(key, *after)
} else {
ctrl.appRefreshQueue.AddRateLimited(key)
ctrl.appOperationQueue.AddRateLimited(key)
}
}
}
Expand Down Expand Up @@ -1548,6 +1546,9 @@ func (ctrl *ApplicationController) processAppRefreshQueueItem() (processNext boo
if r := recover(); r != nil {
log.Errorf("Recovered from panic: %+v\n%s", r, debug.Stack())
}
// We want to have app operation update happen after the sync, so there's no race condition
// and app updates not proceeding. See https://github.com/argoproj/argo-cd/issues/18500.
ctrl.appOperationQueue.AddRateLimited(appKey)
ctrl.appRefreshQueue.Done(appKey)
}()
obj, exists, err := ctrl.appInformer.GetIndexer().GetByKey(appKey.(string))
Expand Down Expand Up @@ -2221,7 +2222,6 @@ func (ctrl *ApplicationController) newApplicationInformerAndLister() (cache.Shar
key, err := cache.MetaNamespaceKeyFunc(obj)
if err == nil {
ctrl.appRefreshQueue.AddRateLimited(key)
ctrl.appOperationQueue.AddRateLimited(key)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think on Application Add/Update/Delete, we should add to the operation queue as soon as possible, to have the sync operation up to date. Refresh/Sync may take a few seconds, so it is nice to have the value reflected as early as possible.

It does make sense to add to the operation queue when a refresh is completed IMO since it needs the latest status of the refresh (and sometimes the refresh is the one that starts the operation with auto-sync).

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My understanding is that with refresh and operation scheduled at the same time it could always make sense to wait for refresh to complete, since otherwise operation can happen on a stale data. In one case on update when refresh and operation were scheduled with a delay and another immediate operation scheduling was present, I’ve kept the later one.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm putting it back, would update the PR shortly.

}
newApp, newOK := obj.(*appv1.Application)
if err == nil && newOK {
Expand Down Expand Up @@ -2256,7 +2256,9 @@ func (ctrl *ApplicationController) newApplicationInformerAndLister() (cache.Shar
}

ctrl.requestAppRefresh(newApp.QualifiedName(), compareWith, delay)
ctrl.appOperationQueue.AddRateLimited(key)
if !newOK || (delay != nil && *delay != time.Duration(0)) {
ctrl.appOperationQueue.AddRateLimited(key)
}
ctrl.clusterSharding.UpdateApp(newApp)
},
DeleteFunc: func(obj interface{}) {
Expand Down
43 changes: 43 additions & 0 deletions test/e2e/app_management_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2879,3 +2879,46 @@ func TestAnnotationTrackingExtraResources(t *testing.T) {
Expect(SyncStatusIs(SyncStatusCodeSynced)).
Expect(HealthIs(health.HealthStatusHealthy))
}

func TestCreateConfigMapsAndWaitForUpdate(t *testing.T) {
Given(t).
Path("config-map").
When().
CreateApp().
Sync().
Then().
And(func(app *Application) {
_, err := RunCli("app", "set", app.Name, "--sync-policy", "automated")
require.NoError(t, err)
}).
When().
AddFile("other-configmap.yaml", `
apiVersion: v1
kind: ConfigMap
metadata:
name: other-map
annotations:
argocd.argoproj.io/sync-wave: "1"
data:
foo2: bar2`).
AddFile("yet-another-configmap.yaml", `
apiVersion: v1
kind: ConfigMap
metadata:
name: yet-another-map
annotations:
argocd.argoproj.io/sync-wave: "2"
data:
foo3: bar3`).
PatchFile("kustomization.yaml", `[{"op": "add", "path": "/resources/-", "value": "other-configmap.yaml"}, {"op": "add", "path": "/resources/-", "value": "yet-another-configmap.yaml"}]`).
Refresh(RefreshTypeNormal).
Wait().
Then().
Expect(OperationPhaseIs(OperationSucceeded)).
Expect(SyncStatusIs(SyncStatusCodeSynced)).
Expect(HealthIs(health.HealthStatusHealthy)).
Expect(ResourceHealthWithNamespaceIs("ConfigMap", "other-map", DeploymentNamespace(), health.HealthStatusHealthy)).
Expect(ResourceSyncStatusWithNamespaceIs("ConfigMap", "other-map", DeploymentNamespace(), SyncStatusCodeSynced)).
Expect(ResourceHealthWithNamespaceIs("ConfigMap", "yet-another-map", DeploymentNamespace(), health.HealthStatusHealthy)).
Expect(ResourceSyncStatusWithNamespaceIs("ConfigMap", "yet-another-map", DeploymentNamespace(), SyncStatusCodeSynced))
}
18 changes: 16 additions & 2 deletions test/e2e/fixture/app/expectation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,28 @@ func ResourceSyncStatusWithNamespaceIs(kind, resource, namespace string, expecte

func ResourceHealthIs(kind, resource string, expected health.HealthStatusCode) Expectation {
return func(c *Consequences) (state, string) {
actual := c.resource(kind, resource, "").Health.Status
var actual health.HealthStatusCode
resourceHealth := c.resource(kind, resource, "").Health
if resourceHealth != nil {
actual = resourceHealth.Status
} else {
// Some resources like ConfigMap may not have health status when they are okay
actual = health.HealthStatusHealthy
}
return simple(actual == expected, fmt.Sprintf("resource '%s/%s' health should be %s, is %s", kind, resource, expected, actual))
}
}

func ResourceHealthWithNamespaceIs(kind, resource, namespace string, expected health.HealthStatusCode) Expectation {
return func(c *Consequences) (state, string) {
actual := c.resource(kind, resource, namespace).Health.Status
var actual health.HealthStatusCode
resourceHealth := c.resource(kind, resource, namespace).Health
if resourceHealth != nil {
actual = resourceHealth.Status
} else {
// Some resources like ConfigMap may not have health status when they are okay
actual = health.HealthStatusHealthy
}
return simple(actual == expected, fmt.Sprintf("resource '%s/%s' health should be %s, is %s", kind, resource, expected, actual))
}
}
Expand Down