fix: improve semaphore concurrency performance

Signed-off-by: Jesse Suen <jesse@akuity.io>

workflow/controller/controller.go

@@ -333,7 +333,11 @@ func (wfc *WorkflowController) createSynchronizationManager(ctx context.Context)
 	}
 
 	nextWorkflow := func(key string) {
-		wfc.wfQueue.AddRateLimited(key)
+		// We now use AddAfter(1s) instead of AddRateLimited() because if we add immediately,
+		// the enqueued workflow will likely be reconciled before we have finished reconciling
+		// *this* workflow. If we reconcile the next workflow too soon (before we have finished
+		// incrementing the semaphore counters), the next workflow will believe it cannot run.
+		wfc.wfQueue.AddAfter(key, 1*time.Second)
 	}
 
 	isWFDeleted := func(key string) bool {

workflow/sync/semaphore.go

@@ -188,6 +188,11 @@ func (s *PrioritySemaphore) tryAcquire(holderKey string) (bool, string) {
 	if s.acquire(holderKey) {
 		s.pending.pop()
 		s.log.Infof("%s acquired by %s ", s.name, nextKey)
+		if s.pending.Len() > 0 && len(s.lockHolder) < s.limit {
+			// We just acquired the semaphore but others are waiting and there is room for the
+			// next one. Enqueue the next workflow in line.
+			s.nextWorkflow(s.pending.peek().key)
+		}
 		return true, ""
 	}
 	s.log.Debugf("Current semaphore Holders. %v", s.lockHolder)