diff --git a/crypto/machine.go b/crypto/machine.go
index 85da2b3b..c130f775 100644
--- a/crypto/machine.go
+++ b/crypto/machine.go
@@ -65,8 +65,9 @@ type OlmMachine struct {
 	megolmEncryptLock sync.Mutex
 	megolmDecryptLock sync.Mutex
 
-	otkUploadLock sync.Mutex
-	lastOTKUpload time.Time
+	otkUploadLock       sync.Mutex
+	lastOTKUpload       time.Time
+	receivedOTKsForSelf bool
 
 	CrossSigningKeys    *CrossSigningKeysCache
 	crossSigningPubkeys *CrossSigningPublicKeysCache
@@ -258,16 +259,15 @@ func (mach *OlmMachine) otkCountIsForCrossSigningKey(otkCount *mautrix.OTKCount)
 
 func (mach *OlmMachine) HandleOTKCounts(ctx context.Context, otkCount *mautrix.OTKCount) {
 	if (len(otkCount.UserID) > 0 && otkCount.UserID != mach.Client.UserID) || (len(otkCount.DeviceID) > 0 && otkCount.DeviceID != mach.Client.DeviceID) {
-		if mach.otkCountIsForCrossSigningKey(otkCount) {
-			return
+		if otkCount.UserID != mach.Client.UserID || (!mach.receivedOTKsForSelf && !mach.otkCountIsForCrossSigningKey(otkCount)) {
+			mach.Log.Warn().
+				Str("target_user_id", otkCount.UserID.String()).
+				Str("target_device_id", otkCount.DeviceID.String()).
+				Msg("Dropping OTK counts targeted to someone else")
 		}
-		// TODO This log probably needs to be silence-able if someone wants to use encrypted appservices with multiple e2ee sessions
-		mach.Log.Warn().
-			Str("target_user_id", otkCount.UserID.String()).
-			Str("target_device_id", otkCount.DeviceID.String()).
-			Msg("Dropping OTK counts targeted to someone else")
 		return
 	}
+	mach.receivedOTKsForSelf = true
 
 	minCount := mach.account.Internal.MaxNumberOfOneTimeKeys() / 2
 	if otkCount.SignedCurve25519 < int(minCount) {