Make it possible to join pseudo ID rooms (#3119)

This should allow us to join pseudo ID rooms over federation.
Also needs matrix-org/gomatrixserverlib#397

federationapi/internal/perform.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"context"
+	"crypto/ed25519"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -170,13 +171,24 @@ func (r *FederationInternalAPI) performJoinUsingServer(
 		UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return r.rsAPI.QueryUserIDForSender(ctx, roomID, senderID)
 		},
-		SenderIDCreator: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID) (spec.SenderID, error) {
+		GetOrCreateSenderID: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) {
+			// assign a roomNID, otherwise we can't create a private key for the user
+			_, nidErr := r.rsAPI.AssignRoomNID(ctx, roomID, gomatrixserverlib.RoomVersion(roomVersion))
+			if nidErr != nil {
+				return "", nil, nidErr
+			}
 			key, keyErr := r.rsAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID)
 			if keyErr != nil {
-				return "", keyErr
+				return "", nil, keyErr
 			}
-
-			return spec.SenderIDFromPseudoIDKey(key), nil
+			return spec.SenderIDFromPseudoIDKey(key), key, nil
+		},
+		StoreSenderIDFromPublicID: func(ctx context.Context, senderID spec.SenderID, userIDRaw string, roomID spec.RoomID) error {
+			storeUserID, userErr := spec.NewUserID(userIDRaw, true)
+			if userErr != nil {
+				return userErr
+			}
+			return r.rsAPI.StoreUserRoomPublicKey(ctx, senderID, *storeUserID, roomID)
 		},
 	}
 	response, joinErr := gomatrixserverlib.PerformJoin(ctx, r, joinInput)

federationapi/routing/join.go

@@ -15,6 +15,7 @@
 package routing
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"sort"
@@ -107,6 +108,10 @@ func MakeJoin(
 		}
 	}
 
+	if senderID == "" {
+		senderID = spec.SenderID(userID.String())
+	}
+
 	input := gomatrixserverlib.HandleMakeJoinInput{
 		Context:           httpReq.Context(),
 		UserID:            userID,
@@ -218,6 +223,13 @@ func SendJoin(
 		UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
 			return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID)
 		},
+		StoreSenderIDFromPublicID: func(ctx context.Context, senderID spec.SenderID, userIDRaw string, roomID spec.RoomID) error {
+			userID, userErr := spec.NewUserID(userIDRaw, true)
+			if userErr != nil {
+				return userErr
+			}
+			return rsAPI.StoreUserRoomPublicKey(ctx, senderID, *userID, roomID)
+		},
 	}
 	response, joinErr := gomatrixserverlib.HandleSendJoin(input)
 	switch e := joinErr.(type) {

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230616092350-99b78e30a272
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230628151943-f6e3c7f7b093
 	github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.16
@@ -43,6 +43,7 @@ require (
 	github.com/yggdrasil-network/yggdrasil-go v0.4.6
 	go.uber.org/atomic v1.10.0
 	golang.org/x/crypto v0.10.0
+	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db
 	golang.org/x/image v0.5.0
 	golang.org/x/mobile v0.0.0-20221020085226-b36e6246172e
 	golang.org/x/sync v0.1.0
@@ -124,7 +125,6 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
-	golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
 	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sys v0.9.0 // indirect

go.sum

@@ -323,8 +323,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230616092350-99b78e30a272 h1:YDpKHyojqwQU/L6I0bjp/BYPfn1aq/D2iBxC0jCSc/0=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230616092350-99b78e30a272/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230628151943-f6e3c7f7b093 h1:FHd3SYhU2ZxZhkssZ/7ms5+M2j+g94lYp8ztvA1E6tA=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230628151943-f6e3c7f7b093/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A=
 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=

roomserver/api/api.go

@@ -74,6 +74,7 @@ type RoomserverInternalAPI interface {
 type UserRoomPrivateKeyCreator interface {
 	// GetOrCreateUserRoomPrivateKey gets the user room key for the specified user. If no key exists yet, a new one is created.
 	GetOrCreateUserRoomPrivateKey(ctx context.Context, userID spec.UserID, roomID spec.RoomID) (ed25519.PrivateKey, error)
+	StoreUserRoomPublicKey(ctx context.Context, senderID spec.SenderID, userID spec.UserID, roomID spec.RoomID) error
 }
 
 type InputRoomEventsAPI interface {
@@ -235,7 +236,7 @@ type FederationRoomserverAPI interface {
 	QueryBulkStateContentAPI
 	QuerySenderIDAPI
 	UserRoomPrivateKeyCreator
-
+	AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error)
 	SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error)
 	// QueryServerBannedFromRoom returns whether a server is banned from a room by server ACLs.
 	QueryServerBannedFromRoom(ctx context.Context, req *QueryServerBannedFromRoomRequest, res *QueryServerBannedFromRoomResponse) error

roomserver/internal/api.go

@@ -289,6 +289,15 @@ func (r *RoomserverInternalAPI) GetOrCreateUserRoomPrivateKey(ctx context.Contex
 	return key, nil
 }
 
+func (r *RoomserverInternalAPI) StoreUserRoomPublicKey(ctx context.Context, senderID spec.SenderID, userID spec.UserID, roomID spec.RoomID) error {
+	pubKeyBytes, err := senderID.RawBytes()
+	if err != nil {
+		return err
+	}
+	_, err = r.DB.InsertUserRoomPublicKey(ctx, userID, roomID, ed25519.PublicKey(pubKeyBytes))
+	return err
+}
+
 func (r *RoomserverInternalAPI) SigningIdentityFor(ctx context.Context, roomID spec.RoomID, senderID spec.UserID) (fclient.SigningIdentity, error) {
 	roomVersion, ok := r.Cache.GetRoomVersion(roomID.String())
 	if !ok {
@@ -307,7 +316,7 @@ func (r *RoomserverInternalAPI) SigningIdentityFor(ctx context.Context, roomID s
 		}
 		return fclient.SigningIdentity{
 			PrivateKey: privKey,
-			KeyID:      "ed25519",
+			KeyID:      "ed25519:1",
 			ServerName: "self",
 		}, nil
 	}
@@ -317,3 +326,7 @@ func (r *RoomserverInternalAPI) SigningIdentityFor(ctx context.Context, roomID s
 	}
 	return *identity, err
 }
+
+func (r *RoomserverInternalAPI) AssignRoomNID(ctx context.Context, roomID spec.RoomID, roomVersion gomatrixserverlib.RoomVersion) (roomNID types.RoomNID, err error) {
+	return r.DB.AssignRoomNID(ctx, roomID, roomVersion)
+}

roomserver/internal/input/input_events.go

@@ -406,7 +406,7 @@ func (r *Inputer) processRoomEvent(
 	)
 	if !isRejected && !isCreateEvent {
 		resolver := state.NewStateResolution(r.DB, roomInfo, r.Queryer)
-		redactionEvent, redactedEvent, err = r.DB.MaybeRedactEvent(ctx, roomInfo, eventNID, event, &resolver)
+		redactionEvent, redactedEvent, err = r.DB.MaybeRedactEvent(ctx, roomInfo, eventNID, event, &resolver, r.Queryer)
 		if err != nil {
 			return err
 		}

roomserver/internal/perform/perform_backfill.go

@@ -647,7 +647,7 @@ func persistEvents(ctx context.Context, db storage.Database, querier api.QuerySe
 
 		resolver := state.NewStateResolution(db, roomInfo, querier)
 
-		_, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev, &resolver)
+		_, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev, &resolver, querier)
 		if err != nil {
 			logrus.WithError(err).WithField("event_id", ev.EventID()).Error("Failed to redact event")
 			continue

roomserver/internal/perform/perform_create_room.go

@@ -196,7 +196,7 @@ func (c *Creator) PerformCreateRoom(ctx context.Context, userID spec.UserID, roo
 		// sign all events with the pseudo ID key
 		identity = &fclient.SigningIdentity{
 			ServerName: "self",
-			KeyID:      "ed25519",
+			KeyID:      "ed25519:1",
 			PrivateKey: pseudoIDKey,
 		}
 	}

roomserver/internal/perform/perform_join.go

@@ -314,7 +314,7 @@ func (r *Joiner) performJoinRoomByID(
 		// sign the event with the pseudo ID key
 		identity = fclient.SigningIdentity{
 			ServerName: "self",
-			KeyID:      "ed25519",
+			KeyID:      "ed25519:1",
 			PrivateKey: pseudoIDKey,
 		}
 	}

roomserver/roomserver_test.go

@@ -35,6 +35,14 @@ import (
 	"github.com/matrix-org/dendrite/test/testrig"
 )
 
+type FakeQuerier struct {
+	api.QuerySenderIDAPI
+}
+
+func (f *FakeQuerier) QueryUserIDForSender(ctx context.Context, roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+	return spec.NewUserID(string(senderID), true)
+}
+
 func TestUsers(t *testing.T) {
 	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
 		cfg, processCtx, close := testrig.CreateConfig(t, dbType)
@@ -566,7 +574,7 @@ func TestRedaction(t *testing.T) {
 					err = updater.Commit()
 					assert.NoError(t, err)
 
-					_, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.PDU, &plResolver)
+					_, redactedEvent, err := db.MaybeRedactEvent(ctx, roomInfo, eventNID, ev.PDU, &plResolver, &FakeQuerier{})
 					assert.NoError(t, err)
 					if redactedEvent != nil {
 						assert.Equal(t, ev.Redacts(), redactedEvent.EventID())

roomserver/storage/interface.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/ed25519"
 
+	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 
@@ -190,7 +191,7 @@ type Database interface {
 	GetOrCreateEventTypeNID(ctx context.Context, eventType string) (eventTypeNID types.EventTypeNID, err error)
 	GetOrCreateEventStateKeyNID(ctx context.Context, eventStateKey *string) (types.EventStateKeyNID, error)
 	MaybeRedactEvent(
-		ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver,
+		ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver, querier api.QuerySenderIDAPI,
 	) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error)
 }
 
@@ -251,7 +252,7 @@ type EventDatabase interface {
 	// MaybeRedactEvent returns the redaction event and the redacted event if this call resulted in a redaction, else an error
 	// (nil if there was nothing to do)
 	MaybeRedactEvent(
-		ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver,
+		ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver, querier api.QuerySenderIDAPI,
 	) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error)
 	StoreEvent(ctx context.Context, event gomatrixserverlib.PDU, roomInfo *types.RoomInfo, eventTypeNID types.EventTypeNID, eventStateKeyNID types.EventStateKeyNID, authEventNIDs []types.EventNID, isRejected bool) (types.EventNID, types.StateAtEvent, error)
 }

roomserver/storage/shared/storage.go

@@ -10,6 +10,7 @@ import (
 	"sort"
 
 	"github.com/matrix-org/dendrite/internal/eventutil"
+	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/gomatrixserverlib"
 	"github.com/matrix-org/gomatrixserverlib/spec"
 	"github.com/matrix-org/util"
@@ -991,6 +992,7 @@ func extractRoomVersionFromCreateEvent(event gomatrixserverlib.PDU) (
 // Returns the redaction event and the redacted event if this call resulted in a redaction.
 func (d *EventDatabase) MaybeRedactEvent(
 	ctx context.Context, roomInfo *types.RoomInfo, eventNID types.EventNID, event gomatrixserverlib.PDU, plResolver state.PowerLevelResolver,
+	querier api.QuerySenderIDAPI,
 ) (gomatrixserverlib.PDU, gomatrixserverlib.PDU, error) {
 	var (
 		redactionEvent, redactedEvent *types.Event
@@ -1030,15 +1032,18 @@ func (d *EventDatabase) MaybeRedactEvent(
 			return nil
 		}
 
-		// TODO: Don't hack senderID into userID here (pseudoIDs)
+		var validRoomID *spec.RoomID
+		validRoomID, err = spec.NewRoomID(redactedEvent.RoomID())
+		if err != nil {
+			return err
+		}
 		sender1Domain := ""
-		sender1, err1 := spec.NewUserID(string(redactedEvent.SenderID()), true)
+		sender1, err1 := querier.QueryUserIDForSender(ctx, *validRoomID, redactedEvent.SenderID())
 		if err1 == nil {
 			sender1Domain = string(sender1.Domain())
 		}
-		// TODO: Don't hack senderID into userID here (pseudoIDs)
 		sender2Domain := ""
-		sender2, err2 := spec.NewUserID(string(redactionEvent.SenderID()), true)
+		sender2, err2 := querier.QueryUserIDForSender(ctx, *validRoomID, redactionEvent.SenderID())
 		if err2 == nil {
 			sender2Domain = string(sender2.Domain())
 		}
@@ -1757,58 +1762,54 @@ func (d *Database) SelectUserRoomPublicKey(ctx context.Context, userID spec.User
 // SelectUserIDsForPublicKeys returns a map from roomID -> map from senderKey -> userID
 func (d *Database) SelectUserIDsForPublicKeys(ctx context.Context, publicKeys map[spec.RoomID][]ed25519.PublicKey) (result map[spec.RoomID]map[string]string, err error) {
 	result = make(map[spec.RoomID]map[string]string, len(publicKeys))
-	err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error {
 
-		// map all roomIDs to roomNIDs
-		query := make(map[types.RoomNID][]ed25519.PublicKey)
-		rooms := make(map[types.RoomNID]spec.RoomID)
-		for roomID, keys := range publicKeys {
-			roomNID, ok := d.Cache.GetRoomServerRoomNID(roomID.String())
-			if !ok {
-				roomInfo, rErr := d.roomInfo(ctx, txn, roomID.String())
-				if rErr != nil {
-					return rErr
-				}
-				if roomInfo == nil {
-					logrus.Warnf("missing room info for %s, there will be missing users in the response", roomID.String())
-					continue
-				}
-				roomNID = roomInfo.RoomNID
+	// map all roomIDs to roomNIDs
+	query := make(map[types.RoomNID][]ed25519.PublicKey)
+	rooms := make(map[types.RoomNID]spec.RoomID)
+	for roomID, keys := range publicKeys {
+		roomNID, ok := d.Cache.GetRoomServerRoomNID(roomID.String())
+		if !ok {
+			roomInfo, rErr := d.roomInfo(ctx, nil, roomID.String())
+			if rErr != nil {
+				return nil, rErr
 			}
-
-			query[roomNID] = keys
-			rooms[roomNID] = roomID
+			if roomInfo == nil {
+				logrus.Warnf("missing room info for %s, there will be missing users in the response", roomID.String())
+				continue
+			}
+			roomNID = roomInfo.RoomNID
 		}
 
-		// get the user room key pars
-		userRoomKeyPairMap, sErr := d.UserRoomKeyTable.BulkSelectUserNIDs(ctx, txn, query)
-		if sErr != nil {
-			return sErr
-		}
-		nids := make([]types.EventStateKeyNID, 0, len(userRoomKeyPairMap))
-		for _, nid := range userRoomKeyPairMap {
-			nids = append(nids, nid.EventStateKeyNID)
-		}
-		// get the userIDs
-		nidMap, seErr := d.EventStateKeys(ctx, nids)
-		if seErr != nil {
-			return seErr
-		}
+		query[roomNID] = keys
+		rooms[roomNID] = roomID
+	}
 
-		// build the result map (roomID -> map publicKey -> userID)
-		for publicKey, userRoomKeyPair := range userRoomKeyPairMap {
-			userID := nidMap[userRoomKeyPair.EventStateKeyNID]
-			roomID := rooms[userRoomKeyPair.RoomNID]
-			resMap, exists := result[roomID]
-			if !exists {
-				resMap = map[string]string{}
-			}
-			resMap[publicKey] = userID
-			result[roomID] = resMap
-		}
+	// get the user room key pars
+	userRoomKeyPairMap, sErr := d.UserRoomKeyTable.BulkSelectUserNIDs(ctx, nil, query)
+	if sErr != nil {
+		return nil, sErr
+	}
+	nids := make([]types.EventStateKeyNID, 0, len(userRoomKeyPairMap))
+	for _, nid := range userRoomKeyPairMap {
+		nids = append(nids, nid.EventStateKeyNID)
+	}
+	// get the userIDs
+	nidMap, seErr := d.EventStateKeys(ctx, nids)
+	if seErr != nil {
+		return nil, seErr
+	}
 
-		return nil
-	})
+	// build the result map (roomID -> map publicKey -> userID)
+	for publicKey, userRoomKeyPair := range userRoomKeyPairMap {
+		userID := nidMap[userRoomKeyPair.EventStateKeyNID]
+		roomID := rooms[userRoomKeyPair.RoomNID]
+		resMap, exists := result[roomID]
+		if !exists {
+			resMap = map[string]string{}
+		}
+		resMap[publicKey] = userID
+		result[roomID] = resMap
+	}
 	return result, err
 }