Merge branch 'matrix-org:main' into main

CHANGES.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## Dendrite 0.13.0 (2023-06-30)
+
+### Features
+
+- Results in responses to `/search` now highlight words more accurately and not only the search terms as before
+- Support for connecting to appservices listening on unix sockets has been added (contributed by [cyberb](https://github.com/cyberb))
+- Admin APIs for token authenticated registration have been added (contributed by [santhoshivan23](https://github.com/santhoshivan23))
+- Initial support for [MSC4014: Pseudonymous Identities](https://github.com/matrix-org/matrix-spec-proposals/blob/kegan/pseudo-ids/proposals/4014-pseudonymous-identities.md)
+  - This is **highly experimental**, things like changing usernames/avatars, inviting users, upgrading rooms isn't working
+
+### Fixes
+
+- `m.upload.size` is now optional, finally allowing uploads with unlimited file size
+- A bug while resolving server names has been fixed (contributed by [anton-molyboha](https://github.com/anton-molyboha))
+- Application services should only receive one invitation instead of 2 (or worse), which could result in state resets previously
+- Several admin endpoints are now using `POST` instead of `GET`
+- `/delete_devices` now uses user-interactive authentication
+- Several "membership" (e.g `/kick`, `/ban`) endpoints are using less heavy database queries to check if the user is allowed to perform this action
+- `/3pid` endpoints are now available on `/v3` instead of the `/unstable` prefix
+- Upgrading rooms ignores state events of other users, which could result in failed upgrades before
+- Uploading key backups with a wrong version now returns `M_WRONG_ROOM_KEYS_VERSION`
+- A potential state reset when joining the same room multiple times in short sequence has been fixed
+- A bug where we returned the full event as `redacted_because` in redaction events has been fixed
+- The `displayname` and `avatar_url` can now be set to empty strings
+- Unsafe hotserving of files has been fixed (contributed by [joshqou](https://github.com/joshqou))
+- Joining new rooms would potentially return "redacted" events, due to history visibility not being set correctly, this could result in events being rejected 
+- Backfilling resulting in `unsuported room version ''` should now be solved
+
+### Other
+
+- Huge refactoring of Dendrite and gomatrixserverlib
+
 ## Dendrite 0.12.0 (2023-03-13)
 
 ### Features

clientapi/routing/membership.go

@@ -22,10 +22,6 @@ import (
 	"time"
 
 	"github.com/getsentry/sentry-go"
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/fclient"
-	"github.com/matrix-org/gomatrixserverlib/spec"
-
 	appserviceAPI "github.com/matrix-org/dendrite/appservice/api"
 	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
 	"github.com/matrix-org/dendrite/clientapi/httputil"
@@ -36,6 +32,9 @@ import (
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/fclient"
+	"github.com/matrix-org/gomatrixserverlib/spec"
 
 	"github.com/matrix-org/util"
 )
@@ -433,11 +432,6 @@ func buildMembershipEvent(
 		return nil, err
 	}
 
-	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
-	if err != nil {
-		return nil, err
-	}
-
 	userID, err := spec.NewUserID(device.UserID, true)
 	if err != nil {
 		return nil, err
@@ -459,6 +453,12 @@ func buildMembershipEvent(
 	if err != nil {
 		return nil, err
 	}
+
+	identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *userID)
+	if err != nil {
+		return nil, err
+	}
+
 	return buildMembershipEventDirect(ctx, targetSenderID, reason, profile.DisplayName, profile.AvatarURL,
 		senderID, device.UserDomain(), membership, roomID, isDirect, identity.KeyID, identity.PrivateKey, evTime, rsAPI)
 }

clientapi/routing/profile.go

@@ -104,12 +104,6 @@ func SetAvatarURL(
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
-	if r.AvatarURL == "" {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("'avatar_url' must be supplied."),
-		}
-	}
 
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
@@ -151,7 +145,7 @@ func SetAvatarURL(
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, evTime)
 	if err != nil {
 		return response
 	}
@@ -199,12 +193,6 @@ func SetDisplayName(
 	if resErr := httputil.UnmarshalJSONRequest(req, &r); resErr != nil {
 		return *resErr
 	}
-	if r.DisplayName == "" {
-		return util.JSONResponse{
-			Code: http.StatusBadRequest,
-			JSON: spec.BadJSON("'displayname' must be supplied."),
-		}
-	}
 
 	localpart, domain, err := gomatrixserverlib.SplitID('@', userID)
 	if err != nil {
@@ -246,7 +234,7 @@ func SetDisplayName(
 		}
 	}
 
-	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, cfg, evTime)
+	response, err := updateProfile(req.Context(), rsAPI, device, profile, userID, evTime)
 	if err != nil {
 		return response
 	}
@@ -260,7 +248,7 @@ func SetDisplayName(
 func updateProfile(
 	ctx context.Context, rsAPI api.ClientRoomserverAPI, device *userapi.Device,
 	profile *authtypes.Profile,
-	userID string, cfg *config.ClientAPI, evTime time.Time,
+	userID string, evTime time.Time,
 ) (util.JSONResponse, error) {
 	var res api.QueryRoomsForUserResponse
 	err := rsAPI.QueryRoomsForUser(ctx, &api.QueryRoomsForUserRequest{
@@ -285,7 +273,7 @@ func updateProfile(
 	}
 
 	events, err := buildMembershipEvents(
-		ctx, device, res.RoomIDs, *profile, userID, cfg, evTime, rsAPI,
+		ctx, res.RoomIDs, *profile, userID, evTime, rsAPI,
 	)
 	switch e := err.(type) {
 	case nil:
@@ -356,9 +344,8 @@ func getProfile(
 
 func buildMembershipEvents(
 	ctx context.Context,
-	device *userapi.Device,
 	roomIDs []string,
-	newProfile authtypes.Profile, userID string, cfg *config.ClientAPI,
+	newProfile authtypes.Profile, userID string,
 	evTime time.Time, rsAPI api.ClientRoomserverAPI,
 ) ([]*types.HeaderedEvent, error) {
 	evs := []*types.HeaderedEvent{}
@@ -395,12 +382,17 @@ func buildMembershipEvents(
 			return nil, err
 		}
 
-		identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+		user, err := spec.NewUserID(userID, true)
+		if err != nil {
+			return nil, err
+		}
+
+		identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *user)
 		if err != nil {
 			return nil, err
 		}
 
-		event, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, rsAPI, nil)
+		event, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, evTime, rsAPI, nil)
 		if err != nil {
 			return nil, err
 		}

clientapi/routing/redaction.go

@@ -150,7 +150,7 @@ func SendRedaction(
 		}
 	}
 
-	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+	identity, err := rsAPI.SigningIdentityFor(req.Context(), *validRoomID, *deviceUserID)
 	if err != nil {
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -159,7 +159,7 @@ func SendRedaction(
 	}
 
 	var queryRes roomserverAPI.QueryLatestEventsAndStateResponse
-	e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, identity, time.Now(), rsAPI, &queryRes)
+	e, err := eventutil.QueryAndBuildEvent(req.Context(), &proto, &identity, time.Now(), rsAPI, &queryRes)
 	if errors.Is(err, eventutil.ErrRoomNoExists{}) {
 		return util.JSONResponse{
 			Code: http.StatusNotFound,

clientapi/routing/sendevent.go

@@ -23,19 +23,18 @@ import (
 	"sync"
 	"time"
 
-	"github.com/matrix-org/gomatrixserverlib"
-	"github.com/matrix-org/gomatrixserverlib/spec"
-	"github.com/matrix-org/util"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
-
 	"github.com/matrix-org/dendrite/clientapi/httputil"
 	"github.com/matrix-org/dendrite/internal/eventutil"
 	"github.com/matrix-org/dendrite/internal/transactions"
 	"github.com/matrix-org/dendrite/roomserver/api"
 	"github.com/matrix-org/dendrite/roomserver/types"
 	"github.com/matrix-org/dendrite/setup/config"
 	userapi "github.com/matrix-org/dendrite/userapi/api"
+	"github.com/matrix-org/gomatrixserverlib"
+	"github.com/matrix-org/gomatrixserverlib/spec"
+	"github.com/matrix-org/util"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
 )
 
 // http://matrix.org/docs/spec/client_server/r0.2.0.html#put-matrix-client-r0-rooms-roomid-send-eventtype-txnid
@@ -68,6 +67,8 @@ var sendEventDuration = prometheus.NewHistogramVec(
 //	/rooms/{roomID}/send/{eventType}
 //	/rooms/{roomID}/send/{eventType}/{txnID}
 //	/rooms/{roomID}/state/{eventType}/{stateKey}
+//
+// nolint: gocyclo
 func SendEvent(
 	req *http.Request,
 	device *userapi.Device,
@@ -121,6 +122,17 @@ func SendEvent(
 		delete(r, "join_authorised_via_users_server")
 	}
 
+	// for power level events we need to replace the userID with the pseudoID
+	if roomVersion == gomatrixserverlib.RoomVersionPseudoIDs && eventType == spec.MRoomPowerLevels {
+		err = updatePowerLevels(req, r, roomID, rsAPI)
+		if err != nil {
+			return util.JSONResponse{
+				Code: http.StatusInternalServerError,
+				JSON: spec.InternalServerError{Err: err.Error()},
+			}
+		}
+	}
+
 	evTime, err := httputil.ParseTSParam(req)
 	if err != nil {
 		return util.JSONResponse{
@@ -129,7 +141,7 @@ func SendEvent(
 		}
 	}
 
-	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, cfg, rsAPI, evTime)
+	e, resErr := generateSendEvent(req.Context(), r, device, roomID, eventType, stateKey, rsAPI, evTime)
 	if resErr != nil {
 		return *resErr
 	}
@@ -225,6 +237,28 @@ func SendEvent(
 	return res
 }
 
+func updatePowerLevels(req *http.Request, r map[string]interface{}, roomID string, rsAPI api.ClientRoomserverAPI) error {
+	userMap := r["users"].(map[string]interface{})
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return err
+	}
+	for user, level := range userMap {
+		uID, err := spec.NewUserID(user, true)
+		if err != nil {
+			continue // we're modifying the map in place, so we're going to have invalid userIDs after the first iteration
+		}
+		senderID, err := rsAPI.QuerySenderIDForUser(req.Context(), *validRoomID, *uID)
+		if err != nil {
+			return err
+		}
+		userMap[string(senderID)] = level
+		delete(userMap, user)
+	}
+	r["users"] = userMap
+	return nil
+}
+
 // stateEqual compares the new and the existing state event content. If they are equal, returns a *util.JSONResponse
 // with the existing event_id, making this an idempotent request.
 func stateEqual(ctx context.Context, rsAPI api.ClientRoomserverAPI, eventType, stateKey, roomID string, newContent map[string]interface{}) *util.JSONResponse {
@@ -261,7 +295,6 @@ func generateSendEvent(
 	r map[string]interface{},
 	device *userapi.Device,
 	roomID, eventType string, stateKey *string,
-	cfg *config.ClientAPI,
 	rsAPI api.ClientRoomserverAPI,
 	evTime time.Time,
 ) (gomatrixserverlib.PDU, *util.JSONResponse) {
@@ -304,7 +337,7 @@ func generateSendEvent(
 		}
 	}
 
-	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
+	identity, err := rsAPI.SigningIdentityFor(ctx, *validRoomID, *fullUserID)
 	if err != nil {
 		return nil, &util.JSONResponse{
 			Code: http.StatusInternalServerError,
@@ -313,7 +346,7 @@ func generateSendEvent(
 	}
 
 	var queryRes api.QueryLatestEventsAndStateResponse
-	e, err := eventutil.QueryAndBuildEvent(ctx, &proto, identity, evTime, rsAPI, &queryRes)
+	e, err := eventutil.QueryAndBuildEvent(ctx, &proto, &identity, evTime, rsAPI, &queryRes)
 	switch specificErr := err.(type) {
 	case nil:
 	case eventutil.ErrRoomNoExists:

clientapi/routing/server_notices.go

@@ -221,7 +221,7 @@ func SendServerNotice(
 		"body":    r.Content.Body,
 		"msgtype": r.Content.MsgType,
 	}
-	e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, cfgClient, rsAPI, time.Now())
+	e, resErr := generateSendEvent(ctx, request, senderDevice, roomID, "m.room.message", nil, rsAPI, time.Now())
 	if resErr != nil {
 		logrus.Errorf("failed to send message: %+v", resErr)
 		return *resErr
@@ -350,7 +350,7 @@ func getSenderDevice(
 	if len(deviceRes.Devices) > 0 {
 		// If there were changes to the profile, create a new membership event
 		if displayNameChanged || avatarChanged {
-			_, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, cfg, time.Now())
+			_, err = updateProfile(ctx, rsAPI, &deviceRes.Devices[0], profile, accRes.Account.UserID, time.Now())
 			if err != nil {
 				return nil, err
 			}

federationapi/consumers/roomserver.go

@@ -192,7 +192,7 @@ func (s *OutputRoomEventConsumer) processMessage(ore api.OutputNewRoomEvent, rew
 		evs[i] = addsStateEvents[i].PDU
 	}
 
-	addsJoinedHosts, err := JoinedHostsFromEvents(evs)
+	addsJoinedHosts, err := JoinedHostsFromEvents(s.ctx, evs, s.rsAPI)
 	if err != nil {
 		return err
 	}
@@ -345,7 +345,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 		return nil, err
 	}
 
-	combinedAddsJoinedHosts, err := JoinedHostsFromEvents(combinedAddsEvents)
+	combinedAddsJoinedHosts, err := JoinedHostsFromEvents(s.ctx, combinedAddsEvents, s.rsAPI)
 	if err != nil {
 		return nil, err
 	}
@@ -394,7 +394,7 @@ func (s *OutputRoomEventConsumer) joinedHostsAtEvent(
 // JoinedHostsFromEvents turns a list of state events into a list of joined hosts.
 // This errors if one of the events was invalid.
 // It should be impossible for an invalid event to get this far in the pipeline.
-func JoinedHostsFromEvents(evs []gomatrixserverlib.PDU) ([]types.JoinedHost, error) {
+func JoinedHostsFromEvents(ctx context.Context, evs []gomatrixserverlib.PDU, rsAPI api.FederationRoomserverAPI) ([]types.JoinedHost, error) {
 	var joinedHosts []types.JoinedHost
 	for _, ev := range evs {
 		if ev.Type() != "m.room.member" || ev.StateKey() == nil {
@@ -407,12 +407,17 @@ func JoinedHostsFromEvents(evs []gomatrixserverlib.PDU) ([]types.JoinedHost, err
 		if membership != spec.Join {
 			continue
 		}
-		_, serverName, err := gomatrixserverlib.SplitID('@', *ev.StateKey())
+		validRoomID, err := spec.NewRoomID(ev.RoomID())
 		if err != nil {
 			return nil, err
 		}
+		userID, err := rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(*ev.StateKey()))
+		if err != nil {
+			return nil, err
+		}
+
 		joinedHosts = append(joinedHosts, types.JoinedHost{
-			MemberEventID: ev.EventID(), ServerName: serverName,
+			MemberEventID: ev.EventID(), ServerName: userID.Domain(),
 		})
 	}
 	return joinedHosts, nil