Merge branch 'matrix-org:main' into main

.github/workflows/docker.yml

@@ -57,8 +57,8 @@ jobs:
         id: docker_build_monolith
         uses: docker/build-push-action@v3
         with:
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache
+          cache-to: type=registry,ref=ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-monolith:buildcache,mode=max
           context: .
           build-args: FLAGS=-X github.com/matrix-org/dendrite/internal.branch=${{ env.BRANCH }} -X github.com/matrix-org/dendrite/internal.build=${{ env.BUILD }}
           platforms: ${{ env.PLATFORMS }}

CHANGES.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## Dendrite 0.13.1 (2023-07-06)
+
+This releases fixes a long-standing "off-by-one" error which could result in state resets. Upgrading to this version is **highly** recommended.
+
+When deduplicating state events, we were checking if the event in question was already in a state snapshot. If it was in a previous state snapshot, we would 
+then remove it from the list of events to store. If this happened, we were, unfortunately, skipping the next event to check. This resulted in 
+events getting stored in state snapshots where they may not be needed. When we now compared two of those state snapshots, one of them
+contained the skipped event, while the other didn't. This difference possibly shouldn't exist, resulting in unexpected state resets and explains
+reports of missing state events as well.
+
+Rooms where a state reset occurred earlier should, hopefully, reconcile over time.
+
+### Fixes:
+
+- A long-standing "off-by-one" error has been fixed, which could result in state resets
+- Roomserver Prometheus Metrics are available again
+
+### Features
+
+- Updated dependencies
+  - Internal NATS Server has been updated from v2.9.15 to v2.9.19
+
 ## Dendrite 0.13.0 (2023-06-30)
 
 ### Features

clientapi/routing/membership.go

@@ -337,22 +337,55 @@ func sendInvite(
 	rsAPI roomserverAPI.ClientRoomserverAPI,
 	asAPI appserviceAPI.AppServiceInternalAPI, evTime time.Time,
 ) (util.JSONResponse, error) {
-	event, err := buildMembershipEvent(
-		ctx, userID, reason, profileAPI, device, spec.Invite,
-		roomID, false, cfg, evTime, rsAPI, asAPI,
-	)
+	validRoomID, err := spec.NewRoomID(roomID)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("RoomID is invalid"),
+		}, err
+	}
+	inviter, err := spec.NewUserID(device.UserID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, err
+	}
+	invitee, err := spec.NewUserID(userID, true)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("UserID is invalid"),
+		}, err
+	}
+	profile, err := loadProfile(ctx, userID, cfg, profileAPI, asAPI)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusInternalServerError,
+			JSON: spec.InternalServerError{},
+		}, err
+	}
+	identity, err := cfg.Matrix.SigningIdentityFor(device.UserDomain())
 	if err != nil {
-		util.GetLogger(ctx).WithError(err).Error("buildMembershipEvent failed")
 		return util.JSONResponse{
 			Code: http.StatusInternalServerError,
 			JSON: spec.InternalServerError{},
 		}, err
 	}
-
 	err = rsAPI.PerformInvite(ctx, &api.PerformInviteRequest{
-		Event:           event,
+		InviteInput: roomserverAPI.InviteInput{
+			RoomID:      *validRoomID,
+			Inviter:     *inviter,
+			Invitee:     *invitee,
+			DisplayName: profile.DisplayName,
+			AvatarURL:   profile.AvatarURL,
+			Reason:      reason,
+			IsDirect:    false,
+			KeyID:       identity.KeyID,
+			PrivateKey:  identity.PrivateKey,
+			EventTime:   evTime,
+		},
 		InviteRoomState: nil, // ask the roomserver to draw up invite room state for us
-		RoomVersion:     event.Version(),
 		SendAsServer:    string(device.UserDomain()),
 	})
 

federationapi/api/api.go

@@ -63,6 +63,8 @@ type RoomserverFederationAPI interface {
 	PerformLeave(ctx context.Context, request *PerformLeaveRequest, response *PerformLeaveResponse) error
 	// Handle sending an invite to a remote server.
 	SendInvite(ctx context.Context, event gomatrixserverlib.PDU, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error)
+	// Handle sending an invite to a remote server.
+	SendInviteV3(ctx context.Context, event gomatrixserverlib.ProtoEvent, invitee spec.UserID, version gomatrixserverlib.RoomVersion, strippedState []gomatrixserverlib.InviteStrippedState) (gomatrixserverlib.PDU, error)
 	// Handle an instruction to peek a room on a remote server.
 	PerformOutboundPeek(ctx context.Context, request *PerformOutboundPeekRequest, response *PerformOutboundPeekResponse) error
 	// Query the server names of the joined hosts in a room.

federationapi/internal/api.go

@@ -54,11 +54,14 @@ func NewFederationInternalAPI(
 			KeyDatabase: serverKeyDB,
 		}
 
+		pubKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey)
 		addDirectFetcher := func() {
 			keyRing.KeyFetchers = append(
 				keyRing.KeyFetchers,
 				&gomatrixserverlib.DirectKeyFetcher{
-					Client: federation,
+					Client:            federation,
+					IsLocalServerName: cfg.Matrix.IsLocalServerName,
+					LocalPublicKey:    []byte(pubKey),
 				},
 			)
 		}

federationapi/internal/perform.go

@@ -599,6 +599,58 @@ func (r *FederationInternalAPI) SendInvite(
 	return inviteEvent, nil
 }
 
+// SendInviteV3 implements api.FederationInternalAPI
+func (r *FederationInternalAPI) SendInviteV3(
+	ctx context.Context,
+	event gomatrixserverlib.ProtoEvent,
+	invitee spec.UserID,
+	version gomatrixserverlib.RoomVersion,
+	strippedState []gomatrixserverlib.InviteStrippedState,
+) (gomatrixserverlib.PDU, error) {
+	validRoomID, err := spec.NewRoomID(event.RoomID)
+	if err != nil {
+		return nil, err
+	}
+	verImpl, err := gomatrixserverlib.GetRoomVersion(version)
+	if err != nil {
+		return nil, err
+	}
+
+	inviter, err := r.rsAPI.QueryUserIDForSender(ctx, *validRoomID, spec.SenderID(event.SenderID))
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO (devon): This should be allowed via a relay. Currently only transactions
+	// can be sent to relays. Would need to extend relays to handle invites.
+	if !r.shouldAttemptDirectFederation(invitee.Domain()) {
+		return nil, fmt.Errorf("relay servers have no meaningful response for invite.")
+	}
+
+	logrus.WithFields(logrus.Fields{
+		"user_id":      invitee.String(),
+		"room_id":      event.RoomID,
+		"room_version": version,
+		"destination":  invitee.Domain(),
+	}).Info("Sending invite")
+
+	inviteReq, err := fclient.NewInviteV3Request(event, version, strippedState)
+	if err != nil {
+		return nil, fmt.Errorf("gomatrixserverlib.NewInviteV3Request: %w", err)
+	}
+
+	inviteRes, err := r.federation.SendInviteV3(ctx, inviter.Domain(), invitee.Domain(), inviteReq, invitee)
+	if err != nil {
+		return nil, fmt.Errorf("r.federation.SendInviteV3: failed to send invite: %w", err)
+	}
+
+	inviteEvent, err := verImpl.NewEventFromUntrustedJSON(inviteRes.Event)
+	if err != nil {
+		return nil, fmt.Errorf("r.federation.SendInviteV3 failed to decode event response: %w", err)
+	}
+	return inviteEvent, nil
+}
+
 // PerformServersAlive implements api.FederationInternalAPI
 func (r *FederationInternalAPI) PerformBroadcastEDU(
 	ctx context.Context,

federationapi/internal/perform_test.go

@@ -16,6 +16,7 @@ package internal
 
 import (
 	"context"
+	"crypto/ed25519"
 	"testing"
 
 	"github.com/matrix-org/dendrite/federationapi/api"
@@ -53,10 +54,14 @@ func TestPerformWakeupServers(t *testing.T) {
 	assert.NoError(t, err)
 	assert.True(t, offline)
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -95,10 +100,14 @@ func TestQueryRelayServers(t *testing.T) {
 	err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers)
 	assert.NoError(t, err)
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -132,10 +141,14 @@ func TestRemoveRelayServers(t *testing.T) {
 	err := testDB.P2PAddRelayServersForServer(context.Background(), server, relayServers)
 	assert.NoError(t, err)
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -168,10 +181,14 @@ func TestRemoveRelayServers(t *testing.T) {
 func TestPerformDirectoryLookup(t *testing.T) {
 	testDB := test.NewInMemoryFederationDatabase()
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
 				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -192,7 +209,7 @@ func TestPerformDirectoryLookup(t *testing.T) {
 		ServerName: "server",
 	}
 	res := api.PerformDirectoryLookupResponse{}
-	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	err = fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
 	assert.NoError(t, err)
 }
 
@@ -203,10 +220,14 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
 	testDB.SetServerAssumedOffline(context.Background(), server)
 	testDB.P2PAddRelayServersForServer(context.Background(), server, []spec.ServerName{"relay"})
 
+	_, key, err := ed25519.GenerateKey(nil)
+	assert.NoError(t, err)
 	cfg := config.FederationAPI{
 		Matrix: &config.Global{
 			SigningIdentity: fclient.SigningIdentity{
-				ServerName: server,
+				ServerName: "relay",
+				KeyID:      "ed25519:1",
+				PrivateKey: key,
 			},
 		},
 	}
@@ -227,6 +248,6 @@ func TestPerformDirectoryLookupRelaying(t *testing.T) {
 		ServerName: server,
 	}
 	res := api.PerformDirectoryLookupResponse{}
-	err := fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
+	err = fedAPI.PerformDirectoryLookup(context.Background(), &req, &res)
 	assert.Error(t, err)
 }

federationapi/routing/invite.go

@@ -16,6 +16,7 @@ package routing
 
 import (
 	"context"
+	"crypto/ed25519"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -29,6 +30,73 @@ import (
 	"github.com/matrix-org/util"
 )
 
+// InviteV3 implements /_matrix/federation/v2/invite/{roomID}/{userID}
+func InviteV3(
+	httpReq *http.Request,
+	request *fclient.FederationRequest,
+	roomID spec.RoomID,
+	invitedUser spec.UserID,
+	cfg *config.FederationAPI,
+	rsAPI api.FederationRoomserverAPI,
+	keys gomatrixserverlib.JSONVerifier,
+) util.JSONResponse {
+	inviteReq := fclient.InviteV3Request{}
+	err := json.Unmarshal(request.Content(), &inviteReq)
+	if err != nil {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.BadJSON(err.Error()),
+		}
+	}
+	if !cfg.Matrix.IsLocalServerName(invitedUser.Domain()) {
+		return util.JSONResponse{
+			Code: http.StatusBadRequest,
+			JSON: spec.InvalidParam("The invited user domain does not belong to this server"),
+		}
+	}
+
+	input := gomatrixserverlib.HandleInviteV3Input{
+		HandleInviteInput: gomatrixserverlib.HandleInviteInput{
+			RoomVersion:       inviteReq.RoomVersion(),
+			RoomID:            roomID,
+			InvitedUser:       invitedUser,
+			KeyID:             cfg.Matrix.KeyID,
+			PrivateKey:        cfg.Matrix.PrivateKey,
+			Verifier:          keys,
+			RoomQuerier:       rsAPI,
+			MembershipQuerier: &api.MembershipQuerier{Roomserver: rsAPI},
+			StateQuerier:      rsAPI.StateQuerier(),
+			InviteEvent:       nil,
+			StrippedState:     inviteReq.InviteRoomState(),
+			UserIDQuerier: func(roomID spec.RoomID, senderID spec.SenderID) (*spec.UserID, error) {
+				return rsAPI.QueryUserIDForSender(httpReq.Context(), roomID, senderID)
+			},
+		},
+		InviteProtoEvent: inviteReq.Event(),
+		GetOrCreateSenderID: func(ctx context.Context, userID spec.UserID, roomID spec.RoomID, roomVersion string) (spec.SenderID, ed25519.PrivateKey, error) {
+			// assign a roomNID, otherwise we can't create a private key for the user
+			_, nidErr := rsAPI.AssignRoomNID(ctx, roomID, gomatrixserverlib.RoomVersion(roomVersion))
+			if nidErr != nil {
+				return "", nil, nidErr
+			}
+			key, keyErr := rsAPI.GetOrCreateUserRoomPrivateKey(ctx, userID, roomID)
+			if keyErr != nil {
+				return "", nil, keyErr
+			}
+
+			return spec.SenderIDFromPseudoIDKey(key), key, nil
+		},
+	}
+	event, jsonErr := handleInviteV3(httpReq.Context(), input, rsAPI)
+	if jsonErr != nil {
+		return *jsonErr
+	}
+	return util.JSONResponse{
+		Code: http.StatusOK,
+		JSON: fclient.RespInviteV2{Event: event.JSON()},
+	}
+}
+
 // InviteV2 implements /_matrix/federation/v2/invite/{roomID}/{eventID}
 func InviteV2(
 	httpReq *http.Request,
@@ -204,6 +272,15 @@ func InviteV1(
 
 func handleInvite(ctx context.Context, input gomatrixserverlib.HandleInviteInput, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) {
 	inviteEvent, err := gomatrixserverlib.HandleInvite(ctx, input)
+	return handleInviteResult(ctx, inviteEvent, err, rsAPI)
+}
+
+func handleInviteV3(ctx context.Context, input gomatrixserverlib.HandleInviteV3Input, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) {
+	inviteEvent, err := gomatrixserverlib.HandleInviteV3(ctx, input)
+	return handleInviteResult(ctx, inviteEvent, err, rsAPI)
+}
+
+func handleInviteResult(ctx context.Context, inviteEvent gomatrixserverlib.PDU, err error, rsAPI api.FederationRoomserverAPI) (gomatrixserverlib.PDU, *util.JSONResponse) {
 	switch e := err.(type) {
 	case nil:
 	case spec.InternalServerError:
@@ -245,4 +322,5 @@ func handleInvite(ctx context.Context, input gomatrixserverlib.HandleInviteInput
 		}
 	}
 	return inviteEvent, nil
+
 }