diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9bfb01667..e0b82e2aa 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,7 +2,7 @@ -* [ ] I have added added tests for PR _or_ I have justified why this PR doesn't need tests. +* [ ] I have added tests for PR _or_ I have justified why this PR doesn't need tests. * [ ] Pull request includes a [sign off](https://github.com/matrix-org/dendrite/blob/main/docs/CONTRIBUTING.md#sign-off) Signed-off-by: `Your Name ` diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 642587924..b4e24e52f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -137,3 +137,63 @@ jobs: ${{ env.DOCKER_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:latest ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-polylith:${{ env.RELEASE_VERSION }} + + demo-pinecone: + name: Pinecone demo image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Get release tag + if: github.event_name == 'release' # Only for GitHub releases + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ env.DOCKER_HUB_USER }} + password: ${{ secrets.DOCKER_TOKEN }} + - name: Login to GitHub Containers + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build main pinecone demo image + if: github.ref_name == 'main' + id: docker_build_demo_pinecone + uses: docker/build-push-action@v2 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + file: ./build/docker/Dockerfile.demo-pinecone + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ github.ref_name }} + + - name: Build release pinecone demo image + if: github.event_name == 'release' # Only for GitHub releases + id: docker_build_demo_pinecone_release + uses: docker/build-push-action@v2 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + context: . + file: ./build/docker/Dockerfile.demo-pinecone + platforms: ${{ env.PLATFORMS }} + push: true + tags: | + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:latest + ${{ env.DOCKER_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:latest + ghcr.io/${{ env.GHCR_NAMESPACE }}/dendrite-demo-pinecone:${{ env.RELEASE_VERSION }} diff --git a/build/docker/Dockerfile.demo-pinecone b/build/docker/Dockerfile.demo-pinecone new file mode 100644 index 000000000..a5d709c7a --- /dev/null +++ b/build/docker/Dockerfile.demo-pinecone @@ -0,0 +1,25 @@ +FROM docker.io/golang:1.18-alpine AS base + +RUN apk --update --no-cache add bash build-base + +WORKDIR /build + +COPY . /build + +RUN mkdir -p bin +RUN go build -trimpath -o bin/ ./cmd/dendrite-demo-pinecone +RUN go build -trimpath -o bin/ ./cmd/create-account +RUN go build -trimpath -o bin/ ./cmd/generate-keys + +FROM alpine:latest +LABEL org.opencontainers.image.title="Dendrite (Pinecone demo)" +LABEL org.opencontainers.image.description="Next-generation Matrix homeserver written in Go" +LABEL org.opencontainers.image.source="https://github.com/matrix-org/dendrite" +LABEL org.opencontainers.image.licenses="Apache-2.0" + +COPY --from=base /build/bin/* /usr/bin/ + +VOLUME /etc/dendrite +WORKDIR /etc/dendrite + +ENTRYPOINT ["/usr/bin/dendrite-demo-pinecone"] diff --git a/build/docker/images-build.sh b/build/docker/images-build.sh index eaed5f6dc..c2c140685 100755 --- a/build/docker/images-build.sh +++ b/build/docker/images-build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash cd $(git rev-parse --show-toplevel) diff --git a/build/docker/images-pull.sh b/build/docker/images-pull.sh index 496e80067..f3f98ce7c 100755 --- a/build/docker/images-pull.sh +++ b/build/docker/images-pull.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash TAG=${1:-latest} diff --git a/build/docker/images-push.sh b/build/docker/images-push.sh index fd9b999ea..248fdee2b 100755 --- a/build/docker/images-push.sh +++ b/build/docker/images-push.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash TAG=${1:-latest} diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 99e597e59..da63f9a2c 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -24,6 +24,7 @@ import ( "net" "net/http" "os" + "path/filepath" "strings" "time" @@ -62,6 +63,7 @@ var ( instancePort = flag.Int("port", 8008, "the port that the client API will listen on") instancePeer = flag.String("peer", "", "the static Pinecone peers to connect to, comma separated-list") instanceListen = flag.String("listen", ":0", "the port Pinecone peers can connect to") + instanceDir = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)") ) // nolint:gocyclo @@ -88,7 +90,7 @@ func main() { cfg = setup.ParseFlags(true) sk = cfg.Global.PrivateKey } else { - keyfile := *instanceName + ".pem" + keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem" if _, err := os.Stat(keyfile); os.IsNotExist(err) { oldkeyfile := *instanceName + ".key" if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) { @@ -129,15 +131,15 @@ func main() { Monolithic: true, }) cfg.Global.PrivateKey = sk - cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) - cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) + cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", filepath.Join(*instanceDir, *instanceName))) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(*instanceDir, *instanceName))) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(*instanceDir, *instanceName))) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(*instanceDir, *instanceName))) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName))) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName))) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName))) cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} - cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) + cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true if err := cfg.Derive(); err != nil { diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 868d9826a..cd0066679 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -24,6 +24,7 @@ import ( "net" "net/http" "os" + "path/filepath" "time" "github.com/matrix-org/gomatrixserverlib" @@ -56,6 +57,7 @@ var ( instancePort = flag.Int("port", 8008, "the port that the client API will listen on") instancePeer = flag.String("peer", "", "the static Yggdrasil peers to connect to, comma separated-list") instanceListen = flag.String("listen", "tcp://:0", "the port Yggdrasil peers can connect to") + instanceDir = flag.String("dir", ".", "the directory to store the databases in (if --config not specified)") ) func main() { @@ -76,7 +78,7 @@ func main() { cfg := &config.Dendrite{} - keyfile := *instanceName + ".pem" + keyfile := filepath.Join(*instanceDir, *instanceName) + ".pem" if _, err := os.Stat(keyfile); os.IsNotExist(err) { oldkeyfile := *instanceName + ".key" if _, err = os.Stat(oldkeyfile); os.IsNotExist(err) { @@ -121,15 +123,15 @@ func main() { Monolithic: true, }) cfg.Global.PrivateKey = sk - cfg.Global.JetStream.StoragePath = config.Path(fmt.Sprintf("%s/", *instanceName)) - cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", *instanceName)) - cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", *instanceName)) - cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", *instanceName)) - cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", *instanceName)) - cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", *instanceName)) - cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", *instanceName)) - cfg.MSCs.MSCs = []string{"msc2836"} - cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", *instanceName)) + cfg.Global.JetStream.StoragePath = config.Path(filepath.Join(*instanceDir, *instanceName)) + cfg.UserAPI.AccountDatabase.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-account.db", filepath.Join(*instanceDir, *instanceName))) + cfg.MediaAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mediaapi.db", filepath.Join(*instanceDir, *instanceName))) + cfg.SyncAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-syncapi.db", filepath.Join(*instanceDir, *instanceName))) + cfg.RoomServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-roomserver.db", filepath.Join(*instanceDir, *instanceName))) + cfg.KeyServer.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-keyserver.db", filepath.Join(*instanceDir, *instanceName))) + cfg.FederationAPI.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-federationapi.db", filepath.Join(*instanceDir, *instanceName))) + cfg.MSCs.MSCs = []string{"msc2836", "msc2946"} + cfg.MSCs.Database.ConnectionString = config.DataSource(fmt.Sprintf("file:%s-mscs.db", filepath.Join(*instanceDir, *instanceName))) cfg.ClientAPI.RegistrationDisabled = false cfg.ClientAPI.OpenRegistrationWithoutVerificationEnabled = true if err := cfg.Derive(); err != nil { diff --git a/docs/installation/4_database.md b/docs/installation/4_database.md index 68f2d44d0..d64ee6615 100644 --- a/docs/installation/4_database.md +++ b/docs/installation/4_database.md @@ -16,6 +16,9 @@ Dendrite can automatically populate the database with the relevant tables and in it is not capable of creating the databases themselves. You will need to create the databases manually. +The databases **must** be created with UTF-8 encoding configured or you will likely run into problems +with your Dendrite deployment. + At this point, you can choose to either use a single database for all Dendrite components, or you can run each component with its own separate database: @@ -65,7 +68,7 @@ sudo -u postgres createuser -P dendrite Create the database itself, using the `dendrite` role from above: ```bash -sudo -u postgres createdb -O dendrite dendrite +sudo -u postgres createdb -O dendrite -E UTF-8 dendrite ``` ### Multiple database creation @@ -85,7 +88,7 @@ The following eight components require a database. In this example they will be ```bash for i in appservice federationapi mediaapi mscs roomserver syncapi keyserver userapi; do - sudo -u postgres createdb -O dendrite dendrite_$i + sudo -u postgres createdb -O dendrite -E UTF-8 dendrite_$i done ``` diff --git a/federationapi/internal/perform.go b/federationapi/internal/perform.go index 9100c8f18..84702f4ce 100644 --- a/federationapi/internal/perform.go +++ b/federationapi/internal/perform.go @@ -217,7 +217,7 @@ func (r *FederationInternalAPI) performJoinUsingServer( var remoteEvent *gomatrixserverlib.Event remoteEvent, err = respSendJoin.Event.UntrustedEvent(respMakeJoin.RoomVersion) if err == nil && isWellFormedMembershipEvent( - remoteEvent, roomID, userID, r.cfg.Matrix.ServerName, + remoteEvent, roomID, userID, ) { event = remoteEvent } @@ -285,7 +285,7 @@ func (r *FederationInternalAPI) performJoinUsingServer( // isWellFormedMembershipEvent returns true if the event looks like a legitimate // membership event. -func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string, origin gomatrixserverlib.ServerName) bool { +func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID string) bool { if membership, err := event.Membership(); err != nil { return false } else if membership != gomatrixserverlib.Join { @@ -294,9 +294,6 @@ func isWellFormedMembershipEvent(event *gomatrixserverlib.Event, roomID, userID if event.RoomID() != roomID { return false } - if event.Origin() != origin { - return false - } if !event.StateKeyEquals(userID) { return false } diff --git a/federationapi/routing/invite.go b/federationapi/routing/invite.go index 4b795018c..504204504 100644 --- a/federationapi/routing/invite.go +++ b/federationapi/routing/invite.go @@ -148,8 +148,15 @@ func processInvite( JSON: jsonerror.BadJSON("The event JSON could not be redacted"), } } + _, serverName, err := gomatrixserverlib.SplitID('@', event.Sender()) + if err != nil { + return util.JSONResponse{ + Code: http.StatusBadRequest, + JSON: jsonerror.BadJSON("The event JSON contains an invalid sender"), + } + } verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ - ServerName: event.Origin(), + ServerName: serverName, Message: redacted, AtTS: event.OriginServerTS(), StrictValidityChecking: true, diff --git a/federationapi/routing/join.go b/federationapi/routing/join.go index 1a1219873..74d065e59 100644 --- a/federationapi/routing/join.go +++ b/federationapi/routing/join.go @@ -203,14 +203,6 @@ func SendJoin( } } - // Check that the event is from the server sending the request. - if event.Origin() != request.Origin() { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("The join must be sent by the server it originated on"), - } - } - // Check that a state key is provided. if event.StateKey() == nil || event.StateKeyEquals("") { return util.JSONResponse{ @@ -228,16 +220,16 @@ func SendJoin( // Check that the sender belongs to the server that is sending us // the request. By this point we've already asserted that the sender // and the state key are equal so we don't need to check both. - var domain gomatrixserverlib.ServerName - if _, domain, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil { + var serverName gomatrixserverlib.ServerName + if _, serverName, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil { return util.JSONResponse{ Code: http.StatusForbidden, JSON: jsonerror.Forbidden("The sender of the join is invalid"), } - } else if domain != request.Origin() { + } else if serverName != request.Origin() { return util.JSONResponse{ Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("The sender of the join must belong to the origin server"), + JSON: jsonerror.Forbidden("The sender does not match the server that originated the request"), } } @@ -292,7 +284,7 @@ func SendJoin( } } verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ - ServerName: event.Origin(), + ServerName: serverName, Message: redacted, AtTS: event.OriginServerTS(), StrictValidityChecking: true, diff --git a/federationapi/routing/leave.go b/federationapi/routing/leave.go index 8e43ce959..a67e4e28b 100644 --- a/federationapi/routing/leave.go +++ b/federationapi/routing/leave.go @@ -118,6 +118,7 @@ func MakeLeave( } // SendLeave implements the /send_leave API +// nolint:gocyclo func SendLeave( httpReq *http.Request, request *gomatrixserverlib.FederationRequest, @@ -167,14 +168,6 @@ func SendLeave( } } - // Check that the event is from the server sending the request. - if event.Origin() != request.Origin() { - return util.JSONResponse{ - Code: http.StatusForbidden, - JSON: jsonerror.Forbidden("The leave must be sent by the server it originated on"), - } - } - if event.StateKey() == nil || event.StateKeyEquals("") { return util.JSONResponse{ Code: http.StatusBadRequest, @@ -188,6 +181,22 @@ func SendLeave( } } + // Check that the sender belongs to the server that is sending us + // the request. By this point we've already asserted that the sender + // and the state key are equal so we don't need to check both. + var serverName gomatrixserverlib.ServerName + if _, serverName, err = gomatrixserverlib.SplitID('@', event.Sender()); err != nil { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("The sender of the join is invalid"), + } + } else if serverName != request.Origin() { + return util.JSONResponse{ + Code: http.StatusForbidden, + JSON: jsonerror.Forbidden("The sender does not match the server that originated the request"), + } + } + // Check if the user has already left. If so, no-op! queryReq := &api.QueryLatestEventsAndStateRequest{ RoomID: roomID, @@ -240,7 +249,7 @@ func SendLeave( } } verifyRequests := []gomatrixserverlib.VerifyJSONRequest{{ - ServerName: event.Origin(), + ServerName: serverName, Message: redacted, AtTS: event.OriginServerTS(), StrictValidityChecking: true, diff --git a/go.mod b/go.mod index 68b5cef84..3d99a71e1 100644 --- a/go.mod +++ b/go.mod @@ -22,8 +22,8 @@ require ( github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3 - github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc + github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 + github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.15 github.com/nats-io/nats-server/v2 v2.9.1-0.20220920152220-52d7b481c4b5 diff --git a/go.sum b/go.sum index 216b4a5fc..32edf8c11 100644 --- a/go.sum +++ b/go.sum @@ -384,10 +384,10 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3 h1:u3FKZmXxfhv3XhD8RziBlt96QTt8eHFhg1upCloBh2g= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220923115829-2217f6c65ce3/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= -github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc h1:NAf6Xec/RoYD+/UT3m2q58yZAu0pYA9C4gc/RbS89ls= -github.com/matrix-org/pinecone v0.0.0-20220915154206-df85cb5026fc/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5 h1:cQMA9hip0WSp6cv7CUfButa9Jl/9E6kqWmQyOjx5A5s= +github.com/matrix-org/gomatrixserverlib v0.0.0-20220926161602-759a8ee7c4d5/go.mod h1:Mtifyr8q8htcBeugvlDnkBcNUy5LO8OzUoplAf1+mb4= +github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89 h1:Ym50Fgn3GiYya4p29k3nJ5nYsalFGev3eIm3DeGNIq4= +github.com/matrix-org/pinecone v0.0.0-20220923151905-0900fceecb89/go.mod h1:K0N1ixHQxXoCyqolDqVxPM3ArrDtcMs8yegOx2Lfv9k= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 h1:eCEHXWDv9Rm335MSuB49mFUK44bwZPFSDde3ORE3syk= github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= diff --git a/roomserver/internal/input/input_events.go b/roomserver/internal/input/input_events.go index 29af649ad..01fd62010 100644 --- a/roomserver/internal/input/input_events.go +++ b/roomserver/internal/input/input_events.go @@ -118,6 +118,10 @@ func (r *Inputer) processRoomEvent( if roomInfo == nil && !isCreateEvent { return fmt.Errorf("room %s does not exist for event %s", event.RoomID(), event.EventID()) } + _, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender()) + if err != nil { + return fmt.Errorf("event has invalid sender %q", input.Event.Sender()) + } // If we already know about this outlier and it hasn't been rejected // then we won't attempt to reprocess it. If it was rejected or has now @@ -145,7 +149,8 @@ func (r *Inputer) processRoomEvent( var missingAuth, missingPrev bool serverRes := &fedapi.QueryJoinedHostServerNamesInRoomResponse{} if !isCreateEvent { - missingAuthIDs, missingPrevIDs, err := r.DB.MissingAuthPrevEvents(ctx, event) + var missingAuthIDs, missingPrevIDs []string + missingAuthIDs, missingPrevIDs, err = r.DB.MissingAuthPrevEvents(ctx, event) if err != nil { return fmt.Errorf("updater.MissingAuthPrevEvents: %w", err) } @@ -158,7 +163,7 @@ func (r *Inputer) processRoomEvent( RoomID: event.RoomID(), ExcludeSelf: true, } - if err := r.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, serverReq, serverRes); err != nil { + if err = r.FSAPI.QueryJoinedHostServerNamesInRoom(ctx, serverReq, serverRes); err != nil { return fmt.Errorf("r.FSAPI.QueryJoinedHostServerNamesInRoom: %w", err) } // Sort all of the servers into a map so that we can randomise @@ -173,9 +178,9 @@ func (r *Inputer) processRoomEvent( serverRes.ServerNames = append(serverRes.ServerNames, input.Origin) delete(servers, input.Origin) } - if origin := event.Origin(); origin != input.Origin { - serverRes.ServerNames = append(serverRes.ServerNames, origin) - delete(servers, origin) + if senderDomain != input.Origin { + serverRes.ServerNames = append(serverRes.ServerNames, senderDomain) + delete(servers, senderDomain) } for server := range servers { serverRes.ServerNames = append(serverRes.ServerNames, server) @@ -188,7 +193,7 @@ func (r *Inputer) processRoomEvent( isRejected := false authEvents := gomatrixserverlib.NewAuthEvents(nil) knownEvents := map[string]*types.Event{} - if err := r.fetchAuthEvents(ctx, logger, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil { + if err = r.fetchAuthEvents(ctx, logger, headered, &authEvents, knownEvents, serverRes.ServerNames); err != nil { return fmt.Errorf("r.fetchAuthEvents: %w", err) } @@ -231,7 +236,6 @@ func (r *Inputer) processRoomEvent( if input.Kind == api.KindNew { // Check that the event passes authentication checks based on the // current room state. - var err error softfail, err = helpers.CheckForSoftFail(ctx, r.DB, headered, input.StateEventIDs) if err != nil { logger.WithError(err).Warn("Error authing soft-failed event") @@ -265,7 +269,8 @@ func (r *Inputer) processRoomEvent( hadEvents: map[string]bool{}, haveEvents: map[string]*gomatrixserverlib.Event{}, } - if stateSnapshot, err := missingState.processEventWithMissingState(ctx, event, headered.RoomVersion); err != nil { + var stateSnapshot *parsedRespState + if stateSnapshot, err = missingState.processEventWithMissingState(ctx, event, headered.RoomVersion); err != nil { // Something went wrong with retrieving the missing state, so we can't // really do anything with the event other than reject it at this point. isRejected = true @@ -302,7 +307,6 @@ func (r *Inputer) processRoomEvent( // burning CPU time. historyVisibility := gomatrixserverlib.HistoryVisibilityShared // Default to shared. if input.Kind != api.KindOutlier && rejectionErr == nil && !isRejected { - var err error historyVisibility, rejectionErr, err = r.processStateBefore(ctx, input, missingPrev) if err != nil { return fmt.Errorf("r.processStateBefore: %w", err) diff --git a/roomserver/internal/perform/perform_backfill.go b/roomserver/internal/perform/perform_backfill.go index 51c66415a..69a075733 100644 --- a/roomserver/internal/perform/perform_backfill.go +++ b/roomserver/internal/perform/perform_backfill.go @@ -468,7 +468,9 @@ FindSuccessor: // Store the server names in a temporary map to avoid duplicates. serverSet := make(map[gomatrixserverlib.ServerName]bool) for _, event := range memberEvents { - serverSet[event.Origin()] = true + if _, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender()); err == nil { + serverSet[senderDomain] = true + } } var servers []gomatrixserverlib.ServerName for server := range serverSet { diff --git a/roomserver/internal/perform/perform_invite.go b/roomserver/internal/perform/perform_invite.go index 483e78c3f..3fbdf332e 100644 --- a/roomserver/internal/perform/perform_invite.go +++ b/roomserver/internal/perform/perform_invite.go @@ -50,6 +50,10 @@ func (r *Inviter) PerformInvite( if event.StateKey() == nil { return nil, fmt.Errorf("invite must be a state event") } + _, senderDomain, err := gomatrixserverlib.SplitID('@', event.Sender()) + if err != nil { + return nil, fmt.Errorf("sender %q is invalid", event.Sender()) + } roomID := event.RoomID() targetUserID := *event.StateKey() @@ -67,7 +71,7 @@ func (r *Inviter) PerformInvite( return nil, nil } isTargetLocal := domain == r.Cfg.Matrix.ServerName - isOriginLocal := event.Origin() == r.Cfg.Matrix.ServerName + isOriginLocal := senderDomain == r.Cfg.Matrix.ServerName if !isOriginLocal && !isTargetLocal { res.Error = &api.PerformError{ Code: api.PerformErrorBadRequest, @@ -235,7 +239,7 @@ func (r *Inviter) PerformInvite( { Kind: api.KindNew, Event: event, - Origin: event.Origin(), + Origin: senderDomain, SendAsServer: req.SendAsServer, }, }, diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index 036404cd2..ada3aab06 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -81,12 +81,11 @@ func (r *Leaver) performLeaveRoomByID( // that. isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID) if err == nil && isInvitePending { - var host gomatrixserverlib.ServerName - _, host, err = gomatrixserverlib.SplitID('@', senderUser) - if err != nil { + _, senderDomain, serr := gomatrixserverlib.SplitID('@', senderUser) + if serr != nil { return nil, fmt.Errorf("sender %q is invalid", senderUser) } - if host != r.Cfg.Matrix.ServerName { + if senderDomain != r.Cfg.Matrix.ServerName { return r.performFederatedRejectInvite(ctx, req, res, senderUser, eventID) } // check that this is not a "server notice room" @@ -172,6 +171,12 @@ func (r *Leaver) performLeaveRoomByID( return nil, fmt.Errorf("eventutil.BuildEvent: %w", err) } + // Get the sender domain. + _, senderDomain, serr := gomatrixserverlib.SplitID('@', event.Sender()) + if serr != nil { + return nil, fmt.Errorf("sender %q is invalid", event.Sender()) + } + // Give our leave event to the roomserver input stream. The // roomserver will process the membership change and notify // downstream automatically. @@ -180,7 +185,7 @@ func (r *Leaver) performLeaveRoomByID( { Kind: api.KindNew, Event: event.Headered(buildRes.RoomVersion), - Origin: event.Origin(), + Origin: senderDomain, SendAsServer: string(r.Cfg.Matrix.ServerName), }, }, diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index 00a17e5cb..593abbea1 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -896,7 +896,7 @@ func (d *Database) handleRedactions( switch { case redactUser >= pl.Redact: // The power level of the redaction event’s sender is greater than or equal to the redact level. - case redactedEvent.Origin() == redactionEvent.Origin() && redactedEvent.Sender() == redactionEvent.Sender(): + case redactedEvent.Sender() == redactionEvent.Sender(): // The domain of the redaction event’s sender matches that of the original event’s sender. default: return nil, "", nil diff --git a/run-sytest.sh b/run-sytest.sh index e23982397..4ed1c8d45 100755 --- a/run-sytest.sh +++ b/run-sytest.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Runs SyTest either from Docker Hub, or from ../sytest. If it's run # locally, the Docker image is rebuilt first. diff --git a/show-expected-fail-tests.sh b/show-expected-fail-tests.sh index 3ed937a0f..b7af8f648 100755 --- a/show-expected-fail-tests.sh +++ b/show-expected-fail-tests.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#!/usr/bin/env bash # # Parses a results.tap file from SyTest output and a file containing test names (a test whitelist) # and checks whether a test name that exists in the whitelist (that should pass), failed or not. diff --git a/syncapi/storage/postgres/current_room_state_table.go b/syncapi/storage/postgres/current_room_state_table.go index 083d10b88..5e6daaaf8 100644 --- a/syncapi/storage/postgres/current_room_state_table.go +++ b/syncapi/storage/postgres/current_room_state_table.go @@ -62,6 +62,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_s CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave'; -- for querying state by event IDs CREATE UNIQUE INDEX IF NOT EXISTS syncapi_current_room_state_eventid_idx ON syncapi_current_room_state(event_id); +-- for improving selectRoomIDsWithAnyMembershipSQL +CREATE INDEX IF NOT EXISTS syncapi_current_room_state_type_state_key_idx ON syncapi_current_room_state(type, state_key); ` const upsertRoomStateSQL = "" + @@ -80,7 +82,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectRoomIDsWithAnyMembershipSQL = "" + - "SELECT DISTINCT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1" + "SELECT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1" const selectCurrentStateSQL = "" + "SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" + diff --git a/syncapi/storage/sqlite3/current_room_state_table.go b/syncapi/storage/sqlite3/current_room_state_table.go index b88c11f88..bd1271dd6 100644 --- a/syncapi/storage/sqlite3/current_room_state_table.go +++ b/syncapi/storage/sqlite3/current_room_state_table.go @@ -51,6 +51,8 @@ CREATE UNIQUE INDEX IF NOT EXISTS syncapi_event_id_idx ON syncapi_current_room_s -- CREATE INDEX IF NOT EXISTS syncapi_membership_idx ON syncapi_current_room_state(type, state_key, membership) WHERE membership IS NOT NULL AND membership != 'leave'; -- for querying state by event IDs CREATE UNIQUE INDEX IF NOT EXISTS syncapi_current_room_state_eventid_idx ON syncapi_current_room_state(event_id); +-- for improving selectRoomIDsWithAnyMembershipSQL +CREATE INDEX IF NOT EXISTS syncapi_current_room_state_type_state_key_idx ON syncapi_current_room_state(type, state_key); ` const upsertRoomStateSQL = "" + @@ -69,7 +71,7 @@ const selectRoomIDsWithMembershipSQL = "" + "SELECT DISTINCT room_id FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1 AND membership = $2" const selectRoomIDsWithAnyMembershipSQL = "" + - "SELECT DISTINCT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1" + "SELECT room_id, membership FROM syncapi_current_room_state WHERE type = 'm.room.member' AND state_key = $1" const selectCurrentStateSQL = "" + "SELECT event_id, headered_event_json FROM syncapi_current_room_state WHERE room_id = $1" diff --git a/userapi/storage/shared/storage.go b/userapi/storage/shared/storage.go index acc65212c..e32a442d0 100644 --- a/userapi/storage/shared/storage.go +++ b/userapi/storage/shared/storage.go @@ -777,7 +777,7 @@ func (d *Database) GetPushers( func (d *Database) RemovePusher( ctx context.Context, appid, pushkey, localpart string, ) error { - return d.Writer.Do(nil, nil, func(txn *sql.Tx) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { err := d.Pushers.DeletePusher(ctx, txn, appid, pushkey, localpart) if err == sql.ErrNoRows { return nil @@ -792,7 +792,7 @@ func (d *Database) RemovePusher( func (d *Database) RemovePushers( ctx context.Context, appid, pushkey string, ) error { - return d.Writer.Do(nil, nil, func(txn *sql.Tx) error { + return d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { return d.Pushers.DeletePushers(ctx, txn, appid, pushkey) }) } diff --git a/userapi/storage/sqlite3/pusher_table.go b/userapi/storage/sqlite3/pusher_table.go index d5bd1617b..dba97c3d4 100644 --- a/userapi/storage/sqlite3/pusher_table.go +++ b/userapi/storage/sqlite3/pusher_table.go @@ -96,7 +96,7 @@ func (s *pushersStatements) InsertPusher( ctx context.Context, txn *sql.Tx, session_id int64, pushkey string, pushkeyTS int64, kind api.PusherKind, appid, appdisplayname, devicedisplayname, profiletag, lang, data, localpart string, ) error { - _, err := s.insertPusherStmt.ExecContext(ctx, localpart, session_id, pushkey, pushkeyTS, kind, appid, appdisplayname, devicedisplayname, profiletag, lang, data) + _, err := sqlutil.TxStmt(txn, s.insertPusherStmt).ExecContext(ctx, localpart, session_id, pushkey, pushkeyTS, kind, appid, appdisplayname, devicedisplayname, profiletag, lang, data) logrus.Debugf("Created pusher %d", session_id) return err } @@ -144,13 +144,13 @@ func (s *pushersStatements) SelectPushers( func (s *pushersStatements) DeletePusher( ctx context.Context, txn *sql.Tx, appid, pushkey, localpart string, ) error { - _, err := s.deletePusherStmt.ExecContext(ctx, appid, pushkey, localpart) + _, err := sqlutil.TxStmt(txn, s.deletePusherStmt).ExecContext(ctx, appid, pushkey, localpart) return err } func (s *pushersStatements) DeletePushers( ctx context.Context, txn *sql.Tx, appid, pushkey string, ) error { - _, err := s.deletePushersByAppIdAndPushKeyStmt.ExecContext(ctx, appid, pushkey) + _, err := sqlutil.TxStmt(txn, s.deletePushersByAppIdAndPushKeyStmt).ExecContext(ctx, appid, pushkey) return err }