diff --git a/CHANGES.md b/CHANGES.md index c909c5715..27356b3cb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,46 @@ # Changelog +## Dendrite 0.4.0 (2021-07-12) + +### Features + +* All-new state storage in the roomserver, which dramatically reduces disk space utilisation + * State snapshots and blocks are now aggressively deduplicated and reused wherever possible, with state blocks being reduced by up to 15x and snapshot references being reduced up to 2x + * Dendrite will upgrade to the new state storage automatically on the first run after upgrade, although this may take some time depending on the size of the state storage +* Appservice support has been improved significantly, with many bridges now working correctly with Dendrite + * Events are now correctly sent to appservices based on room memberships + * Aliases and namespaces are now handled correctly, calling the appservice to query for aliases as needed + * Appservice user registrations are no longer being subject to incorrect validation checks +* Shared secret registration has now been implemented correctly +* The roomserver input API implements a new queuing system to reduce backpressure across rooms +* Checking if the local server is in a room has been optimised substantially, reducing CPU usage +* State resolution v2 has been optimised further by improving the power level checks, reducing CPU usage +* The federation API `/send` endpoint now deduplicates missing auth and prev events more aggressively to reduce memory usage +* The federation API `/send` endpoint now uses workers to reduce backpressure across rooms +* The bcrypt cost for password storage is now configurable with the `user_api.bcrypt_cost` option +* The federation API will now use significantly less memory when calling `/get_missing_events` +* MSC2946 Spaces endpoints have been updated to stable endpoint naming +* The media API can now be configured without a maximum file size +* A new `dendrite-upgrade-test` test has been added for verifying database schema upgrades across versions +* Added Prometheus metrics for roomserver backpressure, excessive device list updates and federation API event processing summaries +* Sentry support has been added for error reporting + +### Fixes + +* Removed the legacy `/v1` register endpoint. Dendrite only implements `/r0` of the CS API, and the legacy `/v1` endpoint had implementation errors which made it possible to bypass shared secret registration (thanks to Jakob Varmose Bentzen for reporting this) +* Attempting to register an account that already exists now returns a sensible error code rather than a HTTP 500 +* Dendrite will no longer attempt to `/make_join` with itself if listed in the request `server_names` +* `/sync` will no longer return immediately if there is nothing to sync, which happened particularly with new accounts, causing high CPU usage +* Malicious media uploads can no longer exhaust all available memory (contributed by [S7evinK](https://github.com/S7evinK)) +* Selecting one-time keys from the database has been optimised (contributed by [S7evinK](https://github.com/S7evinK)) +* The return code when trying to fetch missing account data has been fixed (contributed by [adamgreig](https://github.com/adamgreig)) +* Dendrite will no longer attempt to use `/make_leave` over federation when rejecting a local invite +* A panic has been fixed in `QueryMembershipsForRoom` +* A panic on duplicate membership events has been fixed in the federation sender +* A panic has been fixed in in `IsInterestedInRoomID` (contributed by [bodqhrohro](https://github.com/bodqhrohro)) +* A panic in the roomserver has been fixed when handling empty state sets +* A panic in the federation API has been fixed when handling cached events + ## Dendrite 0.3.11 (2021-03-02) ### Fixes diff --git a/build/docker/README.md b/build/docker/README.md index 6d3cd3dbd..19e4234c5 100644 --- a/build/docker/README.md +++ b/build/docker/README.md @@ -28,8 +28,8 @@ There are three sample `docker-compose` files: The `docker-compose` files refer to the `/etc/dendrite` volume as where the runtime config should come from. The mounted folder must contain: -- `dendrite.yaml` configuration file (based on the sample `dendrite-config.yaml` - in the `docker/config` folder in the [Dendrite repository](https://github.com/matrix-org/dendrite) +- `dendrite.yaml` configuration file (based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml) + sample in the `build/docker/config` folder of this repository.) - `matrix_key.pem` server key, as generated using `cmd/generate-keys` - `server.crt` certificate file - `server.key` private key file for the above certificate @@ -50,8 +50,7 @@ The key files will now exist in your current working directory, and can be mount ## Starting Dendrite as a monolith deployment -Create your config based on the `dendrite.yaml` configuration file in the `docker/config` -folder in the [Dendrite repository](https://github.com/matrix-org/dendrite). +Create your config based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml) configuration file in the `build/docker/config` folder of this repository. And rename the config file to `dendrite.yml` (and put it in your `config` directory). Once in place, start the PostgreSQL dependency: @@ -67,8 +66,7 @@ docker-compose -f docker-compose.monolith.yml up ## Starting Dendrite as a polylith deployment -Create your config based on the `dendrite.yaml` configuration file in the `docker/config` -folder in the [Dendrite repository](https://github.com/matrix-org/dendrite). +Create your config based on the [`dendrite-config.yaml`](https://raw.githubusercontent.com/matrix-org/dendrite/master/dendrite-config.yaml) configuration file in the `build/docker/config` folder of this repository. And rename the config file to `dendrite.yml` (and put it in your `config` directory). Once in place, start all the dependencies: @@ -84,10 +82,10 @@ docker-compose -f docker-compose.polylith.yml up ## Building the images -The `docker/images-build.sh` script will build the base image, followed by +The `build/docker/images-build.sh` script will build the base image, followed by all of the component images. -The `docker/images-push.sh` script will push them to Docker Hub (subject +The `build/docker/images-push.sh` script will push them to Docker Hub (subject to permissions). If you wish to build and push your own images, rename `matrixdotorg/dendrite` to diff --git a/build/gobind-pinecone/monolith.go b/build/gobind-pinecone/monolith.go index 09af80f6c..e30057ed4 100644 --- a/build/gobind-pinecone/monolith.go +++ b/build/gobind-pinecone/monolith.go @@ -334,6 +334,7 @@ func (m *DendriteMonolith) Start() { base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicMediaAPIMux, + base.SynapseAdminMux, ) httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() diff --git a/build/gobind-yggdrasil/monolith.go b/build/gobind-yggdrasil/monolith.go index 5074f6da4..338628049 100644 --- a/build/gobind-yggdrasil/monolith.go +++ b/build/gobind-yggdrasil/monolith.go @@ -173,6 +173,7 @@ func (m *DendriteMonolith) Start() { base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicMediaAPIMux, + base.SynapseAdminMux, ) httpRouter := mux.NewRouter() diff --git a/clientapi/clientapi.go b/clientapi/clientapi.go index 2c4fa5d64..562d89d28 100644 --- a/clientapi/clientapi.go +++ b/clientapi/clientapi.go @@ -35,6 +35,7 @@ import ( // AddPublicRoutes sets up and registers HTTP handlers for the ClientAPI component. func AddPublicRoutes( router *mux.Router, + synapseAdminRouter *mux.Router, cfg *config.ClientAPI, accountsDB accounts.Database, federation *gomatrixserverlib.FederationClient, @@ -56,7 +57,7 @@ func AddPublicRoutes( } routing.Setup( - router, cfg, eduInputAPI, rsAPI, asAPI, + router, synapseAdminRouter, cfg, eduInputAPI, rsAPI, asAPI, accountsDB, userAPI, federation, syncProducer, transactionsCache, fsAPI, keyAPI, extRoomsProvider, mscCfg, ) diff --git a/clientapi/routing/register.go b/clientapi/routing/register.go index 526418669..8823a41e3 100644 --- a/clientapi/routing/register.go +++ b/clientapi/routing/register.go @@ -17,10 +17,7 @@ package routing import ( "context" - "crypto/hmac" - "crypto/sha1" "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -594,7 +591,6 @@ func handleRegistrationFlow( accessToken string, accessTokenErr error, ) util.JSONResponse { - // TODO: Shared secret registration (create new user scripts) // TODO: Enable registration config flag // TODO: Guest account upgrading @@ -643,20 +639,6 @@ func handleRegistrationFlow( // Add Recaptcha to the list of completed registration stages AddCompletedSessionStage(sessionID, authtypes.LoginTypeRecaptcha) - case authtypes.LoginTypeSharedSecret: - // Check shared secret against config - valid, err := isValidMacLogin(cfg, r.Username, r.Password, r.Admin, r.Auth.Mac) - - if err != nil { - util.GetLogger(req.Context()).WithError(err).Error("isValidMacLogin failed") - return jsonerror.InternalServerError() - } else if !valid { - return util.MessageResponse(http.StatusForbidden, "HMAC incorrect") - } - - // Add SharedSecret to the list of completed registration stages - AddCompletedSessionStage(sessionID, authtypes.LoginTypeSharedSecret) - case authtypes.LoginTypeDummy: // there is nothing to do // Add Dummy to the list of completed registration stages @@ -849,49 +831,6 @@ func completeRegistration( } } -// Used for shared secret registration. -// Checks if the username, password and isAdmin flag matches the given mac. -func isValidMacLogin( - cfg *config.ClientAPI, - username, password string, - isAdmin bool, - givenMac []byte, -) (bool, error) { - sharedSecret := cfg.RegistrationSharedSecret - - // Check that shared secret registration isn't disabled. - if cfg.RegistrationSharedSecret == "" { - return false, errors.New("Shared secret registration is disabled") - } - - // Double check that username/password don't contain the HMAC delimiters. We should have - // already checked this. - if strings.Contains(username, "\x00") { - return false, errors.New("Username contains invalid character") - } - if strings.Contains(password, "\x00") { - return false, errors.New("Password contains invalid character") - } - if sharedSecret == "" { - return false, errors.New("Shared secret registration is disabled") - } - - adminString := "notadmin" - if isAdmin { - adminString = "admin" - } - joined := strings.Join([]string{username, password, adminString}, "\x00") - - mac := hmac.New(sha1.New, []byte(sharedSecret)) - _, err := mac.Write([]byte(joined)) - if err != nil { - return false, err - } - expectedMAC := mac.Sum(nil) - - return hmac.Equal(givenMac, expectedMAC), nil -} - // checkFlows checks a single completed flow against another required one. If // one contains at least all of the stages that the other does, checkFlows // returns true. @@ -995,3 +934,34 @@ func RegisterAvailable( }, } } + +func handleSharedSecretRegistration(userAPI userapi.UserInternalAPI, sr *SharedSecretRegistration, req *http.Request) util.JSONResponse { + ssrr, err := NewSharedSecretRegistrationRequest(req.Body) + if err != nil { + return util.JSONResponse{ + Code: 400, + JSON: jsonerror.BadJSON(fmt.Sprintf("malformed json: %s", err)), + } + } + valid, err := sr.IsValidMacLogin(ssrr.Nonce, ssrr.User, ssrr.Password, ssrr.Admin, ssrr.MacBytes) + if err != nil { + return util.ErrorResponse(err) + } + if !valid { + return util.JSONResponse{ + Code: 403, + JSON: jsonerror.Forbidden("bad mac"), + } + } + // downcase capitals + ssrr.User = strings.ToLower(ssrr.User) + + if resErr := validateUsername(ssrr.User); resErr != nil { + return *resErr + } + if resErr := validatePassword(ssrr.Password); resErr != nil { + return *resErr + } + deviceID := "shared_secret_registration" + return completeRegistration(req.Context(), userAPI, ssrr.User, ssrr.Password, "", req.RemoteAddr, req.UserAgent(), false, &ssrr.User, &deviceID) +} diff --git a/clientapi/routing/register_secret.go b/clientapi/routing/register_secret.go new file mode 100644 index 000000000..f0436e322 --- /dev/null +++ b/clientapi/routing/register_secret.go @@ -0,0 +1,99 @@ +package routing + +import ( + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/matrix-org/dendrite/internal" + "github.com/matrix-org/util" + cache "github.com/patrickmn/go-cache" +) + +type SharedSecretRegistrationRequest struct { + User string `json:"username"` + Password string `json:"password"` + Nonce string `json:"nonce"` + MacBytes []byte + MacStr string `json:"mac"` + Admin bool `json:"admin"` +} + +func NewSharedSecretRegistrationRequest(reader io.ReadCloser) (*SharedSecretRegistrationRequest, error) { + defer internal.CloseAndLogIfError(context.Background(), reader, "NewSharedSecretRegistrationRequest: failed to close request body") + var ssrr SharedSecretRegistrationRequest + err := json.NewDecoder(reader).Decode(&ssrr) + if err != nil { + return nil, err + } + ssrr.MacBytes, err = hex.DecodeString(ssrr.MacStr) + return &ssrr, err +} + +type SharedSecretRegistration struct { + sharedSecret string + nonces *cache.Cache +} + +func NewSharedSecretRegistration(sharedSecret string) *SharedSecretRegistration { + return &SharedSecretRegistration{ + sharedSecret: sharedSecret, + // nonces live for 5mins, purge every 10mins + nonces: cache.New(5*time.Minute, 10*time.Minute), + } +} + +func (r *SharedSecretRegistration) GenerateNonce() string { + nonce := util.RandomString(16) + r.nonces.Set(nonce, true, cache.DefaultExpiration) + return nonce +} + +func (r *SharedSecretRegistration) validNonce(nonce string) bool { + _, exists := r.nonces.Get(nonce) + return exists +} + +func (r *SharedSecretRegistration) IsValidMacLogin( + nonce, username, password string, + isAdmin bool, + givenMac []byte, +) (bool, error) { + // Check that shared secret registration isn't disabled. + if r.sharedSecret == "" { + return false, errors.New("Shared secret registration is disabled") + } + if !r.validNonce(nonce) { + return false, fmt.Errorf("Incorrect or expired nonce: %s", nonce) + } + + // Check that username/password don't contain the HMAC delimiters. + if strings.Contains(username, "\x00") { + return false, errors.New("Username contains invalid character") + } + if strings.Contains(password, "\x00") { + return false, errors.New("Password contains invalid character") + } + + adminString := "notadmin" + if isAdmin { + adminString = "admin" + } + joined := strings.Join([]string{nonce, username, password, adminString}, "\x00") + + mac := hmac.New(sha1.New, []byte(r.sharedSecret)) + _, err := mac.Write([]byte(joined)) + if err != nil { + return false, err + } + expectedMAC := mac.Sum(nil) + + return hmac.Equal(givenMac, expectedMAC), nil +} diff --git a/clientapi/routing/register_secret_test.go b/clientapi/routing/register_secret_test.go new file mode 100644 index 000000000..e702b2152 --- /dev/null +++ b/clientapi/routing/register_secret_test.go @@ -0,0 +1,43 @@ +package routing + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/patrickmn/go-cache" +) + +func TestSharedSecretRegister(t *testing.T) { + // these values have come from a local synapse instance to ensure compatibility + jsonStr := []byte(`{"admin":false,"mac":"f1ba8d37123866fd659b40de4bad9b0f8965c565","nonce":"759f047f312b99ff428b21d581256f8592b8976e58bc1b543972dc6147e529a79657605b52d7becd160ff5137f3de11975684319187e06901955f79e5a6c5a79","password":"wonderland","username":"alice"}`) + sharedSecret := "dendritetest" + + req, err := NewSharedSecretRegistrationRequest(ioutil.NopCloser(bytes.NewBuffer(jsonStr))) + if err != nil { + t.Fatalf("failed to read request: %s", err) + } + + r := NewSharedSecretRegistration(sharedSecret) + + // force the nonce to be known + r.nonces.Set(req.Nonce, true, cache.DefaultExpiration) + + valid, err := r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes) + if err != nil { + t.Fatalf("failed to check for valid mac: %s", err) + } + if !valid { + t.Errorf("mac login failed, wanted success") + } + + // modify the mac so it fails + req.MacBytes[0] = 0xff + valid, err = r.IsValidMacLogin(req.Nonce, req.User, req.Password, req.Admin, req.MacBytes) + if err != nil { + t.Fatalf("failed to check for valid mac: %s", err) + } + if valid { + t.Errorf("mac login succeeded, wanted failure") + } +} diff --git a/clientapi/routing/routing.go b/clientapi/routing/routing.go index 9f980e0a9..37279e8ed 100644 --- a/clientapi/routing/routing.go +++ b/clientapi/routing/routing.go @@ -37,6 +37,7 @@ import ( "github.com/matrix-org/dendrite/userapi/storage/accounts" "github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/util" + "github.com/sirupsen/logrus" ) // Setup registers HTTP handlers with the given ServeMux. It also supplies the given http.Client @@ -46,7 +47,7 @@ import ( // applied: // nolint: gocyclo func Setup( - publicAPIMux *mux.Router, cfg *config.ClientAPI, + publicAPIMux, synapseAdminRouter *mux.Router, cfg *config.ClientAPI, eduAPI eduServerAPI.EDUServerInputAPI, rsAPI roomserverAPI.RoomserverInternalAPI, asAPI appserviceAPI.AppServiceQueryAPI, @@ -88,6 +89,32 @@ func Setup( }), ).Methods(http.MethodGet, http.MethodOptions) + if cfg.RegistrationSharedSecret != "" { + logrus.Info("Enabling shared secret registration at /_synapse/admin/v1/register") + sr := NewSharedSecretRegistration(cfg.RegistrationSharedSecret) + synapseAdminRouter.Handle("/admin/v1/register", + httputil.MakeExternalAPI("shared_secret_registration", func(req *http.Request) util.JSONResponse { + if req.Method == http.MethodGet { + return util.JSONResponse{ + Code: 200, + JSON: struct { + Nonce string `json:"nonce"` + }{ + Nonce: sr.GenerateNonce(), + }, + } + } + if req.Method == http.MethodPost { + return handleSharedSecretRegistration(userAPI, sr, req) + } + return util.JSONResponse{ + Code: http.StatusMethodNotAllowed, + JSON: jsonerror.NotFound("unknown method"), + } + }), + ).Methods(http.MethodGet, http.MethodPost, http.MethodOptions) + } + r0mux := publicAPIMux.PathPrefix("/r0").Subrouter() unstableMux := publicAPIMux.PathPrefix("/unstable").Subrouter() diff --git a/cmd/dendrite-demo-libp2p/main.go b/cmd/dendrite-demo-libp2p/main.go index cc7dcf021..6b0e57d8b 100644 --- a/cmd/dendrite-demo-libp2p/main.go +++ b/cmd/dendrite-demo-libp2p/main.go @@ -197,6 +197,7 @@ func main() { base.Base.PublicFederationAPIMux, base.Base.PublicKeyAPIMux, base.Base.PublicMediaAPIMux, + base.Base.SynapseAdminMux, ) if err := mscs.Enable(&base.Base, &monolith); err != nil { logrus.WithError(err).Fatalf("Failed to enable MSCs") diff --git a/cmd/dendrite-demo-pinecone/main.go b/cmd/dendrite-demo-pinecone/main.go index 72936e42e..2712ed4a1 100644 --- a/cmd/dendrite-demo-pinecone/main.go +++ b/cmd/dendrite-demo-pinecone/main.go @@ -210,6 +210,7 @@ func main() { base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicMediaAPIMux, + base.SynapseAdminMux, ) wsUpgrader := websocket.Upgrader{ diff --git a/cmd/dendrite-demo-yggdrasil/main.go b/cmd/dendrite-demo-yggdrasil/main.go index 2d710ae79..abeefbe5a 100644 --- a/cmd/dendrite-demo-yggdrasil/main.go +++ b/cmd/dendrite-demo-yggdrasil/main.go @@ -154,6 +154,7 @@ func main() { base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicMediaAPIMux, + base.SynapseAdminMux, ) if err := mscs.Enable(base, &monolith); err != nil { logrus.WithError(err).Fatalf("Failed to enable MSCs") diff --git a/cmd/dendrite-monolith-server/main.go b/cmd/dendrite-monolith-server/main.go index ef349505c..5efbe8567 100644 --- a/cmd/dendrite-monolith-server/main.go +++ b/cmd/dendrite-monolith-server/main.go @@ -149,6 +149,7 @@ func main() { base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicMediaAPIMux, + base.SynapseAdminMux, ) if len(base.Cfg.MSCs.MSCs) > 0 { diff --git a/cmd/dendrite-polylith-multi/personalities/clientapi.go b/cmd/dendrite-polylith-multi/personalities/clientapi.go index ec445ceb7..5e0c43548 100644 --- a/cmd/dendrite-polylith-multi/personalities/clientapi.go +++ b/cmd/dendrite-polylith-multi/personalities/clientapi.go @@ -33,7 +33,7 @@ func ClientAPI(base *setup.BaseDendrite, cfg *config.Dendrite) { keyAPI := base.KeyServerHTTPClient() clientapi.AddPublicRoutes( - base.PublicClientAPIMux, &base.Cfg.ClientAPI, accountDB, federation, + base.PublicClientAPIMux, base.SynapseAdminMux, &base.Cfg.ClientAPI, accountDB, federation, rsAPI, eduInputAPI, asQuery, transactions.New(), fsAPI, userAPI, keyAPI, nil, &cfg.MSCs, ) diff --git a/cmd/dendrite-upgrade-tests/main.go b/cmd/dendrite-upgrade-tests/main.go index 5b5cf9d7e..a6cc2d3f9 100644 --- a/cmd/dendrite-upgrade-tests/main.go +++ b/cmd/dendrite-upgrade-tests/main.go @@ -35,9 +35,13 @@ var ( flagFrom = flag.String("from", "HEAD-1", "The version to start from e.g '0.3.1'. If 'HEAD-N' then starts N versions behind HEAD.") flagTo = flag.String("to", "HEAD", "The version to end on e.g '0.3.3'.") flagBuildConcurrency = flag.Int("build-concurrency", runtime.NumCPU(), "The amount of build concurrency when building images") + flagHead = flag.String("head", "", "Location to a dendrite repository to treat as HEAD instead of Github") + flagDockerHost = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.") alphaNumerics = regexp.MustCompile("[^a-zA-Z0-9]+") ) +const HEAD = "HEAD" + // Embed the Dockerfile to use when building dendrite versions. // We cannot use the dockerfile associated with the repo with each version sadly due to changes in // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients @@ -104,6 +108,7 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b if resp.StatusCode != 200 { return nil, fmt.Errorf("got HTTP %d", resp.StatusCode) } + _ = os.RemoveAll(tmpDir) if err = os.Mkdir(tmpDir, os.ModePerm); err != nil { return nil, fmt.Errorf("failed to make temporary directory: %s", err) } @@ -134,15 +139,36 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b // buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) { - log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir) - // pull an archive, this contains a top-level directory which screws with the build context - // which we need to fix up post download - u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName) - tarball, err := downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile)) - if err != nil { - return "", fmt.Errorf("failed to download archive %s: %w", u, err) + var tarball *bytes.Buffer + var err error + // If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI + // where we want to use the working directory. + if branchOrTagName == HEAD && *flagHead != "" { + log.Printf("%s: Using %s as HEAD", branchOrTagName, *flagHead) + // add top level Dockerfile + err = ioutil.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm) + if err != nil { + return "", fmt.Errorf("Custom HEAD: failed to inject /Dockerfile: %w", err) + } + // now tarball it + var buffer bytes.Buffer + err = compress(*flagHead, &buffer) + if err != nil { + return "", fmt.Errorf("failed to tarball custom HEAD %s : %s", *flagHead, err) + } + tarball = &buffer + } else { + log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir) + // pull an archive, this contains a top-level directory which screws with the build context + // which we need to fix up post download + u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName) + tarball, err = downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile)) + if err != nil { + return "", fmt.Errorf("failed to download archive %s: %w", u, err) + } + log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len()) } - log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len()) + log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName) res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{ Tags: []string{"dendrite-upgrade"}, @@ -234,7 +260,7 @@ func calculateVersions(cli *http.Client, from, to string) []string { semvers = semvers[i:] } } - if to != "" && to != "HEAD" { + if to != "" && to != HEAD { toVer, err := semver.NewVersion(to) if err != nil { log.Fatalf("invalid --to: %s", err) @@ -252,8 +278,8 @@ func calculateVersions(cli *http.Client, from, to string) []string { for _, sv := range semvers { versions = append(versions, sv.Original()) } - if to == "HEAD" { - versions = append(versions, "HEAD") + if to == HEAD { + versions = append(versions, HEAD) } return versions } @@ -327,7 +353,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string) if !ok { return "", "", fmt.Errorf("port 8008 not exposed - exposed ports: %v", inspect.NetworkSettings.Ports) } - baseURL := fmt.Sprintf("http://localhost:%s", csapiPortInfo[0].HostPort) + baseURL := fmt.Sprintf("http://%s:%s", *flagDockerHost, csapiPortInfo[0].HostPort) versionsURL := fmt.Sprintf("%s/_matrix/client/versions", baseURL) // hit /versions to check it is up var lastErr error diff --git a/cmd/dendrite-upgrade-tests/tar.go b/cmd/dendrite-upgrade-tests/tar.go index 9eadbb3de..fd45424db 100644 --- a/cmd/dendrite-upgrade-tests/tar.go +++ b/cmd/dendrite-upgrade-tests/tar.go @@ -17,7 +17,7 @@ func compress(src string, buf io.Writer) error { tw := tar.NewWriter(zr) // walk through every file in the folder - _ = filepath.Walk(src, func(file string, fi os.FileInfo, e error) error { + err := filepath.Walk(src, func(file string, fi os.FileInfo, e error) error { // generate tar header header, err := tar.FileInfoHeader(fi, file) if err != nil { @@ -37,12 +37,18 @@ func compress(src string, buf io.Writer) error { if err != nil { return err } - if _, err := io.Copy(tw, data); err != nil { + if _, err = io.Copy(tw, data); err != nil { + return err + } + if err = data.Close(); err != nil { return err } } return nil }) + if err != nil { + return err + } // produce tar if err := tw.Close(); err != nil { diff --git a/cmd/dendritejs-pinecone/main.go b/cmd/dendritejs-pinecone/main.go index 433e9bf82..25e496909 100644 --- a/cmd/dendritejs-pinecone/main.go +++ b/cmd/dendritejs-pinecone/main.go @@ -215,6 +215,7 @@ func main() { base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicMediaAPIMux, + base.SynapseAdminMux, ) httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index 7ece94ff0..d5a845ae0 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -236,6 +236,7 @@ func main() { base.PublicFederationAPIMux, base.PublicKeyAPIMux, base.PublicMediaAPIMux, + base.SynapseAdminMux, ) httpRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 1c9e72bff..5f214e0fc 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -573,6 +573,23 @@ func (t *txnReq) processEvent(ctx context.Context, e *gomatrixserverlib.Event) e logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) t.work = "" // reset from previous event + // Ask the roomserver if we know about the room and/or if we're joined + // to it. If we aren't then we won't bother processing the event. + joinedReq := api.QueryServerJoinedToRoomRequest{ + RoomID: e.RoomID(), + } + var joinedRes api.QueryServerJoinedToRoomResponse + if err := t.rsAPI.QueryServerJoinedToRoom(ctx, &joinedReq, &joinedRes); err != nil { + return fmt.Errorf("t.rsAPI.QueryServerJoinedToRoom: %w", err) + } + + if !joinedRes.RoomExists || !joinedRes.IsInRoom { + // We don't believe we're a member of this room, therefore there's + // no point in wasting work trying to figure out what to do with + // missing auth or prev events. Drop the event. + return roomNotFoundError{e.RoomID()} + } + // Work out if the roomserver knows everything it needs to know to auth // the event. This includes the prev_events and auth_events. // NOTE! This is going to include prev_events that have an empty state @@ -589,16 +606,6 @@ func (t *txnReq) processEvent(ctx context.Context, e *gomatrixserverlib.Event) e return fmt.Errorf("t.rsAPI.QueryMissingAuthPrevEvents: %w", err) } - if !stateResp.RoomExists { - // TODO: When synapse receives a message for a room it is not in it - // asks the remote server for the state of the room so that it can - // check if the remote server knows of a join "m.room.member" event - // that this server is unaware of. - // However generally speaking we should reject events for rooms we - // aren't a member of. - return roomNotFoundError{e.RoomID()} - } - // Prepare a map of all the events we already had before this point, so // that we don't send them to the roomserver again. for _, eventID := range append(e.AuthEventIDs(), e.PrevEventIDs()...) { diff --git a/federationapi/routing/send_test.go b/federationapi/routing/send_test.go index 98ff1a0a3..0da06aa95 100644 --- a/federationapi/routing/send_test.go +++ b/federationapi/routing/send_test.go @@ -190,7 +190,9 @@ func (t *testRoomserverAPI) QueryServerJoinedToRoom( request *api.QueryServerJoinedToRoomRequest, response *api.QueryServerJoinedToRoomResponse, ) error { - return fmt.Errorf("not implemented") + response.RoomExists = true + response.IsInRoom = true + return nil } // Query whether a server is allowed to see an event diff --git a/go.mod b/go.mod index 46afd005d..82e066d41 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/ngrok/sqlmw v0.0.0-20200129213757-d5c93a81bec6 github.com/opentracing/opentracing-go v1.2.0 + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/pressly/goose v2.7.0+incompatible diff --git a/go.sum b/go.sum index b770d52c7..e5ca53521 100644 --- a/go.sum +++ b/go.sum @@ -1266,6 +1266,8 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= diff --git a/internal/log.go b/internal/log.go index 0f374bd4a..d2b233c5b 100644 --- a/internal/log.go +++ b/internal/log.go @@ -73,9 +73,9 @@ func callerPrettyfier(f *runtime.Frame) (string, string) { // Append a newline + tab to it to move the actual log content to its own line funcname += "\n\t" - // Surround the filepath in brackets and append line number so IDEs can quickly - // navigate - filename := fmt.Sprintf(" [%s:%d]", f.File, f.Line) + // Use a shortened file path which just has the filename to avoid having lots of redundant + // directories which contribute significantly to overall log sizes! + filename := fmt.Sprintf(" [%s:%d]", path.Base(f.File), f.Line) return funcname, filename } diff --git a/internal/version.go b/internal/version.go index 0d3487799..37f0c30d3 100644 --- a/internal/version.go +++ b/internal/version.go @@ -16,8 +16,8 @@ var build string const ( VersionMajor = 0 - VersionMinor = 3 - VersionPatch = 11 + VersionMinor = 4 + VersionPatch = 0 VersionTag = "" // example: "rc1" ) diff --git a/roomserver/api/query.go b/roomserver/api/query.go index af35f7e72..c70db65c1 100644 --- a/roomserver/api/query.go +++ b/roomserver/api/query.go @@ -170,7 +170,8 @@ type QueryMembershipsForRoomResponse struct { // QueryServerJoinedToRoomRequest is a request to QueryServerJoinedToRoom type QueryServerJoinedToRoomRequest struct { - // Server name of the server to find + // Server name of the server to find. If not specified, we will + // default to checking if the local server is joined. ServerName gomatrixserverlib.ServerName `json:"server_name"` // ID of the room to see if we are still joined to RoomID string `json:"room_id"` @@ -182,7 +183,8 @@ type QueryServerJoinedToRoomResponse struct { RoomExists bool `json:"room_exists"` // True if we still believe that we are participating in the room IsInRoom bool `json:"is_in_room"` - // List of servers that are also in the room + // List of servers that are also in the room. This will not be populated + // if the queried ServerName is the local server name. ServerNames []gomatrixserverlib.ServerName `json:"server_names"` } diff --git a/roomserver/internal/api.go b/roomserver/internal/api.go index c9f92f9ff..f39b26eaf 100644 --- a/roomserver/internal/api.go +++ b/roomserver/internal/api.go @@ -59,6 +59,7 @@ func NewRoomserverAPI( Queryer: &query.Queryer{ DB: roomserverDB, Cache: caches, + ServerName: cfg.Matrix.ServerName, ServerACLs: serverACLs, }, Inputer: &input.Inputer{ @@ -92,6 +93,7 @@ func (r *RoomserverInternalAPI) SetFederationSenderAPI(fsAPI fsAPI.FederationSen FSAPI: r.fsAPI, RSAPI: r, Inputer: r.Inputer, + Queryer: r.Queryer, } r.Peeker = &perform.Peeker{ ServerName: r.Cfg.Matrix.ServerName, diff --git a/roomserver/internal/helpers/helpers.go b/roomserver/internal/helpers/helpers.go index a829bffca..a389cc898 100644 --- a/roomserver/internal/helpers/helpers.go +++ b/roomserver/internal/helpers/helpers.go @@ -50,6 +50,10 @@ func UpdateToInviteMembership( return updates, nil } +// IsServerCurrentlyInRoom checks if a server is in a given room, based on the room +// memberships. If the servername is not supplied then the local server will be +// checked instead using a faster code path. +// TODO: This should probably be replaced by an API call. func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverName gomatrixserverlib.ServerName, roomID string) (bool, error) { info, err := db.RoomInfo(ctx, roomID) if err != nil { @@ -59,6 +63,10 @@ func IsServerCurrentlyInRoom(ctx context.Context, db storage.Database, serverNam return false, fmt.Errorf("unknown room %s", roomID) } + if serverName == "" { + return db.GetLocalServerInRoom(ctx, info.RoomNID) + } + eventNIDs, err := db.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, true, false) if err != nil { return false, err diff --git a/roomserver/internal/perform/perform_join.go b/roomserver/internal/perform/perform_join.go index 048496d45..876888e29 100644 --- a/roomserver/internal/perform/perform_join.go +++ b/roomserver/internal/perform/perform_join.go @@ -28,6 +28,7 @@ import ( rsAPI "github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/internal/helpers" "github.com/matrix-org/dendrite/roomserver/internal/input" + "github.com/matrix-org/dendrite/roomserver/internal/query" "github.com/matrix-org/dendrite/roomserver/storage" "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/gomatrixserverlib" @@ -42,6 +43,7 @@ type Joiner struct { DB storage.Database Inputer *input.Inputer + Queryer *query.Queryer } // PerformJoin handles joining matrix rooms, including over federation by talking to the federationsender. @@ -205,7 +207,14 @@ func (r *Joiner) performJoinRoomByID( // Force a federated join if we aren't in the room and we've been // given some server names to try joining by. - serverInRoom, _ := helpers.IsServerCurrentlyInRoom(ctx, r.DB, r.ServerName, req.RoomIDOrAlias) + inRoomReq := &api.QueryServerJoinedToRoomRequest{ + RoomID: req.RoomIDOrAlias, + } + inRoomRes := &api.QueryServerJoinedToRoomResponse{} + if err = r.Queryer.QueryServerJoinedToRoom(ctx, inRoomReq, inRoomRes); err != nil { + return "", "", fmt.Errorf("r.Queryer.QueryServerJoinedToRoom: %w", err) + } + serverInRoom := inRoomRes.IsInRoom forceFederatedJoin := len(req.ServerNames) > 0 && !serverInRoom // Force a federated join if we're dealing with a pending invite diff --git a/roomserver/internal/perform/perform_leave.go b/roomserver/internal/perform/perform_leave.go index 9d7c0816d..4d10dea67 100644 --- a/roomserver/internal/perform/perform_leave.go +++ b/roomserver/internal/perform/perform_leave.go @@ -64,7 +64,14 @@ func (r *Leaver) performLeaveRoomByID( // that. isInvitePending, senderUser, eventID, err := helpers.IsInvitePending(ctx, r.DB, req.RoomID, req.UserID) if err == nil && isInvitePending { - return r.performRejectInvite(ctx, req, res, senderUser, eventID) + var host gomatrixserverlib.ServerName + _, host, err = gomatrixserverlib.SplitID('@', senderUser) + if err != nil { + return nil, fmt.Errorf("Sender %q is invalid", senderUser) + } + if host != r.Cfg.Matrix.ServerName { + return r.performFederatedRejectInvite(ctx, req, res, senderUser, eventID) + } } // There's no invite pending, so first of all we want to find out @@ -94,9 +101,7 @@ func (r *Leaver) performLeaveRoomByID( if err != nil { return nil, fmt.Errorf("Error getting membership: %w", err) } - if membership != gomatrixserverlib.Join { - // TODO: should be able to handle "invite" in this case too, if - // it's a case of kicking or banning or such + if membership != gomatrixserverlib.Join && membership != gomatrixserverlib.Invite { return nil, fmt.Errorf("User %q is not joined to the room (membership is %q)", req.UserID, membership) } @@ -147,7 +152,7 @@ func (r *Leaver) performLeaveRoomByID( return nil, nil } -func (r *Leaver) performRejectInvite( +func (r *Leaver) performFederatedRejectInvite( ctx context.Context, req *api.PerformLeaveRequest, res *api.PerformLeaveResponse, // nolint:unparam diff --git a/roomserver/internal/query/query.go b/roomserver/internal/query/query.go index 408f9766e..4af0e6397 100644 --- a/roomserver/internal/query/query.go +++ b/roomserver/internal/query/query.go @@ -36,6 +36,7 @@ import ( type Queryer struct { DB storage.Database Cache caching.RoomServerCaches + ServerName gomatrixserverlib.ServerName ServerACLs *acls.ServerACLs } @@ -328,6 +329,16 @@ func (r *Queryer) QueryServerJoinedToRoom( } response.RoomExists = true + if request.ServerName == r.ServerName || request.ServerName == "" { + var joined bool + joined, err = r.DB.GetLocalServerInRoom(ctx, info.RoomNID) + if err != nil { + return fmt.Errorf("r.DB.GetLocalServerInRoom: %w", err) + } + response.IsInRoom = joined + return nil + } + eventNIDs, err := r.DB.GetMembershipEventNIDsForRoom(ctx, info.RoomNID, true, false) if err != nil { return fmt.Errorf("r.DB.GetMembershipEventNIDsForRoom: %w", err) @@ -377,10 +388,16 @@ func (r *Queryer) QueryServerAllowedToSeeEvent( return } roomID := events[0].RoomID() - isServerInRoom, err := helpers.IsServerCurrentlyInRoom(ctx, r.DB, request.ServerName, roomID) - if err != nil { - return + + inRoomReq := &api.QueryServerJoinedToRoomRequest{ + RoomID: roomID, + ServerName: request.ServerName, } + inRoomRes := &api.QueryServerJoinedToRoomResponse{} + if err = r.QueryServerJoinedToRoom(ctx, inRoomReq, inRoomRes); err != nil { + return fmt.Errorf("r.Queryer.QueryServerJoinedToRoom: %w", err) + } + info, err := r.DB.RoomInfo(ctx, roomID) if err != nil { return err @@ -389,7 +406,7 @@ func (r *Queryer) QueryServerAllowedToSeeEvent( return fmt.Errorf("QueryServerAllowedToSeeEvent: no room info for room %s", roomID) } response.AllowedToSeeEvent, err = helpers.CheckServerAllowedToSeeEvent( - ctx, r.DB, *info, request.EventID, request.ServerName, isServerInRoom, + ctx, r.DB, *info, request.EventID, request.ServerName, inRoomRes.IsInRoom, ) return } diff --git a/roomserver/storage/interface.go b/roomserver/storage/interface.go index d2b0e75c9..c25820aac 100644 --- a/roomserver/storage/interface.go +++ b/roomserver/storage/interface.go @@ -154,6 +154,8 @@ type Database interface { GetBulkStateContent(ctx context.Context, roomIDs []string, tuples []gomatrixserverlib.StateKeyTuple, allowWildcards bool) ([]tables.StrippedEvent, error) // JoinedUsersSetInRooms returns all joined users in the rooms given, along with the count of how many times they appear. JoinedUsersSetInRooms(ctx context.Context, roomIDs []string) (map[string]int, error) + // GetLocalServerInRoom returns true if we think we're in a given room or false otherwise. + GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) // GetKnownUsers searches all users that userID knows about. GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) // GetKnownRooms returns a list of all rooms we know about. diff --git a/roomserver/storage/postgres/membership_table.go b/roomserver/storage/postgres/membership_table.go index 3466da6d2..9102f26a3 100644 --- a/roomserver/storage/postgres/membership_table.go +++ b/roomserver/storage/postgres/membership_table.go @@ -124,6 +124,14 @@ var selectKnownUsersSQL = "" + " SELECT DISTINCT room_nid FROM roomserver_membership WHERE target_nid=$1 AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + ") AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " AND event_state_key LIKE $2 LIMIT $3" +// selectLocalServerInRoomSQL is an optimised case for checking if we, the local server, +// are in the room by using the target_local column of the membership table. Normally when +// we want to know if a server is in a room, we have to unmarshal the entire room state which +// is expensive. The presence of a single row from this query suggests we're still in the +// room, no rows returned suggests we aren't. +const selectLocalServerInRoomSQL = "" + + "SELECT room_nid FROM roomserver_membership WHERE target_local = true AND membership_nid = $1 AND room_nid = $2 LIMIT 1" + type membershipStatements struct { insertMembershipStmt *sql.Stmt selectMembershipForUpdateStmt *sql.Stmt @@ -137,6 +145,7 @@ type membershipStatements struct { selectJoinedUsersSetForRoomsStmt *sql.Stmt selectKnownUsersStmt *sql.Stmt updateMembershipForgetRoomStmt *sql.Stmt + selectLocalServerInRoomStmt *sql.Stmt } func createMembershipTable(db *sql.DB) error { @@ -160,6 +169,7 @@ func prepareMembershipTable(db *sql.DB) (tables.Membership, error) { {&s.selectJoinedUsersSetForRoomsStmt, selectJoinedUsersSetForRoomsSQL}, {&s.selectKnownUsersStmt, selectKnownUsersSQL}, {&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom}, + {&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL}, }.Prepare(db) } @@ -324,3 +334,16 @@ func (s *membershipStatements) UpdateForgetMembership( ) return err } + +func (s *membershipStatements) SelectLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) { + var nid types.RoomNID + err := s.selectLocalServerInRoomStmt.QueryRowContext(ctx, tables.MembershipStateJoin, roomNID).Scan(&nid) + if err != nil { + if err == sql.ErrNoRows { + return false, nil + } + return false, err + } + found := nid > 0 + return found, nil +} diff --git a/roomserver/storage/postgres/state_block_table.go b/roomserver/storage/postgres/state_block_table.go index 4523d18bb..7ae987d6d 100644 --- a/roomserver/storage/postgres/state_block_table.go +++ b/roomserver/storage/postgres/state_block_table.go @@ -64,7 +64,7 @@ const insertStateDataSQL = "" + const bulkSelectStateBlockEntriesSQL = "" + "SELECT state_block_nid, event_nids" + - " FROM roomserver_state_block WHERE state_block_nid = ANY($1)" + " FROM roomserver_state_block WHERE state_block_nid = ANY($1) ORDER BY state_block_nid ASC" type stateBlockStatements struct { insertStateDataStmt *sql.Stmt diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index e77d62e06..9d9434cbb 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -1059,6 +1059,11 @@ func (d *Database) JoinedUsersSetInRooms(ctx context.Context, roomIDs []string) return result, nil } +// GetLocalServerInRoom returns true if we think we're in a given room or false otherwise. +func (d *Database) GetLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) { + return d.MembershipTable.SelectLocalServerInRoom(ctx, roomNID) +} + // GetKnownUsers searches all users that userID knows about. func (d *Database) GetKnownUsers(ctx context.Context, userID, searchString string, limit int) ([]string, error) { stateKeyNID, err := d.EventStateKeysTable.SelectEventStateKeyNID(ctx, nil, userID) diff --git a/roomserver/storage/sqlite3/membership_table.go b/roomserver/storage/sqlite3/membership_table.go index d9fe32cf8..82babe0d2 100644 --- a/roomserver/storage/sqlite3/membership_table.go +++ b/roomserver/storage/sqlite3/membership_table.go @@ -100,6 +100,14 @@ var selectKnownUsersSQL = "" + " SELECT DISTINCT room_nid FROM roomserver_membership WHERE target_nid=$1 AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + ") AND membership_nid = " + fmt.Sprintf("%d", tables.MembershipStateJoin) + " AND event_state_key LIKE $2 LIMIT $3" +// selectLocalServerInRoomSQL is an optimised case for checking if we, the local server, +// are in the room by using the target_local column of the membership table. Normally when +// we want to know if a server is in a room, we have to unmarshal the entire room state which +// is expensive. The presence of a single row from this query suggests we're still in the +// room, no rows returned suggests we aren't. +const selectLocalServerInRoomSQL = "" + + "SELECT room_nid FROM roomserver_membership WHERE target_local = 1 AND membership_nid = $1 AND room_nid = $2 LIMIT 1" + type membershipStatements struct { db *sql.DB insertMembershipStmt *sql.Stmt @@ -113,6 +121,7 @@ type membershipStatements struct { updateMembershipStmt *sql.Stmt selectKnownUsersStmt *sql.Stmt updateMembershipForgetRoomStmt *sql.Stmt + selectLocalServerInRoomStmt *sql.Stmt } func createMembershipTable(db *sql.DB) error { @@ -137,6 +146,7 @@ func prepareMembershipTable(db *sql.DB) (tables.Membership, error) { {&s.selectRoomsWithMembershipStmt, selectRoomsWithMembershipSQL}, {&s.selectKnownUsersStmt, selectKnownUsersSQL}, {&s.updateMembershipForgetRoomStmt, updateMembershipForgetRoom}, + {&s.selectLocalServerInRoomStmt, selectLocalServerInRoomSQL}, }.Prepare(db) } @@ -304,3 +314,16 @@ func (s *membershipStatements) UpdateForgetMembership( ) return err } + +func (s *membershipStatements) SelectLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) { + var nid types.RoomNID + err := s.selectLocalServerInRoomStmt.QueryRowContext(ctx, tables.MembershipStateJoin, roomNID).Scan(&nid) + if err != nil { + if err == sql.ErrNoRows { + return false, nil + } + return false, err + } + found := nid > 0 + return found, nil +} diff --git a/roomserver/storage/sqlite3/state_block_table.go b/roomserver/storage/sqlite3/state_block_table.go index cfb2a49e5..5cb21e91c 100644 --- a/roomserver/storage/sqlite3/state_block_table.go +++ b/roomserver/storage/sqlite3/state_block_table.go @@ -57,7 +57,7 @@ const insertStateDataSQL = ` const bulkSelectStateBlockEntriesSQL = "" + "SELECT state_block_nid, event_nids" + - " FROM roomserver_state_block WHERE state_block_nid IN ($1)" + " FROM roomserver_state_block WHERE state_block_nid IN ($1) ORDER BY state_block_nid ASC" type stateBlockStatements struct { db *sql.DB diff --git a/roomserver/storage/tables/interface.go b/roomserver/storage/tables/interface.go index dd486873a..4a893663f 100644 --- a/roomserver/storage/tables/interface.go +++ b/roomserver/storage/tables/interface.go @@ -135,6 +135,7 @@ type Membership interface { SelectJoinedUsersSetForRooms(ctx context.Context, roomNIDs []types.RoomNID) (map[types.EventStateKeyNID]int, error) SelectKnownUsers(ctx context.Context, userID types.EventStateKeyNID, searchString string, limit int) ([]string, error) UpdateForgetMembership(ctx context.Context, txn *sql.Tx, roomNID types.RoomNID, targetUserNID types.EventStateKeyNID, forget bool) error + SelectLocalServerInRoom(ctx context.Context, roomNID types.RoomNID) (bool, error) } type Published interface { diff --git a/setup/base.go b/setup/base.go index 6bdeb80f7..7b691608d 100644 --- a/setup/base.go +++ b/setup/base.go @@ -77,6 +77,7 @@ type BaseDendrite struct { PublicKeyAPIMux *mux.Router PublicMediaAPIMux *mux.Router InternalAPIMux *mux.Router + SynapseAdminMux *mux.Router UseHTTPAPIs bool apiHttpClient *http.Client httpClient *http.Client @@ -199,6 +200,7 @@ func NewBaseDendrite(cfg *config.Dendrite, componentName string, useHTTPAPIs boo PublicKeyAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicKeyPathPrefix).Subrouter().UseEncodedPath(), PublicMediaAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.PublicMediaPathPrefix).Subrouter().UseEncodedPath(), InternalAPIMux: mux.NewRouter().SkipClean(true).PathPrefix(httputil.InternalPathPrefix).Subrouter().UseEncodedPath(), + SynapseAdminMux: mux.NewRouter().SkipClean(true).PathPrefix("/_synapse/").Subrouter().UseEncodedPath(), apiHttpClient: &apiClient, httpClient: &client, } @@ -391,6 +393,7 @@ func (b *BaseDendrite) SetupAndServeHTTP( externalRouter.PathPrefix(httputil.PublicKeyPathPrefix).Handler(b.PublicKeyAPIMux) externalRouter.PathPrefix(httputil.PublicFederationPathPrefix).Handler(federationHandler) } + externalRouter.PathPrefix("/_synapse/").Handler(b.SynapseAdminMux) externalRouter.PathPrefix(httputil.PublicMediaPathPrefix).Handler(b.PublicMediaAPIMux) if internalAddr != NoListener && internalAddr != externalAddr { diff --git a/setup/monolith.go b/setup/monolith.go index 235be4474..5ceb4ed30 100644 --- a/setup/monolith.go +++ b/setup/monolith.go @@ -57,9 +57,9 @@ type Monolith struct { } // AddAllPublicRoutes attaches all public paths to the given router -func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, mediaMux *mux.Router) { +func (m *Monolith) AddAllPublicRoutes(process *process.ProcessContext, csMux, ssMux, keyMux, mediaMux, synapseMux *mux.Router) { clientapi.AddPublicRoutes( - csMux, &m.Config.ClientAPI, m.AccountDB, + csMux, synapseMux, &m.Config.ClientAPI, m.AccountDB, m.FedClient, m.RoomserverAPI, m.EDUInternalAPI, m.AppserviceAPI, transactions.New(), m.FederationSenderAPI, m.UserAPI, m.KeyAPI, m.ExtPublicRoomsProvider, diff --git a/sytest-whitelist b/sytest-whitelist index 8c4585716..55922f943 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -520,3 +520,7 @@ Inviting an AS-hosted user asks the AS server Can generate a openid access_token that can be exchanged for information about a user Invalid openid access tokens are rejected Requests to userinfo without access tokens are rejected +POST /_synapse/admin/v1/register with shared secret +POST /_synapse/admin/v1/register admin with shared secret +POST /_synapse/admin/v1/register with shared secret downcases capitals +POST /_synapse/admin/v1/register with shared secret disallows symbols