diff --git a/appservice/storage/storage.go b/appservice/storage/storage.go index b0df2b7dc..97b8501e2 100644 --- a/appservice/storage/storage.go +++ b/appservice/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/build/gobind-pinecone/platform_ios.go b/build/gobind-pinecone/platform_ios.go index 01f8a6a04..802d7faca 100644 --- a/build/gobind-pinecone/platform_ios.go +++ b/build/gobind-pinecone/platform_ios.go @@ -1,3 +1,4 @@ +//go:build ios // +build ios package gobind diff --git a/build/gobind-pinecone/platform_other.go b/build/gobind-pinecone/platform_other.go index fdfb13bc0..2e81e2f43 100644 --- a/build/gobind-pinecone/platform_other.go +++ b/build/gobind-pinecone/platform_other.go @@ -1,3 +1,4 @@ +//go:build !ios // +build !ios package gobind diff --git a/build/gobind-yggdrasil/platform_ios.go b/build/gobind-yggdrasil/platform_ios.go index 01f8a6a04..802d7faca 100644 --- a/build/gobind-yggdrasil/platform_ios.go +++ b/build/gobind-yggdrasil/platform_ios.go @@ -1,3 +1,4 @@ +//go:build ios // +build ios package gobind diff --git a/build/gobind-yggdrasil/platform_other.go b/build/gobind-yggdrasil/platform_other.go index fdfb13bc0..2e81e2f43 100644 --- a/build/gobind-yggdrasil/platform_other.go +++ b/build/gobind-yggdrasil/platform_other.go @@ -1,3 +1,4 @@ +//go:build !ios // +build !ios package gobind diff --git a/cmd/create-account/main.go b/cmd/create-account/main.go index a1e254f8d..3ac077705 100644 --- a/cmd/create-account/main.go +++ b/cmd/create-account/main.go @@ -22,7 +22,6 @@ import ( "io/ioutil" "os" "strings" - "syscall" "github.com/matrix-org/dendrite/setup" "github.com/matrix-org/dendrite/setup/config" @@ -121,13 +120,13 @@ func getPassword(password, pwdFile *string, pwdStdin, askPass *bool, r io.Reader // ask the user to provide the password if *askPass { fmt.Print("Enter Password: ") - bytePassword, err := term.ReadPassword(syscall.Stdin) + bytePassword, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { logrus.Fatalln("Unable to read password:", err) } fmt.Println() fmt.Print("Confirm Password: ") - bytePassword2, err := term.ReadPassword(syscall.Stdin) + bytePassword2, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { logrus.Fatalln("Unable to read password:", err) } diff --git a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go index 4d2da55cb..8b3be72c1 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_elementweb.go @@ -1,3 +1,4 @@ +//go:build elementweb // +build elementweb package embed diff --git a/cmd/dendrite-demo-pinecone/embed/embed_other.go b/cmd/dendrite-demo-pinecone/embed/embed_other.go index 04c2188c3..a4b223452 100644 --- a/cmd/dendrite-demo-pinecone/embed/embed_other.go +++ b/cmd/dendrite-demo-pinecone/embed/embed_other.go @@ -1,3 +1,4 @@ +//go:build !elementweb // +build !elementweb package embed diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go b/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go index 8d49c553a..e7725ec83 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_elementweb.go @@ -1,3 +1,4 @@ +//go:build elementweb // +build elementweb package embed diff --git a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go index 04c2188c3..a4b223452 100644 --- a/cmd/dendrite-demo-yggdrasil/embed/embed_other.go +++ b/cmd/dendrite-demo-yggdrasil/embed/embed_other.go @@ -1,3 +1,4 @@ +//go:build !elementweb // +build !elementweb package embed diff --git a/cmd/dendritejs-pinecone/jsServer.go b/cmd/dendritejs-pinecone/jsServer.go index 074d20cba..4298c2ae9 100644 --- a/cmd/dendritejs-pinecone/jsServer.go +++ b/cmd/dendritejs-pinecone/jsServer.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs-pinecone/main_noop.go b/cmd/dendritejs-pinecone/main_noop.go index dcea032f2..0cc7e47e5 100644 --- a/cmd/dendritejs-pinecone/main_noop.go +++ b/cmd/dendritejs-pinecone/main_noop.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package main diff --git a/cmd/dendritejs-pinecone/main_test.go b/cmd/dendritejs-pinecone/main_test.go index 751700cb2..17fea6cce 100644 --- a/cmd/dendritejs-pinecone/main_test.go +++ b/cmd/dendritejs-pinecone/main_test.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/jsServer.go b/cmd/dendritejs/jsServer.go index 074d20cba..4298c2ae9 100644 --- a/cmd/dendritejs/jsServer.go +++ b/cmd/dendritejs/jsServer.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/keyfetcher.go b/cmd/dendritejs/keyfetcher.go index cef045372..cdf937649 100644 --- a/cmd/dendritejs/keyfetcher.go +++ b/cmd/dendritejs/keyfetcher.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/main.go b/cmd/dendritejs/main.go index d8fc8b837..10aadb6e5 100644 --- a/cmd/dendritejs/main.go +++ b/cmd/dendritejs/main.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/cmd/dendritejs/main_noop.go b/cmd/dendritejs/main_noop.go index dcea032f2..0cc7e47e5 100644 --- a/cmd/dendritejs/main_noop.go +++ b/cmd/dendritejs/main_noop.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package main diff --git a/cmd/dendritejs/publicrooms.go b/cmd/dendritejs/publicrooms.go index a4623ba32..19afc5bcf 100644 --- a/cmd/dendritejs/publicrooms.go +++ b/cmd/dendritejs/publicrooms.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package main diff --git a/federationapi/routing/send.go b/federationapi/routing/send.go index 3cae837c9..4b5f0d660 100644 --- a/federationapi/routing/send.go +++ b/federationapi/routing/send.go @@ -148,7 +148,8 @@ type inputWorker struct { input *sendFIFOQueue } -var inputWorkers sync.Map // room ID -> *inputWorker +var inFlightTxnsPerOrigin sync.Map // transaction ID -> chan util.JSONResponse +var inputWorkers sync.Map // room ID -> *inputWorker // Send implements /_matrix/federation/v1/send/{txnID} func Send( @@ -164,6 +165,37 @@ func Send( mu *internal.MutexByRoom, servers federationAPI.ServersInRoomProvider, ) util.JSONResponse { + // First we should check if this origin has already submitted this + // txn ID to us. If they have and the txnIDs map contains an entry, + // the transaction is still being worked on. The new client can wait + // for it to complete rather than creating more work. + index := string(request.Origin()) + "\000" + string(txnID) + v, ok := inFlightTxnsPerOrigin.LoadOrStore(index, make(chan util.JSONResponse, 1)) + ch := v.(chan util.JSONResponse) + if ok { + // This origin already submitted this txn ID to us, and the work + // is still taking place, so we'll just wait for it to finish. + ctx, cancel := context.WithTimeout(httpReq.Context(), time.Minute*5) + defer cancel() + select { + case <-ctx.Done(): + // If the caller gives up then return straight away. We don't + // want to attempt to process what they sent us any further. + return util.JSONResponse{Code: http.StatusRequestTimeout} + case res := <-ch: + // The original task just finished processing so let's return + // the result of it. + if res.Code == 0 { + return util.JSONResponse{Code: http.StatusAccepted} + } + return res + } + } + // Otherwise, store that we're currently working on this txn from + // this origin. When we're done processing, close the channel. + defer close(ch) + defer inFlightTxnsPerOrigin.Delete(index) + t := txnReq{ rsAPI: rsAPI, eduAPI: eduAPI, @@ -205,7 +237,7 @@ func Send( util.GetLogger(httpReq.Context()).Infof("Received transaction %q from %q containing %d PDUs, %d EDUs", txnID, request.Origin(), len(t.PDUs), len(t.EDUs)) - resp, jsonErr := t.processTransaction(httpReq.Context()) + resp, jsonErr := t.processTransaction(context.Background()) if jsonErr != nil { util.GetLogger(httpReq.Context()).WithField("jsonErr", jsonErr).Error("t.processTransaction failed") return *jsonErr @@ -215,10 +247,12 @@ func Send( // Status code 200: // The result of processing the transaction. The server is to use this response // even in the event of one or more PDUs failing to be processed. - return util.JSONResponse{ + res := util.JSONResponse{ Code: http.StatusOK, JSON: resp, } + ch <- res + return res } type txnReq struct { diff --git a/federationsender/storage/storage.go b/federationsender/storage/storage.go index 5462c3523..46e01f256 100644 --- a/federationsender/storage/storage.go +++ b/federationsender/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/go.mod b/go.mod index 0a223f5af..662114d08 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/matrix-org/go-http-js-libp2p v0.0.0-20200518170932-783164aeeda4 github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 - github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1 + github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82 github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 diff --git a/go.sum b/go.sum index 21f4807b4..998f8429b 100644 --- a/go.sum +++ b/go.sum @@ -993,8 +993,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1 h1:Pv7+98sreiHltpamJ4em6RCX/WPVN1wl53Gli9Cz744= -github.com/matrix-org/gomatrixserverlib v0.0.0-20211102131912-13366e7985e1/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82 h1:hULlw0rjPgoH53BGSNQZNASJIzYgoW3LH4yJ4Cx8Bak= +github.com/matrix-org/gomatrixserverlib v0.0.0-20211108101507-7a3ac5aa8c82/go.mod h1:rB8tBUUUo1rzUqpzklRDSooxZ6YMhoaEPx4SO5fGeUc= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0 h1:HZCzy4oVzz55e+cOMiX/JtSF2UOY1evBl2raaE7ACcU= github.com/matrix-org/naffka v0.0.0-20210623111924-14ff508b58e0/go.mod h1:sjyPyRxKM5uw1nD2cJ6O2OxI6GOqyVBfNXqKjBZTBZE= github.com/matrix-org/pinecone v0.0.0-20211022090602-08a50945ac89 h1:6JkIymZ1vxfI0shSpg6gNPTJaF4/95Evy34slPVZGKM= @@ -1884,7 +1884,6 @@ gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20160105164936-4f90aeace3a2/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/log.go b/internal/log.go index f0656d7d0..bba0ac6e6 100644 --- a/internal/log.go +++ b/internal/log.go @@ -18,7 +18,6 @@ import ( "context" "fmt" "io" - "log/syslog" "net/http" "os" "path" @@ -31,7 +30,6 @@ import ( "github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dugong" "github.com/sirupsen/logrus" - lSyslog "github.com/sirupsen/logrus/hooks/syslog" ) type utcFormatter struct { @@ -108,37 +106,6 @@ func SetupStdLogging() { }) } -// SetupHookLogging configures the logging hooks defined in the configuration. -// If something fails here it means that the logging was improperly configured, -// so we just exit with the error -func SetupHookLogging(hooks []config.LogrusHook, componentName string) { - logrus.SetReportCaller(true) - for _, hook := range hooks { - // Check we received a proper logging level - level, err := logrus.ParseLevel(hook.Level) - if err != nil { - logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) - } - - // Perform a first filter on the logs according to the lowest level of all - // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) - if logrus.GetLevel() < level { - logrus.SetLevel(level) - } - - switch hook.Type { - case "file": - checkFileHookParams(hook.Params) - setupFileHook(hook, level, componentName) - case "syslog": - checkSyslogHookParams(hook.Params) - setupSyslogHook(hook, level, componentName) - default: - logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) - } - } -} - // File type hooks should be provided a path to a directory to store log files func checkFileHookParams(params map[string]interface{}) { path, ok := params["path"] @@ -178,34 +145,6 @@ func setupFileHook(hook config.LogrusHook, level logrus.Level, componentName str }) } -func checkSyslogHookParams(params map[string]interface{}) { - addr, ok := params["address"] - if !ok { - logrus.Fatalf("Expecting a parameter \"address\" for logging hook of type \"syslog\"") - } - - if _, ok := addr.(string); !ok { - logrus.Fatalf("Parameter \"address\" for logging hook of type \"syslog\" should be a string") - } - - proto, ok2 := params["protocol"] - if !ok2 { - logrus.Fatalf("Expecting a parameter \"protocol\" for logging hook of type \"syslog\"") - } - - if _, ok2 := proto.(string); !ok2 { - logrus.Fatalf("Parameter \"protocol\" for logging hook of type \"syslog\" should be a string") - } - -} - -func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { - syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) - if err == nil { - logrus.AddHook(&logLevelHook{level, syslogHook}) - } -} - //CloseAndLogIfError Closes io.Closer and logs the error if any func CloseAndLogIfError(ctx context.Context, closer io.Closer, message string) { if closer == nil { diff --git a/internal/log_unix.go b/internal/log_unix.go new file mode 100644 index 000000000..25ad04205 --- /dev/null +++ b/internal/log_unix.go @@ -0,0 +1,84 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package internal + +import ( + "log/syslog" + + "github.com/matrix-org/dendrite/setup/config" + "github.com/sirupsen/logrus" + lSyslog "github.com/sirupsen/logrus/hooks/syslog" +) + +// SetupHookLogging configures the logging hooks defined in the configuration. +// If something fails here it means that the logging was improperly configured, +// so we just exit with the error +func SetupHookLogging(hooks []config.LogrusHook, componentName string) { + logrus.SetReportCaller(true) + for _, hook := range hooks { + // Check we received a proper logging level + level, err := logrus.ParseLevel(hook.Level) + if err != nil { + logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) + } + + // Perform a first filter on the logs according to the lowest level of all + // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) + if logrus.GetLevel() < level { + logrus.SetLevel(level) + } + + switch hook.Type { + case "file": + checkFileHookParams(hook.Params) + setupFileHook(hook, level, componentName) + case "syslog": + checkSyslogHookParams(hook.Params) + setupSyslogHook(hook, level, componentName) + default: + logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) + } + } +} + +func checkSyslogHookParams(params map[string]interface{}) { + addr, ok := params["address"] + if !ok { + logrus.Fatalf("Expecting a parameter \"address\" for logging hook of type \"syslog\"") + } + + if _, ok := addr.(string); !ok { + logrus.Fatalf("Parameter \"address\" for logging hook of type \"syslog\" should be a string") + } + + proto, ok2 := params["protocol"] + if !ok2 { + logrus.Fatalf("Expecting a parameter \"protocol\" for logging hook of type \"syslog\"") + } + + if _, ok2 := proto.(string); !ok2 { + logrus.Fatalf("Parameter \"protocol\" for logging hook of type \"syslog\" should be a string") + } + +} + +func setupSyslogHook(hook config.LogrusHook, level logrus.Level, componentName string) { + syslogHook, err := lSyslog.NewSyslogHook(hook.Params["protocol"].(string), hook.Params["address"].(string), syslog.LOG_INFO, componentName) + if err == nil { + logrus.AddHook(&logLevelHook{level, syslogHook}) + } +} diff --git a/internal/log_windows.go b/internal/log_windows.go new file mode 100644 index 000000000..39562328c --- /dev/null +++ b/internal/log_windows.go @@ -0,0 +1,48 @@ +// Copyright 2021 The Matrix.org Foundation C.I.C. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "github.com/matrix-org/dendrite/setup/config" + "github.com/sirupsen/logrus" +) + +// SetupHookLogging configures the logging hooks defined in the configuration. +// If something fails here it means that the logging was improperly configured, +// so we just exit with the error +func SetupHookLogging(hooks []config.LogrusHook, componentName string) { + logrus.SetReportCaller(true) + for _, hook := range hooks { + // Check we received a proper logging level + level, err := logrus.ParseLevel(hook.Level) + if err != nil { + logrus.Fatalf("Unrecognised logging level %s: %q", hook.Level, err) + } + + // Perform a first filter on the logs according to the lowest level of all + // (Eg: If we have hook for info and above, prevent logrus from processing debug logs) + if logrus.GetLevel() < level { + logrus.SetLevel(level) + } + + switch hook.Type { + case "file": + checkFileHookParams(hook.Params) + setupFileHook(hook, level, componentName) + default: + logrus.Fatalf("Unrecognised logging hook type: %s", hook.Type) + } + } +} diff --git a/internal/sqlutil/postgres.go b/internal/sqlutil/postgres.go index 41a5508a1..5e656b1da 100644 --- a/internal/sqlutil/postgres.go +++ b/internal/sqlutil/postgres.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package sqlutil diff --git a/internal/sqlutil/postgres_wasm.go b/internal/sqlutil/postgres_wasm.go index c45842f0c..34086f450 100644 --- a/internal/sqlutil/postgres_wasm.go +++ b/internal/sqlutil/postgres_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package sqlutil diff --git a/internal/sqlutil/trace_driver.go b/internal/sqlutil/trace_driver.go index f123b1e4d..b7bb36764 100644 --- a/internal/sqlutil/trace_driver.go +++ b/internal/sqlutil/trace_driver.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package sqlutil diff --git a/internal/sqlutil/trace_driver_wasm.go b/internal/sqlutil/trace_driver_wasm.go index a3c163f50..51b60c3c8 100644 --- a/internal/sqlutil/trace_driver_wasm.go +++ b/internal/sqlutil/trace_driver_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package sqlutil diff --git a/keyserver/storage/storage.go b/keyserver/storage/storage.go index 8f05d0030..742e8463a 100644 --- a/keyserver/storage/storage.go +++ b/keyserver/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/mediaapi/routing/download.go b/mediaapi/routing/download.go index 39b86b443..2358915ee 100644 --- a/mediaapi/routing/download.go +++ b/mediaapi/routing/download.go @@ -737,7 +737,7 @@ func (r *downloadRequest) fetchRemoteFile( return "", false, parseErr } - if contentLength > int64(maxFileSizeBytes) { + if maxFileSizeBytes > 0 && contentLength > int64(maxFileSizeBytes) { // TODO: Bubble up this as a 413 return "", false, fmt.Errorf("remote file is too large (%v > %v bytes)", contentLength, maxFileSizeBytes) } diff --git a/mediaapi/storage/storage.go b/mediaapi/storage/storage.go index a976f795b..56059f1c8 100644 --- a/mediaapi/storage/storage.go +++ b/mediaapi/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/mediaapi/thumbnailer/thumbnailer_bimg.go b/mediaapi/thumbnailer/thumbnailer_bimg.go index 087385a76..6ca533176 100644 --- a/mediaapi/thumbnailer/thumbnailer_bimg.go +++ b/mediaapi/thumbnailer/thumbnailer_bimg.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build bimg // +build bimg package thumbnailer diff --git a/mediaapi/thumbnailer/thumbnailer_nfnt.go b/mediaapi/thumbnailer/thumbnailer_nfnt.go index d7aa294ac..beae88c5c 100644 --- a/mediaapi/thumbnailer/thumbnailer_nfnt.go +++ b/mediaapi/thumbnailer/thumbnailer_nfnt.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !bimg // +build !bimg package thumbnailer diff --git a/roomserver/state/state.go b/roomserver/state/state.go index bae8b24c5..78398fc7c 100644 --- a/roomserver/state/state.go +++ b/roomserver/state/state.go @@ -778,7 +778,8 @@ func (v *StateResolution) resolveConflictsV2( ctx context.Context, notConflicted, conflicted []types.StateEntry, ) ([]types.StateEntry, error) { - eventIDMap := make(map[string]types.StateEntry) + estimate := len(conflicted) + len(notConflicted) + eventIDMap := make(map[string]types.StateEntry, estimate) // Load the conflicted events conflictedEvents, conflictedEventMap, err := v.loadStateEvents(ctx, conflicted) @@ -800,18 +801,20 @@ func (v *StateResolution) resolveConflictsV2( // For each conflicted event, we will add a new set of auth events. Auth // events may be duplicated across these sets but that's OK. - authSets := make(map[string][]*gomatrixserverlib.Event) - var authEvents []*gomatrixserverlib.Event - var authDifference []*gomatrixserverlib.Event + authSets := make(map[string][]*gomatrixserverlib.Event, len(conflicted)) + authEvents := make([]*gomatrixserverlib.Event, 0, estimate*3) + authDifference := make([]*gomatrixserverlib.Event, 0, estimate) // For each conflicted event, let's try and get the needed auth events. + neededStateKeys := make([]string, 16) + authEntries := make([]types.StateEntry, 16) for _, conflictedEvent := range conflictedEvents { // Work out which auth events we need to load. key := conflictedEvent.EventID() needed := gomatrixserverlib.StateNeededForAuth([]*gomatrixserverlib.Event{conflictedEvent}) // Find the numeric IDs for the necessary state keys. - var neededStateKeys []string + neededStateKeys = neededStateKeys[:0] neededStateKeys = append(neededStateKeys, needed.Member...) neededStateKeys = append(neededStateKeys, needed.ThirdPartyInvite...) stateKeyNIDMap, err := v.db.EventStateKeyNIDs(ctx, neededStateKeys) @@ -821,7 +824,7 @@ func (v *StateResolution) resolveConflictsV2( // Load the necessary auth events. tuplesNeeded := v.stateKeyTuplesNeeded(stateKeyNIDMap, needed) - var authEntries []types.StateEntry + authEntries = authEntries[:0] for _, tuple := range tuplesNeeded { if eventNID, ok := stateEntryMap(notConflicted).lookup(tuple); ok { authEntries = append(authEntries, types.StateEntry{ diff --git a/roomserver/storage/shared/storage.go b/roomserver/storage/shared/storage.go index b4fc15f1d..3685332fd 100644 --- a/roomserver/storage/shared/storage.go +++ b/roomserver/storage/shared/storage.go @@ -165,14 +165,21 @@ func (d *Database) AddState( if berr != nil { return 0, fmt.Errorf("d.StateBlockTable.BulkSelectStateBlockEntries: %w", berr) } + var found bool for i := len(state) - 1; i >= 0; i-- { + found = false for _, events := range blocks { for _, event := range events { if state[i].EventNID == event { - state = append(state[:i], state[i+1:]...) + found = true + break } } } + if found { + state = append(state[:i], state[i+1:]...) + i-- + } } } err = d.Writer.Do(d.DB, nil, func(txn *sql.Tx) error { diff --git a/roomserver/storage/storage.go b/roomserver/storage/storage.go index 9359312db..9f98ea3ed 100644 --- a/roomserver/storage/storage.go +++ b/roomserver/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/signingkeyserver/storage/keydb.go b/signingkeyserver/storage/keydb.go index aa247f1d8..82b6d0ad5 100644 --- a/signingkeyserver/storage/keydb.go +++ b/signingkeyserver/storage/keydb.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/signingkeyserver/storage/keydb_wasm.go b/signingkeyserver/storage/keydb_wasm.go index 187d9669f..b112993a1 100644 --- a/signingkeyserver/storage/keydb_wasm.go +++ b/signingkeyserver/storage/keydb_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package storage diff --git a/syncapi/storage/postgres/output_room_events_table.go b/syncapi/storage/postgres/output_room_events_table.go index bd7aa018d..44de02c92 100644 --- a/syncapi/storage/postgres/output_room_events_table.go +++ b/syncapi/storage/postgres/output_room_events_table.go @@ -116,7 +116,7 @@ const updateEventJSONSQL = "" + // In order for us to apply the state updates correctly, rows need to be ordered in the order they were received (id). const selectStateInRangeSQL = "" + - "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2) AND (add_state_ids IS NOT NULL OR remove_state_ids IS NOT NULL)" + " AND ( $3::text[] IS NULL OR sender = ANY($3) )" + @@ -221,13 +221,14 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for rows.Next() { var ( + eventID string streamPos types.StreamPosition eventBytes []byte excludeFromSync bool addIDs pq.StringArray delIDs pq.StringArray ) - if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDs, &delIDs); err != nil { return nil, nil, err } // Sanity check for deleted state and whine if we see it. We don't need to do anything @@ -243,7 +244,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( // TODO: Handle redacted events var ev gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal(eventBytes, &ev); err != nil { + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -258,7 +259,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( } stateNeeded[ev.RoomID()] = needSet - eventIDToEvent[ev.EventID()] = types.StreamEvent{ + eventIDToEvent[eventID] = types.StreamEvent{ HeaderedEvent: &ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, diff --git a/syncapi/storage/sqlite3/output_room_events_table.go b/syncapi/storage/sqlite3/output_room_events_table.go index 37f7ea003..afdbe55ce 100644 --- a/syncapi/storage/sqlite3/output_room_events_table.go +++ b/syncapi/storage/sqlite3/output_room_events_table.go @@ -81,7 +81,7 @@ const updateEventJSONSQL = "" + "UPDATE syncapi_output_room_events SET headered_event_json=$1 WHERE event_id=$2" const selectStateInRangeSQL = "" + - "SELECT id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + + "SELECT event_id, id, headered_event_json, exclude_from_sync, add_state_ids, remove_state_ids" + " FROM syncapi_output_room_events" + " WHERE (id > $1 AND id <= $2)" + " AND ((add_state_ids IS NOT NULL AND add_state_ids != '') OR (remove_state_ids IS NOT NULL AND remove_state_ids != ''))" @@ -173,13 +173,14 @@ func (s *outputRoomEventsStatements) SelectStateInRange( for rows.Next() { var ( + eventID string streamPos types.StreamPosition eventBytes []byte excludeFromSync bool addIDsJSON string delIDsJSON string ) - if err := rows.Scan(&streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil { + if err := rows.Scan(&eventID, &streamPos, &eventBytes, &excludeFromSync, &addIDsJSON, &delIDsJSON); err != nil { return nil, nil, err } @@ -201,7 +202,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( // TODO: Handle redacted events var ev gomatrixserverlib.HeaderedEvent - if err := json.Unmarshal(eventBytes, &ev); err != nil { + if err := ev.UnmarshalJSONWithEventID(eventBytes, eventID); err != nil { return nil, nil, err } needSet := stateNeeded[ev.RoomID()] @@ -216,7 +217,7 @@ func (s *outputRoomEventsStatements) SelectStateInRange( } stateNeeded[ev.RoomID()] = needSet - eventIDToEvent[ev.EventID()] = types.StreamEvent{ + eventIDToEvent[eventID] = types.StreamEvent{ HeaderedEvent: &ev, StreamPosition: streamPos, ExcludeFromSync: excludeFromSync, diff --git a/syncapi/storage/storage.go b/syncapi/storage/storage.go index 15386c338..7f9c28e9d 100644 --- a/syncapi/storage/storage.go +++ b/syncapi/storage/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package storage diff --git a/userapi/storage/accounts/sqlite3/constraint_wasm.go b/userapi/storage/accounts/sqlite3/constraint_wasm.go index 0dd5b1fea..6c4ee762f 100644 --- a/userapi/storage/accounts/sqlite3/constraint_wasm.go +++ b/userapi/storage/accounts/sqlite3/constraint_wasm.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build wasm // +build wasm package sqlite3 diff --git a/userapi/storage/accounts/storage.go b/userapi/storage/accounts/storage.go index 3489c9d07..a21f7d94e 100644 --- a/userapi/storage/accounts/storage.go +++ b/userapi/storage/accounts/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package accounts diff --git a/userapi/storage/devices/storage.go b/userapi/storage/devices/storage.go index bfce924d9..3c2034300 100644 --- a/userapi/storage/devices/storage.go +++ b/userapi/storage/devices/storage.go @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build !wasm // +build !wasm package devices