From 00b3545b14ca5e6987be93cabafc51e771f8bcc7 Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 2 Mar 2022 10:36:20 +0000 Subject: [PATCH 1/5] Update NATS Server to v2.7.3 --- go.mod | 5 ++--- go.sum | 16 ++++++---------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index ddf3f2175..dbcae5d54 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/matrix-org/dendrite -replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.7.2-0.20220217100407-087330ed46ad +replace github.com/nats-io/nats-server/v2 => github.com/neilalexander/nats-server/v2 v2.7.4-0.20220302103432-6b04b9f12740 replace github.com/nats-io/nats.go => github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c @@ -24,7 +24,6 @@ require ( github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/golang-lru v0.5.4 github.com/juju/testing v0.0.0-20211215003918-77eb13d6cad2 // indirect - github.com/klauspost/compress v1.14.2 // indirect github.com/lib/pq v1.10.4 github.com/libp2p/go-libp2p v0.13.0 github.com/libp2p/go-libp2p-circuit v0.4.0 @@ -45,7 +44,7 @@ require ( github.com/matrix-org/util v0.0.0-20200807132607-55161520e1d4 github.com/mattn/go-sqlite3 v1.14.10 github.com/morikuni/aec v1.0.0 // indirect - github.com/nats-io/nats-server/v2 v2.3.2 + github.com/nats-io/nats-server/v2 v2.7.3 github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d github.com/neilalexander/utp v0.1.1-0.20210727203401-54ae7b1cd5f9 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 diff --git a/go.sum b/go.sum index 74a181528..25697f59c 100644 --- a/go.sum +++ b/go.sum @@ -480,7 +480,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gologme/log v1.2.0/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U= github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo= github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4= @@ -712,9 +711,8 @@ github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw= -github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4= +github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -983,8 +981,6 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20210709140738-b0d1ba599a6d/go.mod h1 github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220301125114-e6012a13a6e6 h1:CqbM5tPbF1mV2h1J6N0NSF8et6HvFlwA98TLCUcjiSI= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220301125114-e6012a13a6e6/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= github.com/matrix-org/gomatrixserverlib v0.0.0-20220301141554-e124bd7d7902 h1:WHlrE8BYh/hzn1RKwq3YMAlhHivX47jQKAjZFtkJyPE= github.com/matrix-org/gomatrixserverlib v0.0.0-20220301141554-e124bd7d7902/go.mod h1:+WF5InseAMgi1fTnU46JH39IDpEvLep0fDzx9LDf2Bo= github.com/matrix-org/pinecone v0.0.0-20220223104432-0f0afd1a46aa h1:rMYFNVto66gp+eWS8XAUzgp4m0qmUBid6l1HX3mHstk= @@ -1031,8 +1027,8 @@ github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= -github.com/minio/highwayhash v1.0.1 h1:dZ6IIu8Z14VlC0VpfKofAhCy74wu/Qb5gcn52yWoz/0= -github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= +github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= @@ -1134,8 +1130,8 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uY github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/neilalexander/nats-server/v2 v2.7.2-0.20220217100407-087330ed46ad h1:Z2nWMQsXWWqzj89nW6OaLJSdkFknqhaR5whEOz4++Y8= -github.com/neilalexander/nats-server/v2 v2.7.2-0.20220217100407-087330ed46ad/go.mod h1:tckmrt0M6bVaDT3kmh9UrIq/CBOBBse+TpXQi5ldaa8= +github.com/neilalexander/nats-server/v2 v2.7.4-0.20220302103432-6b04b9f12740 h1:RJrc+z35RHZlrjR6UBt9UmVRAlFh4SgYyEA0YpQdPHM= +github.com/neilalexander/nats-server/v2 v2.7.4-0.20220302103432-6b04b9f12740/go.mod h1:eJUrA5gm0ch6sJTEv85xmXIgQWsB0OyjkTsKXvlHbYc= github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c h1:G2qsv7D0rY94HAu8pXmElMluuMHQ85waxIDQBhIzV2Q= github.com/neilalexander/nats.go v1.11.1-0.20220104162523-f4ddebe1061c/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/neilalexander/utp v0.1.1-0.20210622132614-ee9a34a30488/go.mod h1:NPHGhPc0/wudcaCqL/H5AOddkRf8GPRhzOujuUKGQu8= From a4c918ee17d342e704aee738e4f0d5712b98eaa6 Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 2 Mar 2022 10:49:29 +0000 Subject: [PATCH 2/5] Fix data race in unit tests --- keyserver/internal/device_list_update_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go index e8d1bfe8b..59a66ec8e 100644 --- a/keyserver/internal/device_list_update_test.go +++ b/keyserver/internal/device_list_update_test.go @@ -87,6 +87,12 @@ func (d *mockDeviceListUpdaterDatabase) MarkDeviceListStale(ctx context.Context, return nil } +func (d *mockDeviceListUpdaterDatabase) isStale(userID string) bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.staleUsers[userID] +} + // StoreRemoteDeviceKeys persists the given keys. Keys with the same user ID and device ID will be replaced. An empty KeyJSON removes the key // for this (user, device). Does not modify the stream ID for keys. func (d *mockDeviceListUpdaterDatabase) StoreRemoteDeviceKeys(ctx context.Context, keys []api.DeviceMessage, clear []string) error { @@ -169,7 +175,7 @@ func TestUpdateHavePrevID(t *testing.T) { if !reflect.DeepEqual(db.storedKeys, []api.DeviceMessage{want}) { t.Errorf("DB didn't store correct event, got %v want %v", db.storedKeys, want) } - if db.staleUsers[event.UserID] { + if db.isStale(event.UserID) { t.Errorf("%s incorrectly marked as stale", event.UserID) } } @@ -243,7 +249,7 @@ func TestUpdateNoPrevID(t *testing.T) { }, } // Now we should have a fresh list and the keys and emitted something - if db.staleUsers[event.UserID] { + if db.isStale(event.UserID) { t.Errorf("%s still marked as stale", event.UserID) } if !reflect.DeepEqual(producer.events, []api.DeviceMessage{want}) { @@ -304,7 +310,7 @@ func TestDebounce(t *testing.T) { } // user should be marked as stale - if !db.staleUsers[userID] { + if !db.isStale(userID) { t.Errorf("user %s not marked as stale", userID) } // now send the response over federation @@ -330,7 +336,7 @@ func TestDebounce(t *testing.T) { wg.Wait() // user is no longer stale now - if db.staleUsers[userID] { + if db.isStale(userID) { t.Errorf("user %s is marked as stale", userID) } } From 8996cc805913e9927f57cd3c4b8247a09a47472b Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Wed, 2 Mar 2022 11:35:35 +0000 Subject: [PATCH 3/5] Media endpoints on `/v3` (#2242) * Media endpoints on `/v3` * Keep `/v1` too? --- mediaapi/routing/routing.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/mediaapi/routing/routing.go b/mediaapi/routing/routing.go index 44f9a9d69..fc2136bb6 100644 --- a/mediaapi/routing/routing.go +++ b/mediaapi/routing/routing.go @@ -53,8 +53,7 @@ func Setup( ) { rateLimits := httputil.NewRateLimits(rateLimit) - r0mux := publicAPIMux.PathPrefix("/r0").Subrouter() - v1mux := publicAPIMux.PathPrefix("/v1").Subrouter() + v3mux := publicAPIMux.PathPrefix("/{apiversion:(?:r0|v1|v3)}/").Subrouter() activeThumbnailGeneration := &types.ActiveThumbnailGeneration{ PathToResult: map[string]*types.ThumbnailGenerationResult{}, @@ -80,21 +79,18 @@ func Setup( } }) - r0mux.Handle("/upload", uploadHandler).Methods(http.MethodPost, http.MethodOptions) - r0mux.Handle("/config", configHandler).Methods(http.MethodGet, http.MethodOptions) - v1mux.Handle("/upload", uploadHandler).Methods(http.MethodPost, http.MethodOptions) + v3mux.Handle("/upload", uploadHandler).Methods(http.MethodPost, http.MethodOptions) + v3mux.Handle("/config", configHandler).Methods(http.MethodGet, http.MethodOptions) activeRemoteRequests := &types.ActiveRemoteRequests{ MXCToResult: map[string]*types.RemoteRequestResult{}, } downloadHandler := makeDownloadAPI("download", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration) - r0mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) - v1mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed - v1mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) // TODO: remove when synapse is fixed + v3mux.Handle("/download/{serverName}/{mediaId}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) + v3mux.Handle("/download/{serverName}/{mediaId}/{downloadName}", downloadHandler).Methods(http.MethodGet, http.MethodOptions) - r0mux.Handle("/thumbnail/{serverName}/{mediaId}", + v3mux.Handle("/thumbnail/{serverName}/{mediaId}", makeDownloadAPI("thumbnail", cfg, rateLimits, db, client, activeRemoteRequests, activeThumbnailGeneration), ).Methods(http.MethodGet, http.MethodOptions) } From e46a61c49e5a0116b5df3df34a95cf5b8c811a8d Mon Sep 17 00:00:00 2001 From: Kegan Dougal Date: Wed, 2 Mar 2022 11:38:13 +0000 Subject: [PATCH 4/5] Skip flakey test for now --- keyserver/internal/device_list_update_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/keyserver/internal/device_list_update_test.go b/keyserver/internal/device_list_update_test.go index 59a66ec8e..ff939355f 100644 --- a/keyserver/internal/device_list_update_test.go +++ b/keyserver/internal/device_list_update_test.go @@ -265,6 +265,7 @@ func TestUpdateNoPrevID(t *testing.T) { // Test that if we make N calls to ManualUpdate for the same user, we only do it once, assuming the // update is still ongoing. func TestDebounce(t *testing.T) { + t.Skipf("panic on closed channel on GHA") db := &mockDeviceListUpdaterDatabase{ staleUsers: make(map[string]bool), prevIDsExist: func(string, []int) bool { From 111f01ddc81d775dfdaab6e6a3a6afa6fa5608ea Mon Sep 17 00:00:00 2001 From: Neil Alexander Date: Thu, 3 Mar 2022 09:18:40 +0000 Subject: [PATCH 5/5] Update `sytest-whitelist` for changes in matrix-org/sytest#1200 --- sytest-whitelist | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sytest-whitelist b/sytest-whitelist index 0cb80b7b6..3e38176f4 100644 --- a/sytest-whitelist +++ b/sytest-whitelist @@ -78,7 +78,7 @@ Room creation reports m.room.member to myself Outbound federation rejects send_join responses with no m.room.create event Outbound federation rejects m.room.create events with an unknown room version Invited user can see room metadata -# Blacklisted because these tests call /r0/events which we don't implement +# Blacklisted because these tests call /v3/events which we don't implement # New room members see their own join event # Existing members see new members' join events setting 'm.room.power_levels' respects room powerlevel @@ -257,7 +257,7 @@ Guest non-joined users cannot send messages to guest_access rooms if not joined Real non-joined users cannot room initalSync for non-world_readable rooms Push rules come down in an initial /sync Regular users can add and delete aliases in the default room configuration -GET /r0/capabilities is not public +GET /v3/capabilities is not public GET /joined_rooms lists newly-created room /joined_rooms returns only joined rooms Message history can be paginated over federation @@ -366,8 +366,8 @@ Outbound federation will ignore a missing event with bad JSON for room version 6 Server correctly handles transactions that break edu limits Server rejects invalid JSON in a version 6 room Can download without a file name over federation -POST /media/r0/upload can create an upload -GET /media/r0/download can fetch the value again +POST /media/v3/upload can create an upload +GET /media/v3/download can fetch the value again Remote users can join room by alias Alias creators can delete alias with no ops Alias creators can delete canonical alias with no ops