mirrors/dendrite
1
0
Fork 0
mirror of https://github.com/matrix-org/dendrite.git synced 2026-01-16 10:33:11 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into patch-1

Browse source
This commit is contained in:
Till 2023-04-03 09:01:27 +02:00 committed by GitHub
parent b9bb0f795e 01dd02dad2
commit 81f35687a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 1224 additions and 193 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
.github/workflows/schedules.yaml vendored
View file

@ -219,7 +219,7 @@ jobs:
flags: complement
fail_ci_if_error: true
element_web:
element-web:
timeout-minutes: 120
runs-on: ubuntu-latest
steps:
@ -257,3 +257,42 @@ jobs:
env:
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
TMPDIR: ${{ runner.temp }}
element-web-pinecone:
timeout-minutes: 120
runs-on: ubuntu-latest
steps:
- uses: tecolicom/actions-use-apt-tools@v1
with:
# Our test suite includes some screenshot tests with unusual diacritics, which are
# supposed to be covered by STIXGeneral.
tools: fonts-stix
- uses: actions/checkout@v2
with:
repository: matrix-org/matrix-react-sdk
- uses: actions/setup-node@v3
with:
cache: 'yarn'
- name: Fetch layered build
run: scripts/ci/layered.sh
- name: Copy config
run: cp element.io/develop/config.json config.json
working-directory: ./element-web
- name: Build
env:
CI_PACKAGE: true
NODE_OPTIONS: "--openssl-legacy-provider"
run: yarn build
working-directory: ./element-web
- name: Edit Test Config
run: |
sed -i '/HOMESERVER/c\ HOMESERVER: "dendritePinecone",' cypress.config.ts
- name: "Run cypress tests"
uses: cypress-io/github-action@v4.1.1
with:
browser: chrome
start: npx serve -p 8080 ./element-web/webapp
wait-on: 'http://localhost:8080'
env:
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
TMPDIR: ${{ runner.temp }}

85
appservice/appservice_test.go
View file

@ -16,10 +16,12 @@ import (
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver"
rsapi "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/userapi"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/dendrite/test/testrig"
)
@ -212,3 +214,86 @@ func testProtocol(t *testing.T, asAPI api.AppServiceInternalAPI, proto string, w
t.Errorf("unexpected result for Protocols(%s): %+v, expected %+v", proto, protoResp.Protocols[proto], wantResult)
}
}
// Tests that the roomserver consumer only receives one invite
func TestRoomserverConsumerOneInvite(t *testing.T) {
alice := test.NewUser(t)
bob := test.NewUser(t)
room := test.NewRoom(t, alice)
// Invite Bob
room.CreateAndInsert(t, alice, gomatrixserverlib.MRoomMember, map[string]interface{}{
"membership": "invite",
}, test.WithStateKey(bob.ID))
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
defer closeDB()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
natsInstance := &jetstream.NATSInstance{}
evChan := make(chan struct{})
// create a dummy AS url, handling the events
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var txn gomatrixserverlib.ApplicationServiceTransaction
err := json.NewDecoder(r.Body).Decode(&txn)
if err != nil {
t.Fatal(err)
}
for _, ev := range txn.Events {
if ev.Type != gomatrixserverlib.MRoomMember {
continue
}
// Usually we would check the event content for the membership, but since
// we only invited bob, this should be fine for this test.
if ev.StateKey != nil && *ev.StateKey == bob.ID {
evChan <- struct{}{}
}
}
}))
defer srv.Close()
// Create a dummy application service
cfg.AppServiceAPI.Derived.ApplicationServices = []config.ApplicationService{
{
ID: "someID",
URL: srv.URL,
ASToken: "",
HSToken: "",
SenderLocalpart: "senderLocalPart",
NamespaceMap: map[string][]config.ApplicationServiceNamespace{
"users": {{RegexpObject: regexp.MustCompile(bob.ID)}},
"aliases": {{RegexpObject: regexp.MustCompile(room.ID)}},
},
},
}
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
// Create required internal APIs
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, natsInstance, caches, caching.DisableMetrics)
rsAPI.SetFederationAPI(nil, nil)
usrAPI := userapi.NewInternalAPI(processCtx, cfg, cm, natsInstance, rsAPI, nil)
// start the consumer
appservice.NewInternalAPI(processCtx, cfg, natsInstance, usrAPI, rsAPI)
// Create the room
if err := rsapi.SendEvents(context.Background(), rsAPI, rsapi.KindNew, room.Events(), "test", "test", "test", nil, false); err != nil {
t.Fatalf("failed to send events: %v", err)
}
var seenInvitesForBob int
waitLoop:
for {
select {
case <-time.After(time.Millisecond * 50): // wait for the AS to process the events
break waitLoop
case <-evChan:
seenInvitesForBob++
if seenInvitesForBob != 1 {
t.Fatalf("received unexpected invites: %d", seenInvitesForBob)
}
}
}
close(evChan)
})
}

6
appservice/consumers/roomserver.go
View file

@ -140,12 +140,6 @@ func (s *OutputRoomEventConsumer) onMessage(
}
}
case api.OutputTypeNewInviteEvent:
if output.NewInviteEvent == nil || !s.appserviceIsInterestedInEvent(ctx, output.NewInviteEvent.Event, state.ApplicationService) {
continue
}
events = append(events, output.NewInviteEvent.Event)
default:
continue
}

311
clientapi/admin_test.go
View file

@ -4,16 +4,17 @@ import (
"context"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"time"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/federationapi"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/roomserver/api"
basepkg "github.com/matrix-org/dendrite/setup/base"
"github.com/matrix-org/dendrite/setup/config"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/syncapi"
@ -52,39 +53,12 @@ func TestAdminResetPassword(t *testing.T) {
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
// Create the users in the userapi and login
accessTokens := map[*test.User]string{
aliceAdmin: "",
bob: "",
vhUser: "",
}
for u := range accessTokens {
localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
userRes := &uapi.PerformAccountCreationResponse{}
password := util.RandomString(8)
if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
AccountType: u.AccountType,
Localpart: localpart,
ServerName: serverName,
Password: password,
}, userRes); err != nil {
t.Errorf("failed to create account: %s", err)
}
req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
"type": authtypes.LoginTypePassword,
"identifier": map[string]interface{}{
"type": "m.id.user",
"user": u.ID,
},
"password": password,
}))
rec := httptest.NewRecorder()
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("failed to login: %s", rec.Body.String())
}
accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
accessTokens := map[*test.User]userDevice{
aliceAdmin: {},
bob: {},
vhUser: {},
}
createAccessTokens(t, accessTokens, userAPI, ctx, routers)
testCases := []struct {
name string
@ -128,7 +102,7 @@ func TestAdminResetPassword(t *testing.T) {
}
if tc.withHeader {
req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser])
req.Header.Set("Authorization", "Bearer "+accessTokens[tc.requestingUser].accessToken)
}
rec := httptest.NewRecorder()
@ -179,37 +153,10 @@ func TestPurgeRoom(t *testing.T) {
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
// Create the users in the userapi and login
accessTokens := map[*test.User]string{
aliceAdmin: "",
}
for u := range accessTokens {
localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
userRes := &uapi.PerformAccountCreationResponse{}
password := util.RandomString(8)
if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
AccountType: u.AccountType,
Localpart: localpart,
ServerName: serverName,
Password: password,
}, userRes); err != nil {
t.Errorf("failed to create account: %s", err)
}
req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
"type": authtypes.LoginTypePassword,
"identifier": map[string]interface{}{
"type": "m.id.user",
"user": u.ID,
},
"password": password,
}))
rec := httptest.NewRecorder()
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("failed to login: %s", rec.Body.String())
}
accessTokens[u] = gjson.GetBytes(rec.Body.Bytes(), "access_token").String()
accessTokens := map[*test.User]userDevice{
aliceAdmin: {},
}
createAccessTokens(t, accessTokens, userAPI, ctx, routers)
testCases := []struct {
name string
@ -226,7 +173,7 @@ func TestPurgeRoom(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/purgeRoom/"+tc.roomID)
req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin])
req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
rec := httptest.NewRecorder()
routers.DendriteAdmin.ServeHTTP(rec, req)
@ -239,3 +186,237 @@ func TestPurgeRoom(t *testing.T) {
})
}
func TestAdminEvacuateRoom(t *testing.T) {
aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
bob := test.NewUser(t)
room := test.NewRoom(t, aliceAdmin)
// Join Bob
room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
"membership": "join",
}, test.WithStateKey(bob.ID))
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, close := testrig.CreateConfig(t, dbType)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
natsInstance := jetstream.NATSInstance{}
defer close()
routers := httputil.NewRouters()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
// this starts the JetStream consumers
fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, nil, rsAPI, caches, nil, true)
rsAPI.SetFederationAPI(fsAPI, nil)
// Create the room
if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
t.Fatalf("failed to send events: %v", err)
}
// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
// Create the users in the userapi and login
accessTokens := map[*test.User]userDevice{
aliceAdmin: {},
}
createAccessTokens(t, accessTokens, userAPI, ctx, routers)
testCases := []struct {
name string
roomID string
wantOK bool
wantAffected []string
}{
{name: "Can evacuate existing room", wantOK: true, roomID: room.ID, wantAffected: []string{aliceAdmin.ID, bob.ID}},
{name: "Can not evacuate non-existent room", wantOK: false, roomID: "!doesnotexist:localhost", wantAffected: []string{}},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateRoom/"+tc.roomID)
req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
rec := httptest.NewRecorder()
routers.DendriteAdmin.ServeHTTP(rec, req)
t.Logf("%s", rec.Body.String())
if tc.wantOK && rec.Code != http.StatusOK {
t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
}
affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array()
affected := make([]string, 0, len(affectedArr))
for _, x := range affectedArr {
affected = append(affected, x.Str)
}
if !reflect.DeepEqual(affected, tc.wantAffected) {
t.Fatalf("expected affected %#v, but got %#v", tc.wantAffected, affected)
}
})
}
})
}
func TestAdminEvacuateUser(t *testing.T) {
aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
bob := test.NewUser(t)
room := test.NewRoom(t, aliceAdmin)
room2 := test.NewRoom(t, aliceAdmin)
// Join Bob
room.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
"membership": "join",
}, test.WithStateKey(bob.ID))
room2.CreateAndInsert(t, bob, gomatrixserverlib.MRoomMember, map[string]interface{}{
"membership": "join",
}, test.WithStateKey(bob.ID))
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, close := testrig.CreateConfig(t, dbType)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
natsInstance := jetstream.NATSInstance{}
defer close()
routers := httputil.NewRouters()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
// this starts the JetStream consumers
fsAPI := federationapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, basepkg.CreateFederationClient(cfg, nil), rsAPI, caches, nil, true)
rsAPI.SetFederationAPI(fsAPI, nil)
// Create the room
if err := api.SendEvents(ctx, rsAPI, api.KindNew, room.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
t.Fatalf("failed to send events: %v", err)
}
if err := api.SendEvents(ctx, rsAPI, api.KindNew, room2.Events(), "test", "test", api.DoNotSendToOtherServers, nil, false); err != nil {
t.Fatalf("failed to send events: %v", err)
}
// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
// Create the users in the userapi and login
accessTokens := map[*test.User]userDevice{
aliceAdmin: {},
}
createAccessTokens(t, accessTokens, userAPI, ctx, routers)
testCases := []struct {
name string
userID string
wantOK bool
wantAffectedRooms []string
}{
{name: "Can evacuate existing user", wantOK: true, userID: bob.ID, wantAffectedRooms: []string{room.ID, room2.ID}},
{name: "invalid userID is rejected", wantOK: false, userID: "!notauserid:test", wantAffectedRooms: []string{}},
{name: "Can not evacuate user from different server", wantOK: false, userID: "@doesnotexist:localhost", wantAffectedRooms: []string{}},
{name: "Can not evacuate non-existent user", wantOK: false, userID: "@doesnotexist:test", wantAffectedRooms: []string{}},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/evacuateUser/"+tc.userID)
req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
rec := httptest.NewRecorder()
routers.DendriteAdmin.ServeHTTP(rec, req)
t.Logf("%s", rec.Body.String())
if tc.wantOK && rec.Code != http.StatusOK {
t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
}
affectedArr := gjson.GetBytes(rec.Body.Bytes(), "affected").Array()
affected := make([]string, 0, len(affectedArr))
for _, x := range affectedArr {
affected = append(affected, x.Str)
}
if !reflect.DeepEqual(affected, tc.wantAffectedRooms) {
t.Fatalf("expected affected %#v, but got %#v", tc.wantAffectedRooms, affected)
}
})
}
// Wait for the FS API to have consumed every message
js, _ := natsInstance.Prepare(processCtx, &cfg.Global.JetStream)
timeout := time.After(time.Second)
for {
select {
case <-timeout:
t.Fatalf("FS API didn't process all events in time")
default:
}
info, err := js.ConsumerInfo(cfg.Global.JetStream.Prefixed(jetstream.OutputRoomEvent), cfg.Global.JetStream.Durable("FederationAPIRoomServerConsumer")+"Pull")
if err != nil {
time.Sleep(time.Millisecond * 10)
continue
}
if info.NumPending == 0 && info.NumAckPending == 0 {
break
}
}
})
}
func TestAdminMarkAsStale(t *testing.T) {
aliceAdmin := test.NewUser(t, test.WithAccountType(uapi.AccountTypeAdmin))
ctx := context.Background()
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, close := testrig.CreateConfig(t, dbType)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
natsInstance := jetstream.NATSInstance{}
defer close()
routers := httputil.NewRouters()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
// Create the users in the userapi and login
accessTokens := map[*test.User]userDevice{
aliceAdmin: {},
}
createAccessTokens(t, accessTokens, userAPI, ctx, routers)
testCases := []struct {
name string
userID string
wantOK bool
}{
{name: "local user is not allowed", userID: aliceAdmin.ID},
{name: "invalid userID", userID: "!notvalid:test"},
{name: "remote user is allowed", userID: "@alice:localhost", wantOK: true},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := test.NewRequest(t, http.MethodPost, "/_dendrite/admin/refreshDevices/"+tc.userID)
req.Header.Set("Authorization", "Bearer "+accessTokens[aliceAdmin].accessToken)
rec := httptest.NewRecorder()
routers.DendriteAdmin.ServeHTTP(rec, req)
t.Logf("%s", rec.Body.String())
if tc.wantOK && rec.Code != http.StatusOK {
t.Fatalf("expected http status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String())
}
})
}
})
}

373
clientapi/clientapi_test.go Normal file
View file

@ -0,0 +1,373 @@
package clientapi
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/matrix-org/dendrite/clientapi/auth/authtypes"
"github.com/matrix-org/dendrite/internal/caching"
"github.com/matrix-org/dendrite/internal/httputil"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/roomserver"
"github.com/matrix-org/dendrite/setup/jetstream"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
"github.com/matrix-org/dendrite/userapi"
uapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/matrix-org/util"
"github.com/tidwall/gjson"
)
type userDevice struct {
accessToken string
deviceID string
password string
}
func TestGetPutDevices(t *testing.T) {
alice := test.NewUser(t)
bob := test.NewUser(t)
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
testCases := []struct {
name string
requestUser *test.User
deviceUser *test.User
request *http.Request
wantStatusCode int
validateFunc func(t *testing.T, device userDevice, routers httputil.Routers)
}{
{
name: "can get all devices",
requestUser: alice,
request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices", strings.NewReader("")),
wantStatusCode: http.StatusOK,
},
{
name: "can get specific own device",
requestUser: alice,
deviceUser: alice,
request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices/", strings.NewReader("")),
wantStatusCode: http.StatusOK,
},
{
name: "can not get device for different user",
requestUser: alice,
deviceUser: bob,
request: httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices/", strings.NewReader("")),
wantStatusCode: http.StatusNotFound,
},
{
name: "can update own device",
requestUser: alice,
deviceUser: alice,
request: httptest.NewRequest(http.MethodPut, "/_matrix/client/v3/devices/", strings.NewReader(`{"display_name":"my new displayname"}`)),
wantStatusCode: http.StatusOK,
validateFunc: func(t *testing.T, device userDevice, routers httputil.Routers) {
req := httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices/"+device.deviceID, strings.NewReader(""))
req.Header.Set("Authorization", "Bearer "+device.accessToken)
rec := httptest.NewRecorder()
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
}
gotDisplayName := gjson.GetBytes(rec.Body.Bytes(), "display_name").Str
if gotDisplayName != "my new displayname" {
t.Fatalf("expected displayname '%s', got '%s'", "my new displayname", gotDisplayName)
}
},
},
{
// this should return "device does not exist"
name: "can not update device for different user",
requestUser: alice,
deviceUser: bob,
request: httptest.NewRequest(http.MethodPut, "/_matrix/client/v3/devices/", strings.NewReader(`{"display_name":"my new displayname"}`)),
wantStatusCode: http.StatusNotFound,
},
}
cfg, processCtx, close := testrig.CreateConfig(t, dbType)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
natsInstance := jetstream.NATSInstance{}
defer close()
routers := httputil.NewRouters()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
// We mostly need the rsAPI for this test, so nil for other APIs/caches etc.
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
accessTokens := map[*test.User]userDevice{
alice: {},
bob: {},
}
createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
dev := accessTokens[tc.requestUser]
if tc.deviceUser != nil {
tc.request = httptest.NewRequest(tc.request.Method, tc.request.RequestURI+accessTokens[tc.deviceUser].deviceID, tc.request.Body)
}
tc.request.Header.Set("Authorization", "Bearer "+dev.accessToken)
rec := httptest.NewRecorder()
routers.Client.ServeHTTP(rec, tc.request)
if rec.Code != tc.wantStatusCode {
t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
}
if tc.wantStatusCode != http.StatusOK && rec.Code != http.StatusOK {
return
}
if tc.validateFunc != nil {
tc.validateFunc(t, dev, routers)
}
})
}
})
}
// Deleting devices requires the UIA dance, so do this in a different test
func TestDeleteDevice(t *testing.T) {
alice := test.NewUser(t)
localpart, serverName, _ := gomatrixserverlib.SplitID('@', alice.ID)
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
defer closeDB()
natsInstance := jetstream.NATSInstance{}
routers := httputil.NewRouters()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
accessTokens := map[*test.User]userDevice{
alice: {},
}
// create the account and an initial device
createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
// create some more devices
accessToken := util.RandomString(8)
devRes := &uapi.PerformDeviceCreationResponse{}
if err := userAPI.PerformDeviceCreation(processCtx.Context(), &uapi.PerformDeviceCreationRequest{
Localpart: localpart,
ServerName: serverName,
AccessToken: accessToken,
NoDeviceListUpdate: true,
}, devRes); err != nil {
t.Fatal(err)
}
if !devRes.DeviceCreated {
t.Fatalf("failed to create device")
}
secondDeviceID := devRes.Device.ID
// initiate UIA for the second device
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodDelete, "/_matrix/client/v3/devices/"+secondDeviceID, strings.NewReader(""))
req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Fatalf("expected HTTP 401, got %d: %s", rec.Code, rec.Body.String())
}
// get the session ID
sessionID := gjson.GetBytes(rec.Body.Bytes(), "session").Str
// prepare UIA request body
reqBody := bytes.Buffer{}
if err := json.NewEncoder(&reqBody).Encode(map[string]interface{}{
"auth": map[string]string{
"session": sessionID,
"type": authtypes.LoginTypePassword,
"user": alice.ID,
"password": accessTokens[alice].password,
},
}); err != nil {
t.Fatal(err)
}
// copy the request body, so we can use it again for the successful delete
reqBody2 := reqBody
// do the same request again, this time with our UIA, but for a different device ID, this should fail
rec = httptest.NewRecorder()
req = httptest.NewRequest(http.MethodDelete, "/_matrix/client/v3/devices/"+accessTokens[alice].deviceID, &reqBody)
req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("expected HTTP 403, got %d: %s", rec.Code, rec.Body.String())
}
// do the same request again, this time with our UIA, but for the correct device ID, this should be fine
rec = httptest.NewRecorder()
req = httptest.NewRequest(http.MethodDelete, "/_matrix/client/v3/devices/"+secondDeviceID, &reqBody2)
req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
}
// verify devices are deleted
rec = httptest.NewRecorder()
req = httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices", strings.NewReader(""))
req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
}
for _, device := range gjson.GetBytes(rec.Body.Bytes(), "devices.#.device_id").Array() {
if device.Str == secondDeviceID {
t.Fatalf("expected device %s to be deleted, but wasn't", secondDeviceID)
}
}
})
}
// Deleting devices requires the UIA dance, so do this in a different test
func TestDeleteDevices(t *testing.T) {
alice := test.NewUser(t)
localpart, serverName, _ := gomatrixserverlib.SplitID('@', alice.ID)
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
defer closeDB()
natsInstance := jetstream.NATSInstance{}
routers := httputil.NewRouters()
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
caches := caching.NewRistrettoCache(128*1024*1024, time.Hour, caching.DisableMetrics)
rsAPI := roomserver.NewInternalAPI(processCtx, cfg, cm, &natsInstance, caches, caching.DisableMetrics)
userAPI := userapi.NewInternalAPI(processCtx, cfg, cm, &natsInstance, rsAPI, nil)
// We mostly need the rsAPI/ for this test, so nil for other APIs/caches etc.
AddPublicRoutes(processCtx, routers, cfg, &natsInstance, nil, rsAPI, nil, nil, nil, userAPI, nil, nil, caching.DisableMetrics)
accessTokens := map[*test.User]userDevice{
alice: {},
}
// create the account and an initial device
createAccessTokens(t, accessTokens, userAPI, processCtx.Context(), routers)
// create some more devices
var devices []string
for i := 0; i < 10; i++ {
accessToken := util.RandomString(8)
devRes := &uapi.PerformDeviceCreationResponse{}
if err := userAPI.PerformDeviceCreation(processCtx.Context(), &uapi.PerformDeviceCreationRequest{
Localpart: localpart,
ServerName: serverName,
AccessToken: accessToken,
NoDeviceListUpdate: true,
}, devRes); err != nil {
t.Fatal(err)
}
if !devRes.DeviceCreated {
t.Fatalf("failed to create device")
}
devices = append(devices, devRes.Device.ID)
}
// initiate UIA
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/delete_devices", strings.NewReader(""))
req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusUnauthorized {
t.Fatalf("expected HTTP 401, got %d: %s", rec.Code, rec.Body.String())
}
// get the session ID
sessionID := gjson.GetBytes(rec.Body.Bytes(), "session").Str
// prepare UIA request body
reqBody := bytes.Buffer{}
if err := json.NewEncoder(&reqBody).Encode(map[string]interface{}{
"auth": map[string]string{
"session": sessionID,
"type": authtypes.LoginTypePassword,
"user": alice.ID,
"password": accessTokens[alice].password,
},
"devices": devices[5:],
}); err != nil {
t.Fatal(err)
}
// do the same request again, this time with our UIA,
rec = httptest.NewRecorder()
req = httptest.NewRequest(http.MethodPost, "/_matrix/client/v3/delete_devices", &reqBody)
req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
}
// verify devices are deleted
rec = httptest.NewRecorder()
req = httptest.NewRequest(http.MethodGet, "/_matrix/client/v3/devices", strings.NewReader(""))
req.Header.Set("Authorization", "Bearer "+accessTokens[alice].accessToken)
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("expected HTTP 200, got %d: %s", rec.Code, rec.Body.String())
}
for _, device := range gjson.GetBytes(rec.Body.Bytes(), "devices.#.device_id").Array() {
for _, deletedDevice := range devices[5:] {
if device.Str == deletedDevice {
t.Fatalf("expected device %s to be deleted, but wasn't", deletedDevice)
}
}
}
})
}
func createAccessTokens(t *testing.T, accessTokens map[*test.User]userDevice, userAPI uapi.UserInternalAPI, ctx context.Context, routers httputil.Routers) {
t.Helper()
for u := range accessTokens {
localpart, serverName, _ := gomatrixserverlib.SplitID('@', u.ID)
userRes := &uapi.PerformAccountCreationResponse{}
password := util.RandomString(8)
if err := userAPI.PerformAccountCreation(ctx, &uapi.PerformAccountCreationRequest{
AccountType: u.AccountType,
Localpart: localpart,
ServerName: serverName,
Password: password,
}, userRes); err != nil {
t.Errorf("failed to create account: %s", err)
}
req := test.NewRequest(t, http.MethodPost, "/_matrix/client/v3/login", test.WithJSONBody(t, map[string]interface{}{
"type": authtypes.LoginTypePassword,
"identifier": map[string]interface{}{
"type": "m.id.user",
"user": u.ID,
},
"password": password,
}))
rec := httptest.NewRecorder()
routers.Client.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("failed to login: %s", rec.Body.String())
}
accessTokens[u] = userDevice{
accessToken: gjson.GetBytes(rec.Body.Bytes(), "access_token").String(),
deviceID: gjson.GetBytes(rec.Body.Bytes(), "device_id").String(),
password: password,
}
}
}

31
clientapi/routing/admin.go
View file

@ -22,23 +22,16 @@ import (
"github.com/matrix-org/dendrite/userapi/api"
)
func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
func AdminEvacuateRoom(req *http.Request, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
roomID, ok := vars["roomID"]
if !ok {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting room ID."),
}
}
res := &roomserverAPI.PerformAdminEvacuateRoomResponse{}
if err := rsAPI.PerformAdminEvacuateRoom(
req.Context(),
&roomserverAPI.PerformAdminEvacuateRoomRequest{
RoomID: roomID,
RoomID: vars["roomID"],
},
res,
); err != nil {
@ -55,18 +48,13 @@ func AdminEvacuateRoom(req *http.Request, cfg *config.ClientAPI, device *api.Dev
}
}
func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, device *api.Device, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
func AdminEvacuateUser(req *http.Request, cfg *config.ClientAPI, rsAPI roomserverAPI.ClientRoomserverAPI) util.JSONResponse {
vars, err := httputil.URLDecodeMapValues(mux.Vars(req))
if err != nil {
return util.ErrorResponse(err)
}
userID, ok := vars["userID"]
if !ok {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting user ID."),
}
}
userID := vars["userID"]
_, domain, err := gomatrixserverlib.SplitID('@', userID)
if err != nil {
return util.MessageResponse(http.StatusBadRequest, err.Error())
@ -103,13 +91,8 @@ func AdminPurgeRoom(req *http.Request, cfg *config.ClientAPI, device *api.Device
if err != nil {
return util.ErrorResponse(err)
}
roomID, ok := vars["roomID"]
if !ok {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.MissingArgument("Expecting room ID."),
}
}
roomID := vars["roomID"]
res := &roomserverAPI.PerformAdminPurgeRoomResponse{}
if err := rsAPI.PerformAdminPurgeRoom(
context.Background(),

5
clientapi/routing/deactivate.go
View file

@ -33,7 +33,7 @@ func Deactivate(
return *errRes
}
localpart, _, err := gomatrixserverlib.SplitID('@', login.Username())
localpart, serverName, err := gomatrixserverlib.SplitID('@', login.Username())
if err != nil {
util.GetLogger(req.Context()).WithError(err).Error("gomatrixserverlib.SplitID failed")
return jsonerror.InternalServerError()
@ -41,7 +41,8 @@ func Deactivate(
var res api.PerformAccountDeactivationResponse
err = accountAPI.PerformAccountDeactivation(ctx, &api.PerformAccountDeactivationRequest{
Localpart: localpart,
Localpart: localpart,
ServerName: serverName,
}, &res)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("userAPI.PerformAccountDeactivation failed")

40
clientapi/routing/device.go
View file

@ -15,6 +15,7 @@
package routing
import (
"encoding/json"
"io"
"net"
"net/http"
@ -146,12 +147,6 @@ func UpdateDeviceByID(
JSON: jsonerror.Forbidden("device does not exist"),
}
}
if performRes.Forbidden {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("device not owned by current user"),
}
}
return util.JSONResponse{
Code: http.StatusOK,
@ -189,7 +184,7 @@ func DeleteDeviceById(
if dev != deviceID {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("session & device mismatch"),
JSON: jsonerror.Forbidden("session and device mismatch"),
}
}
}
@ -242,16 +237,37 @@ func DeleteDeviceById(
// DeleteDevices handles POST requests to /delete_devices
func DeleteDevices(
req *http.Request, userAPI api.ClientUserAPI, device *api.Device,
req *http.Request, userInteractiveAuth *auth.UserInteractive, userAPI api.ClientUserAPI, device *api.Device,
) util.JSONResponse {
ctx := req.Context()
payload := devicesDeleteJSON{}
if resErr := httputil.UnmarshalJSONRequest(req, &payload); resErr != nil {
return *resErr
bodyBytes, err := io.ReadAll(req.Body)
if err != nil {
return util.JSONResponse{
Code: http.StatusBadRequest,
JSON: jsonerror.BadJSON("The request body could not be read: " + err.Error()),
}
}
defer req.Body.Close() // nolint:errcheck
// initiate UIA
login, errRes := userInteractiveAuth.Verify(ctx, bodyBytes, device)
if errRes != nil {
return *errRes
}
defer req.Body.Close() // nolint: errcheck
if login.Username() != device.UserID {
return util.JSONResponse{
Code: http.StatusForbidden,
JSON: jsonerror.Forbidden("unable to delete devices for other user"),
}
}
payload := devicesDeleteJSON{}
if err = json.Unmarshal(bodyBytes, &payload); err != nil {
util.GetLogger(ctx).WithError(err).Error("unable to unmarshal device deletion request")
return jsonerror.InternalServerError()
}
var res api.PerformDeviceDeletionResponse
if err := userAPI.PerformDeviceDeletion(ctx, &api.PerformDeviceDeletionRequest{

10
clientapi/routing/routing.go
View file

@ -155,15 +155,15 @@ func Setup(
dendriteAdminRouter.Handle("/admin/evacuateRoom/{roomID}",
httputil.MakeAdminAPI("admin_evacuate_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminEvacuateRoom(req, cfg, device, rsAPI)
return AdminEvacuateRoom(req, rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
).Methods(http.MethodPost, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/evacuateUser/{userID}",
httputil.MakeAdminAPI("admin_evacuate_user", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return AdminEvacuateUser(req, cfg, device, rsAPI)
return AdminEvacuateUser(req, cfg, rsAPI)
}),
).Methods(http.MethodGet, http.MethodOptions)
).Methods(http.MethodPost, http.MethodOptions)
dendriteAdminRouter.Handle("/admin/purgeRoom/{roomID}",
httputil.MakeAdminAPI("admin_purge_room", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
@ -1115,7 +1115,7 @@ func Setup(
v3mux.Handle("/delete_devices",
httputil.MakeAuthAPI("delete_devices", userAPI, func(req *http.Request, device *userapi.Device) util.JSONResponse {
return DeleteDevices(req, userAPI, device)
return DeleteDevices(req, userInteractiveAuth, userAPI, device)
}),
).Methods(http.MethodPost, http.MethodOptions)

2
cmd/dendrite-demo-pinecone/monolith/monolith.go
View file

@ -213,7 +213,7 @@ func (p *P2PMonolith) Stop() {
}
func (p *P2PMonolith) WaitForShutdown() {
p.ProcessCtx.WaitForShutdown()
base.WaitForShutdown(p.ProcessCtx)
p.closeAllResources()
}

4
docs/administration/4_adminapi.md
View file

@ -32,7 +32,7 @@ UPDATE userapi_accounts SET account_type = 3 WHERE localpart = '$localpart';
Where `$localpart` is the username only (e.g. `alice`).
## GET `/_dendrite/admin/evacuateRoom/{roomID}`
## POST `/_dendrite/admin/evacuateRoom/{roomID}`
This endpoint will instruct Dendrite to part all local users from the given `roomID`
in the URL. It may take some time to complete. A JSON body will be returned containing
@ -41,7 +41,7 @@ the user IDs of all affected users.
If the room has an alias set (e.g. is published), the room's ID will not be visible in the URL, but it can
be found as the room's "internal ID" in Element Web (Settings -> Advanced)
## GET `/_dendrite/admin/evacuateUser/{userID}`
## POST `/_dendrite/admin/evacuateUser/{userID}`
This endpoint will instruct Dendrite to part the given local `userID` in the URL from
all rooms which they are currently joined. A JSON body will be returned containing

2
helm/dendrite/Chart.yaml
View file

@ -1,6 +1,6 @@
apiVersion: v2
name: dendrite
version: "0.12.1"
version: "0.12.2"
appVersion: "0.12.0"
description: Dendrite Matrix Homeserver
type: application

14
helm/dendrite/templates/_helpers.tpl
View file

@ -1,15 +1,9 @@
{{- define "validate.config" }}
{{- if not .Values.signing_key.create -}}
{{- fail "You must create a signing key for configuration.signing_key. (see https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#server-key-generation)" -}}
{{- if and (not .Values.signing_key.create) (eq .Values.signing_key.existingSecret "") -}}
{{- fail "You must create a signing key for configuration.signing_key OR specify an existing secret name in .Values.signing_key.existingSecret to mount it. (see https://github.com/matrix-org/dendrite/blob/master/docs/INSTALL.md#server-key-generation)" -}}
{{- end -}}
{{- if not (or .Values.dendrite_config.global.database.host .Values.postgresql.enabled) -}}
{{- fail "Database server must be set." -}}
{{- end -}}
{{- if not (or .Values.dendrite_config.global.database.user .Values.postgresql.enabled) -}}
{{- fail "Database user must be set." -}}
{{- end -}}
{{- if not (or .Values.dendrite_config.global.database.password .Values.postgresql.enabled) -}}
{{- fail "Database password must be set." -}}
{{- if and (not .Values.postgresql.enabled) (eq .Values.dendrite_config.global.database.connection_string "") -}}
{{- fail "Database connection string must be set." -}}
{{- end -}}
{{- end -}}

21
helm/dendrite/templates/deployment.yaml
View file

@ -12,16 +12,19 @@ spec:
matchLabels:
{{- include "dendrite.selectorLabels" . | nindent 6 }}
replicas: 1
strategy:
type: {{ $.Values.strategy.type }}
{{- if eq $.Values.strategy.type "RollingUpdate" }}
rollingUpdate:
maxSurge: {{ $.Values.strategy.rollingUpdate.maxSurge }}
maxUnavailable: {{ $.Values.strategy.rollingUpdate.maxUnavailable }}
{{- end }}
template:
metadata:
labels:
{{- include "dendrite.selectorLabels" . | nindent 8 }}
annotations:
confighash-global: secret-{{ .Values.global | toYaml | sha256sum | trunc 32 }}
confighash-clientapi: clientapi-{{ .Values.clientapi | toYaml | sha256sum | trunc 32 }}
confighash-federationapi: federationapi-{{ .Values.federationapi | toYaml | sha256sum | trunc 32 }}
confighash-mediaapi: mediaapi-{{ .Values.mediaapi | toYaml | sha256sum | trunc 32 }}
confighash-syncapi: syncapi-{{ .Values.syncapi | toYaml | sha256sum | trunc 32 }}
confighash: secret-{{ .Values.dendrite_config | toYaml | sha256sum | trunc 32 }}
spec:
volumes:
- name: {{ include "dendrite.fullname" . }}-conf-vol
@ -44,6 +47,9 @@ spec:
- name: {{ include "dendrite.fullname" . }}-search
persistentVolumeClaim:
claimName: {{ default (print ( include "dendrite.fullname" . ) "-search-pvc") $.Values.persistence.search.existingClaim | quote }}
{{- with .Values.extraVolumes }}
{{ . | toYaml | nindent 6 }}
{{- end }}
containers:
- name: {{ .Chart.Name }}
{{- include "image.name" . | nindent 8 }}
@ -57,7 +63,7 @@ spec:
{{- if $.Values.dendrite_config.global.profiling.enabled }}
env:
- name: PPROFLISTEN
value: "localhost:{{- $.Values.global.profiling.port -}}"
value: "localhost:{{- $.Values.dendrite_config.global.profiling.port -}}"
{{- end }}
resources:
{{- toYaml $.Values.resources | nindent 10 }}
@ -77,6 +83,9 @@ spec:
name: {{ include "dendrite.fullname" . }}-jetstream
- mountPath: {{ .Values.dendrite_config.sync_api.search.index_path }}
name: {{ include "dendrite.fullname" . }}-search
{{- with .Values.extraVolumeMounts }}
{{ . | toYaml | nindent 8 }}
{{- end }}
livenessProbe:
initialDelaySeconds: 10
periodSeconds: 10

24
helm/dendrite/values.yaml
View file

@ -43,6 +43,30 @@ persistence:
# -- PVC Storage Request for the search volume
capacity: "1Gi"
# Add additional volumes to the Dendrite Pod
extraVolumes: []
# ex.
# - name: extra-config
# secret:
# secretName: extra-config
# Configure additional mount points volumes in the Dendrite Pod
extraVolumeMounts: []
# ex.
# - mountPath: /etc/dendrite/extra-config
# name: extra-config
strategy:
# -- Strategy to use for rolling updates (e.g. Recreate, RollingUpdate)
# If you are using ReadWriteOnce volumes, you should probably use Recreate
type: RollingUpdate
rollingUpdate:
# -- Maximum number of pods that can be unavailable during the update process
maxUnavailable: 25%
# -- Maximum number of pods that can be scheduled above the desired number of pods
maxSurge: 25%
dendrite_config:
version: 2
global:

47
internal/fulltext/bleve.go
View file

@ -18,6 +18,7 @@
package fulltext
import (
"regexp"
"strings"
"github.com/blevesearch/bleve/v2"
@ -60,6 +61,7 @@ type Indexer interface {
Index(elements ...IndexElement) error
Delete(eventID string) error
Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (*bleve.SearchResult, error)
GetHighlights(result *bleve.SearchResult) []string
Close() error
}
@ -124,6 +126,47 @@ func (f *Search) Delete(eventID string) error {
return f.FulltextIndex.Delete(eventID)
}
var highlightMatcher = regexp.MustCompile("<mark>(.*?)</mark>")
// GetHighlights extracts the highlights from a SearchResult.
func (f *Search) GetHighlights(result *bleve.SearchResult) []string {
if result == nil {
return []string{}
}
seenMatches := make(map[string]struct{})
for _, hit := range result.Hits {
if hit.Fragments == nil {
continue
}
fragments, ok := hit.Fragments["Content"]
if !ok {
continue
}
for _, x := range fragments {
substringMatches := highlightMatcher.FindAllStringSubmatch(x, -1)
for _, matches := range substringMatches {
for i := range matches {
if i == 0 { // skip first match, this is the complete substring match
continue
}
if _, ok := seenMatches[matches[i]]; ok {
continue
}
seenMatches[matches[i]] = struct{}{}
}
}
}
}
res := make([]string, 0, len(seenMatches))
for m := range seenMatches {
res = append(res, m)
}
return res
}
// Search searches the index given a search term, roomIDs and keys.
func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (*bleve.SearchResult, error) {
qry := bleve.NewConjunctionQuery()
@ -163,6 +206,10 @@ func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, or
s.SortBy([]string{"-StreamPosition"})
}
// Highlight some words
s.Highlight = bleve.NewHighlight()
s.Highlight.Fields = []string{"Content"}
return f.FulltextIndex.Search(s)
}

46
internal/fulltext/bleve_test.go
View file

@ -160,14 +160,16 @@ func TestSearch(t *testing.T) {
roomIndex []int
}
tests := []struct {
name string
args args
wantCount int
wantErr bool
name string
args args
wantCount int
wantErr bool
wantHighlights []string
}{
{
name: "Can search for many results in one room",
wantCount: 16,
name: "Can search for many results in one room",
wantCount: 16,
wantHighlights: []string{"lorem"},
args: args{
term: "lorem",
roomIndex: []int{0},
@ -175,8 +177,9 @@ func TestSearch(t *testing.T) {
},
},
{
name: "Can search for one result in one room",
wantCount: 1,
name: "Can search for one result in one room",
wantCount: 1,
wantHighlights: []string{"lorem"},
args: args{
term: "lorem",
roomIndex: []int{16},
@ -184,8 +187,9 @@ func TestSearch(t *testing.T) {
},
},
{
name: "Can search for many results in multiple rooms",
wantCount: 17,
name: "Can search for many results in multiple rooms",
wantCount: 17,
wantHighlights: []string{"lorem"},
args: args{
term: "lorem",
roomIndex: []int{0, 16},
@ -193,8 +197,9 @@ func TestSearch(t *testing.T) {
},
},
{
name: "Can search for many results in all rooms, reversed",
wantCount: 30,
name: "Can search for many results in all rooms, reversed",
wantCount: 30,
wantHighlights: []string{"lorem"},
args: args{
term: "lorem",
limit: 30,
@ -202,8 +207,9 @@ func TestSearch(t *testing.T) {
},
},
{
name: "Can search for specific search room name",
wantCount: 1,
name: "Can search for specific search room name",
wantCount: 1,
wantHighlights: []string{"testing"},
args: args{
term: "testing",
roomIndex: []int{},
@ -212,8 +218,9 @@ func TestSearch(t *testing.T) {
},
},
{
name: "Can search for specific search room topic",
wantCount: 1,
name: "Can search for specific search room topic",
wantCount: 1,
wantHighlights: []string{"fulltext"},
args: args{
term: "fulltext",
roomIndex: []int{},
@ -222,6 +229,7 @@ func TestSearch(t *testing.T) {
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, ctx := mustOpenIndex(t, "")
@ -238,6 +246,12 @@ func TestSearch(t *testing.T) {
t.Errorf("Search() error = %v, wantErr %v", err, tt.wantErr)
return
}
highlights := f.GetHighlights(got)
if !reflect.DeepEqual(highlights, tt.wantHighlights) {
t.Errorf("Search() got highligts = %v, want %v", highlights, tt.wantHighlights)
}
if !reflect.DeepEqual(len(got.Hits), tt.wantCount) {
t.Errorf("Search() got = %v, want %v", len(got.Hits), tt.wantCount)
}

5
internal/fulltext/bleve_wasm.go
View file

@ -33,6 +33,7 @@ type Indexer interface {
Index(elements ...IndexElement) error
Delete(eventID string) error
Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (SearchResult, error)
GetHighlights(result SearchResult) []string
Close() error
}
@ -71,3 +72,7 @@ func (f *Search) Delete(eventID string) error {
func (f *Search) Search(term string, roomIDs, keys []string, limit, from int, orderByStreamPos bool) (SearchResult, error) {
return SearchResult{}, nil
}
func (f *Search) GetHighlights(result SearchResult) []string {
return []string{}
}

3
roomserver/internal/perform/perform_admin.go
View file

@ -227,6 +227,7 @@ func (r *Admin) PerformAdminEvacuateUser(
}
return nil
}
res.Affected = append(res.Affected, roomID)
if len(outputEvents) == 0 {
continue
}
@ -237,8 +238,6 @@ func (r *Admin) PerformAdminEvacuateUser(
}
return nil
}
res.Affected = append(res.Affected, roomID)
}
return nil
}

77
syncapi/routing/search.go
View file

@ -19,7 +19,6 @@ import (
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/blevesearch/bleve/v2/search"
@ -123,8 +122,8 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
return util.JSONResponse{
Code: http.StatusOK,
JSON: SearchResponse{
SearchCategories: SearchCategories{
RoomEvents: RoomEvents{
SearchCategories: SearchCategoriesResponse{
RoomEvents: RoomEventsResponse{
Count: int(result.Total),
NextBatch: nil,
},
@ -158,7 +157,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
}
groups := make(map[string]RoomResult)
knownUsersProfiles := make(map[string]ProfileInfo)
knownUsersProfiles := make(map[string]ProfileInfoResponse)
// Sort the events by depth, as the returned values aren't ordered
if orderByTime {
@ -180,7 +179,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
return jsonerror.InternalServerError()
}
profileInfos := make(map[string]ProfileInfo)
profileInfos := make(map[string]ProfileInfoResponse)
for _, ev := range append(eventsBefore, eventsAfter...) {
profile, ok := knownUsersProfiles[event.Sender()]
if !ok {
@ -192,7 +191,7 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
if stateEvent == nil {
continue
}
profile = ProfileInfo{
profile = ProfileInfoResponse{
AvatarURL: gjson.GetBytes(stateEvent.Content(), "avatar_url").Str,
DisplayName: gjson.GetBytes(stateEvent.Content(), "displayname").Str,
}
@ -237,13 +236,13 @@ func Search(req *http.Request, device *api.Device, syncDB storage.Database, fts
}
res := SearchResponse{
SearchCategories: SearchCategories{
RoomEvents: RoomEvents{
SearchCategories: SearchCategoriesResponse{
RoomEvents: RoomEventsResponse{
Count: int(result.Total),
Groups: Groups{RoomID: groups},
Results: results,
NextBatch: nextBatchResult,
Highlights: strings.Split(searchReq.SearchCategories.RoomEvents.SearchTerm, " "),
Highlights: fts.GetHighlights(result),
State: stateForRooms,
},
},
@ -286,30 +285,40 @@ func contextEvents(
return eventsBefore, eventsAfter, err
}
type EventContext struct {
AfterLimit int `json:"after_limit,omitempty"`
BeforeLimit int `json:"before_limit,omitempty"`
IncludeProfile bool `json:"include_profile,omitempty"`
}
type GroupBy struct {
Key string `json:"key"`
}
type Groupings struct {
GroupBy []GroupBy `json:"group_by"`
}
type RoomEvents struct {
EventContext EventContext `json:"event_context"`
Filter gomatrixserverlib.RoomEventFilter `json:"filter"`
Groupings Groupings `json:"groupings"`
IncludeState bool `json:"include_state"`
Keys []string `json:"keys"`
OrderBy string `json:"order_by"`
SearchTerm string `json:"search_term"`
}
type SearchCategories struct {
RoomEvents RoomEvents `json:"room_events"`
}
type SearchRequest struct {
SearchCategories struct {
RoomEvents struct {
EventContext struct {
AfterLimit int `json:"after_limit,omitempty"`
BeforeLimit int `json:"before_limit,omitempty"`
IncludeProfile bool `json:"include_profile,omitempty"`
} `json:"event_context"`
Filter gomatrixserverlib.RoomEventFilter `json:"filter"`
Groupings struct {
GroupBy []struct {
Key string `json:"key"`
} `json:"group_by"`
} `json:"groupings"`
IncludeState bool `json:"include_state"`
Keys []string `json:"keys"`
OrderBy string `json:"order_by"`
SearchTerm string `json:"search_term"`
} `json:"room_events"`
} `json:"search_categories"`
SearchCategories SearchCategories `json:"search_categories"`
}
type SearchResponse struct {
SearchCategories SearchCategories `json:"search_categories"`
SearchCategories SearchCategoriesResponse `json:"search_categories"`
}
type RoomResult struct {
NextBatch *string `json:"next_batch,omitempty"`
@ -332,15 +341,15 @@ type SearchContextResponse struct {
EventsAfter []gomatrixserverlib.ClientEvent `json:"events_after"`
EventsBefore []gomatrixserverlib.ClientEvent `json:"events_before"`
Start string `json:"start"`
ProfileInfo map[string]ProfileInfo `json:"profile_info"`
ProfileInfo map[string]ProfileInfoResponse `json:"profile_info"`
}
type ProfileInfo struct {
type ProfileInfoResponse struct {
AvatarURL string `json:"avatar_url"`
DisplayName string `json:"display_name"`
}
type RoomEvents struct {
type RoomEventsResponse struct {
Count int `json:"count"`
Groups Groups `json:"groups"`
Highlights []string `json:"highlights"`
@ -348,6 +357,6 @@ type RoomEvents struct {
Results []Result `json:"results"`
State map[string][]gomatrixserverlib.ClientEvent `json:"state,omitempty"`
}
type SearchCategories struct {
RoomEvents RoomEvents `json:"room_events"`
type SearchCategoriesResponse struct {
RoomEvents RoomEventsResponse `json:"room_events"`
}

264
syncapi/routing/search_test.go Normal file
View file

@ -0,0 +1,264 @@
package routing
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/matrix-org/dendrite/internal/fulltext"
"github.com/matrix-org/dendrite/internal/sqlutil"
"github.com/matrix-org/dendrite/syncapi/storage"
"github.com/matrix-org/dendrite/syncapi/types"
"github.com/matrix-org/dendrite/test"
"github.com/matrix-org/dendrite/test/testrig"
userapi "github.com/matrix-org/dendrite/userapi/api"
"github.com/matrix-org/gomatrixserverlib"
"github.com/stretchr/testify/assert"
)
func TestSearch(t *testing.T) {
alice := test.NewUser(t)
aliceDevice := userapi.Device{UserID: alice.ID}
room := test.NewRoom(t, alice)
room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "context before"})
room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "hello world3!"})
room.CreateAndInsert(t, alice, "m.room.message", map[string]interface{}{"body": "context after"})
roomsFilter := []string{room.ID}
roomsFilterUnknown := []string{"!unknown"}
emptyFromString := ""
fromStringValid := "1"
fromStringInvalid := "iCantBeParsed"
testCases := []struct {
name string
wantOK bool
searchReq SearchRequest
device *userapi.Device
wantResponseCount int
from *string
}{
{
name: "no user ID",
searchReq: SearchRequest{},
device: &userapi.Device{},
},
{
name: "with alice ID",
wantOK: true,
searchReq: SearchRequest{},
device: &aliceDevice,
},
{
name: "searchTerm specified, found at the beginning",
wantOK: true,
searchReq: SearchRequest{
SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "hello"}},
},
device: &aliceDevice,
wantResponseCount: 1,
},
{
name: "searchTerm specified, found at the end",
wantOK: true,
searchReq: SearchRequest{
SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "world3"}},
},
device: &aliceDevice,
wantResponseCount: 1,
},
/* the following would need matchQuery.SetFuzziness(1) in bleve.go
{
name: "searchTerm fuzzy search",
wantOK: true,
searchReq: SearchRequest{
SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "hell"}}, // this still should find hello world
},
device: &aliceDevice,
wantResponseCount: 1,
},
*/
{
name: "searchTerm specified but no result",
wantOK: true,
searchReq: SearchRequest{
SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "i don't match"}},
},
device: &aliceDevice,
},
{
name: "filter on room",
wantOK: true,
searchReq: SearchRequest{
SearchCategories: SearchCategories{
RoomEvents: RoomEvents{
SearchTerm: "hello",
Filter: gomatrixserverlib.RoomEventFilter{
Rooms: &roomsFilter,
},
},
},
},
device: &aliceDevice,
wantResponseCount: 1,
},
{
name: "filter on unknown room",
searchReq: SearchRequest{
SearchCategories: SearchCategories{
RoomEvents: RoomEvents{
SearchTerm: "hello",
Filter: gomatrixserverlib.RoomEventFilter{
Rooms: &roomsFilterUnknown,
},
},
},
},
device: &aliceDevice,
},
{
name: "include state",
wantOK: true,
searchReq: SearchRequest{
SearchCategories: SearchCategories{
RoomEvents: RoomEvents{
SearchTerm: "hello",
Filter: gomatrixserverlib.RoomEventFilter{
Rooms: &roomsFilter,
},
IncludeState: true,
},
},
},
device: &aliceDevice,
wantResponseCount: 1,
},
{
name: "empty from does not error",
wantOK: true,
searchReq: SearchRequest{
SearchCategories: SearchCategories{
RoomEvents: RoomEvents{
SearchTerm: "hello",
Filter: gomatrixserverlib.RoomEventFilter{
Rooms: &roomsFilter,
},
},
},
},
wantResponseCount: 1,
device: &aliceDevice,
from: &emptyFromString,
},
{
name: "valid from does not error",
wantOK: true,
searchReq: SearchRequest{
SearchCategories: SearchCategories{
RoomEvents: RoomEvents{
SearchTerm: "hello",
Filter: gomatrixserverlib.RoomEventFilter{
Rooms: &roomsFilter,
},
},
},
},
wantResponseCount: 1,
device: &aliceDevice,
from: &fromStringValid,
},
{
name: "invalid from does error",
searchReq: SearchRequest{
SearchCategories: SearchCategories{
RoomEvents: RoomEvents{
SearchTerm: "hello",
Filter: gomatrixserverlib.RoomEventFilter{
Rooms: &roomsFilter,
},
},
},
},
device: &aliceDevice,
from: &fromStringInvalid,
},
{
name: "order by stream position",
wantOK: true,
searchReq: SearchRequest{
SearchCategories: SearchCategories{RoomEvents: RoomEvents{SearchTerm: "hello", OrderBy: "recent"}},
},
device: &aliceDevice,
wantResponseCount: 1,
},
}
test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
cfg, processCtx, closeDB := testrig.CreateConfig(t, dbType)
defer closeDB()
// create requisites
fts, err := fulltext.New(processCtx, cfg.SyncAPI.Fulltext)
assert.NoError(t, err)
assert.NotNil(t, fts)
cm := sqlutil.NewConnectionManager(processCtx, cfg.Global.DatabaseOptions)
db, err := storage.NewSyncServerDatasource(processCtx.Context(), cm, &cfg.SyncAPI.Database)
assert.NoError(t, err)
elements := []fulltext.IndexElement{}
// store the events in the database
var sp types.StreamPosition
for _, x := range room.Events() {
var stateEvents []*gomatrixserverlib.HeaderedEvent
var stateEventIDs []string
if x.Type() == gomatrixserverlib.MRoomMember {
stateEvents = append(stateEvents, x)
stateEventIDs = append(stateEventIDs, x.EventID())
}
sp, err = db.WriteEvent(processCtx.Context(), x, stateEvents, stateEventIDs, nil, nil, false, gomatrixserverlib.HistoryVisibilityShared)
assert.NoError(t, err)
if x.Type() != "m.room.message" {
continue
}
elements = append(elements, fulltext.IndexElement{
EventID: x.EventID(),
RoomID: x.RoomID(),
Content: string(x.Content()),
ContentType: x.Type(),
StreamPosition: int64(sp),
})
}
// Index the events
err = fts.Index(elements...)
assert.NoError(t, err)
// run the tests
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
reqBody := &bytes.Buffer{}
err = json.NewEncoder(reqBody).Encode(tc.searchReq)
assert.NoError(t, err)
req := httptest.NewRequest(http.MethodPost, "/", reqBody)
res := Search(req, tc.device, db, fts, tc.from)
if !tc.wantOK && !res.Is2xx() {
return
}
resp, ok := res.JSON.(SearchResponse)
if !ok && !tc.wantOK {
t.Fatalf("not a SearchResponse: %T: %s", res.JSON, res.JSON)
}
assert.Equal(t, tc.wantResponseCount, resp.SearchCategories.RoomEvents.Count)
// if we requested state, it should not be empty
if tc.searchReq.SearchCategories.RoomEvents.IncludeState {
assert.NotEmpty(t, resp.SearchCategories.RoomEvents.State)
}
})
}
})
}

1
userapi/api/api.go
View file

@ -224,7 +224,6 @@ type PerformDeviceUpdateRequest struct {
}
type PerformDeviceUpdateResponse struct {
DeviceExists bool
Forbidden bool
}
type PerformDeviceDeletionRequest struct {

5
userapi/internal/user_api.go
View file

@ -386,11 +386,6 @@ func (a *UserInternalAPI) PerformDeviceUpdate(ctx context.Context, req *api.Perf
}
res.DeviceExists = true
if dev.UserID != req.RequestingUserID {
res.Forbidden = true
return nil
}
err = a.DB.UpdateDevice(ctx, localpart, domain, req.DeviceID, req.DisplayName)
if err != nil {
util.GetLogger(ctx).WithError(err).Error("deviceDB.UpdateDevice failed")